diff --git a/src/DonloaderService.cs b/src/DonloaderService.cs new file mode 100644 index 0000000..4ec6fc4 --- /dev/null +++ b/src/DonloaderService.cs @@ -0,0 +1,203 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace EpicMorg.Atlassian.Downloader +{ + class DonloaderService : IHostedService + { + private readonly ILogger logger; + private readonly DownloaderOptions options; + private readonly HttpClient client; + private readonly IHostApplicationLifetime hostApplicationLifetime; + + public DonloaderService(IHostApplicationLifetime hostApplicationLifetime, ILogger logger, HttpClient client, DownloaderOptions options) + { + this.logger = logger; + this.client = client; + this.options = options; + this.hostApplicationLifetime = hostApplicationLifetime; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + this.SetConsoleTitle(); + var feedUrls = this.GetFeedUrls(); + + logger.LogTrace($"Task started"); + foreach (var feedUrl in feedUrls) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + var (json, versions) = await this.GetJson(feedUrl, cancellationToken).ConfigureAwait(false); + + switch (options.Action) + { + case DownloadAction.ShowRawJson: + Console.Out.WriteLine(json); + break; + case DownloadAction.Download: + await this.DownloadFilesFromFreed(feedUrl, versions, cancellationToken).ConfigureAwait(false); + break; + case DownloadAction.ListURLs: + foreach (var versionProg in versions) + { + foreach (var file in versionProg.Value) + { + Console.Out.WriteLine(file.ZipUrl); + } + } + break; + case DownloadAction.ListVersions: + foreach (var versionProg in versions) + { + foreach (var file in versionProg.Value) + { + Console.Out.WriteLine(file.Version); + } + } + break; + } + } + logger.LogInformation($"Complete"); + this.hostApplicationLifetime.StopApplication(); + } + + private async Task<(string json, IDictionary versions)> GetJson(string feedUrl, CancellationToken cancellationToken) + { + var atlassianJson = await client.GetStringAsync(feedUrl, cancellationToken).ConfigureAwait(false); + var json = atlassianJson["downloads(".Length..^1]; + var parsed = JsonSerializer.Deserialize(json, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }); + var versions = parsed.GroupBy(a => a.Version).ToDictionary(a => a.Key, a => a.ToArray()); + + return (json, versions); + } + + private string[] GetFeedUrls() => options.CustomFeed != null + ? options.CustomFeed.Select(a => a.ToString()).ToArray() + : new[] { + "https://my.atlassian.com/download/feeds/archived/bamboo.json", + "https://my.atlassian.com/download/feeds/archived/clover.json", + "https://my.atlassian.com/download/feeds/archived/confluence.json", + "https://my.atlassian.com/download/feeds/archived/crowd.json", + "https://my.atlassian.com/download/feeds/archived/crucible.json", + "https://my.atlassian.com/download/feeds/archived/fisheye.json", + "https://my.atlassian.com/download/feeds/archived/jira-core.json", + "https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json", + "https://my.atlassian.com/download/feeds/archived/jira-software.json", + "https://my.atlassian.com/download/feeds/archived/jira.json", + "https://my.atlassian.com/download/feeds/archived/stash.json", + + "https://my.atlassian.com/download/feeds/current/bamboo.json", + "https://my.atlassian.com/download/feeds/current/clover.json", + "https://my.atlassian.com/download/feeds/current/confluence.json", + "https://my.atlassian.com/download/feeds/current/crowd.json", + "https://my.atlassian.com/download/feeds/current/crucible.json", + "https://my.atlassian.com/download/feeds/current/fisheye.json", + "https://my.atlassian.com/download/feeds/current/jira-core.json", + "https://my.atlassian.com/download/feeds/current/jira-servicedesk.json", + "https://my.atlassian.com/download/feeds/current/jira-software.json", + "https://my.atlassian.com/download/feeds/current/stash.json", + + "https://my.atlassian.com/download/feeds/eap/bamboo.json", + "https://my.atlassian.com/download/feeds/eap/confluence.json", + "https://my.atlassian.com/download/feeds/eap/jira.json", + "https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json", + "https://my.atlassian.com/download/feeds/eap/stash.json" + }; + + private void SetConsoleTitle() + { + const string appBuildType = +#if DEBUG + "[Debug]" +#else + + "[Release]" +#endif + ; + var assemblyName = System.Reflection.Assembly.GetExecutingAssembly().GetName(); + Console.Title = $@"{assemblyName.Name} {assemblyName.Version} {appBuildType}"; + } + + private async Task DownloadFilesFromFreed(string feedUrl, IDictionary versions, CancellationToken cancellationToken) + { + var feedDir = Path.Combine(options.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..(feedUrl.LastIndexOf('.'))]); + logger.LogInformation($"Download from JSON \"{feedUrl}\" started"); + foreach (var version in versions) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + var directory = Path.Combine(feedDir, version.Key); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + foreach (var file in version.Value) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + if (file.ZipUrl == null) { continue; } + var serverPath = file.ZipUrl.PathAndQuery; + var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf("/") + 1)..]); + if (!File.Exists(outputFile)) + { + await DownloadFile(file, outputFile, cancellationToken).ConfigureAwait(false); + } + else + { + logger.LogWarning($"File \"{outputFile}\" already exists. Download from \"{file.ZipUrl}\" skipped."); + } + } + } + logger.LogTrace($"All files from \"{feedUrl}\" successfully downloaded."); + } + + private async Task DownloadFile(ResponseItem file, string outputFile, CancellationToken cancellationToken) + { + if (!string.IsNullOrEmpty(file.Md5)) + { + File.WriteAllText(outputFile + ".md5", file.Md5); + } + try + { + using var outputStream = File.OpenWrite(outputFile); + using var request = await client.GetStreamAsync(file.ZipUrl, cancellationToken).ConfigureAwait(false); + await request.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); + } + catch (Exception downloadEx) + { + logger.LogError(downloadEx, $"Failed to download file {file.ZipUrl} to {outputFile}."); + try + { + File.Delete(outputFile); + } + catch (Exception removeEx) + { + logger.LogError(removeEx, $"Failed to remove incomplete file {outputFile}."); + } + } + logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\"."); + } + + public async Task StopAsync(CancellationToken cancellationToken) { } + } +} \ No newline at end of file diff --git a/src/Models/DownloadAction.cs b/src/Models/DownloadAction.cs new file mode 100644 index 0000000..97b3c5e --- /dev/null +++ b/src/Models/DownloadAction.cs @@ -0,0 +1,22 @@ +namespace EpicMorg.Atlassian.Downloader +{ + public enum DownloadAction + { + /// + /// Download application files + /// + Download, + /// + /// Print download URLs and exit + /// + ListURLs, + /// + /// Print available application versions and exit + /// + ListVersions, + /// + /// Print feed JSONs to stdout and exit + /// + ShowRawJson, + } +} \ No newline at end of file diff --git a/src/Models/DownloaderOptions.cs b/src/Models/DownloaderOptions.cs new file mode 100644 index 0000000..ea43012 --- /dev/null +++ b/src/Models/DownloaderOptions.cs @@ -0,0 +1,6 @@ +using System; + +namespace EpicMorg.Atlassian.Downloader +{ + public record DownloaderOptions(string OutputDir, Uri[] CustomFeed, DownloadAction Action) { } +} \ No newline at end of file diff --git a/src/Models/ResponseItem.cs b/src/Models/ResponseItem.cs new file mode 100644 index 0000000..939a3da --- /dev/null +++ b/src/Models/ResponseItem.cs @@ -0,0 +1,31 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace EpicMorg.Atlassian.Downloader +{ + + public partial class ResponseItem + { + public string Description { get; set; } + public string Edition { get; set; } + public Uri ZipUrl { get; set; } + public object TarUrl { get; set; } + public string Md5 { get; set; } + public string Size { get; set; } + public string Released { get; set; } + public string Type { get; set; } + public string Platform { get; set; } + public string Version { get; set; } + public Uri ReleaseNotes { get; set; } + public Uri UpgradeNotes { get; set; } + } +} \ No newline at end of file diff --git a/src/Program.cs b/src/Program.cs index f304279..9559bd2 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,202 +1,48 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Serilog; +using Microsoft.Extensions.Logging; + +using Serilog; + using System; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text.Json; -using System.Threading; using System.Threading.Tasks; -namespace EpicMorg.Atlassian.Downloader { - class Program : IHostedService { - - private readonly ILogger logger; - private readonly Arguments arguments; - - public Program(ILogger logger,Arguments arguments) { - this.logger = logger; - this.arguments = arguments; - } - /// - /// - /// +namespace EpicMorg.Atlassian.Downloader +{ + public class Program + { + /// + /// Atlassian archive downloader. See https://github.com/EpicMorg/atlassian-downloader for more info + /// /// Override output directory to download. - /// Show all download links from feed(s) without downloading. - /// Show all versions from feed(s) without downloading. - /// Show raw json content from feed(s) downloading. - /// Override URIs to import. - /// - static async Task Main(string OutputDir = "atlassian", bool ListURL = false, bool ListVersions = false, Uri[] customFeed = null, bool ShowRawJson = false) => await + /// Override URIs to import. + /// Action to perform + static async Task Main(string OutputDir = "atlassian", Uri[] customFeed = null, DownloadAction Action = DownloadAction.Download) => await Host .CreateDefaultBuilder() .ConfigureHostConfiguration(configHost => configHost.AddEnvironmentVariables()) - .ConfigureAppConfiguration((ctx, configuration) => { + .ConfigureAppConfiguration((ctx, configuration) => configuration .SetBasePath(Environment.CurrentDirectory) .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile($"appsettings.{ctx.HostingEnvironment.EnvironmentName}.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables(); - }) - .ConfigureServices((ctx, services) => { - - services - .AddOptions() - .AddLogging(builder => { - builder.ClearProviders(); - Log.Logger = new LoggerConfiguration() - .ReadFrom.Configuration(ctx.Configuration) - .CreateLogger(); - builder.AddSerilog(dispose: true); - }); - services.AddHostedService(); - services.AddSingleton(new Arguments(OutputDir, ListURL, ListVersions, customFeed, ShowRawJson)); - }) - .RunConsoleAsync(); - - public record Arguments(string OutputDir = "atlassian", bool ListURL = false, bool ListVersions = false, Uri[] CustomFeed = null, bool ShowRawJson = false); - - public async Task StartAsync(CancellationToken cancellationToken) { - - var appTitle = System.Reflection.Assembly.GetExecutingAssembly().GetName().Name; - var appVersion = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; - var appStartupDate = DateTime.Now; - var appBuildType = "[Release]"; -#if DEBUG - appBuildType = "[Debug]"; -#endif - - var feedUrls = arguments.CustomFeed != null - ? arguments.CustomFeed.Select(a => a.ToString()).ToArray() - : new[] { - "https://my.atlassian.com/download/feeds/archived/bamboo.json", - "https://my.atlassian.com/download/feeds/archived/clover.json", - "https://my.atlassian.com/download/feeds/archived/confluence.json", - "https://my.atlassian.com/download/feeds/archived/crowd.json", - "https://my.atlassian.com/download/feeds/archived/crucible.json", - "https://my.atlassian.com/download/feeds/archived/fisheye.json", - "https://my.atlassian.com/download/feeds/archived/jira-core.json", - "https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json", - "https://my.atlassian.com/download/feeds/archived/jira-software.json", - "https://my.atlassian.com/download/feeds/archived/jira.json", - "https://my.atlassian.com/download/feeds/archived/stash.json", - - "https://my.atlassian.com/download/feeds/current/bamboo.json", - "https://my.atlassian.com/download/feeds/current/clover.json", - "https://my.atlassian.com/download/feeds/current/confluence.json", - "https://my.atlassian.com/download/feeds/current/crowd.json", - "https://my.atlassian.com/download/feeds/current/crucible.json", - "https://my.atlassian.com/download/feeds/current/fisheye.json", - "https://my.atlassian.com/download/feeds/current/jira-core.json", - "https://my.atlassian.com/download/feeds/current/jira-servicedesk.json", - "https://my.atlassian.com/download/feeds/current/jira-software.json", - "https://my.atlassian.com/download/feeds/current/stash.json", - - "https://my.atlassian.com/download/feeds/eap/bamboo.json", - "https://my.atlassian.com/download/feeds/eap/confluence.json", - "https://my.atlassian.com/download/feeds/eap/jira.json", - "https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json", - "https://my.atlassian.com/download/feeds/eap/stash.json" - }; - - Console.Title = $"{appTitle} {appVersion} {appBuildType}"; - logger.LogTrace($"Task started at {appStartupDate}."); - - var client = new HttpClient(); - - foreach (var feedUrl in feedUrls) { - var feedDir = Path.Combine(arguments.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..(feedUrl.LastIndexOf('.'))]); - var atlassianJson = await client.GetStringAsync(feedUrl); - var callString = "downloads("; - var json = atlassianJson[callString.Length..^1]; - var parsed = JsonSerializer.Deserialize(json, new JsonSerializerOptions { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - }); - var versionsProg = parsed.GroupBy(a => a.Version).ToDictionary(a => a.Key, a => a.ToArray()); - if (arguments.ShowRawJson) - { - Console.WriteLine("Not released yet."); - return; - //foreach (var versionProg in versionsProg) - //{ - // foreach (var file in versionProg.Value) - // { - // - // } - //} - } - else if (arguments.ListVersions) - { - foreach (var versionProg in versionsProg) - { - foreach (var file in versionProg.Value) - { - Console.WriteLine(file.Version); - } - } - } else if (arguments.ListURL) { - foreach (var versionProg in versionsProg) { - foreach (var file in versionProg.Value) { - Console.WriteLine(file.ZipUrl); - } - } - } else { - logger.LogInformation($"Download from JSON \"{feedUrl}\" started at {appStartupDate}."); - foreach (var versionProg in versionsProg) { - var directory = Path.Combine(feedDir, versionProg.Key); - if (!Directory.Exists(directory)) { - Directory.CreateDirectory(directory); - } - foreach (var file in versionProg.Value) { - if (file.ZipUrl == null) { continue; } - var serverPath = file.ZipUrl.PathAndQuery; - var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf("/") + 1)..]); - if (!File.Exists(outputFile)) { - if (!string.IsNullOrEmpty(file.Md5)) { - File.WriteAllText(outputFile + ".md5", file.Md5); - } - using var outputStream = File.OpenWrite(outputFile); - using var request = await client.GetStreamAsync(file.ZipUrl).ConfigureAwait(false); - await request.CopyToAsync(outputStream).ConfigureAwait(false); - //Console.ForegroundColor = ConsoleColor.Green; - logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\"."); - // Console.ResetColor(); - } else { - // Console.ForegroundColor = ConsoleColor.Yellow; - logger.LogWarning($"File \"{outputFile}\" already exists. Download from \"{file.ZipUrl}\" skipped."); - // Console.ResetColor(); - } - } - } - logger.LogTrace($"All files from \"{feedUrl}\" successfully downloaded."); - } - } - - logger.LogTrace($"Download complete at {appStartupDate}."); - } - - - public Task StopAsync(CancellationToken cancellationToken) { - throw new NotImplementedException(); - } + .AddEnvironmentVariables()) + .ConfigureServices((ctx, services) => services + .AddOptions() + .AddLogging(builder => + { + Log.Logger = new LoggerConfiguration() + .ReadFrom.Configuration(ctx.Configuration) + .CreateLogger(); + builder + .ClearProviders() + .AddSerilog(dispose: true); + }) + .AddHostedService() + .AddSingleton(new DownloaderOptions(OutputDir, customFeed, Action)) + .AddHttpClient()) + .RunConsoleAsync() + .ConfigureAwait(false); } - - public partial class ResponseArray { - public string Description { get; set; } - public string Edition { get; set; } - public Uri ZipUrl { get; set; } - public object TarUrl { get; set; } - public string Md5 { get; set; } - public string Size { get; set; } - public string Released { get; set; } - public string Type { get; set; } - public string Platform { get; set; } - public string Version { get; set; } - public Uri ReleaseNotes { get; set; } - public Uri UpgradeNotes { get; set; } - } - } \ No newline at end of file diff --git a/src/Properties/launchSettings.json b/src/Properties/launchSettings.json new file mode 100644 index 0000000..1ea6c9f --- /dev/null +++ b/src/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "atlassian-downloader": { + "commandName": "Project", + "commandLineArgs": "--help" + } + } +} \ No newline at end of file diff --git a/src/atlassian-downloader.csproj b/src/atlassian-downloader.csproj index c9dc513..af77227 100644 --- a/src/atlassian-downloader.csproj +++ b/src/atlassian-downloader.csproj @@ -26,6 +26,7 @@ +