Merge pull request #14 from EpicMorg/feature/show-raw-json

Fixes for #7, #10, #12, #13 and general improvements

src/DonloaderService.cs

@@ -0,0 +1,203 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace EpicMorg.Atlassian.Downloader
+{
+    class DonloaderService : IHostedService
+    {
+        private readonly ILogger<DonloaderService> logger;
+        private readonly DownloaderOptions options;
+        private readonly HttpClient client;
+        private readonly IHostApplicationLifetime hostApplicationLifetime;
+
+        public DonloaderService(IHostApplicationLifetime hostApplicationLifetime, ILogger<DonloaderService> logger, HttpClient client, DownloaderOptions options)
+        {
+            this.logger = logger;
+            this.client = client;
+            this.options = options;
+            this.hostApplicationLifetime = hostApplicationLifetime;
+        }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            this.SetConsoleTitle();
+            var feedUrls = this.GetFeedUrls();
+
+            logger.LogTrace($"Task started");
+            foreach (var feedUrl in feedUrls)
+            {
+                if (cancellationToken.IsCancellationRequested)
+                {
+                    return;
+                }
+                var (json, versions) = await this.GetJson(feedUrl, cancellationToken).ConfigureAwait(false);
+
+                switch (options.Action)
+                {
+                    case DownloadAction.ShowRawJson:
+                        Console.Out.WriteLine(json);
+                        break;
+                    case DownloadAction.Download:
+                        await this.DownloadFilesFromFreed(feedUrl, versions, cancellationToken).ConfigureAwait(false);
+                        break;
+                    case DownloadAction.ListURLs:
+                        foreach (var versionProg in versions)
+                        {
+                            foreach (var file in versionProg.Value)
+                            {
+                                Console.Out.WriteLine(file.ZipUrl);
+                            }
+                        }
+                        break;
+                    case DownloadAction.ListVersions:
+                        foreach (var versionProg in versions)
+                        {
+                            foreach (var file in versionProg.Value)
+                            {
+                                Console.Out.WriteLine(file.Version);
+                            }
+                        }
+                        break;
+                }
+            }
+            logger.LogInformation($"Complete");
+            this.hostApplicationLifetime.StopApplication();
+        }
+
+        private async Task<(string json, IDictionary<string, ResponseItem[]> versions)> GetJson(string feedUrl, CancellationToken cancellationToken)
+        {
+            var atlassianJson = await client.GetStringAsync(feedUrl, cancellationToken).ConfigureAwait(false);
+            var json = atlassianJson["downloads(".Length..^1];
+            var parsed = JsonSerializer.Deserialize<ResponseItem[]>(json, new JsonSerializerOptions
+            {
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+            });
+            var versions = parsed.GroupBy(a => a.Version).ToDictionary(a => a.Key, a => a.ToArray());
+
+            return (json, versions);
+        }
+
+        private string[] GetFeedUrls() => options.CustomFeed != null
+            ? options.CustomFeed.Select(a => a.ToString()).ToArray()
+            : new[] {
+                "https://my.atlassian.com/download/feeds/archived/bamboo.json",
+                "https://my.atlassian.com/download/feeds/archived/clover.json",
+                "https://my.atlassian.com/download/feeds/archived/confluence.json",
+                "https://my.atlassian.com/download/feeds/archived/crowd.json",
+                "https://my.atlassian.com/download/feeds/archived/crucible.json",
+                "https://my.atlassian.com/download/feeds/archived/fisheye.json",
+                "https://my.atlassian.com/download/feeds/archived/jira-core.json",
+                "https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json",
+                "https://my.atlassian.com/download/feeds/archived/jira-software.json",
+                "https://my.atlassian.com/download/feeds/archived/jira.json",
+                "https://my.atlassian.com/download/feeds/archived/stash.json",
+
+                "https://my.atlassian.com/download/feeds/current/bamboo.json",
+                "https://my.atlassian.com/download/feeds/current/clover.json",
+                "https://my.atlassian.com/download/feeds/current/confluence.json",
+                "https://my.atlassian.com/download/feeds/current/crowd.json",
+                "https://my.atlassian.com/download/feeds/current/crucible.json",
+                "https://my.atlassian.com/download/feeds/current/fisheye.json",
+                "https://my.atlassian.com/download/feeds/current/jira-core.json",
+                "https://my.atlassian.com/download/feeds/current/jira-servicedesk.json",
+                "https://my.atlassian.com/download/feeds/current/jira-software.json",
+                "https://my.atlassian.com/download/feeds/current/stash.json",
+
+                "https://my.atlassian.com/download/feeds/eap/bamboo.json",
+                "https://my.atlassian.com/download/feeds/eap/confluence.json",
+                "https://my.atlassian.com/download/feeds/eap/jira.json",
+                "https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json",
+                "https://my.atlassian.com/download/feeds/eap/stash.json"
+            };
+
+        private void SetConsoleTitle()
+        {
+            const string appBuildType =
+#if DEBUG
+                "[Debug]"
+#else
+            
+                "[Release]"
+#endif
+            ;
+            var assemblyName = System.Reflection.Assembly.GetExecutingAssembly().GetName();
+            Console.Title = $@"{assemblyName.Name} {assemblyName.Version} {appBuildType}";
+        }
+
+        private async Task DownloadFilesFromFreed(string feedUrl, IDictionary<string, ResponseItem[]> versions, CancellationToken cancellationToken)
+        {
+            var feedDir = Path.Combine(options.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..(feedUrl.LastIndexOf('.'))]);
+            logger.LogInformation($"Download from JSON \"{feedUrl}\" started");
+            foreach (var version in versions)
+            {
+                if (cancellationToken.IsCancellationRequested)
+                {
+                    return;
+                }
+
+                var directory = Path.Combine(feedDir, version.Key);
+                if (!Directory.Exists(directory))
+                {
+                    Directory.CreateDirectory(directory);
+                }
+                foreach (var file in version.Value)
+                {
+                    if (cancellationToken.IsCancellationRequested)
+                    {
+                        return;
+                    }
+                    if (file.ZipUrl == null) { continue; }
+                    var serverPath = file.ZipUrl.PathAndQuery;
+                    var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf("/") + 1)..]);
+                    if (!File.Exists(outputFile))
+                    {
+                        await DownloadFile(file, outputFile, cancellationToken).ConfigureAwait(false);
+                    }
+                    else
+                    {
+                        logger.LogWarning($"File \"{outputFile}\" already exists. Download from \"{file.ZipUrl}\" skipped.");
+                    }
+                }
+            }
+            logger.LogTrace($"All files from \"{feedUrl}\" successfully downloaded.");
+        }
+
+        private async Task DownloadFile(ResponseItem file, string outputFile, CancellationToken cancellationToken)
+        {
+            if (!string.IsNullOrEmpty(file.Md5))
+            {
+                File.WriteAllText(outputFile + ".md5", file.Md5);
+            }
+            try
+            {
+                using var outputStream = File.OpenWrite(outputFile);
+                using var request = await client.GetStreamAsync(file.ZipUrl, cancellationToken).ConfigureAwait(false);
+                await request.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false);
+            }
+            catch (Exception downloadEx)
+            {
+                logger.LogError(downloadEx, $"Failed to download file {file.ZipUrl} to {outputFile}.");
+                try
+                {
+                    File.Delete(outputFile);
+                }
+                catch (Exception removeEx)
+                {
+                    logger.LogError(removeEx, $"Failed to remove incomplete file {outputFile}.");
+                }
+            }
+            logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\".");
+        }
+
+        public async Task StopAsync(CancellationToken cancellationToken) { }
+    }
+}

src/Models/DownloadAction.cs

@@ -0,0 +1,22 @@
+namespace EpicMorg.Atlassian.Downloader
+{
+    public enum DownloadAction
+    {
+        /// <summary>
+        /// Download application files
+        /// </summary>
+        Download,
+        /// <summary>
+        /// Print download URLs and exit
+        /// </summary>
+        ListURLs,
+        /// <summary>
+        /// Print available application versions and exit
+        /// </summary>
+        ListVersions,
+        /// <summary>
+        /// Print feed JSONs to stdout and exit
+        /// </summary>
+        ShowRawJson,
+    }
+}

src/Models/DownloaderOptions.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace EpicMorg.Atlassian.Downloader
+{
+    public record DownloaderOptions(string OutputDir, Uri[] CustomFeed, DownloadAction Action) { }
+}

src/Models/ResponseItem.cs

@@ -0,0 +1,31 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace EpicMorg.Atlassian.Downloader
+{
+
+    public partial class ResponseItem
+    {
+        public string Description { get; set; }
+        public string Edition { get; set; }
+        public Uri ZipUrl { get; set; }
+        public object TarUrl { get; set; }
+        public string Md5 { get; set; }
+        public string Size { get; set; }
+        public string Released { get; set; }
+        public string Type { get; set; }
+        public string Platform { get; set; }
+        public string Version { get; set; }
+        public Uri ReleaseNotes { get; set; }
+        public Uri UpgradeNotes { get; set; }
+    }
+}

src/Program.cs

@@ -2,201 +2,47 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+
 using Serilog;
+
 using System;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
-using System.Text.Json;
-using System.Threading;
 using System.Threading.Tasks;
 
-namespace EpicMorg.Atlassian.Downloader {
+namespace EpicMorg.Atlassian.Downloader
-    class Program : IHostedService {
+{
-
+    public class Program
-        private readonly ILogger<Program> logger;
+    {
-        private readonly Arguments arguments;
-
-        public Program(ILogger<Program> logger,Arguments arguments) {
-            this.logger = logger;
-            this.arguments = arguments;
-        }
         /// <summary>
-        /// 
+        /// Atlassian archive downloader. See https://github.com/EpicMorg/atlassian-downloader for more info
         /// </summary>
         /// <param name="OutputDir">Override output directory to download.</param>
-        /// <param name="ListURL">Show all download links from feed(s) without downloading.</param>
-        /// <param name="ListVersions">Show all versions from feed(s) without downloading.</param>
-        /// <param name="ShowRawJson">Show raw json content from feed(s) downloading.</param>
         /// <param name="customFeed">Override URIs to import.</param>
-        /// <returns></returns>
+        /// <param name="Action">Action to perform</param>
-        static async Task Main(string OutputDir = "atlassian", bool ListURL = false, bool ListVersions = false, Uri[] customFeed = null, bool ShowRawJson = false) => await
+        static async Task Main(string OutputDir = "atlassian", Uri[] customFeed = null, DownloadAction Action = DownloadAction.Download) => await
             Host
                 .CreateDefaultBuilder()
                 .ConfigureHostConfiguration(configHost => configHost.AddEnvironmentVariables())
-                .ConfigureAppConfiguration((ctx, configuration) => {
+                .ConfigureAppConfiguration((ctx, configuration) =>
                     configuration
                         .SetBasePath(Environment.CurrentDirectory)
                         .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
                         .AddJsonFile($"appsettings.{ctx.HostingEnvironment.EnvironmentName}.json", optional: true, reloadOnChange: true)
-                        .AddEnvironmentVariables();
+                        .AddEnvironmentVariables())
-                })
+                .ConfigureServices((ctx, services) => services
-                .ConfigureServices((ctx, services) => {
-                  
-                    services
                        .AddOptions()
-                   .AddLogging(builder => {
+                       .AddLogging(builder =>
-                       builder.ClearProviders();
+                       {
                             Log.Logger = new LoggerConfiguration()
                                    .ReadFrom.Configuration(ctx.Configuration)
                                    .CreateLogger();
-                       builder.AddSerilog(dispose: true);
+                            builder
-                   });
+                                .ClearProviders()
-                    services.AddHostedService<Program>();
+                                .AddSerilog(dispose: true);
-                    services.AddSingleton(new Arguments(OutputDir, ListURL, ListVersions, customFeed, ShowRawJson));
                        })
-                .RunConsoleAsync();
+                       .AddHostedService<DonloaderService>()
-
+                       .AddSingleton(new DownloaderOptions(OutputDir, customFeed, Action))
-        public record Arguments(string OutputDir = "atlassian", bool ListURL = false, bool ListVersions = false, Uri[] CustomFeed = null, bool ShowRawJson = false);
+                       .AddHttpClient())
-
+                .RunConsoleAsync()
-        public async Task StartAsync(CancellationToken cancellationToken) {
+                .ConfigureAwait(false);
-
-            var appTitle = System.Reflection.Assembly.GetExecutingAssembly().GetName().Name;
-            var appVersion = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
-            var appStartupDate = DateTime.Now;
-            var appBuildType = "[Release]";
-#if DEBUG
-            appBuildType = "[Debug]";
-#endif
-
-            var feedUrls = arguments.CustomFeed != null
-                ? arguments.CustomFeed.Select(a => a.ToString()).ToArray()
-                : new[] {
-        "https://my.atlassian.com/download/feeds/archived/bamboo.json",
-		"https://my.atlassian.com/download/feeds/archived/clover.json",
-        "https://my.atlassian.com/download/feeds/archived/confluence.json",
-        "https://my.atlassian.com/download/feeds/archived/crowd.json",
-        "https://my.atlassian.com/download/feeds/archived/crucible.json",
-        "https://my.atlassian.com/download/feeds/archived/fisheye.json",
-        "https://my.atlassian.com/download/feeds/archived/jira-core.json",
-        "https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json",
-        "https://my.atlassian.com/download/feeds/archived/jira-software.json",
-        "https://my.atlassian.com/download/feeds/archived/jira.json",
-        "https://my.atlassian.com/download/feeds/archived/stash.json",
-
-        "https://my.atlassian.com/download/feeds/current/bamboo.json",
-		"https://my.atlassian.com/download/feeds/current/clover.json",
-        "https://my.atlassian.com/download/feeds/current/confluence.json",
-        "https://my.atlassian.com/download/feeds/current/crowd.json",
-        "https://my.atlassian.com/download/feeds/current/crucible.json",
-        "https://my.atlassian.com/download/feeds/current/fisheye.json",
-        "https://my.atlassian.com/download/feeds/current/jira-core.json",
-        "https://my.atlassian.com/download/feeds/current/jira-servicedesk.json",
-        "https://my.atlassian.com/download/feeds/current/jira-software.json",
-        "https://my.atlassian.com/download/feeds/current/stash.json",
-
-        "https://my.atlassian.com/download/feeds/eap/bamboo.json",
-        "https://my.atlassian.com/download/feeds/eap/confluence.json",
-        "https://my.atlassian.com/download/feeds/eap/jira.json",
-        "https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json",
-		"https://my.atlassian.com/download/feeds/eap/stash.json"
-                };
-
-            Console.Title = $"{appTitle} {appVersion} {appBuildType}";
-            logger.LogTrace($"Task started at {appStartupDate}.");
-
-            var client = new HttpClient();
-
-            foreach (var feedUrl in feedUrls) {
-                var feedDir = Path.Combine(arguments.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..(feedUrl.LastIndexOf('.'))]);
-                var atlassianJson = await client.GetStringAsync(feedUrl);
-                var callString = "downloads(";
-                var json = atlassianJson[callString.Length..^1];
-                var parsed = JsonSerializer.Deserialize<ResponseArray[]>(json, new JsonSerializerOptions {
-                    PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
-                });
-                var versionsProg = parsed.GroupBy(a => a.Version).ToDictionary(a => a.Key, a => a.ToArray());
-                if (arguments.ShowRawJson)
-                {
-                    Console.WriteLine("Not released yet.");
-                    return;
-                    //foreach (var versionProg in versionsProg)
-                    //{
-                    //    foreach (var file in versionProg.Value)
-                    //    {
-                    //        
-                    //    }
-                    //}
     }
-                else if (arguments.ListVersions)
-                {
-                    foreach (var versionProg in versionsProg)
-                    {
-                        foreach (var file in versionProg.Value)
-                        {
-                            Console.WriteLine(file.Version);
-                        }
-                    }
-                }  else if (arguments.ListURL) {
-                    foreach (var versionProg in versionsProg) {
-                        foreach (var file in versionProg.Value) {
-                            Console.WriteLine(file.ZipUrl);
-                        }
-                    }
-                } else {
-                    logger.LogInformation($"Download from JSON \"{feedUrl}\" started at {appStartupDate}.");
-                    foreach (var versionProg in versionsProg) {
-                        var directory = Path.Combine(feedDir, versionProg.Key);
-                        if (!Directory.Exists(directory)) {
-                            Directory.CreateDirectory(directory);
-                        }
-                        foreach (var file in versionProg.Value) {
-                            if (file.ZipUrl == null) { continue; }
-                            var serverPath = file.ZipUrl.PathAndQuery;
-                            var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf("/") + 1)..]);
-                            if (!File.Exists(outputFile)) {
-                                if (!string.IsNullOrEmpty(file.Md5)) {
-                                    File.WriteAllText(outputFile + ".md5", file.Md5);
-                                }
-                                using var outputStream = File.OpenWrite(outputFile);
-                                using var request = await client.GetStreamAsync(file.ZipUrl).ConfigureAwait(false);
-                                await request.CopyToAsync(outputStream).ConfigureAwait(false);
-                                //Console.ForegroundColor = ConsoleColor.Green;
-                                logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\".");
-                                // Console.ResetColor();
-                            } else {
-                                // Console.ForegroundColor = ConsoleColor.Yellow;
-                                logger.LogWarning($"File \"{outputFile}\" already exists. Download from \"{file.ZipUrl}\" skipped.");
-                                //  Console.ResetColor();
-                            }
-                        }
-                    }
-                    logger.LogTrace($"All files from \"{feedUrl}\" successfully downloaded.");
-                }
-            }
-
-            logger.LogTrace($"Download complete at {appStartupDate}.");
-        }
-
-
-        public Task StopAsync(CancellationToken cancellationToken) {
-            throw new NotImplementedException();
-        }
-    }
-
-    public partial class ResponseArray {
-        public string Description { get; set; }
-        public string Edition { get; set; }
-        public Uri ZipUrl { get; set; }
-        public object TarUrl { get; set; }
-        public string Md5 { get; set; }
-        public string Size { get; set; }
-        public string Released { get; set; }
-        public string Type { get; set; }
-        public string Platform { get; set; }
-        public string Version { get; set; }
-        public Uri ReleaseNotes { get; set; }
-        public Uri UpgradeNotes { get; set; }
-    }
-
 }

src/Properties/launchSettings.json

@@ -0,0 +1,8 @@
+{
+  "profiles": {
+    "atlassian-downloader": {
+      "commandName": "Project",
+      "commandLineArgs": "--help"
+    }
+  }
+}

src/appSettings.json

@@ -2,7 +2,13 @@
   "Serilog": {
     "MinimumLevel": "Verbose",
     "WriteTo": [
-      "Console",
+      {
+        "Name": "Console",
+        "Args": {
+          "theme": "Serilog.Sinks.SystemConsole.Themes.AnsiConsoleTheme::Code, Serilog.Sinks.Console"
+        }
+      }
+      ,
       {
         "Name": "Logger",
         "Args": {

src/atlassian-downloader.csproj

@@ -26,6 +26,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="5.0.1" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />