Merge pull request #14 from EpicMorg/feature/show-raw-json

Fixes for #7, #10, #12, #13 and general improvements
This commit is contained in:
STAM 2021-01-16 19:51:25 +03:00 committed by GitHub
commit e0f4bde3e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 310 additions and 187 deletions

203
src/DonloaderService.cs Normal file
View File

@ -0,0 +1,203 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace EpicMorg.Atlassian.Downloader
{
class DonloaderService : IHostedService
{
private readonly ILogger<DonloaderService> logger;
private readonly DownloaderOptions options;
private readonly HttpClient client;
private readonly IHostApplicationLifetime hostApplicationLifetime;
public DonloaderService(IHostApplicationLifetime hostApplicationLifetime, ILogger<DonloaderService> logger, HttpClient client, DownloaderOptions options)
{
this.logger = logger;
this.client = client;
this.options = options;
this.hostApplicationLifetime = hostApplicationLifetime;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
this.SetConsoleTitle();
var feedUrls = this.GetFeedUrls();
logger.LogTrace($"Task started");
foreach (var feedUrl in feedUrls)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var (json, versions) = await this.GetJson(feedUrl, cancellationToken).ConfigureAwait(false);
switch (options.Action)
{
case DownloadAction.ShowRawJson:
Console.Out.WriteLine(json);
break;
case DownloadAction.Download:
await this.DownloadFilesFromFreed(feedUrl, versions, cancellationToken).ConfigureAwait(false);
break;
case DownloadAction.ListURLs:
foreach (var versionProg in versions)
{
foreach (var file in versionProg.Value)
{
Console.Out.WriteLine(file.ZipUrl);
}
}
break;
case DownloadAction.ListVersions:
foreach (var versionProg in versions)
{
foreach (var file in versionProg.Value)
{
Console.Out.WriteLine(file.Version);
}
}
break;
}
}
logger.LogInformation($"Complete");
this.hostApplicationLifetime.StopApplication();
}
private async Task<(string json, IDictionary<string, ResponseItem[]> versions)> GetJson(string feedUrl, CancellationToken cancellationToken)
{
var atlassianJson = await client.GetStringAsync(feedUrl, cancellationToken).ConfigureAwait(false);
var json = atlassianJson["downloads(".Length..^1];
var parsed = JsonSerializer.Deserialize<ResponseItem[]>(json, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
});
var versions = parsed.GroupBy(a => a.Version).ToDictionary(a => a.Key, a => a.ToArray());
return (json, versions);
}
private string[] GetFeedUrls() => options.CustomFeed != null
? options.CustomFeed.Select(a => a.ToString()).ToArray()
: new[] {
"https://my.atlassian.com/download/feeds/archived/bamboo.json",
"https://my.atlassian.com/download/feeds/archived/clover.json",
"https://my.atlassian.com/download/feeds/archived/confluence.json",
"https://my.atlassian.com/download/feeds/archived/crowd.json",
"https://my.atlassian.com/download/feeds/archived/crucible.json",
"https://my.atlassian.com/download/feeds/archived/fisheye.json",
"https://my.atlassian.com/download/feeds/archived/jira-core.json",
"https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/archived/jira-software.json",
"https://my.atlassian.com/download/feeds/archived/jira.json",
"https://my.atlassian.com/download/feeds/archived/stash.json",
"https://my.atlassian.com/download/feeds/current/bamboo.json",
"https://my.atlassian.com/download/feeds/current/clover.json",
"https://my.atlassian.com/download/feeds/current/confluence.json",
"https://my.atlassian.com/download/feeds/current/crowd.json",
"https://my.atlassian.com/download/feeds/current/crucible.json",
"https://my.atlassian.com/download/feeds/current/fisheye.json",
"https://my.atlassian.com/download/feeds/current/jira-core.json",
"https://my.atlassian.com/download/feeds/current/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/current/jira-software.json",
"https://my.atlassian.com/download/feeds/current/stash.json",
"https://my.atlassian.com/download/feeds/eap/bamboo.json",
"https://my.atlassian.com/download/feeds/eap/confluence.json",
"https://my.atlassian.com/download/feeds/eap/jira.json",
"https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/eap/stash.json"
};
private void SetConsoleTitle()
{
const string appBuildType =
#if DEBUG
"[Debug]"
#else
"[Release]"
#endif
;
var assemblyName = System.Reflection.Assembly.GetExecutingAssembly().GetName();
Console.Title = $@"{assemblyName.Name} {assemblyName.Version} {appBuildType}";
}
private async Task DownloadFilesFromFreed(string feedUrl, IDictionary<string, ResponseItem[]> versions, CancellationToken cancellationToken)
{
var feedDir = Path.Combine(options.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..(feedUrl.LastIndexOf('.'))]);
logger.LogInformation($"Download from JSON \"{feedUrl}\" started");
foreach (var version in versions)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var directory = Path.Combine(feedDir, version.Key);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
foreach (var file in version.Value)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (file.ZipUrl == null) { continue; }
var serverPath = file.ZipUrl.PathAndQuery;
var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf("/") + 1)..]);
if (!File.Exists(outputFile))
{
await DownloadFile(file, outputFile, cancellationToken).ConfigureAwait(false);
}
else
{
logger.LogWarning($"File \"{outputFile}\" already exists. Download from \"{file.ZipUrl}\" skipped.");
}
}
}
logger.LogTrace($"All files from \"{feedUrl}\" successfully downloaded.");
}
private async Task DownloadFile(ResponseItem file, string outputFile, CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(file.Md5))
{
File.WriteAllText(outputFile + ".md5", file.Md5);
}
try
{
using var outputStream = File.OpenWrite(outputFile);
using var request = await client.GetStreamAsync(file.ZipUrl, cancellationToken).ConfigureAwait(false);
await request.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false);
}
catch (Exception downloadEx)
{
logger.LogError(downloadEx, $"Failed to download file {file.ZipUrl} to {outputFile}.");
try
{
File.Delete(outputFile);
}
catch (Exception removeEx)
{
logger.LogError(removeEx, $"Failed to remove incomplete file {outputFile}.");
}
}
logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\".");
}
public async Task StopAsync(CancellationToken cancellationToken) { }
}
}

View File

@ -0,0 +1,22 @@
namespace EpicMorg.Atlassian.Downloader
{
public enum DownloadAction
{
/// <summary>
/// Download application files
/// </summary>
Download,
/// <summary>
/// Print download URLs and exit
/// </summary>
ListURLs,
/// <summary>
/// Print available application versions and exit
/// </summary>
ListVersions,
/// <summary>
/// Print feed JSONs to stdout and exit
/// </summary>
ShowRawJson,
}
}

View File

@ -0,0 +1,6 @@
using System;
namespace EpicMorg.Atlassian.Downloader
{
public record DownloaderOptions(string OutputDir, Uri[] CustomFeed, DownloadAction Action) { }
}

View File

@ -0,0 +1,31 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace EpicMorg.Atlassian.Downloader
{
public partial class ResponseItem
{
public string Description { get; set; }
public string Edition { get; set; }
public Uri ZipUrl { get; set; }
public object TarUrl { get; set; }
public string Md5 { get; set; }
public string Size { get; set; }
public string Released { get; set; }
public string Type { get; set; }
public string Platform { get; set; }
public string Version { get; set; }
public Uri ReleaseNotes { get; set; }
public Uri UpgradeNotes { get; set; }
}
}

View File

@ -1,202 +1,48 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Serilog;
using Microsoft.Extensions.Logging;
using Serilog;
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace EpicMorg.Atlassian.Downloader {
class Program : IHostedService {
private readonly ILogger<Program> logger;
private readonly Arguments arguments;
public Program(ILogger<Program> logger,Arguments arguments) {
this.logger = logger;
this.arguments = arguments;
}
/// <summary>
///
/// </summary>
namespace EpicMorg.Atlassian.Downloader
{
public class Program
{
/// <summary>
/// Atlassian archive downloader. See https://github.com/EpicMorg/atlassian-downloader for more info
/// </summary>
/// <param name="OutputDir">Override output directory to download.</param>
/// <param name="ListURL">Show all download links from feed(s) without downloading.</param>
/// <param name="ListVersions">Show all versions from feed(s) without downloading.</param>
/// <param name="ShowRawJson">Show raw json content from feed(s) downloading.</param>
/// <param name="customFeed">Override URIs to import.</param>
/// <returns></returns>
static async Task Main(string OutputDir = "atlassian", bool ListURL = false, bool ListVersions = false, Uri[] customFeed = null, bool ShowRawJson = false) => await
/// <param name="customFeed">Override URIs to import.</param>
/// <param name="Action">Action to perform</param>
static async Task Main(string OutputDir = "atlassian", Uri[] customFeed = null, DownloadAction Action = DownloadAction.Download) => await
Host
.CreateDefaultBuilder()
.ConfigureHostConfiguration(configHost => configHost.AddEnvironmentVariables())
.ConfigureAppConfiguration((ctx, configuration) => {
.ConfigureAppConfiguration((ctx, configuration) =>
configuration
.SetBasePath(Environment.CurrentDirectory)
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{ctx.HostingEnvironment.EnvironmentName}.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables();
})
.ConfigureServices((ctx, services) => {
services
.AddOptions()
.AddLogging(builder => {
builder.ClearProviders();
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(ctx.Configuration)
.CreateLogger();
builder.AddSerilog(dispose: true);
});
services.AddHostedService<Program>();
services.AddSingleton(new Arguments(OutputDir, ListURL, ListVersions, customFeed, ShowRawJson));
})
.RunConsoleAsync();
public record Arguments(string OutputDir = "atlassian", bool ListURL = false, bool ListVersions = false, Uri[] CustomFeed = null, bool ShowRawJson = false);
public async Task StartAsync(CancellationToken cancellationToken) {
var appTitle = System.Reflection.Assembly.GetExecutingAssembly().GetName().Name;
var appVersion = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
var appStartupDate = DateTime.Now;
var appBuildType = "[Release]";
#if DEBUG
appBuildType = "[Debug]";
#endif
var feedUrls = arguments.CustomFeed != null
? arguments.CustomFeed.Select(a => a.ToString()).ToArray()
: new[] {
"https://my.atlassian.com/download/feeds/archived/bamboo.json",
"https://my.atlassian.com/download/feeds/archived/clover.json",
"https://my.atlassian.com/download/feeds/archived/confluence.json",
"https://my.atlassian.com/download/feeds/archived/crowd.json",
"https://my.atlassian.com/download/feeds/archived/crucible.json",
"https://my.atlassian.com/download/feeds/archived/fisheye.json",
"https://my.atlassian.com/download/feeds/archived/jira-core.json",
"https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/archived/jira-software.json",
"https://my.atlassian.com/download/feeds/archived/jira.json",
"https://my.atlassian.com/download/feeds/archived/stash.json",
"https://my.atlassian.com/download/feeds/current/bamboo.json",
"https://my.atlassian.com/download/feeds/current/clover.json",
"https://my.atlassian.com/download/feeds/current/confluence.json",
"https://my.atlassian.com/download/feeds/current/crowd.json",
"https://my.atlassian.com/download/feeds/current/crucible.json",
"https://my.atlassian.com/download/feeds/current/fisheye.json",
"https://my.atlassian.com/download/feeds/current/jira-core.json",
"https://my.atlassian.com/download/feeds/current/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/current/jira-software.json",
"https://my.atlassian.com/download/feeds/current/stash.json",
"https://my.atlassian.com/download/feeds/eap/bamboo.json",
"https://my.atlassian.com/download/feeds/eap/confluence.json",
"https://my.atlassian.com/download/feeds/eap/jira.json",
"https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/eap/stash.json"
};
Console.Title = $"{appTitle} {appVersion} {appBuildType}";
logger.LogTrace($"Task started at {appStartupDate}.");
var client = new HttpClient();
foreach (var feedUrl in feedUrls) {
var feedDir = Path.Combine(arguments.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..(feedUrl.LastIndexOf('.'))]);
var atlassianJson = await client.GetStringAsync(feedUrl);
var callString = "downloads(";
var json = atlassianJson[callString.Length..^1];
var parsed = JsonSerializer.Deserialize<ResponseArray[]>(json, new JsonSerializerOptions {
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
});
var versionsProg = parsed.GroupBy(a => a.Version).ToDictionary(a => a.Key, a => a.ToArray());
if (arguments.ShowRawJson)
{
Console.WriteLine("Not released yet.");
return;
//foreach (var versionProg in versionsProg)
//{
// foreach (var file in versionProg.Value)
// {
//
// }
//}
}
else if (arguments.ListVersions)
{
foreach (var versionProg in versionsProg)
{
foreach (var file in versionProg.Value)
{
Console.WriteLine(file.Version);
}
}
} else if (arguments.ListURL) {
foreach (var versionProg in versionsProg) {
foreach (var file in versionProg.Value) {
Console.WriteLine(file.ZipUrl);
}
}
} else {
logger.LogInformation($"Download from JSON \"{feedUrl}\" started at {appStartupDate}.");
foreach (var versionProg in versionsProg) {
var directory = Path.Combine(feedDir, versionProg.Key);
if (!Directory.Exists(directory)) {
Directory.CreateDirectory(directory);
}
foreach (var file in versionProg.Value) {
if (file.ZipUrl == null) { continue; }
var serverPath = file.ZipUrl.PathAndQuery;
var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf("/") + 1)..]);
if (!File.Exists(outputFile)) {
if (!string.IsNullOrEmpty(file.Md5)) {
File.WriteAllText(outputFile + ".md5", file.Md5);
}
using var outputStream = File.OpenWrite(outputFile);
using var request = await client.GetStreamAsync(file.ZipUrl).ConfigureAwait(false);
await request.CopyToAsync(outputStream).ConfigureAwait(false);
//Console.ForegroundColor = ConsoleColor.Green;
logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\".");
// Console.ResetColor();
} else {
// Console.ForegroundColor = ConsoleColor.Yellow;
logger.LogWarning($"File \"{outputFile}\" already exists. Download from \"{file.ZipUrl}\" skipped.");
// Console.ResetColor();
}
}
}
logger.LogTrace($"All files from \"{feedUrl}\" successfully downloaded.");
}
}
logger.LogTrace($"Download complete at {appStartupDate}.");
}
public Task StopAsync(CancellationToken cancellationToken) {
throw new NotImplementedException();
}
.AddEnvironmentVariables())
.ConfigureServices((ctx, services) => services
.AddOptions()
.AddLogging(builder =>
{
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(ctx.Configuration)
.CreateLogger();
builder
.ClearProviders()
.AddSerilog(dispose: true);
})
.AddHostedService<DonloaderService>()
.AddSingleton(new DownloaderOptions(OutputDir, customFeed, Action))
.AddHttpClient())
.RunConsoleAsync()
.ConfigureAwait(false);
}
public partial class ResponseArray {
public string Description { get; set; }
public string Edition { get; set; }
public Uri ZipUrl { get; set; }
public object TarUrl { get; set; }
public string Md5 { get; set; }
public string Size { get; set; }
public string Released { get; set; }
public string Type { get; set; }
public string Platform { get; set; }
public string Version { get; set; }
public Uri ReleaseNotes { get; set; }
public Uri UpgradeNotes { get; set; }
}
}

View File

@ -0,0 +1,8 @@
{
"profiles": {
"atlassian-downloader": {
"commandName": "Project",
"commandLineArgs": "--help"
}
}
}

View File

@ -2,7 +2,13 @@
"Serilog": {
"MinimumLevel": "Verbose",
"WriteTo": [
"Console",
{
"Name": "Console",
"Args": {
"theme": "Serilog.Sinks.SystemConsole.Themes.AnsiConsoleTheme::Code, Serilog.Sinks.Console"
}
}
,
{
"Name": "Logger",
"Args": {

View File

@ -26,6 +26,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="5.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />