EpicMorg/atlassian-downloader
2
1
Fork 0
mirror of https://github.com/EpicMorg/atlassian-downloader.git synced 2025-03-14 06:10:19 +03:00
Code Issues Projects Releases Wiki Activity

Fix #36: restore args parsing.

Browse Source 
Move bells and whistles from downloader service
Apply code styles.
Fix exception when parsing json
Move sources to a separate type
This commit is contained in:
Konstantin Safonov 2023-10-12 23:33:26 +03:00
parent c2d6260d61
commit 00000a2119
No known key found for this signature in database
GPG Key ID: EBC01B465FCF8665
10 changed files with 531 additions and 514 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
src/Core/BellsAndWhistles.cs Normal file
View File

@ -0,0 +1,116 @@
namespace EpicMorg.Atlassian.Downloader.Models;
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
internal class BellsAndWhistles
{
private static readonly string assemblyEnvironment = string.Format("[{1}, {0}]", RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(), RuntimeInformation.FrameworkDescription);
private static readonly Assembly entryAssembly = Assembly.GetEntryAssembly();
private static readonly string assemblyVersion = entryAssembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>().InformationalVersion;
private static readonly string fileVersion = entryAssembly.GetCustomAttribute<AssemblyFileVersionAttribute>().Version;
private static readonly string assemblyName = entryAssembly.GetCustomAttribute<AssemblyProductAttribute>().Product;
const string assemblyBuildType =
#if DEBUG
"[Debug]"
#else
"[Release]"
#endif
;
private const ConsoleColor DEFAULT = ConsoleColor.Blue;
public static void ShowVersionInfo(ILogger logger)
{
logger.LogInformation($"{assemblyName} {assemblyVersion} {assemblyEnvironment} {assemblyBuildType}");
Console.BackgroundColor = ConsoleColor.Black;
WriteColorLine("%╔═╦═══════════════════════════════════════════════════════════════════════════════════════╦═╗");
WriteColorLine("%╠═╝ .''. %╚═%╣");
WriteColorLine("%║ .:cc;. %║");
WriteColorLine("%║ .;cccc;. %║");
WriteColorLine("%║ .;cccccc;. !╔══════════════════════════════════════════════╗ %║");
WriteColorLine($"%║ .:ccccccc;. !║ {assemblyName} !║ %║");
WriteColorLine("%║ 'ccccccccc;. !╠══════════════════════════════════════════════╣ %║");
WriteColorLine("%║ ,cccccccccc;. !║ &Code: @kasthack !║ %║");
WriteColorLine("%║ ,ccccccccccc;. !║ &GFX: @stam !║ %║");
WriteColorLine("%║ .... .:ccccccccccc;. !╠══════════════════════════════════════════════╣ %║");
WriteColorLine($"%║ .',,'..;cccccccccccc;. !║ &Version: {fileVersion} !║ %║");
WriteColorLine("%║ .,,,,,'.';cccccccccccc;. !║ &GitHub: $EpicMorg/atlassian-downloader !║ %║");
WriteColorLine("%║ .,;;;;;,'.':cccccccccccc;. !╚══════════════════════════════════════════════╝ %║");
WriteColorLine("%║ .;:;;;;;;,...:cccccccccccc;. %║");
WriteColorLine("%║ .;:::::;;;;'. .;:ccccccccccc;. %║");
WriteColorLine("%║ .:cc::::::::,. ..:ccccccccccc;. %║");
WriteColorLine("%║ .:cccccc:::::' .:ccccccccccc;. %║");
WriteColorLine("%║ .;:::::::::::,. .;:::::::::::,. %║");
WriteColorLine("%╠═╗ ............ ............ %╔═╣");
WriteColorLine("%╚═╩═══════════════════════════════════════════════════════════════════════════════════════╩═╝");
Console.ResetColor();
}
public static void SetConsoleTitle() => Console.Title = $@"{assemblyName} {assemblyVersion} {assemblyEnvironment} - {assemblyBuildType}";
private static void WriteColorLine(string text, params object[] args)
{
Dictionary<char, ConsoleColor> colors = new()
{
{ '!', ConsoleColor.Red },
{ '@', ConsoleColor.Green },
{ '#', ConsoleColor.Blue },
{ '$', ConsoleColor.Magenta },
{ '&', ConsoleColor.Yellow },
{ '%', ConsoleColor.Cyan }
};
// TODO: word wrap, backslash escapes
text = string.Format(text, args);
var chunk = "";
var paren = false;
for (var i = 0; i < text.Length; i++)
{
var c = text[i];
if (colors.ContainsKey(c) && StringNext(text, i) != ' ')
{
Console.Write(chunk);
chunk = "";
if (StringNext(text, i) == '(')
{
i++; // skip past the paren
paren = true;
}
Console.ForegroundColor = colors[c];
}
else if (paren && c == ')')
{
paren = false;
Console.ForegroundColor = DEFAULT;
}
else if (Console.ForegroundColor != DEFAULT)
{
Console.Write(c);
if (c == ' ' && !paren)
{
Console.ForegroundColor = DEFAULT;
}
}
else
{
chunk += c;
}
}
Console.WriteLine(chunk);
Console.ForegroundColor = DEFAULT;
}
private static char StringNext(string text, int index) => index < text.Length ? text[index + 1] : '\0';
}

233
src/Core/DownloaderService.cs Normal file
View File

@ -0,0 +1,233 @@
namespace EpicMorg.Atlassian.Downloader.Core;
using EpicMorg.Atlassian.Downloader.Models;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
internal class DownloaderService : IHostedService
{
private readonly string UserAgentString = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0";
private readonly ILogger<DownloaderService> logger;
private readonly DownloaderOptions options;
private readonly HttpClient client;
private readonly IHostApplicationLifetime hostApplicationLifetime;
public DownloaderService(IHostApplicationLifetime hostApplicationLifetime, ILogger<DownloaderService> logger, HttpClient client, DownloaderOptions options)
{
this.logger = logger;
this.client = client;
client.DefaultRequestHeaders.Add("User-Agent", this.UserAgentString);
this.options = options;
this.hostApplicationLifetime = hostApplicationLifetime;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
BellsAndWhistles.SetConsoleTitle();
BellsAndWhistles.ShowVersionInfo(this.logger);
if (!this.options.Version && !string.IsNullOrWhiteSpace(this.options.OutputDir))
{
var feedUrls = this.GetFeedUrls();
this.logger.LogInformation($"Task started");
foreach (var feedUrl in feedUrls)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var (json, versions) = await this.GetJson(feedUrl, this.options.ProductVersion, cancellationToken).ConfigureAwait(false);
switch (this.options.Action)
{
case DownloadAction.ShowRawJson:
Console.Out.WriteLine(json);
break;
case DownloadAction.Download:
await this.DownloadFilesFromFeed(feedUrl, versions, cancellationToken).ConfigureAwait(false);
break;
case DownloadAction.ListURLs:
foreach (var versionProg in versions)
{
foreach (var file in versionProg.Value)
{
Console.Out.WriteLine(file.ZipUrl);
}
}
break;
case DownloadAction.ListVersions:
foreach (var versionProg in versions)
{
foreach (var file in versionProg.Value)
{
Console.Out.WriteLine(file.Version);
}
}
break;
}
}
}
this.logger.LogInformation($"Complete");
this.hostApplicationLifetime.StopApplication();
}
private async Task<(string json, IDictionary<string, ResponseItem[]> versions)> GetJson(string feedUrl, string productVersion = null, CancellationToken cancellationToken = default)
{
var atlassianJson = await this.client.GetStringAsync(feedUrl, cancellationToken).ConfigureAwait(false);
var json = atlassianJson.Trim()["downloads(".Length..^1];
this.logger.LogTrace("Downloaded json: {0}", json);
var parsed = JsonSerializer.Deserialize<ResponseItem[]>(json, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
});
this.logger.LogDebug("Found {releaseCount} releases", parsed.Length);
var versions = parsed
.GroupBy(a => a.Version)
.Where(a => productVersion is null || a.Key == productVersion)
.ToDictionary(a => a.Key, a => a.ToArray());
this.logger.LogDebug("Found {releaseCount} releases", versions.Count);
return (json, versions);
}
private IReadOnlyList<string> GetFeedUrls() => this.options.CustomFeed != null
? this.options.CustomFeed.Select(a => a.ToString()).ToArray()
: SourceInformation.AtlassianSources;
private async Task DownloadFilesFromFeed(string feedUrl, IDictionary<string, ResponseItem[]> versions, CancellationToken cancellationToken)
{
var feedDir = Path.Combine(this.options.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..feedUrl.LastIndexOf('.')]);
this.logger.LogInformation("Download from JSON \"{feedUrl}\" started", feedUrl);
foreach (var version in versions)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var directory = Path.Combine(feedDir, version.Key);
if (!Directory.Exists(directory))
{
_ = Directory.CreateDirectory(directory);
}
foreach (var file in version.Value)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (file.ZipUrl == null)
{
this.logger.LogWarning($"Empty ZipUrl found for version '{version.Key}' in {feedUrl}");
continue;
}
var serverPath = file.ZipUrl.PathAndQuery;
var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf('/') + 1)..]);
if (!File.Exists(outputFile))
{
await this.DownloadFile(file, outputFile, cancellationToken).ConfigureAwait(false);
}
else
{
this.logger.LogWarning($"File \"{outputFile}\" already exists. File sizes will be compared.");
var localFileSize = new FileInfo(outputFile).Length;
this.logger.LogInformation($"Size of local file is {localFileSize} bytes.");
try
{
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Add("User-Agent", this.UserAgentString);
var response = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Head, file.ZipUrl));
if (response.IsSuccessStatusCode)
{
if (response.Content.Headers.ContentLength.HasValue)
{
var remoteFileSize = response.Content.Headers.ContentLength.Value;
this.logger.LogInformation($"Size of remote file is \"{remoteFileSize}\" bytes.");
if (remoteFileSize == localFileSize)
{
this.logger.LogInformation($"Size of remote and local files and are same ({remoteFileSize} bytes and {localFileSize} bytes). Nothing to download. Operation skipped.");
}
else
{
this.logger.LogWarning($"Size of remote and local files and are not same ({remoteFileSize} bytes and {localFileSize} bytes). Download started.");
File.Delete(outputFile);
await this.DownloadFile(file, outputFile, cancellationToken).ConfigureAwait(false);
}
}
else
{
this.logger.LogWarning($"Cant get size of remote file \"{file.ZipUrl}\". May be server not support it feature. Sorry.");
continue;
}
}
else
{
this.logger.LogCritical($"Request execution error: \"{response.StatusCode}\". Sorry.");
}
}
catch (HttpRequestException ex)
{
this.logger.LogCritical($"HTTP request error: \"{ex.Message}\", \"{ex.StackTrace}\", \"{ex.StatusCode}\". Sorry.");
}
}
}
}
this.logger.LogInformation($"All files from \"{feedUrl}\" successfully downloaded.");
}
private async Task DownloadFile(ResponseItem file, string outputFile, CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(file.Md5))
{
File.WriteAllText(outputFile + ".md5", file.Md5);
}
try
{
using var outputStream = File.OpenWrite(outputFile);
using var request = await this.client.GetStreamAsync(file.ZipUrl, cancellationToken).ConfigureAwait(false);
await request.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false);
}
catch (Exception downloadEx)
{
this.logger.LogError(downloadEx, "Failed to download file \"{uri}\" to \"{outputFile}\".", file.ZipUrl, outputFile);
try
{
File.Delete(outputFile);
}
catch (Exception removeEx)
{
this.logger.LogError(removeEx, $"Failed to remove incomplete file \"{outputFile}\".");
}
}
this.logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\".");
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public async Task StopAsync(CancellationToken cancellationToken) { }
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
}

410
src/DonloaderService.cs
View File

@ -1,410 +0,0 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace EpicMorg.Atlassian.Downloader
{
class DonloaderService : IHostedService
{
private readonly string UserAgentString = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0";
private readonly ILogger<DonloaderService> logger;
private readonly DownloaderOptions options;
private readonly HttpClient client;
private readonly IHostApplicationLifetime hostApplicationLifetime;
private readonly string assemblyEnvironment = string.Format("[{1}, {0}]",
System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(),
System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription);
private readonly string assemblyVersion = Assembly.GetEntryAssembly().GetCustomAttribute<AssemblyInformationalVersionAttribute>().InformationalVersion;
private readonly string fileVersion = Assembly.GetEntryAssembly().GetCustomAttribute<AssemblyFileVersionAttribute>().Version;
private readonly string assemblyName = Assembly.GetEntryAssembly().GetCustomAttribute<AssemblyProductAttribute>().Product;
const string assemblyBuildType =
#if DEBUG
"[Debug]"
#else
"[Release]"
#endif
;
public DonloaderService(IHostApplicationLifetime hostApplicationLifetime, ILogger<DonloaderService> logger, HttpClient client, DownloaderOptions options)
{
this.logger = logger;
this.client = client;
client.DefaultRequestHeaders.Add("User-Agent", UserAgentString);
this.options = options;
this.hostApplicationLifetime = hostApplicationLifetime;
}
public const ConsoleColor DEFAULT = ConsoleColor.Blue;
public static void WriteColorLine(string text, params object[] args)
{
Dictionary<char, ConsoleColor> colors = new()
{
{ '!', ConsoleColor.Red },
{ '@', ConsoleColor.Green },
{ '#', ConsoleColor.Blue },
{ '$', ConsoleColor.Magenta },
{ '&', ConsoleColor.Yellow },
{ '%', ConsoleColor.Cyan }
};
// TODO: word wrap, backslash escapes
text = string.Format(text, args);
string chunk = "";
bool paren = false;
for (int i = 0; i < text.Length; i++)
{
char c = text[i];
if (colors.ContainsKey(c) && StringNext(text, i) != ' ')
{
Console.Write(chunk);
chunk = "";
if (StringNext(text, i) == '(')
{
i++; // skip past the paren
paren = true;
}
Console.ForegroundColor = colors[c];
}
else if (paren && c == ')')
{
paren = false;
Console.ForegroundColor = DEFAULT;
}
else if (Console.ForegroundColor != DEFAULT)
{
Console.Write(c);
if (c == ' ' && !paren)
Console.ForegroundColor = DEFAULT;
}
else
chunk += c;
}
Console.WriteLine(chunk);
Console.ForegroundColor = DEFAULT;
}
public static char StringPrev(string text, int index)
{
return (index == 0 || text.Length == 0) ? '\0' : text[index - 1];
}
public static char StringNext(string text, int index)
{
return (index < text.Length) ? text[index + 1] : '\0';
}
public async Task StartAsync(CancellationToken cancellationToken)
{
SetConsoleTitle();
if (options.Version || string.IsNullOrWhiteSpace(options.OutputDir))
{
ShowVersionInfo();
}
else
{
var feedUrls = this.GetFeedUrls();
ShowVersionInfo();
logger.LogInformation($"Task started");
foreach (var feedUrl in feedUrls)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var (json, versions) = await this.GetJson(feedUrl, options.ProductVersion, cancellationToken).ConfigureAwait(false);
switch (options.Action)
{
case DownloadAction.ShowRawJson:
Console.Out.WriteLine(json);
break;
case DownloadAction.Download:
await this.DownloadFilesFromFeed(feedUrl, versions, cancellationToken).ConfigureAwait(false);
break;
case DownloadAction.ListURLs:
foreach (var versionProg in versions)
{
foreach (var file in versionProg.Value)
{
Console.Out.WriteLine(file.ZipUrl);
}
}
break;
case DownloadAction.ListVersions:
foreach (var versionProg in versions)
{
foreach (var file in versionProg.Value)
{
Console.Out.WriteLine(file.Version);
}
}
break;
}
}
}
logger.LogInformation($"Complete");
this.hostApplicationLifetime.StopApplication();
}
private void ShowVersionInfo()
{
logger.LogInformation($"{assemblyName} {assemblyVersion} {assemblyEnvironment} {assemblyBuildType}");
Console.BackgroundColor = ConsoleColor.Black;
WriteColorLine("%╔═╦═══════════════════════════════════════════════════════════════════════════════════════╦═╗");
WriteColorLine("%╠═╝ .''. %╚═%╣");
WriteColorLine("%║ .:cc;. %║");
WriteColorLine("%║ .;cccc;. %║");
WriteColorLine("%║ .;cccccc;. !╔══════════════════════════════════════════════╗ %║");
WriteColorLine("%║ .:ccccccc;. !║ " + assemblyName + " !║ %║");
WriteColorLine("%║ 'ccccccccc;. !╠══════════════════════════════════════════════╣ %║");
WriteColorLine("%║ ,cccccccccc;. !║ &Code: @kasthack !║ %║");
WriteColorLine("%║ ,ccccccccccc;. !║ &GFX: @stam !║ %║");
WriteColorLine("%║ .... .:ccccccccccc;. !╠══════════════════════════════════════════════╣ %║");
WriteColorLine("%║ .',,'..;cccccccccccc;. !║ &Version: " + fileVersion + " !║ %║");
WriteColorLine("%║ .,,,,,'.';cccccccccccc;. !║ &GitHub: $EpicMorg/atlassian-downloader !║ %║");
WriteColorLine("%║ .,;;;;;,'.':cccccccccccc;. !╚══════════════════════════════════════════════╝ %║");
WriteColorLine("%║ .;:;;;;;;,...:cccccccccccc;. %║");
WriteColorLine("%║ .;:::::;;;;'. .;:ccccccccccc;. %║");
WriteColorLine("%║ .:cc::::::::,. ..:ccccccccccc;. %║");
WriteColorLine("%║ .:cccccc:::::' .:ccccccccccc;. %║");
WriteColorLine("%║ .;:::::::::::,. .;:::::::::::,. %║");
WriteColorLine("%╠═╗ ............ ............ %╔═╣");
WriteColorLine("%╚═╩═══════════════════════════════════════════════════════════════════════════════════════╩═╝");
Console.ResetColor();
}
private async Task<(string json, IDictionary<string, ResponseItem[]> versions)> GetJson(string feedUrl, string? productVersion = null, CancellationToken cancellationToken = default)
{
var atlassianJson = await client.GetStringAsync(feedUrl, cancellationToken).ConfigureAwait(false);
var json = atlassianJson.Trim()["downloads(".Length..^1];
logger.LogTrace("Downloaded json: {0}", json);
var parsed = JsonSerializer.Deserialize<ResponseItem[]>(json, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
});
logger.LogDebug("Found {0} releases", parsed.Length);
var versions = parsed
.GroupBy(a => a.Version)
.Where(a => productVersion is null || a.Key == productVersion)
.ToDictionary(a => a.Key, a => a.ToArray());
logger.LogDebug("Found {0} releases", versions.Count);
return (json, versions);
}
private string[] GetFeedUrls() => options.CustomFeed != null
? options.CustomFeed.Select(a => a.ToString()).ToArray()
: new[] {
//official links
"https://my.atlassian.com/download/feeds/archived/bamboo.json",
"https://my.atlassian.com/download/feeds/archived/clover.json",
"https://my.atlassian.com/download/feeds/archived/confluence.json",
"https://my.atlassian.com/download/feeds/archived/crowd.json",
"https://my.atlassian.com/download/feeds/archived/crucible.json",
"https://my.atlassian.com/download/feeds/archived/fisheye.json",
"https://my.atlassian.com/download/feeds/archived/jira-core.json",
"https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/archived/jira-software.json",
"https://my.atlassian.com/download/feeds/archived/jira.json",
"https://my.atlassian.com/download/feeds/archived/stash.json",
"https://my.atlassian.com/download/feeds/archived/mesh.json",
//cdn mirror of official links
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/bamboo.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/clover.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/confluence.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/crowd.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/crucible.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/fisheye.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira-core.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira-servicedesk.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira-software.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/stash.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/mesh.json",
//official links
"https://my.atlassian.com/download/feeds/current/bamboo.json",
"https://my.atlassian.com/download/feeds/current/clover.json",
"https://my.atlassian.com/download/feeds/current/confluence.json",
"https://my.atlassian.com/download/feeds/current/crowd.json",
"https://my.atlassian.com/download/feeds/current/crucible.json",
"https://my.atlassian.com/download/feeds/current/fisheye.json",
"https://my.atlassian.com/download/feeds/current/jira-core.json",
"https://my.atlassian.com/download/feeds/current/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/current/jira-software.json",
"https://my.atlassian.com/download/feeds/current/stash.json",
"https://my.atlassian.com/download/feeds/current/mesh.json",
//cdn mirror of official links
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/bamboo.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/clover.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/confluence.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/crowd.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/crucible.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/fisheye.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/jira-core.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/jira-servicedesk.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/jira-software.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/stash.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/mesh.json",
//official links
"https://my.atlassian.com/download/feeds/eap/bamboo.json",
"https://my.atlassian.com/download/feeds/eap/confluence.json",
"https://my.atlassian.com/download/feeds/eap/jira.json",
"https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/eap/stash.json",
//"https://my.atlassian.com/download/feeds/eap/mesh.json", //404
//cdn mirror of official links
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/bamboo.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/confluence.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/jira.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/jira-servicedesk.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/stash.json",
//"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/mesh.json", //404
//https://raw.githubusercontent.com/EpicMorg/atlassian-json/master/json-backups/archived/sourcetree.json //unstable link with r\l
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/sourcetree.json",
//https://raw.githubusercontent.com/EpicMorg/atlassian-json/master/json-backups/current/sourcetree.json //unstable link with r\l
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/sourcetree.json"
};
private void SetConsoleTitle()
{
Console.Title = $@"{assemblyName} {assemblyVersion} {assemblyEnvironment} - {assemblyBuildType}";
}
private async Task DownloadFilesFromFeed(string feedUrl, IDictionary<string, ResponseItem[]> versions, CancellationToken cancellationToken)
{
var feedDir = Path.Combine(options.OutputDir, feedUrl[(feedUrl.LastIndexOf('/') + 1)..(feedUrl.LastIndexOf('.'))]);
logger.LogInformation($"Download from JSON \"{feedUrl}\" started");
foreach (var version in versions)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var directory = Path.Combine(feedDir, version.Key);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
foreach (var file in version.Value)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (file.ZipUrl == null)
{
logger.LogWarning($"Empty ZipUrl found for version '{version.Key}' in {feedUrl}");
continue;
}
var serverPath = file.ZipUrl.PathAndQuery;
var outputFile = Path.Combine(directory, serverPath[(serverPath.LastIndexOf("/") + 1)..]);
if (!File.Exists(outputFile))
{
await DownloadFile(file, outputFile, cancellationToken).ConfigureAwait(false);
}
else
{
logger.LogWarning($"File \"{outputFile}\" already exists. File sizes will be compared.");
long localFileSize = new System.IO.FileInfo(outputFile).Length;
logger.LogInformation($"Size of local file is {localFileSize} bytes.");
try
{
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgentString);
HttpResponseMessage response = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Head, file.ZipUrl));
if (response.IsSuccessStatusCode)
{
if (response.Content.Headers.ContentLength.HasValue)
{
long remoteFileSize = response.Content.Headers.ContentLength.Value;
logger.LogInformation($"Size of remote file is \"{remoteFileSize}\" bytes.");
if (remoteFileSize == localFileSize) {
logger.LogInformation($"Size of remote and local files and are same ({remoteFileSize} bytes and {localFileSize} bytes). Nothing to download. Operation skipped.");
}
else
{
logger.LogWarning($"Size of remote and local files and are not same ({remoteFileSize} bytes and {localFileSize} bytes). Download started.");
File.Delete(outputFile);
await DownloadFile(file, outputFile, cancellationToken).ConfigureAwait(false);
}
}
else
{
logger.LogWarning($"Cant get size of remote file \"{file.ZipUrl}\". May be server not support it feature. Sorry.");
continue;
}
}
else
{
logger.LogCritical($"Request execution error: \"{response.StatusCode}\". Sorry.");
}
}
catch (HttpRequestException ex)
{
logger.LogCritical($"HTTP request error: \"{ex.Message}\", \"{ex.StackTrace}\", \"{ex.StatusCode}\". Sorry.");
}
}
}
}
logger.LogInformation($"All files from \"{feedUrl}\" successfully downloaded.");
}
private async Task DownloadFile(ResponseItem file, string outputFile, CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(file.Md5))
{
File.WriteAllText(outputFile + ".md5", file.Md5);
}
try
{
using var outputStream = File.OpenWrite(outputFile);
using var request = await client.GetStreamAsync(file.ZipUrl, cancellationToken).ConfigureAwait(false);
await request.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false);
}
catch (Exception downloadEx)
{
logger.LogError(downloadEx, $"Failed to download file \"{file.ZipUrl}\" to \"{outputFile}\".");
try
{
File.Delete(outputFile);
}
catch (Exception removeEx)
{
logger.LogError(removeEx, $"Failed to remove incomplete file \"{outputFile}\".");
}
}
logger.LogInformation($"File \"{file.ZipUrl}\" successfully downloaded to \"{outputFile}\".");
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public async Task StopAsync(CancellationToken cancellationToken) { }
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
}
}

41
src/Models/DownloadAction.cs
View File

@ -1,22 +1,21 @@
namespace EpicMorg.Atlassian.Downloader
{
public enum DownloadAction
{
/// <summary>
/// Download application files
/// </summary>
Download,
/// <summary>
/// Print download URLs and exit
/// </summary>
ListURLs,
/// <summary>
/// Print available application versions and exit
/// </summary>
ListVersions,
/// <summary>
/// Print feed JSONs to stdout and exit
/// </summary>
ShowRawJson,
}
namespace EpicMorg.Atlassian.Downloader;
public enum DownloadAction
{
/// <summary>
/// Download application files
/// </summary>
Download,
/// <summary>
/// Print download URLs and exit
/// </summary>
ListURLs,
/// <summary>
/// Print available application versions and exit
/// </summary>
ListVersions,
/// <summary>
/// Print feed JSONs to stdout and exit
/// </summary>
ShowRawJson,
}

8
src/Models/DownloaderOptions.cs
View File

@ -1,6 +1,4 @@
using System;
namespace EpicMorg.Atlassian.Downloader;
using System;
namespace EpicMorg.Atlassian.Downloader
{
public record DownloaderOptions(string OutputDir, Uri[] CustomFeed, DownloadAction Action,bool Version, string ProductVersion) { }
}
public record DownloaderOptions(string OutputDir, Uri[] CustomFeed, DownloadAction Action,bool Version, string ProductVersion) { }

40
src/Models/ResponseItem.cs
View File

@ -1,31 +1,19 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace EpicMorg.Atlassian.Downloader;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace EpicMorg.Atlassian.Downloader
public partial class ResponseItem
{
public partial class ResponseItem
{
public string Description { get; set; }
public string Edition { get; set; }
public Uri ZipUrl { get; set; }
public object TarUrl { get; set; }
public string Md5 { get; set; }
public string Size { get; set; }
public string Released { get; set; }
public string Type { get; set; }
public string Platform { get; set; }
public string Version { get; set; }
public Uri ReleaseNotes { get; set; }
public Uri UpgradeNotes { get; set; }
}
public string Description { get; set; }
public string Edition { get; set; }
public Uri ZipUrl { get; set; }
public object TarUrl { get; set; }
public string Md5 { get; set; }
public string Size { get; set; }
public string Released { get; set; }
public string Type { get; set; }
public string Platform { get; set; }
public string Version { get; set; }
public Uri ReleaseNotes { get; set; }
public Uri UpgradeNotes { get; set; }
}

85
src/Models/SourceInformation.cs Normal file
View File

@ -0,0 +1,85 @@
namespace EpicMorg.Atlassian.Downloader.Models;
using System.Collections.Generic;
internal static class SourceInformation
{
public static IReadOnlyList<string> AtlassianSources { get; } = new[] {
//official links
"https://my.atlassian.com/download/feeds/archived/bamboo.json",
"https://my.atlassian.com/download/feeds/archived/clover.json",
"https://my.atlassian.com/download/feeds/archived/confluence.json",
"https://my.atlassian.com/download/feeds/archived/crowd.json",
"https://my.atlassian.com/download/feeds/archived/crucible.json",
"https://my.atlassian.com/download/feeds/archived/fisheye.json",
"https://my.atlassian.com/download/feeds/archived/jira-core.json",
"https://my.atlassian.com/download/feeds/archived/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/archived/jira-software.json",
"https://my.atlassian.com/download/feeds/archived/jira.json",
"https://my.atlassian.com/download/feeds/archived/stash.json",
"https://my.atlassian.com/download/feeds/archived/mesh.json",
//cdn mirror of official links
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/bamboo.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/clover.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/confluence.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/crowd.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/crucible.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/fisheye.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira-core.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira-servicedesk.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira-software.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/jira.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/stash.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/mesh.json",
//official links
"https://my.atlassian.com/download/feeds/current/bamboo.json",
"https://my.atlassian.com/download/feeds/current/clover.json",
"https://my.atlassian.com/download/feeds/current/confluence.json",
"https://my.atlassian.com/download/feeds/current/crowd.json",
"https://my.atlassian.com/download/feeds/current/crucible.json",
"https://my.atlassian.com/download/feeds/current/fisheye.json",
"https://my.atlassian.com/download/feeds/current/jira-core.json",
"https://my.atlassian.com/download/feeds/current/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/current/jira-software.json",
"https://my.atlassian.com/download/feeds/current/stash.json",
"https://my.atlassian.com/download/feeds/current/mesh.json",
//cdn mirror of official links
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/bamboo.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/clover.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/confluence.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/crowd.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/crucible.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/fisheye.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/jira-core.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/jira-servicedesk.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/jira-software.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/stash.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/mesh.json",
//official links
"https://my.atlassian.com/download/feeds/eap/bamboo.json",
"https://my.atlassian.com/download/feeds/eap/confluence.json",
"https://my.atlassian.com/download/feeds/eap/jira.json",
"https://my.atlassian.com/download/feeds/eap/jira-servicedesk.json",
"https://my.atlassian.com/download/feeds/eap/stash.json",
//"https://my.atlassian.com/download/feeds/eap/mesh.json", //404
//cdn mirror of official links
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/bamboo.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/confluence.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/jira.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/jira-servicedesk.json",
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/stash.json",
//"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/eap/mesh.json", //404
//https://raw.githubusercontent.com/EpicMorg/atlassian-json/master/json-backups/archived/sourcetree.json //unstable link with r\l
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/archived/sourcetree.json",
//https://raw.githubusercontent.com/EpicMorg/atlassian-json/master/json-backups/current/sourcetree.json //unstable link with r\l
"https://raw.githack.com/EpicMorg/atlassian-json/master/json-backups/current/sourcetree.json"
};
}

81
src/Program.cs
View File

@ -1,4 +1,8 @@
using Microsoft.Extensions.Configuration;
namespace EpicMorg.Atlassian.Downloader;
using EpicMorg.Atlassian.Downloader.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@ -6,47 +10,42 @@ using Microsoft.Extensions.Logging;
using Serilog;
using System;
using System.IO;
using System.Reflection;
using System.Threading.Tasks;
namespace EpicMorg.Atlassian.Downloader
public class Program
{
public class Program
{
/// <summary>
/// Atlassian archive downloader. See https://github.com/EpicMorg/atlassian-downloader for more info
/// </summary>
/// <param name="Action">Action to perform</param>
/// <param name="OutputDir">Override output directory to download</param>
/// <param name="customFeed">Override URIs to import</param>
/// <param name="Version">Show credits banner</param>
/// <param name="productVersion">Override target version to download some product. Advice: Use it with "customFeed".</param>
static async Task Main(string OutputDir, Uri[] customFeed = null, DownloadAction Action = DownloadAction.Download, bool Version = false, string productVersion = null) => await
Host
.CreateDefaultBuilder()
.ConfigureHostConfiguration(configHost => configHost.AddEnvironmentVariables())
.ConfigureAppConfiguration((ctx, configuration) =>
configuration
.SetBasePath(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location))
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{ctx.HostingEnvironment.EnvironmentName}.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables())
.ConfigureServices((ctx, services) => services
.AddOptions()
.AddLogging(builder =>
{
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(ctx.Configuration)
.CreateLogger();
builder
.ClearProviders()
.AddSerilog(dispose: true);
})
.AddHostedService<DonloaderService>()
.AddSingleton(new DownloaderOptions(OutputDir, customFeed, Action, Version, productVersion))
.AddHttpClient())
.RunConsoleAsync()
.ConfigureAwait(false);
}
/// <summary>
/// Atlassian archive downloader. See https://github.com/EpicMorg/atlassian-downloader for more info
/// </summary>
/// <param name="action">Action to perform</param>
/// <param name="outputDir">Override output directory to download</param>
/// <param name="customFeed">Override URIs to import</param>
/// <param name="about">Show credits banner</param>
/// <param name="productVersion">Override target version to download some product. Advice: Use it with "customFeed".</param>
static async Task Main(string outputDir, Uri[] customFeed = null, DownloadAction action = DownloadAction.Download, bool about = false, string productVersion = null) => await
Host
.CreateDefaultBuilder()
.ConfigureHostConfiguration(configHost => configHost.AddEnvironmentVariables())
.ConfigureAppConfiguration((ctx, configuration) =>
configuration
.SetBasePath(System.AppContext.BaseDirectory)
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{ctx.HostingEnvironment.EnvironmentName}.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables())
.ConfigureServices((ctx, services) => services
.AddOptions()
.AddLogging(builder =>
{
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(ctx.Configuration)
.CreateLogger();
_ = builder
.ClearProviders()
.AddSerilog(dispose: true);
})
.AddHostedService<DownloaderService>()
.AddSingleton(new DownloaderOptions(outputDir, customFeed, action, about, productVersion))
.AddHttpClient())
.RunConsoleAsync()
.ConfigureAwait(false);
}

3
src/Properties/launchSettings.json
View File

@ -2,7 +2,8 @@
"profiles": {
"atlassian-downloader": {
"commandName": "Project",
"commandLineArgs": "--help"
//"commandLineArgs": "--help"
"commandLineArgs": "--action=ShowRawJson --output-dir=F:\\temp\\atlassian\\test1"
}
}
}

28
src/atlassian-downloader.csproj
View File

@ -1,14 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<PublishSingleFile>true</PublishSingleFile>
<!--publish config-->
<PublishSingleFile>false</PublishSingleFile>
<PublishTrimmed>false</PublishTrimmed>
<PublishAot>false</PublishAot>
<SelfContained>true</SelfContained>
<PublishReadyToRun>false</PublishReadyToRun>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<PublishTrimmed>true</PublishTrimmed>
<PublishReadyToRun>true</PublishReadyToRun>
<!--build props-->
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
<ApplicationIcon>favicon.ico</ApplicationIcon>
<!--package metadata-->
<PackageId>EpicMorg.Atlassian.Downloader</PackageId>
<Authors>EpicMorg, kasthack, stam</Authors>
<Description>Atlassian Downloader by EpicMorg</Description>
@ -22,11 +30,16 @@
<Version>1.1.0.0</Version>
<Copyright>EpicMorg 2023</Copyright>
<Product>Atlassian Downloader</Product>
<IsPackable>true</IsPackable>
<Company>EpicMorg</Company>
<PackageReadmeFile>README.md</PackageReadmeFile>
<RootNamespace>EpicMorg.Atlassian.Downloader</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Serilog.Extensions.Logging" Version="7.0.1-dev-10354" />
<PackageReference Include="Serilog.Settings.Configuration" Version="7.0.2-dev-00546" />
<PackageReference Include="Serilog.Sinks.Console" Version="4.2.0-dev-00918" />
@ -41,11 +54,6 @@
<PackagePath>
</PackagePath>
</None>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0-rc.2.23479.6" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0-rc.2.23479.6" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">