diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs new file mode 100644 index 000000000..5bad549ef --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs @@ -0,0 +1,50 @@ +using System; +using Calamari.Common.Plumbing.Logging; +using Octopus.Versioning; + +namespace Calamari.Integration.Packages.Download.Oci +{ + public class OciArtifactManifestRetriever + { + readonly OciRegistryClient ociRegistryClient; + readonly ILog log; + + public OciArtifactManifestRetriever(OciRegistryClient ociRegistryClient, ILog log) + { + this.ociRegistryClient = ociRegistryClient; + this.log = log; + } + + public OciArtifactTypes TryGetArtifactType( + string packageId, + IVersion version, + Uri feedUri, + string? feedUsername, + string? feedPassword) + { + try + { + var jsonManifest = ociRegistryClient.GetManifest(feedUri, packageId, version, feedUsername, feedPassword); + + // Check for Helm chart annotations + var isHelmChart = + jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.OciImageMediaTypeValue) + || jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.HelmChartMediaTypeValue); + + if (isHelmChart) + return OciArtifactTypes.HelmChart; + + var isDockerImage = jsonManifest.HasMediaTypeContaining(OciConstants.Manifest.DockerImageMediaTypeValue) + || jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.DockerImageMediaTypeValue) + || jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.DockerImageMediaTypeValue); + + return isDockerImage ? OciArtifactTypes.DockerImage : OciArtifactTypes.Unknown; + } + catch (Exception ex) + { + log.ErrorFormat("Failed to get artifact type: {Message}", ex.Message); + return OciArtifactTypes.Unknown; + } + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs new file mode 100644 index 000000000..fcddff8d7 --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs @@ -0,0 +1,37 @@ +using System; + +namespace Calamari.Integration.Packages.Download.Oci +{ + public static class OciConstants + { + public class Manifest + { + internal const string MediaTypePropertyName = "mediaType"; + internal const string DockerImageMediaTypeValue = "application/vnd.docker.distribution.manifest.v2+json"; + + public class Config + { + internal const string PropertyName = "config"; + internal const string MediaTypePropertyName = "mediaType"; + internal const string OciImageMediaTypeValue = "application/vnd.oci.image.config.v1+json"; + internal const string DockerImageMediaTypeValue = "application/vnd.docker.container.image.v1+json"; + } + + public class Image + { + internal const string TitleAnnotationKey = "org.opencontainers.image.title"; + } + + public class Layers + { + internal const string PropertyName = "layers"; + internal const string DigestPropertyName = "digest"; + internal const string SizePropertyName = "size"; + internal const string MediaTypePropertyName = "mediaType"; + internal const string AnnotationsPropertyName = "annotations"; + internal const string HelmChartMediaTypeValue = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v370 + internal const string DockerImageMediaTypeValue = "application/vnd.docker.image.rootfs.diff.tar.gzip"; + } + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs new file mode 100644 index 000000000..9ded1669b --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs @@ -0,0 +1,44 @@ +using System; +using Newtonsoft.Json.Linq; + +namespace Calamari.Integration.Packages.Download.Oci +{ + static class OciJObjectExtensions + { + public static bool HasMediaTypeContaining(this JObject manifest, string value) + { + var mediaType = manifest[OciConstants.Manifest.MediaTypePropertyName]; + + return mediaType != null + && mediaType.ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; + } + + public static bool HasConfigMediaTypeContaining(this JObject manifest, string value) + { + var config = manifest[OciConstants.Manifest.Config.PropertyName]; + + return config is { Type: JTokenType.Object } + && config[OciConstants.Manifest.Config.MediaTypePropertyName] != null + && config[OciConstants.Manifest.Config.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; + } + + public static bool HasLayersMediaTypeContaining(this JObject manifest, string value) + { + var layers = manifest[OciConstants.Manifest.Layers.PropertyName]; + + if (layers is { Type: JTokenType.Array }) + { + foreach (var layer in layers) + { + if (layer[OciConstants.Manifest.Layers.MediaTypePropertyName] != null + && layer[OciConstants.Manifest.Layers.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + } + + return false; + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciRegistryClient.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciRegistryClient.cs new file mode 100644 index 000000000..816eee668 --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciRegistryClient.cs @@ -0,0 +1,232 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.RegularExpressions; +using Calamari.Common.Commands; +using Calamari.Common.Plumbing.FileSystem; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Octopus.Versioning; + +namespace Calamari.Integration.Packages.Download.Oci +{ + public class OciRegistryClient + { + readonly HttpClient httpClient = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); + readonly ICalamariFileSystem fileSystem; + + public OciRegistryClient(ICalamariFileSystem fileSystem) + { + this.fileSystem = fileSystem; + } + + public JObject? GetManifest(Uri feedUri, string packageId, IVersion version, string? feedUsername, string? feedPassword) + { + var url = GetApiUri(feedUri); + var fixedVersion = FixVersion(version); + using var response = Get(new Uri($"{url}/{packageId}/manifests/{fixedVersion}"), new NetworkCredential(feedUsername, feedPassword), ApplyAcceptHeaderFunc); + return JsonConvert.DeserializeObject(response.Content.ReadAsStringAsync().Result); + + void ApplyAcceptHeaderFunc(HttpRequestMessage request) => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciConstants.Manifest.Config.OciImageMediaTypeValue)); + } + + public void DownloadPackage( + Uri feedUri, + string packageId, + string digest, + string? feedUsername, + string? feedPassword, + string downloadPath) + { + var url = GetApiUri(feedUri); + using var fileStream = fileSystem.OpenFile(downloadPath, FileAccess.Write); + using var response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); + + if (!response.IsSuccessStatusCode) + { + throw new CommandException($"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); + } + + response.Content.CopyToAsync(fileStream).GetAwaiter().GetResult(); + } + + HttpResponseMessage Get(Uri url, ICredentials credentials, Action? customAcceptHeader = null) + { + var request = new HttpRequestMessage(HttpMethod.Get, url); + try + { + var networkCredential = credentials.GetCredential(url, "Basic"); + + if (!string.IsNullOrWhiteSpace(networkCredential?.UserName) || !string.IsNullOrWhiteSpace(networkCredential?.Password)) + { + request.Headers.Authorization = CreateAuthenticationHeader(networkCredential); + } + + customAcceptHeader?.Invoke(request); + var response = SendRequest(httpClient, request); + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + var tokenFromAuthService = GetAuthRequestHeader(response, networkCredential); + request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Authorization = tokenFromAuthService; + customAcceptHeader?.Invoke(request); + response = SendRequest(httpClient, request); + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + throw new CommandException($"Authorization to `{url}` failed."); + } + } + + if (response.StatusCode == HttpStatusCode.NotFound) + { + // Some registries do not support the Docker HTTP APIs + // For example GitHub: https://github.community/t/ghcr-io-docker-http-api/130121 + throw new CommandException($"Docker registry located at `{url}` does not support this action."); + } + + if (!response.IsSuccessStatusCode) + { + var errorMessage = $"Request to Docker registry located at `{url}` failed with {response.StatusCode}:{response.ReasonPhrase}."; + + var responseBody = GetContent(response); + if (!string.IsNullOrWhiteSpace(responseBody)) errorMessage += $" {responseBody}"; + + throw new CommandException(errorMessage); + } + + return response; + } + finally + { + request.Dispose(); + } + } + + AuthenticationHeaderValue GetAuthRequestHeader(HttpResponseMessage response, NetworkCredential credential) + { + var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer"); + if (auth != null) + { + var authToken = RetrieveAuthenticationToken(auth, credential); + return new AuthenticationHeaderValue("Bearer", authToken); + } + + if (response.Headers.WwwAuthenticate.Any(a => a.Scheme == "Basic")) + { + return CreateAuthenticationHeader(credential); + } + + throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`"); + } + + string RetrieveAuthenticationToken(AuthenticationHeaderValue auth, NetworkCredential credential) + { + HttpResponseMessage? response = null; + + try + { + var authUrl = GetOAuthServiceUrl(auth); + using (var msg = new HttpRequestMessage(HttpMethod.Get, authUrl)) + { + if (credential.UserName != null) + { + msg.Headers.Authorization = CreateAuthenticationHeader(credential); + } + + response = SendRequest(httpClient, msg); + } + + if (response.IsSuccessStatusCode) + { + return ExtractTokenFromResponse(response); + } + } + finally + { + response?.Dispose(); + } + + throw new CommandException("Unable to retrieve authentication token required to perform operation."); + } + + static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) + { + var details = auth.Parameter.Split(',').ToDictionary(x => x.Substring(0, x.IndexOf('=')), y => y.Substring(y.IndexOf('=') + 1, y.Length - y.IndexOf('=') - 1).Trim('"')); + var oathUrl = new UriBuilder(details["realm"]); + var queryStringValues = new Dictionary(); + if (details.TryGetValue("service", out var service)) + { + var encodedService = WebUtility.UrlEncode(service); + queryStringValues.Add("service", encodedService); + } + + if (details.TryGetValue("scope", out var scope)) + { + var encodedScope = WebUtility.UrlEncode(scope); + queryStringValues.Add("scope", encodedScope); + } + + oathUrl.Query = "?" + string.Join("&", queryStringValues.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray()); + return oathUrl.ToString(); + } + + static string ExtractTokenFromResponse(HttpResponseMessage response) + { + var token = GetContent(response); + return (string)JObject.Parse(token).SelectToken("token") + ?? throw new CommandException("Unable to retrieve authentication token required to perform operation."); + } + + static Uri GetApiUri(Uri feedUri) + { + var httpScheme = IsPlainHttp(feedUri) ? Uri.UriSchemeHttp : Uri.UriSchemeHttps; + + var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); + var uri = new Uri(r); + + const string versionPath = "v2"; + if (!r.EndsWith("/" + versionPath)) + { + uri = new Uri(uri, versionPath); + } + + return uri; + + static bool IsPlainHttp(Uri uri) + => uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase); + } + + // oci registries don't support the '+' tagging + // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380 + static string FixVersion(IVersion version) + => version.ToString().Replace("+", "_"); + + static readonly Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); + + internal static string? GetPackageHashFromDigest(string digest) + => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; + + static AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential) + { + var byteArray = Encoding.ASCII.GetBytes($"{credential.UserName}:{credential.Password}"); + return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); + } + + static HttpResponseMessage SendRequest(HttpClient client, HttpRequestMessage request) + { + return client.SendAsync(request).GetAwaiter().GetResult(); + } + + static string? GetContent(HttpResponseMessage response) + { + return response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs new file mode 100644 index 000000000..cc4bf4246 --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs @@ -0,0 +1,64 @@ +using System; +using Calamari.Common.Features.Packages; +using Calamari.Common.Plumbing.Logging; +using Calamari.Integration.Packages.Download.Oci; +using Octopus.Versioning; + +namespace Calamari.Integration.Packages.Download +{ + public class OciOrDockerImagePackageDownloader : IPackageDownloader + { + readonly OciPackageDownloader ociPackageDownloader; + readonly DockerImagePackageDownloader dockerImagePackageDownloader; + readonly OciRegistryClient ociRegistryClient; + readonly ILog log; + + public OciOrDockerImagePackageDownloader( + OciPackageDownloader ociPackageDownloader, + DockerImagePackageDownloader dockerImagePackageDownloader, + OciRegistryClient ociRegistryClient, + ILog log) + { + this.ociPackageDownloader = ociPackageDownloader; + this.dockerImagePackageDownloader = dockerImagePackageDownloader; + this.log = log; + this.ociRegistryClient = ociRegistryClient; + } + + public PackagePhysicalFileMetadata DownloadPackage( + string packageId, + IVersion version, + string feedId, + Uri feedUri, + string? feedUsername, + string? feedPassword, + bool forcePackageDownload, + int maxDownloadAttempts, + TimeSpan downloadAttemptBackoff) + { + var downloader = GetInnerDownloader(packageId, version, feedUri, feedUsername, feedPassword); + + return downloader.DownloadPackage( + packageId, + version, + feedId, + feedUri, + feedUsername, + feedPassword, + forcePackageDownload, + maxDownloadAttempts, + downloadAttemptBackoff); + } + + IPackageDownloader GetInnerDownloader(string packageId, IVersion version, Uri feedUri, string? feedUsername, string? feedPassword) + { + var ociArtifactManifestRetriever = new OciArtifactManifestRetriever(ociRegistryClient, log); + if (ociArtifactManifestRetriever.TryGetArtifactType(packageId, version, feedUri, feedUsername, feedPassword) == OciArtifactTypes.HelmChart) + { + return ociPackageDownloader; + } + + return dockerImagePackageDownloader; + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs index f72b6368f..cf7eb5677 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs @@ -1,53 +1,45 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Text.RegularExpressions; -using System.Web; using Calamari.Common.Commands; using Calamari.Common.Features.Packages; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; -using Newtonsoft.Json; +using Calamari.Integration.Packages.Download.Oci; using Newtonsoft.Json.Linq; using Octopus.Versioning; namespace Calamari.Integration.Packages.Download { - public class OciPackageDownloader : IPackageDownloader + public enum OciArtifactTypes { - const string VersionPath = "v2"; - const string OciImageManifestAcceptHeader = "application/vnd.oci.image.manifest.v1+json"; - const string ManifestImageTitleAnnotationKey = "org.opencontainers.image.title"; - const string ManifestLayerPropertyName = "layers"; - const string ManifestLayerDigestPropertyName = "digest"; - const string ManifestLayerSizePropertyName = "size"; - const string ManifestLayerAnnotationsPropertyName = "annotations"; - const string ManifestLayerMediaTypePropertyName = "mediaType"; + DockerImage, + HelmChart, + Unknown + } - static Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); + public class OciPackageDownloader : IPackageDownloader + { static readonly IPackageDownloaderUtils PackageDownloaderUtils = new PackageDownloaderUtils(); readonly ICalamariFileSystem fileSystem; readonly ICombinedPackageExtractor combinedPackageExtractor; readonly ILog log; - readonly HttpClient client; + readonly OciRegistryClient ociRegistryClient; public OciPackageDownloader( ICalamariFileSystem fileSystem, ICombinedPackageExtractor combinedPackageExtractor, + OciRegistryClient ociRegistryClient, ILog log) { this.fileSystem = fileSystem; this.combinedPackageExtractor = combinedPackageExtractor; + this.ociRegistryClient = ociRegistryClient; this.log = log; - client = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); } - public PackagePhysicalFileMetadata DownloadPackage(string packageId, + public PackagePhysicalFileMetadata DownloadPackage( + string packageId, IVersion version, string feedId, Uri feedUri, @@ -86,23 +78,14 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, Directory.CreateDirectory(stagingDir); } - var versionString = FixVersion(version); - - var apiUrl = GetApiUri(feedUri); - var (digest, size, extension) = GetPackageDetails(apiUrl, packageId, versionString, feedUsername, feedPassword); - var hash = GetPackageHashFromDigest(digest); + var (digest, size, extension) = GetPackageDetails(feedUri, packageId, version, feedUsername, feedPassword); + var hash = OciRegistryClient.GetPackageHashFromDigest(digest); var cachedFileName = PackageName.ToCachedFileName(packageId, version, extension); var downloadPath = Path.Combine(Path.Combine(stagingDir, cachedFileName)); var retryStrategy = PackageDownloaderRetryUtils.CreateRetryStrategy(maxDownloadAttempts, downloadAttemptBackoff, log); - retryStrategy.Execute(() => DownloadPackage(apiUrl, - packageId, - digest, - feedUsername, - feedPassword, - downloadPath)); -; + retryStrategy.Execute(() => ociRegistryClient.DownloadPackage(feedUri, packageId, digest, feedUsername, feedPassword, downloadPath)); var localDownloadName = Path.Combine(cacheDirectory, cachedFileName); fileSystem.MoveFile(downloadPath, localDownloadName); @@ -113,32 +96,23 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, localDownloadName, hash, size) - : PackagePhysicalFileMetadata.Build(localDownloadName) + : PackagePhysicalFileMetadata.Build(localDownloadName) ?? throw new CommandException($"Unable to retrieve metadata for package {packageId}, version {version}"); } } - // oci registries don't support the '+' tagging - // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380 - static string FixVersion(IVersion version) - => version.ToString().Replace("+", "_"); - - static string? GetPackageHashFromDigest(string digest) - => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; - (string digest, int size, string extension) GetPackageDetails( - Uri url, + Uri feedUri, string packageId, - string version, - string? feedUserName, + IVersion version, + string? feedUserName, string? feedPassword) { - using var response = Get(new Uri($"{url}/{packageId}/manifests/{version}"), new NetworkCredential(feedUserName, feedPassword), ApplyAccept); - var manifest = JsonConvert.DeserializeObject(response.Content.ReadAsStringAsync().Result); + var manifest = ociRegistryClient.GetManifest(feedUri, packageId, version, feedUserName, feedPassword); - var layer = manifest.Value(ManifestLayerPropertyName)[0]; - var digest = layer.Value(ManifestLayerDigestPropertyName); - var size = layer.Value(ManifestLayerSizePropertyName); + var layer = manifest.Value(OciConstants.Manifest.Layers.PropertyName)[0]; + var digest = layer.Value(OciConstants.Manifest.Layers.DigestPropertyName); + var size = layer.Value(OciConstants.Manifest.Layers.SizePropertyName); var extension = GetExtensionFromManifest(layer); return (digest, size, extension); @@ -146,54 +120,14 @@ static string FixVersion(IVersion version) string GetExtensionFromManifest(JToken layer) { - var artifactTitle = layer.Value(ManifestLayerAnnotationsPropertyName)?[ManifestImageTitleAnnotationKey]?.Value() ?? ""; + var artifactTitle = layer.Value(OciConstants.Manifest.Layers.AnnotationsPropertyName)?[OciConstants.Manifest.Image.TitleAnnotationKey]?.Value() ?? ""; var extension = combinedPackageExtractor - .Extensions - .FirstOrDefault(ext => - Path.GetExtension(artifactTitle).Equals(ext, StringComparison.OrdinalIgnoreCase)); - - return extension ?? (layer.Value(ManifestLayerMediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); - } - - void DownloadPackage( - Uri url, - string packageId, - string digest, - string? feedUsername, - string? feedPassword, - string downloadPath) - { - using var fileStream = fileSystem.OpenFile(downloadPath, FileAccess.Write); - using var response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); - if (!response.IsSuccessStatusCode) - { - throw new CommandException( - $"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); - } - - response.Content.CopyToAsync(fileStream).GetAwaiter().GetResult(); - } - - static void ApplyAccept(HttpRequestMessage request) - => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciImageManifestAcceptHeader)); - - static Uri GetApiUri(Uri feedUri) - { - var httpScheme = BuildScheme(IsPlainHttp(feedUri)); - var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); - var uri = new Uri(r); - if (!r.EndsWith("/" + VersionPath)) - { - uri = new Uri(uri, VersionPath); - } + .Extensions + .FirstOrDefault( + ext => + Path.GetExtension(artifactTitle).Equals(ext, StringComparison.OrdinalIgnoreCase)); - return uri; - - static bool IsPlainHttp(Uri uri) - => uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase); - - static string BuildScheme(bool isPlainHttp) - => isPlainHttp ? Uri.UriSchemeHttp : Uri.UriSchemeHttps; + return extension ?? (layer.Value(OciConstants.Manifest.Layers.MediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); } PackagePhysicalFileMetadata? SourceFromCache(string packageId, IVersion version, string cacheDirectory) @@ -218,155 +152,5 @@ static string BuildScheme(bool isPlainHttp) return null; } - - HttpResponseMessage Get(Uri url, ICredentials credentials, Action? customAcceptHeader = null) - { - var request = new HttpRequestMessage(HttpMethod.Get, url); - try - { - var networkCredential = credentials.GetCredential(url, "Basic"); - - if (!string.IsNullOrWhiteSpace(networkCredential?.UserName) || !string.IsNullOrWhiteSpace(networkCredential?.Password)) - { - request.Headers.Authorization = CreateAuthenticationHeader(networkCredential); - } - - customAcceptHeader?.Invoke(request); - var response = SendRequest(request); - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - var tokenFromAuthService = GetAuthRequestHeader(response, networkCredential); - request = new HttpRequestMessage(HttpMethod.Get, url); - request.Headers.Authorization = tokenFromAuthService; - customAcceptHeader?.Invoke(request); - response = SendRequest(request); - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - throw new CommandException($"Authorization to `{url}` failed."); - } - } - - if (response.StatusCode == HttpStatusCode.NotFound) - { - // Some registries do not support the Docker HTTP APIs - // For example GitHub: https://github.community/t/ghcr-io-docker-http-api/130121 - throw new CommandException($"Docker registry located at `{url}` does not support this action."); - } - - if (!response.IsSuccessStatusCode) - { - var errorMessage = $"Request to Docker registry located at `{url}` failed with {response.StatusCode}:{response.ReasonPhrase}."; - - var responseBody = GetContent(response); - if (!string.IsNullOrWhiteSpace(responseBody)) errorMessage += $" {responseBody}"; - - throw new CommandException(errorMessage); - } - - return response; - } - finally - { - request.Dispose(); - } - } - - string RetrieveAuthenticationToken(string authUrl, NetworkCredential credential) - { - HttpResponseMessage? response = null; - - try - { - using (var msg = new HttpRequestMessage(HttpMethod.Get, authUrl)) - { - if (credential?.UserName != null) - { - msg.Headers.Authorization = CreateAuthenticationHeader(credential); - } - - response = SendRequest(msg); - } - - if (response.IsSuccessStatusCode) - { - return ExtractTokenFromResponse(response); - } - } - finally - { - response?.Dispose(); - } - - throw new CommandException("Unable to retrieve authentication token required to perform operation."); - } - - AuthenticationHeaderValue GetAuthRequestHeader(HttpResponseMessage response, NetworkCredential credential) - { - var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer"); - if (auth != null) - { - var authToken = RetrieveAuthenticationToken(GetOAuthServiceUrl(auth), credential); - return new AuthenticationHeaderValue("Bearer", authToken); - } - - if (response.Headers.WwwAuthenticate.Any(a => a.Scheme == "Basic")) - { - return CreateAuthenticationHeader(credential); - } - - throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`"); - } - - static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) - { - var details = auth.Parameter.Split(',').ToDictionary(x => x.Substring(0, x.IndexOf('=')), y => y.Substring(y.IndexOf('=') + 1, y.Length - y.IndexOf('=') - 1).Trim('"')); - var oathUrl = new UriBuilder(details["realm"]); - var queryStringValues = new Dictionary(); - if (details.TryGetValue("service", out var service)) - { - var encodedService = WebUtility.UrlEncode(service); - queryStringValues.Add("service", encodedService); - } - - if (details.TryGetValue("scope", out var scope)) - { - var encodedScope = WebUtility.UrlEncode(scope); - queryStringValues.Add("scope", encodedScope); - } - - oathUrl.Query = "?" + string.Join("&", queryStringValues.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray()); - return oathUrl.ToString(); - } - - static string ExtractTokenFromResponse(HttpResponseMessage response) - { - var token = GetContent(response); - - var lastItem = (string) JObject.Parse(token).SelectToken("token"); - if (lastItem != null) - { - return lastItem; - } - - throw new CommandException("Unable to retrieve authentication token required to perform operation."); - } - - AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential) - { - var byteArray = Encoding.ASCII.GetBytes($"{credential.UserName}:{credential.Password}"); - return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); - } - - HttpResponseMessage SendRequest(HttpRequestMessage request) - { - return client.SendAsync(request).GetAwaiter().GetResult(); - } - - static string? GetContent(HttpResponseMessage response) - { - return response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - } } } \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index 23174594c..7a92f84aa 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -2,9 +2,11 @@ using Calamari.Common.Features.Packages; using Calamari.Common.Features.Processes; using Calamari.Common.Features.Scripting; +using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; +using Calamari.Integration.Packages.Download.Oci; using Octopus.Versioning; namespace Calamari.Integration.Packages.Download @@ -35,16 +37,17 @@ public PackageDownloaderStrategy( this.variables = variables; } - public PackagePhysicalFileMetadata DownloadPackage(string packageId, - IVersion version, - string feedId, - Uri feedUri, - FeedType feedType, - string feedUsername, - string feedPassword, - bool forcePackageDownload, - int maxDownloadAttempts, - TimeSpan downloadAttemptBackoff) + public PackagePhysicalFileMetadata DownloadPackage( + string packageId, + IVersion version, + string feedId, + Uri feedUri, + FeedType feedType, + string feedUsername, + string feedPassword, + bool forcePackageDownload, + int maxDownloadAttempts, + TimeSpan downloadAttemptBackoff) { IPackageDownloader? downloader = null; switch (feedType) @@ -62,13 +65,15 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, downloader = new HelmChartPackageDownloader(fileSystem, log); break; case FeedType.OciRegistry: - downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), log); + downloader = OciPackageDownloader(); break; - case FeedType.Docker: case FeedType.AwsElasticContainerRegistry: + downloader = new OciOrDockerImagePackageDownloader(OciPackageDownloader(), DockerImagePackageDownloader(), new OciRegistryClient(fileSystem), log); + break; + case FeedType.Docker: case FeedType.AzureContainerRegistry: case FeedType.GoogleContainerRegistry: - downloader = new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); + downloader = DockerImagePackageDownloader(); break; case FeedType.S3: downloader = new S3PackageDownloader(log, fileSystem); @@ -79,6 +84,7 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, default: throw new NotImplementedException($"No Calamari downloader exists for feed type `{feedType}`."); } + Log.Verbose($"Feed type provided `{feedType}` using {downloader.GetType().Name}"); return downloader.DownloadPackage( @@ -92,5 +98,11 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, maxDownloadAttempts, downloadAttemptBackoff); } + + OciPackageDownloader OciPackageDownloader() + => new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), new OciRegistryClient(fileSystem), log); + + DockerImagePackageDownloader DockerImagePackageDownloader() + => new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); } } \ No newline at end of file diff --git a/source/Calamari.Tests/Calamari.Tests.csproj b/source/Calamari.Tests/Calamari.Tests.csproj index 70f697fbc..2d9e46023 100644 --- a/source/Calamari.Tests/Calamari.Tests.csproj +++ b/source/Calamari.Tests/Calamari.Tests.csproj @@ -19,6 +19,7 @@ $(DefineConstants);NETFX;IIS_SUPPORT;USE_NUGET_V2_LIBS;USE_OCTODIFF_EXE; + diff --git a/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs b/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs new file mode 100644 index 000000000..b4629f903 --- /dev/null +++ b/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs @@ -0,0 +1,115 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon; +using Calamari.Common.Features.Packages; +using Calamari.Common.Features.Processes; +using Calamari.Common.Features.Scripting; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Logging; +using Calamari.Common.Plumbing.Variables; +using Calamari.Integration.Packages.Download; +using Calamari.Testing; +using Calamari.Testing.Requirements; +using Calamari.Tests.Helpers; +using FluentAssertions; +using FluentAssertions.Execution; +using NSubstitute; +using NUnit.Framework; +using Octopus.Versioning.Semver; + +namespace Calamari.Tests.Fixtures.PackageDownload +{ + [TestFixture] + public class AwsEcrDownloadFixture : CalamariFixture + { + static readonly string Home = Path.GetTempPath(); + readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + + [OneTimeSetUp] + public void TestFixtureSetUp() + { + Environment.SetEnvironmentVariable("TentacleHome", Home); + } + + [Test] + public async Task HelmChartIsSuccessfullyDownloaded() + { + var regionEndpoint = RegionEndpoint.USWest2; + const string repositoryName = "calamari-testing-helm-chart"; + const string imageTag = "0.1.0"; + + var packagePhysicalFileMetadata = await DoDownload(regionEndpoint, repositoryName, imageTag, cancellationTokenSource.Token); + + packagePhysicalFileMetadata.Should().NotBeNull(); + + using (new AssertionScope()) + { + packagePhysicalFileMetadata.PackageId.Should().Be(repositoryName); + packagePhysicalFileMetadata.Version.Should().Be(new SemanticVersion(imageTag)); + packagePhysicalFileMetadata.Extension.Should().Be(".tgz"); + packagePhysicalFileMetadata.FullFilePath.Should().Contain(repositoryName); + } + } + + [Test] + [RequiresDockerInstalled] + public async Task DockerImageIsSuccessfullyDownloaded() + { + const string repositoryName = "calamari-testing-container-image"; + const string imageTag = "1.0.0"; + + var regionEndpoint = RegionEndpoint.USWest2; + + var packagePhysicalFileMetadata = await DoDownload(regionEndpoint, repositoryName, imageTag, cancellationTokenSource.Token); + + packagePhysicalFileMetadata.Should().NotBeNull(); + + using (new AssertionScope()) + { + packagePhysicalFileMetadata.PackageId.Should().Be(repositoryName); + packagePhysicalFileMetadata.Version.ToString().Should().Be(imageTag); + packagePhysicalFileMetadata.FullFilePath.Should().BeEmpty(); + } + } + + static async Task DoDownload(RegionEndpoint regionEndpoint, string repositoryName, string imageTag, CancellationToken cancellationToken) + { + var log = Substitute.For(); + var runner = new CommandLineRunner(log, new CalamariVariables()); + var engine = new ScriptEngine(Enumerable.Empty(), log); + var strategy = new PackageDownloaderStrategy( + log, + engine, + CalamariPhysicalFileSystem.GetPhysicalFileSystem(), + runner, + new CalamariVariables()); + + var authDetails = await GetEcrAuthDetails(regionEndpoint, cancellationToken); + var registryUri = new Uri(authDetails.RegistryUri); + + var packagePhysicalFileMetadata = strategy.DownloadPackage( + repositoryName, + SemVerFactory.CreateVersion(imageTag), + "", + registryUri, + FeedType.AwsElasticContainerRegistry, + authDetails.Username, + authDetails.Password, + true, + 1, + TimeSpan.FromSeconds(30)); + return packagePhysicalFileMetadata; + } + + static async Task GetEcrAuthDetails(RegionEndpoint regionEndpoint, CancellationToken cancellationToken) + { + return new AwsElasticContainerRegistryCredentials().RetrieveTemporaryCredentials( + await ExternalVariables.Get(ExternalVariable.AwsCloudFormationAndS3AccessKey, cancellationToken), + await ExternalVariables.Get(ExternalVariable.AwsCloudFormationAndS3SecretKey, cancellationToken), + regionEndpoint.SystemName); + } + } +} \ No newline at end of file diff --git a/source/Calamari.Tests/Fixtures/PackageDownload/AwsElasticContainerRegistryCredentials.cs b/source/Calamari.Tests/Fixtures/PackageDownload/AwsElasticContainerRegistryCredentials.cs new file mode 100644 index 000000000..1b5f2f73a --- /dev/null +++ b/source/Calamari.Tests/Fixtures/PackageDownload/AwsElasticContainerRegistryCredentials.cs @@ -0,0 +1,110 @@ +#pragma warning disable CS8601 // Possible null reference assignment +#pragma warning disable CS8618 // Non-nullable property {0} is uninitialized +using System; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Security.Authentication; +using System.Text; +using Amazon; +using Amazon.ECR; +using Amazon.ECR.Model; +using Amazon.Runtime; +using JetBrains.Annotations; +using SharpCompress; + +namespace Calamari.Tests.Fixtures.PackageDownload +{ + // Taken from Server + public class AwsElasticContainerRegistryCredentials + { + /// + /// AWS credentials expire after 12 hours: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html + /// Rather than hit the AWS service every time we want to use the registry, we can cache them locally based on the expiry time in the response + /// + static readonly ConcurrentDictionary<(string accessKey, string region), TemporaryCredentials> CachedCreds = new ConcurrentDictionary<(string accessKey, string region), TemporaryCredentials>(); + + public virtual TemporaryCredentials RetrieveTemporaryCredentials(string accessKey, string secretKey, string region, bool bypassCache = false) + { + if (!bypassCache) + { + if (TryUseCache(accessKey, region, out var cachedCreds)) return cachedCreds; + } + + var authToken = GetAuthorizationData(accessKey, secretKey, region); + var creds = DecodeCredentials(authToken); + var tempCreds = new TemporaryCredentials + { + Expiry = authToken.ExpiresAt, + RegistryUri = authToken.ProxyEndpoint, + Password = creds.Password, + Username = creds.Username + }; + + return CachedCreds.AddOrUpdate((accessKey, region), tempCreds, (tuple, credentials) => tempCreds); + } + + static bool TryUseCache(string accessKey, string region, out TemporaryCredentials temporaryCredentials) + { + // AWS Creds have 12 hour expiry + // to ensure it doesnt expire mid-deploy lets make sure we have at least 2 hours left + CachedCreds.ToArray() + .Where(kvp => kvp.Value.Expiry.AddHours(-2) < DateTime.UtcNow) + .ForEach(k => CachedCreds.TryRemove(k.Key, out var _)); + + return CachedCreds.TryGetValue((accessKey, region), out temporaryCredentials); + } + + (string Username, string Password) DecodeCredentials(AuthorizationData authToken) + { + try + { + var decodedToken = Encoding.UTF8.GetString(Convert.FromBase64String(authToken.AuthorizationToken)); + var parts = decodedToken.Split(':'); + if (parts.Length != 2) + { + throw new AuthenticationException("Token returned by AWS is in an unexpected format"); + } + + return (parts[0], parts[1]); + } + catch (Exception) + { + throw new AuthenticationException("Token returned by AWS is in an unexpected format"); + } + } + + static AuthorizationData GetAuthorizationData(string accessKey, string secretKey, string region) + { + var regionEndpoint = RegionEndpoint.GetBySystemName(region); + var credentials = new BasicAWSCredentials(accessKey, secretKey); + var client = new AmazonECRClient(credentials, regionEndpoint); + try + { + var token = client.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()).Result; + var authToken = token.AuthorizationData.FirstOrDefault(); + if (authToken == null) + { + throw new Exception("No AuthToken found"); + } + + return authToken; + } + catch (Exception ex) + { + throw new AuthenticationException($"Unable to retrieve AWS Authorization token:\r\n\t{ex.Message}"); + } + } + + public class TemporaryCredentials + { + public string Username { get; set; } + + [CanBeNull] + public string Password { get; set; } + + public string RegistryUri { get; set; } + public DateTime Expiry { get; set; } + } + } +}