From 684ff084be6cccc248409b8961c2c118d57866ed Mon Sep 17 00:00:00 2001 From: Mark Coafield <46470837+MissedTheMark@users.noreply.github.com> Date: Thu, 23 May 2024 16:56:35 +1000 Subject: [PATCH 1/7] First go --- .../Integration/Packages/Download/Oci.cs | 230 +++++++++++++++++ .../Download/OciArtifactManifestRetriever.cs | 98 +++++++ .../Packages/Download/OciPackageDownloader.cs | 244 ++---------------- .../Download/PackageDownloaderStrategy.cs | 18 +- 4 files changed, 365 insertions(+), 225 deletions(-) create mode 100644 source/Calamari.Shared/Integration/Packages/Download/Oci.cs create mode 100644 source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci.cs new file mode 100644 index 000000000..dcf9c65ab --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci.cs @@ -0,0 +1,230 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.RegularExpressions; +using Calamari.Common.Commands; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Octopus.Versioning; + +namespace Calamari.Integration.Packages.Download +{ + // TODO: make less static, and store things like the client. + class Oci + { + const string VersionPath = "v2"; + const string OciImageManifestAcceptHeader = "application/vnd.oci.image.manifest.v1+json"; + + internal class Manifest + { + internal class Image + { + internal const string TitleAnnotationKey = "org.opencontainers.image.title"; + } + + internal class Layer + { + internal const string PropertyName = "layers"; + internal const string DigestPropertyName = "digest"; + internal const string SizePropertyName = "size"; + internal const string MediaTypePropertyName = "mediaType"; + internal const string AnnotationsPropertyName = "annotations"; + } + } + + static readonly Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); + + internal static JObject? GetManifest(HttpClient client, Uri url, string packageId, string version, string? feedUsername, string? feedPassword) + { + using var response = Get(client, new Uri($"{url}/{packageId}/manifests/{version}"), new NetworkCredential(feedUsername, feedPassword), ApplyAcceptHeader); + var manifest = JsonConvert.DeserializeObject(response.Content.ReadAsStringAsync().Result); + + return manifest; + } + + static void ApplyAcceptHeader(HttpRequestMessage request) + => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciImageManifestAcceptHeader)); + + internal static Uri GetApiUri(Uri feedUri) + { + var httpScheme = BuildScheme(IsPlainHttp(feedUri)); + var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); + var uri = new Uri(r); + if (!r.EndsWith("/" + VersionPath)) + { + uri = new Uri(uri, VersionPath); + } + + return uri; + + static bool IsPlainHttp(Uri uri) + => uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase); + + static string BuildScheme(bool isPlainHttp) + => isPlainHttp ? Uri.UriSchemeHttp : Uri.UriSchemeHttps; + } + + // oci registries don't support the '+' tagging + // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380 + internal static string FixVersion(IVersion version) + => version.ToString().Replace("+", "_"); + + internal static string? GetPackageHashFromDigest(string digest) + => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; + + internal static HttpResponseMessage Get(HttpClient client, Uri url, ICredentials credentials, Action? customAcceptHeader = null) + { + var request = new HttpRequestMessage(HttpMethod.Get, url); + try + { + var networkCredential = credentials.GetCredential(url, "Basic"); + + if (!string.IsNullOrWhiteSpace(networkCredential?.UserName) || !string.IsNullOrWhiteSpace(networkCredential?.Password)) + { + request.Headers.Authorization = CreateAuthenticationHeader(networkCredential); + } + + customAcceptHeader?.Invoke(request); + var response = SendRequest(client, request); + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + var tokenFromAuthService = GetAuthRequestHeader(client, response, networkCredential); + request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Authorization = tokenFromAuthService; + customAcceptHeader?.Invoke(request); + response = SendRequest(client, request); + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + throw new CommandException($"Authorization to `{url}` failed."); + } + } + + if (response.StatusCode == HttpStatusCode.NotFound) + { + // Some registries do not support the Docker HTTP APIs + // For example GitHub: https://github.community/t/ghcr-io-docker-http-api/130121 + throw new CommandException($"Docker registry located at `{url}` does not support this action."); + } + + if (!response.IsSuccessStatusCode) + { + var errorMessage = $"Request to Docker registry located at `{url}` failed with {response.StatusCode}:{response.ReasonPhrase}."; + + var responseBody = GetContent(response); + if (!string.IsNullOrWhiteSpace(responseBody)) errorMessage += $" {responseBody}"; + + throw new CommandException(errorMessage); + } + + return response; + } + finally + { + request.Dispose(); + } + } + + static AuthenticationHeaderValue GetAuthRequestHeader(HttpClient client, HttpResponseMessage response, NetworkCredential credential) + { + var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer"); + if (auth != null) + { + var authToken = RetrieveAuthenticationToken(client, GetOAuthServiceUrl(auth), credential); + return new AuthenticationHeaderValue("Bearer", authToken); + } + + if (response.Headers.WwwAuthenticate.Any(a => a.Scheme == "Basic")) + { + return CreateAuthenticationHeader(credential); + } + + throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`"); + } + + + static string RetrieveAuthenticationToken(HttpClient client, string authUrl, NetworkCredential credential) + { + HttpResponseMessage? response = null; + + try + { + using (var msg = new HttpRequestMessage(HttpMethod.Get, authUrl)) + { + if (credential?.UserName != null) + { + msg.Headers.Authorization = CreateAuthenticationHeader(credential); + } + + response = SendRequest(client, msg); + } + + if (response.IsSuccessStatusCode) + { + return ExtractTokenFromResponse(response); + } + } + finally + { + response?.Dispose(); + } + + throw new CommandException("Unable to retrieve authentication token required to perform operation."); + } + + static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) + { + var details = auth.Parameter.Split(',').ToDictionary(x => x.Substring(0, x.IndexOf('=')), y => y.Substring(y.IndexOf('=') + 1, y.Length - y.IndexOf('=') - 1).Trim('"')); + var oathUrl = new UriBuilder(details["realm"]); + var queryStringValues = new Dictionary(); + if (details.TryGetValue("service", out var service)) + { + var encodedService = WebUtility.UrlEncode(service); + queryStringValues.Add("service", encodedService); + } + + if (details.TryGetValue("scope", out var scope)) + { + var encodedScope = WebUtility.UrlEncode(scope); + queryStringValues.Add("scope", encodedScope); + } + + oathUrl.Query = "?" + string.Join("&", queryStringValues.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray()); + return oathUrl.ToString(); + } + + static string ExtractTokenFromResponse(HttpResponseMessage response) + { + var token = GetContent(response); + + var lastItem = (string) JObject.Parse(token).SelectToken("token"); + if (lastItem != null) + { + return lastItem; + } + + throw new CommandException("Unable to retrieve authentication token required to perform operation."); + } + + static AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential) + { + var byteArray = Encoding.ASCII.GetBytes($"{credential.UserName}:{credential.Password}"); + return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); + } + + static HttpResponseMessage SendRequest(HttpClient client, HttpRequestMessage request) + { + return client.SendAsync(request).GetAwaiter().GetResult(); + } + + static string? GetContent(HttpResponseMessage response) + { + return response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs b/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs new file mode 100644 index 000000000..981a876f5 --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs @@ -0,0 +1,98 @@ +using System; +using System.Net; +using System.Net.Http; +using Newtonsoft.Json.Linq; +using Octopus.Versioning; + +namespace Calamari.Integration.Packages.Download +{ + + public class OciArtifactManifestRetriever + { + public OciArtifactTypes TryGetArtifactType(string packageId, + IVersion version, + string feedId, + Uri feedUri, + string? feedUsername, + string? feedPassword) + { + try + { + var versionString = Oci.FixVersion(version); + var apiUrl = Oci.GetApiUri(feedUri); + + var client = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); + + var jsonManifest = Oci.GetManifest(client, + apiUrl, + packageId, + versionString, + feedUsername, + feedPassword); + + // Check for Helm chart annotations + var annotations = jsonManifest["annotations"]; + if (annotations != null && annotations.Type == JTokenType.Object) + { + var chartAnnotation = annotations["io.helm.sh/chart"]; + if (chartAnnotation != null) + { + return OciArtifactTypes.HelmChart; + } + } + + // Check the media type in the config section for Docker images and other OCI artifacts + var config = jsonManifest["config"]; + if (config != null && config["mediaType"] != null) + { + var mediaType = config["mediaType"].ToString(); + if (mediaType.Contains("docker")) + { + return OciArtifactTypes.DockerImage; + } + } + + return OciArtifactTypes.Unknown; + } + catch + { + return OciArtifactTypes.Unknown; + } + } + + public bool HasAnnotationContaining(JObject manifest, string name) + { + var annotations = manifest["annotations"]; + return annotations is { Type: JTokenType.Object } + && annotations[name] != null; + } + + public bool HasConfigMediaTypeContaining(JObject manifest, string name) + { + var config = manifest["config"]; + + return config is { Type: JTokenType.Object } + && config["mediaType"] != null + && config["mediaType"].ToString().Contains(name); + } + + public bool HasLayersMediaTypeContaining(JObject manifest, string name) + { + var layers = manifest["layers"]; + + if (layers is { Type: JTokenType.Array }) + { + foreach (var layer in layers) + { + if (layer["mediaType"] != null && layer["mediaType"].ToString().Contains(name)) + { + return true; + } + } + } + + return false; + } + + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs index 8201eb835..22ef323e6 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs @@ -18,18 +18,15 @@ namespace Calamari.Integration.Packages.Download { + public enum OciArtifactTypes + { + DockerImage, + HelmChart, + Unknown + } + public class OciPackageDownloader : IPackageDownloader { - const string VersionPath = "v2"; - const string OciImageManifestAcceptHeader = "application/vnd.oci.image.manifest.v1+json"; - const string ManifestImageTitleAnnotationKey = "org.opencontainers.image.title"; - const string ManifestLayerPropertyName = "layers"; - const string ManifestLayerDigestPropertyName = "digest"; - const string ManifestLayerSizePropertyName = "size"; - const string ManifestLayerAnnotationsPropertyName = "annotations"; - const string ManifestLayerMediaTypePropertyName = "mediaType"; - - static Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); static readonly IPackageDownloaderUtils PackageDownloaderUtils = new PackageDownloaderUtils(); readonly ICalamariFileSystem fileSystem; readonly ICombinedPackageExtractor combinedPackageExtractor; @@ -43,7 +40,7 @@ public OciPackageDownloader( this.combinedPackageExtractor = combinedPackageExtractor; client = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); } - + public PackagePhysicalFileMetadata DownloadPackage(string packageId, IVersion version, string feedId, @@ -83,11 +80,11 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, Directory.CreateDirectory(stagingDir); } - var versionString = FixVersion(version); + var versionString = Oci.FixVersion(version); - var apiUrl = GetApiUri(feedUri); + var apiUrl = Oci.GetApiUri(feedUri); var (digest, size, extension) = GetPackageDetails(apiUrl, packageId, versionString, feedUsername, feedPassword); - var hash = GetPackageHashFromDigest(digest); + var hash = Oci.GetPackageHashFromDigest(digest); var cachedFileName = PackageName.ToCachedFileName(packageId, version, extension); var downloadPath = Path.Combine(Path.Combine(stagingDir, cachedFileName)); @@ -108,14 +105,6 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, } } - // oci registries don't support the '+' tagging - // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380 - static string FixVersion(IVersion version) - => version.ToString().Replace("+", "_"); - - static string? GetPackageHashFromDigest(string digest) - => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; - (string digest, int size, string extension) GetPackageDetails( Uri url, string packageId, @@ -123,12 +112,11 @@ static string FixVersion(IVersion version) string? feedUserName, string? feedPassword) { - using var response = Get(new Uri($"{url}/{packageId}/manifests/{version}"), new NetworkCredential(feedUserName, feedPassword), ApplyAccept); - var manifest = JsonConvert.DeserializeObject(response.Content.ReadAsStringAsync().Result); + var manifest = Oci.GetManifest(client, url, packageId, version, feedUserName, feedPassword); - var layer = manifest.Value(ManifestLayerPropertyName)[0]; - var digest = layer.Value(ManifestLayerDigestPropertyName); - var size = layer.Value(ManifestLayerSizePropertyName); + var layer = manifest.Value(Oci.Manifest.Layer.PropertyName)[0]; + var digest = layer.Value(Oci.Manifest.Layer.DigestPropertyName); + var size = layer.Value(Oci.Manifest.Layer.SizePropertyName); var extension = GetExtensionFromManifest(layer); return (digest, size, extension); @@ -136,13 +124,13 @@ static string FixVersion(IVersion version) string GetExtensionFromManifest(JToken layer) { - var artifactTitle = layer.Value(ManifestLayerAnnotationsPropertyName)?[ManifestImageTitleAnnotationKey]?.Value() ?? ""; + var artifactTitle = layer.Value(Oci.Manifest.Layer.AnnotationsPropertyName)?[Oci.Manifest.Image.TitleAnnotationKey]?.Value() ?? ""; var extension = combinedPackageExtractor .Extensions .FirstOrDefault(ext => Path.GetExtension(artifactTitle).Equals(ext, StringComparison.OrdinalIgnoreCase)); - return extension ?? (layer.Value(ManifestLayerMediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); + return extension ?? (layer.Value(Oci.Manifest.Layer.MediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); } void DownloadPackage( @@ -154,42 +142,16 @@ void DownloadPackage( string downloadPath) { using var fileStream = fileSystem.OpenFile(downloadPath, FileAccess.Write); - using var response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); + using var response = Oci.Get(client, new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); if (!response.IsSuccessStatusCode) { throw new CommandException( $"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); } - -#if NET40 - response.Content.CopyToAsync(fileStream).Wait(); -#else + response.Content.CopyToAsync(fileStream).GetAwaiter().GetResult(); -#endif } - - static void ApplyAccept(HttpRequestMessage request) - => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciImageManifestAcceptHeader)); - - static Uri GetApiUri(Uri feedUri) - { - var httpScheme = BuildScheme(IsPlainHttp(feedUri)); - var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); - var uri = new Uri(r); - if (!r.EndsWith("/" + VersionPath)) - { - uri = new Uri(uri, VersionPath); - } - - return uri; - - static bool IsPlainHttp(Uri uri) - => uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase); - - static string BuildScheme(bool isPlainHttp) - => isPlainHttp ? Uri.UriSchemeHttp : Uri.UriSchemeHttps; - } - + PackagePhysicalFileMetadata? SourceFromCache(string packageId, IVersion version, string cacheDirectory) { Log.VerboseFormat("Checking package cache for package {0} v{1}", packageId, version.ToString()); @@ -212,171 +174,5 @@ static string BuildScheme(bool isPlainHttp) return null; } - - HttpResponseMessage Get(Uri url, ICredentials credentials, Action? customAcceptHeader = null) - { - var request = new HttpRequestMessage(HttpMethod.Get, url); - try - { - var networkCredential = credentials.GetCredential(url, "Basic"); - - if (!string.IsNullOrWhiteSpace(networkCredential?.UserName) || !string.IsNullOrWhiteSpace(networkCredential?.Password)) - { - request.Headers.Authorization = CreateAuthenticationHeader(networkCredential); - } - - customAcceptHeader?.Invoke(request); - var response = SendRequest(request); - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - var tokenFromAuthService = GetAuthRequestHeader(response, networkCredential); - request = new HttpRequestMessage(HttpMethod.Get, url); - request.Headers.Authorization = tokenFromAuthService; - customAcceptHeader?.Invoke(request); - response = SendRequest(request); - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - throw new CommandException($"Authorization to `{url}` failed."); - } - } - - if (response.StatusCode == HttpStatusCode.NotFound) - { - // Some registries do not support the Docker HTTP APIs - // For example GitHub: https://github.community/t/ghcr-io-docker-http-api/130121 - throw new CommandException($"Docker registry located at `{url}` does not support this action."); - } - - if (!response.IsSuccessStatusCode) - { - var errorMessage = $"Request to Docker registry located at `{url}` failed with {response.StatusCode}:{response.ReasonPhrase}."; - - var responseBody = GetContent(response); - if (!string.IsNullOrWhiteSpace(responseBody)) errorMessage += $" {responseBody}"; - - throw new CommandException(errorMessage); - } - - return response; - } - finally - { - request.Dispose(); - } - } - - string RetrieveAuthenticationToken(string authUrl, NetworkCredential credential) - { - HttpResponseMessage? response = null; - - try - { - using (var msg = new HttpRequestMessage(HttpMethod.Get, authUrl)) - { - if (credential?.UserName != null) - { - msg.Headers.Authorization = CreateAuthenticationHeader(credential); - } - - response = SendRequest(msg); - } - - if (response.IsSuccessStatusCode) - { - return ExtractTokenFromResponse(response); - } - } - finally - { - response?.Dispose(); - } - - throw new CommandException("Unable to retrieve authentication token required to perform operation."); - } - - AuthenticationHeaderValue GetAuthRequestHeader(HttpResponseMessage response, NetworkCredential credential) - { - var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer"); - if (auth != null) - { - var authToken = RetrieveAuthenticationToken(GetOAuthServiceUrl(auth), credential); - return new AuthenticationHeaderValue("Bearer", authToken); - } - - if (response.Headers.WwwAuthenticate.Any(a => a.Scheme == "Basic")) - { - return CreateAuthenticationHeader(credential); - } - - throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`"); - } - - static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) - { - var details = auth.Parameter.Split(',').ToDictionary(x => x.Substring(0, x.IndexOf('=')), y => y.Substring(y.IndexOf('=') + 1, y.Length - y.IndexOf('=') - 1).Trim('"')); - var oathUrl = new UriBuilder(details["realm"]); - var queryStringValues = new Dictionary(); - if (details.TryGetValue("service", out var service)) - { -#if NET40 - var encodedService = System.Web.HttpUtility.UrlEncode(service); -#else - var encodedService = WebUtility.UrlEncode(service); -#endif - queryStringValues.Add("service", encodedService); - } - - if (details.TryGetValue("scope", out var scope)) - { -#if NET40 - var encodedScope = System.Web.HttpUtility.UrlEncode(scope); -#else - var encodedScope = WebUtility.UrlEncode(scope); -#endif - queryStringValues.Add("scope", encodedScope); - } - - oathUrl.Query = "?" + string.Join("&", queryStringValues.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray()); - return oathUrl.ToString(); - } - - static string ExtractTokenFromResponse(HttpResponseMessage response) - { - var token = GetContent(response); - - var lastItem = (string) JObject.Parse(token).SelectToken("token"); - if (lastItem != null) - { - return lastItem; - } - - throw new CommandException("Unable to retrieve authentication token required to perform operation."); - } - - AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential) - { - var byteArray = Encoding.ASCII.GetBytes($"{credential.UserName}:{credential.Password}"); - return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); - } - - HttpResponseMessage SendRequest(HttpRequestMessage request) - { -#if NET40 - return client.SendAsync(request).Result; -#else - return client.SendAsync(request).GetAwaiter().GetResult(); -#endif - } - - static string? GetContent(HttpResponseMessage response) - { -#if NET40 - return response.Content.ReadAsStringAsync().Result; -#else - return response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); -#endif - } } } \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index 9e1dc7864..b8ffacbc0 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -64,8 +64,24 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, case FeedType.OciRegistry: downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, variables, commandLineRunner)); break; - case FeedType.Docker: case FeedType.AwsElasticContainerRegistry: + var x = new OciArtifactManifestRetriever(); + if (x.TryGetArtifactType(packageId, + version, + feedId, + feedUri, + feedUsername, + feedPassword) + == OciArtifactTypes.HelmChart) + { + downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, variables, commandLineRunner)); + } + else + { + downloader = new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); + } + break; + case FeedType.Docker: case FeedType.AzureContainerRegistry: case FeedType.GoogleContainerRegistry: downloader = new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); From 0ba45c8d877695b9237f70113b71a819f73be60c Mon Sep 17 00:00:00 2001 From: Mark Coafield <46470837+MissedTheMark@users.noreply.github.com> Date: Fri, 24 May 2024 14:41:24 +1000 Subject: [PATCH 2/7] Added tests --- .../Integration/Packages/Download/Oci.cs | 97 +++++++++++--- .../Download/OciArtifactManifestRetriever.cs | 74 ++--------- .../Packages/Download/OciPackageDownloader.cs | 10 +- .../Download/PackageDownloaderStrategy.cs | 1 - source/Calamari.Tests/Calamari.Tests.csproj | 1 + .../PackageDownload/AwsEcrDownloadFixture.cs | 123 ++++++++++++++++++ 6 files changed, 222 insertions(+), 84 deletions(-) create mode 100644 source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci.cs index dcf9c65ab..ed1bdd6bb 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/Oci.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci.cs @@ -12,46 +12,65 @@ using Octopus.Versioning; namespace Calamari.Integration.Packages.Download -{ +{ // TODO: make less static, and store things like the client. class Oci { const string VersionPath = "v2"; - const string OciImageManifestAcceptHeader = "application/vnd.oci.image.manifest.v1+json"; + // const string OciImageManifestAcceptHeader = "application/vnd.oci.image.manifest.v1+json"; internal class Manifest { + internal const string MediaTypePropertyName = "mediaType"; + internal const string DockerImageMediaTypeValue = "application/vnd.docker.distribution.manifest.v2+json"; + + internal class Config + { + internal const string PropertyName = "config"; + internal const string MediaTypePropertyName = "mediaType"; + internal const string OciImageMediaTypeValue = "application/vnd.oci.image.config.v1+json"; + internal const string DockerImageMediaTypeValue = "application/vnd.docker.container.image.v1+json"; + } + internal class Image { internal const string TitleAnnotationKey = "org.opencontainers.image.title"; } - internal class Layer + internal class Layers { internal const string PropertyName = "layers"; internal const string DigestPropertyName = "digest"; - internal const string SizePropertyName = "size"; - internal const string MediaTypePropertyName = "mediaType"; + internal const string SizePropertyName = "size"; + internal const string MediaTypePropertyName = "mediaType"; internal const string AnnotationsPropertyName = "annotations"; - } + internal const string HelmChartMediaTypeValue = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v370 + internal const string DockerImageMediaTypeValue = "application/vnd.docker.image.rootfs.diff.tar.gzip"; + } } static readonly Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); - internal static JObject? GetManifest(HttpClient client, Uri url, string packageId, string version, string? feedUsername, string? feedPassword) + internal static JObject? GetManifest(HttpClient client, + Uri url, + string packageId, + string version, + string? feedUsername, + string? feedPassword) { using var response = Get(client, new Uri($"{url}/{packageId}/manifests/{version}"), new NetworkCredential(feedUsername, feedPassword), ApplyAcceptHeader); var manifest = JsonConvert.DeserializeObject(response.Content.ReadAsStringAsync().Result); return manifest; - } - + } + static void ApplyAcceptHeader(HttpRequestMessage request) - => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciImageManifestAcceptHeader)); + => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(Manifest.Config.OciImageMediaTypeValue)); internal static Uri GetApiUri(Uri feedUri) { var httpScheme = BuildScheme(IsPlainHttp(feedUri)); + var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); var uri = new Uri(r); if (!r.EndsWith("/" + VersionPath)) @@ -75,7 +94,7 @@ internal static string FixVersion(IVersion version) internal static string? GetPackageHashFromDigest(string digest) => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; - + internal static HttpResponseMessage Get(HttpClient client, Uri url, ICredentials credentials, Action? customAcceptHeader = null) { var request = new HttpRequestMessage(HttpMethod.Get, url); @@ -129,7 +148,7 @@ internal static HttpResponseMessage Get(HttpClient client, Uri url, ICredentials request.Dispose(); } } - + static AuthenticationHeaderValue GetAuthRequestHeader(HttpClient client, HttpResponseMessage response, NetworkCredential credential) { var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer"); @@ -146,8 +165,7 @@ static AuthenticationHeaderValue GetAuthRequestHeader(HttpClient client, HttpRes throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`"); } - - + static string RetrieveAuthenticationToken(HttpClient client, string authUrl, NetworkCredential credential) { HttpResponseMessage? response = null; @@ -176,7 +194,7 @@ static string RetrieveAuthenticationToken(HttpClient client, string authUrl, Net throw new CommandException("Unable to retrieve authentication token required to perform operation."); } - + static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) { var details = auth.Parameter.Split(',').ToDictionary(x => x.Substring(0, x.IndexOf('=')), y => y.Substring(y.IndexOf('=') + 1, y.Length - y.IndexOf('=') - 1).Trim('"')); @@ -197,12 +215,57 @@ static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) oathUrl.Query = "?" + string.Join("&", queryStringValues.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray()); return oathUrl.ToString(); } + + /* + public static bool HasAnnotationContaining(JObject manifest, string key, string value) + { + var annotations = manifest[Manifest.Annotations.PropertyName]; + return annotations is { Type: JTokenType.Object } + && annotations[key] != null + && annotations[key].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; // ~equiv to case insensitive contains, for non-net standard 2.0+ + } */ + public static bool HasMediaTypeContaining(JObject manifest, string value) + { + var mediaType = manifest[Manifest.MediaTypePropertyName]; + + return mediaType != null + && mediaType.ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; + } + + public static bool HasConfigMediaTypeContaining(JObject manifest, string value) + { + var config = manifest[Manifest.Config.PropertyName]; + + return config is { Type: JTokenType.Object } + && config[Manifest.Config.MediaTypePropertyName] != null + && config[Manifest.Config.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; + } + + public static bool HasLayersMediaTypeContaining(JObject manifest, string value) + { + var layers = manifest[Manifest.Layers.PropertyName]; + + if (layers is { Type: JTokenType.Array }) + { + foreach (var layer in layers) + { + if (layer[Manifest.Layers.MediaTypePropertyName] != null + && layer[Manifest.Layers.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + } + + return false; + } + static string ExtractTokenFromResponse(HttpResponseMessage response) { var token = GetContent(response); - var lastItem = (string) JObject.Parse(token).SelectToken("token"); + var lastItem = (string)JObject.Parse(token).SelectToken("token"); if (lastItem != null) { return lastItem; @@ -210,7 +273,7 @@ static string ExtractTokenFromResponse(HttpResponseMessage response) throw new CommandException("Unable to retrieve authentication token required to perform operation."); } - + static AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential) { var byteArray = Encoding.ASCII.GetBytes($"{credential.UserName}:{credential.Password}"); diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs b/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs index 981a876f5..72a77778f 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs @@ -1,17 +1,14 @@ using System; using System.Net; using System.Net.Http; -using Newtonsoft.Json.Linq; using Octopus.Versioning; namespace Calamari.Integration.Packages.Download { - public class OciArtifactManifestRetriever { public OciArtifactTypes TryGetArtifactType(string packageId, IVersion version, - string feedId, Uri feedUri, string? feedUsername, string? feedPassword) @@ -31,68 +28,23 @@ public OciArtifactTypes TryGetArtifactType(string packageId, feedPassword); // Check for Helm chart annotations - var annotations = jsonManifest["annotations"]; - if (annotations != null && annotations.Type == JTokenType.Object) - { - var chartAnnotation = annotations["io.helm.sh/chart"]; - if (chartAnnotation != null) - { - return OciArtifactTypes.HelmChart; - } - } - - // Check the media type in the config section for Docker images and other OCI artifacts - var config = jsonManifest["config"]; - if (config != null && config["mediaType"] != null) - { - var mediaType = config["mediaType"].ToString(); - if (mediaType.Contains("docker")) - { - return OciArtifactTypes.DockerImage; - } - } - - return OciArtifactTypes.Unknown; + var isHelmChart = //Oci.HasAnnotationContaining(jsonManifest, Oci.Manifest.Annotations.HelmChartAnnotationKey, "helm") + Oci.HasConfigMediaTypeContaining(jsonManifest, Oci.Manifest.Config.OciImageMediaTypeValue) + || Oci.HasLayersMediaTypeContaining(jsonManifest, Oci.Manifest.Layers.HelmChartMediaTypeValue); + + if (isHelmChart) + return OciArtifactTypes.HelmChart; + + var isDockerImage = Oci.HasMediaTypeContaining(jsonManifest, Oci.Manifest.DockerImageMediaTypeValue) + || Oci.HasConfigMediaTypeContaining(jsonManifest, Oci.Manifest.Config.DockerImageMediaTypeValue) + || Oci.HasLayersMediaTypeContaining(jsonManifest, Oci.Manifest.Layers.DockerImageMediaTypeValue); + + return isDockerImage ? OciArtifactTypes.DockerImage : OciArtifactTypes.Unknown; } - catch + catch (Exception ex) { return OciArtifactTypes.Unknown; } } - - public bool HasAnnotationContaining(JObject manifest, string name) - { - var annotations = manifest["annotations"]; - return annotations is { Type: JTokenType.Object } - && annotations[name] != null; - } - - public bool HasConfigMediaTypeContaining(JObject manifest, string name) - { - var config = manifest["config"]; - - return config is { Type: JTokenType.Object } - && config["mediaType"] != null - && config["mediaType"].ToString().Contains(name); - } - - public bool HasLayersMediaTypeContaining(JObject manifest, string name) - { - var layers = manifest["layers"]; - - if (layers is { Type: JTokenType.Array }) - { - foreach (var layer in layers) - { - if (layer["mediaType"] != null && layer["mediaType"].ToString().Contains(name)) - { - return true; - } - } - } - - return false; - } - } } \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs index 22ef323e6..11c89b7e5 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs @@ -114,9 +114,9 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, { var manifest = Oci.GetManifest(client, url, packageId, version, feedUserName, feedPassword); - var layer = manifest.Value(Oci.Manifest.Layer.PropertyName)[0]; - var digest = layer.Value(Oci.Manifest.Layer.DigestPropertyName); - var size = layer.Value(Oci.Manifest.Layer.SizePropertyName); + var layer = manifest.Value(Oci.Manifest.Layers.PropertyName)[0]; + var digest = layer.Value(Oci.Manifest.Layers.DigestPropertyName); + var size = layer.Value(Oci.Manifest.Layers.SizePropertyName); var extension = GetExtensionFromManifest(layer); return (digest, size, extension); @@ -124,13 +124,13 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, string GetExtensionFromManifest(JToken layer) { - var artifactTitle = layer.Value(Oci.Manifest.Layer.AnnotationsPropertyName)?[Oci.Manifest.Image.TitleAnnotationKey]?.Value() ?? ""; + var artifactTitle = layer.Value(Oci.Manifest.Layers.AnnotationsPropertyName)?[Oci.Manifest.Image.TitleAnnotationKey]?.Value() ?? ""; var extension = combinedPackageExtractor .Extensions .FirstOrDefault(ext => Path.GetExtension(artifactTitle).Equals(ext, StringComparison.OrdinalIgnoreCase)); - return extension ?? (layer.Value(Oci.Manifest.Layer.MediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); + return extension ?? (layer.Value(Oci.Manifest.Layers.MediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); } void DownloadPackage( diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index b8ffacbc0..bfc99992d 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -68,7 +68,6 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, var x = new OciArtifactManifestRetriever(); if (x.TryGetArtifactType(packageId, version, - feedId, feedUri, feedUsername, feedPassword) diff --git a/source/Calamari.Tests/Calamari.Tests.csproj b/source/Calamari.Tests/Calamari.Tests.csproj index 0aeed4500..d291ffbb0 100644 --- a/source/Calamari.Tests/Calamari.Tests.csproj +++ b/source/Calamari.Tests/Calamari.Tests.csproj @@ -19,6 +19,7 @@ $(DefineConstants);NETFX;AWS;IIS_SUPPORT;USE_NUGET_V2_LIBS;USE_OCTODIFF_EXE;WINDOWS_CERTIFICATE_STORE_SUPPORT;WINDOWS_USER_ACCOUNT_SUPPORT;WINDOWS_REGISTRY_SUPPORT + diff --git a/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs b/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs new file mode 100644 index 000000000..52dade37c --- /dev/null +++ b/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs @@ -0,0 +1,123 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Amazon; +using Amazon.ECR; +using Amazon.ECR.Model; +using Calamari.Common.Features.Packages; +using Calamari.Common.Features.Processes; +using Calamari.Common.Features.Scripting; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Logging; +using Calamari.Common.Plumbing.Variables; +using Calamari.Integration.Packages.Download; +using Calamari.Testing.Helpers; +using Calamari.Testing.Requirements; +using Calamari.Tests.Helpers; +using FluentAssertions; +using FluentAssertions.Execution; +using NSubstitute; +using NUnit.Framework; +using Octopus.Versioning.Semver; + +namespace Calamari.Tests.Fixtures.PackageDownload +{ + [TestFixture] + public class AwsEcrDownloadFixture : CalamariFixture + { + static readonly string Home = Path.GetTempPath(); + + [OneTimeSetUp] + public void TestFixtureSetUp() + { + Environment.SetEnvironmentVariable("TentacleHome", Home); + } + + [Test] + public void HelmChartIsSuccessfullyDownloaded() + { + var regionEndpoint = RegionEndpoint.USEast1; + const string repositoryName = "markc-test"; + const string imageTag = "0.1.0"; + + var packagePhysicalFileMetadata = DoDownload(regionEndpoint, repositoryName, imageTag); + + packagePhysicalFileMetadata.Should().NotBeNull(); + + using (new AssertionScope()) + { + packagePhysicalFileMetadata.PackageId.Should().Be(repositoryName); + packagePhysicalFileMetadata.Version.Should().Be(imageTag); + packagePhysicalFileMetadata.Extension.Should().Be(".tgz"); + packagePhysicalFileMetadata.FullFilePath.Should().Contain(repositoryName); + } + } + + [Test] + [RequiresDockerInstalled] + public void DockerImageIsSuccessfullyDownloaded() + { + const string repositoryName = "markc-test"; + const string imageTag = "1.0.0"; + + var regionEndpoint = RegionEndpoint.USEast1; + + var packagePhysicalFileMetadata = DoDownload(regionEndpoint, repositoryName, imageTag); + + packagePhysicalFileMetadata.Should().NotBeNull(); + + using (new AssertionScope()) + { + packagePhysicalFileMetadata.PackageId.Should().Be(repositoryName); + packagePhysicalFileMetadata.Version.ToString().Should().Be(imageTag); + packagePhysicalFileMetadata.FullFilePath.Should().BeEmpty(); + } + } + + static PackagePhysicalFileMetadata DoDownload(RegionEndpoint regionEndpoint, string repositoryName, string imageTag) + { + var log = Substitute.For(); + var runner = new CommandLineRunner(log, new CalamariVariables()); + var engine = new ScriptEngine(Enumerable.Empty(), log); + var strategy = new PackageDownloaderStrategy(log, + engine, + CalamariPhysicalFileSystem.GetPhysicalFileSystem(), + runner, + new CalamariVariables()); + + var authDetails = GetEcrAuthDetails(regionEndpoint); + var registryUri = new Uri(authDetails.Registry); + + var packagePhysicalFileMetadata = strategy.DownloadPackage(repositoryName, + SemVerFactory.CreateVersion(imageTag), + "", + registryUri, + FeedType.AwsElasticContainerRegistry, + authDetails.Username, + authDetails.Password, + true, + 1, + TimeSpan.FromSeconds(30)); + return packagePhysicalFileMetadata; + } + + static (string Username, string Password, string Registry) GetEcrAuthDetails(RegionEndpoint regionEndpoint) + { + var ecrClient = new AmazonECRClient(regionEndpoint); + var authTokenRequest = new GetAuthorizationTokenRequest(); + var authTokenResponse = ecrClient.GetAuthorizationTokenAsync(authTokenRequest).Result; + + var authorizationData = authTokenResponse.AuthorizationData[0]; + var token = authorizationData.AuthorizationToken; + var decodedToken = Encoding.UTF8.GetString(Convert.FromBase64String(token)); + var usernamePassword = decodedToken.Split(':'); + var username = usernamePassword[0]; + var password = usernamePassword[1]; + var registry = authorizationData.ProxyEndpoint; + + return (username, password, registry); + } + } +} \ No newline at end of file From 38431d35225ea1493e568fa8d6c40609af8ddc31 Mon Sep 17 00:00:00 2001 From: Eddy Moulton Date: Mon, 16 Dec 2024 12:56:29 +1100 Subject: [PATCH 3/7] Fix merge error --- .../Download/PackageDownloaderStrategy.cs | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index fc091ce89..6c4e6edd7 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -35,16 +35,17 @@ public PackageDownloaderStrategy( this.variables = variables; } - public PackagePhysicalFileMetadata DownloadPackage(string packageId, - IVersion version, - string feedId, - Uri feedUri, - FeedType feedType, - string feedUsername, - string feedPassword, - bool forcePackageDownload, - int maxDownloadAttempts, - TimeSpan downloadAttemptBackoff) + public PackagePhysicalFileMetadata DownloadPackage( + string packageId, + IVersion version, + string feedId, + Uri feedUri, + FeedType feedType, + string feedUsername, + string feedPassword, + bool forcePackageDownload, + int maxDownloadAttempts, + TimeSpan downloadAttemptBackoff) { IPackageDownloader? downloader = null; switch (feedType) @@ -66,19 +67,16 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, break; case FeedType.AwsElasticContainerRegistry: var x = new OciArtifactManifestRetriever(); - if (x.TryGetArtifactType(packageId, - version, - feedUri, - feedUsername, - feedPassword) + if (x.TryGetArtifactType(packageId, version, feedUri, feedUsername, feedPassword) == OciArtifactTypes.HelmChart) { - downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, variables, commandLineRunner)); + downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), log); } else - { + { downloader = new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); } + break; case FeedType.Docker: case FeedType.AzureContainerRegistry: @@ -94,6 +92,7 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, default: throw new NotImplementedException($"No Calamari downloader exists for feed type `{feedType}`."); } + Log.Verbose($"Feed type provided `{feedType}` using {downloader.GetType().Name}"); return downloader.DownloadPackage( From bb188f0f08e7336e78453f8eb4cbaf1b71539751 Mon Sep 17 00:00:00 2001 From: Eddy Moulton Date: Mon, 16 Dec 2024 17:25:21 +1100 Subject: [PATCH 4/7] Clean up and get tests working --- .../Oci/OciArtifactManifestRetriever.cs | 50 +++++ .../Download/{Oci.cs => Oci/OciClient.cs} | 179 ++++++------------ .../Packages/Download/Oci/OciConstants.cs | 37 ++++ .../Download/Oci/OciJObjectExtensions.cs | 44 +++++ .../Download/OciArtifactManifestRetriever.cs | 50 ----- .../Packages/Download/OciPackageDownloader.cs | 67 +++---- .../Download/PackageDownloaderStrategy.cs | 11 +- .../PackageDownload/AwsEcrDownloadFixture.cs | 88 ++++----- .../AwsElasticContainerRegistryCredentials.cs | 110 +++++++++++ 9 files changed, 369 insertions(+), 267 deletions(-) create mode 100644 source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs rename source/Calamari.Shared/Integration/Packages/Download/{Oci.cs => Oci/OciClient.cs} (52%) create mode 100644 source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs create mode 100644 source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs delete mode 100644 source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs create mode 100644 source/Calamari.Tests/Fixtures/PackageDownload/AwsElasticContainerRegistryCredentials.cs diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs new file mode 100644 index 000000000..6d5be5b6b --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs @@ -0,0 +1,50 @@ +using System; +using Calamari.Common.Plumbing.Logging; +using Octopus.Versioning; + +namespace Calamari.Integration.Packages.Download.Oci +{ + public class OciArtifactManifestRetriever + { + readonly OciClient ociClient; + readonly ILog log; + + public OciArtifactManifestRetriever(OciClient ociClient, ILog log) + { + this.ociClient = ociClient; + this.log = log; + } + + public OciArtifactTypes TryGetArtifactType( + string packageId, + IVersion version, + Uri feedUri, + string? feedUsername, + string? feedPassword) + { + try + { + var jsonManifest = ociClient.GetManifest(feedUri, packageId, version, feedUsername, feedPassword); + + // Check for Helm chart annotations + var isHelmChart = + jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.OciImageMediaTypeValue) + || jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.HelmChartMediaTypeValue); + + if (isHelmChart) + return OciArtifactTypes.HelmChart; + + var isDockerImage = jsonManifest.HasMediaTypeContaining(OciConstants.Manifest.DockerImageMediaTypeValue) + || jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.DockerImageMediaTypeValue) + || jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.DockerImageMediaTypeValue); + + return isDockerImage ? OciArtifactTypes.DockerImage : OciArtifactTypes.Unknown; + } + catch (Exception ex) + { + log.ErrorFormat("Failed to get artifact type: {Message}", ex.Message); + return OciArtifactTypes.Unknown; + } + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciClient.cs similarity index 52% rename from source/Calamari.Shared/Integration/Packages/Download/Oci.cs rename to source/Calamari.Shared/Integration/Packages/Download/Oci/OciClient.cs index ed1bdd6bb..234490c8d 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/Oci.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciClient.cs @@ -11,91 +11,45 @@ using Newtonsoft.Json.Linq; using Octopus.Versioning; -namespace Calamari.Integration.Packages.Download +namespace Calamari.Integration.Packages.Download.Oci { - // TODO: make less static, and store things like the client. - class Oci + public class OciClient { - const string VersionPath = "v2"; - // const string OciImageManifestAcceptHeader = "application/vnd.oci.image.manifest.v1+json"; + readonly HttpClient httpClient = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); - internal class Manifest + public JObject? GetManifest(Uri feedUri, string packageId, IVersion version, string? feedUsername, string? feedPassword) { - internal const string MediaTypePropertyName = "mediaType"; - internal const string DockerImageMediaTypeValue = "application/vnd.docker.distribution.manifest.v2+json"; - - internal class Config - { - internal const string PropertyName = "config"; - internal const string MediaTypePropertyName = "mediaType"; - internal const string OciImageMediaTypeValue = "application/vnd.oci.image.config.v1+json"; - internal const string DockerImageMediaTypeValue = "application/vnd.docker.container.image.v1+json"; - } - - internal class Image - { - internal const string TitleAnnotationKey = "org.opencontainers.image.title"; - } + var url = GetApiUri(feedUri); + var fixedVersion = FixVersion(version); + using var response = Get(new Uri($"{url}/{packageId}/manifests/{fixedVersion}"), new NetworkCredential(feedUsername, feedPassword), ApplyAcceptHeaderFunc); + return JsonConvert.DeserializeObject(response.Content.ReadAsStringAsync().Result); - internal class Layers - { - internal const string PropertyName = "layers"; - internal const string DigestPropertyName = "digest"; - internal const string SizePropertyName = "size"; - internal const string MediaTypePropertyName = "mediaType"; - internal const string AnnotationsPropertyName = "annotations"; - internal const string HelmChartMediaTypeValue = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v370 - internal const string DockerImageMediaTypeValue = "application/vnd.docker.image.rootfs.diff.tar.gzip"; - } + void ApplyAcceptHeaderFunc(HttpRequestMessage request) => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciConstants.Manifest.Config.OciImageMediaTypeValue)); } - static readonly Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); - - internal static JObject? GetManifest(HttpClient client, - Uri url, - string packageId, - string version, - string? feedUsername, - string? feedPassword) + public HttpResponseMessage GetPackage(Uri feedUri, string packageId, string digest, string? feedUsername, string? feedPassword) { - using var response = Get(client, new Uri($"{url}/{packageId}/manifests/{version}"), new NetworkCredential(feedUsername, feedPassword), ApplyAcceptHeader); - var manifest = JsonConvert.DeserializeObject(response.Content.ReadAsStringAsync().Result); - - return manifest; - } - - static void ApplyAcceptHeader(HttpRequestMessage request) - => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(Manifest.Config.OciImageMediaTypeValue)); + var url = GetApiUri(feedUri); + HttpResponseMessage? response = null; + try + { + response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); - internal static Uri GetApiUri(Uri feedUri) - { - var httpScheme = BuildScheme(IsPlainHttp(feedUri)); + if (!response.IsSuccessStatusCode) + { + throw new CommandException($"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); + } - var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); - var uri = new Uri(r); - if (!r.EndsWith("/" + VersionPath)) + return response; + } + catch { - uri = new Uri(uri, VersionPath); + response?.Dispose(); + throw; } - - return uri; - - static bool IsPlainHttp(Uri uri) - => uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase); - - static string BuildScheme(bool isPlainHttp) - => isPlainHttp ? Uri.UriSchemeHttp : Uri.UriSchemeHttps; } - // oci registries don't support the '+' tagging - // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380 - internal static string FixVersion(IVersion version) - => version.ToString().Replace("+", "_"); - - internal static string? GetPackageHashFromDigest(string digest) - => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; - - internal static HttpResponseMessage Get(HttpClient client, Uri url, ICredentials credentials, Action? customAcceptHeader = null) + HttpResponseMessage Get(Uri url, ICredentials credentials, Action? customAcceptHeader = null) { var request = new HttpRequestMessage(HttpMethod.Get, url); try @@ -108,15 +62,15 @@ internal static HttpResponseMessage Get(HttpClient client, Uri url, ICredentials } customAcceptHeader?.Invoke(request); - var response = SendRequest(client, request); + var response = SendRequest(httpClient, request); if (response.StatusCode == HttpStatusCode.Unauthorized) { - var tokenFromAuthService = GetAuthRequestHeader(client, response, networkCredential); + var tokenFromAuthService = GetAuthRequestHeader(response, networkCredential); request = new HttpRequestMessage(HttpMethod.Get, url); request.Headers.Authorization = tokenFromAuthService; customAcceptHeader?.Invoke(request); - response = SendRequest(client, request); + response = SendRequest(httpClient, request); if (response.StatusCode == HttpStatusCode.Unauthorized) { @@ -149,12 +103,12 @@ internal static HttpResponseMessage Get(HttpClient client, Uri url, ICredentials } } - static AuthenticationHeaderValue GetAuthRequestHeader(HttpClient client, HttpResponseMessage response, NetworkCredential credential) + AuthenticationHeaderValue GetAuthRequestHeader(HttpResponseMessage response, NetworkCredential credential) { var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer"); if (auth != null) { - var authToken = RetrieveAuthenticationToken(client, GetOAuthServiceUrl(auth), credential); + var authToken = RetrieveAuthenticationToken(auth, credential); return new AuthenticationHeaderValue("Bearer", authToken); } @@ -166,20 +120,21 @@ static AuthenticationHeaderValue GetAuthRequestHeader(HttpClient client, HttpRes throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`"); } - static string RetrieveAuthenticationToken(HttpClient client, string authUrl, NetworkCredential credential) + string RetrieveAuthenticationToken(AuthenticationHeaderValue auth, NetworkCredential credential) { HttpResponseMessage? response = null; try { + var authUrl = GetOAuthServiceUrl(auth); using (var msg = new HttpRequestMessage(HttpMethod.Get, authUrl)) { - if (credential?.UserName != null) + if (credential.UserName != null) { msg.Headers.Authorization = CreateAuthenticationHeader(credential); } - response = SendRequest(client, msg); + response = SendRequest(httpClient, msg); } if (response.IsSuccessStatusCode) @@ -216,63 +171,41 @@ static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) return oathUrl.ToString(); } - /* - public static bool HasAnnotationContaining(JObject manifest, string key, string value) - { - var annotations = manifest[Manifest.Annotations.PropertyName]; - return annotations is { Type: JTokenType.Object } - && annotations[key] != null - && annotations[key].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; // ~equiv to case insensitive contains, for non-net standard 2.0+ - } */ - - public static bool HasMediaTypeContaining(JObject manifest, string value) + static string ExtractTokenFromResponse(HttpResponseMessage response) { - var mediaType = manifest[Manifest.MediaTypePropertyName]; - - return mediaType != null - && mediaType.ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; + var token = GetContent(response); + return (string)JObject.Parse(token).SelectToken("token") + ?? throw new CommandException("Unable to retrieve authentication token required to perform operation."); } - public static bool HasConfigMediaTypeContaining(JObject manifest, string value) + static Uri GetApiUri(Uri feedUri) { - var config = manifest[Manifest.Config.PropertyName]; - - return config is { Type: JTokenType.Object } - && config[Manifest.Config.MediaTypePropertyName] != null - && config[Manifest.Config.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; - } + var httpScheme = IsPlainHttp(feedUri) ? Uri.UriSchemeHttp : Uri.UriSchemeHttps; - public static bool HasLayersMediaTypeContaining(JObject manifest, string value) - { - var layers = manifest[Manifest.Layers.PropertyName]; + var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); + var uri = new Uri(r); - if (layers is { Type: JTokenType.Array }) + const string versionPath = "v2"; + if (!r.EndsWith("/" + versionPath)) { - foreach (var layer in layers) - { - if (layer[Manifest.Layers.MediaTypePropertyName] != null - && layer[Manifest.Layers.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0) - { - return true; - } - } + uri = new Uri(uri, versionPath); } - return false; + return uri; + + static bool IsPlainHttp(Uri uri) + => uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase); } - static string ExtractTokenFromResponse(HttpResponseMessage response) - { - var token = GetContent(response); + // oci registries don't support the '+' tagging + // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380 + static string FixVersion(IVersion version) + => version.ToString().Replace("+", "_"); - var lastItem = (string)JObject.Parse(token).SelectToken("token"); - if (lastItem != null) - { - return lastItem; - } + static readonly Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); - throw new CommandException("Unable to retrieve authentication token required to perform operation."); - } + internal static string? GetPackageHashFromDigest(string digest) + => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; static AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential) { diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs new file mode 100644 index 000000000..fcddff8d7 --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs @@ -0,0 +1,37 @@ +using System; + +namespace Calamari.Integration.Packages.Download.Oci +{ + public static class OciConstants + { + public class Manifest + { + internal const string MediaTypePropertyName = "mediaType"; + internal const string DockerImageMediaTypeValue = "application/vnd.docker.distribution.manifest.v2+json"; + + public class Config + { + internal const string PropertyName = "config"; + internal const string MediaTypePropertyName = "mediaType"; + internal const string OciImageMediaTypeValue = "application/vnd.oci.image.config.v1+json"; + internal const string DockerImageMediaTypeValue = "application/vnd.docker.container.image.v1+json"; + } + + public class Image + { + internal const string TitleAnnotationKey = "org.opencontainers.image.title"; + } + + public class Layers + { + internal const string PropertyName = "layers"; + internal const string DigestPropertyName = "digest"; + internal const string SizePropertyName = "size"; + internal const string MediaTypePropertyName = "mediaType"; + internal const string AnnotationsPropertyName = "annotations"; + internal const string HelmChartMediaTypeValue = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v370 + internal const string DockerImageMediaTypeValue = "application/vnd.docker.image.rootfs.diff.tar.gzip"; + } + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs new file mode 100644 index 000000000..9ded1669b --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs @@ -0,0 +1,44 @@ +using System; +using Newtonsoft.Json.Linq; + +namespace Calamari.Integration.Packages.Download.Oci +{ + static class OciJObjectExtensions + { + public static bool HasMediaTypeContaining(this JObject manifest, string value) + { + var mediaType = manifest[OciConstants.Manifest.MediaTypePropertyName]; + + return mediaType != null + && mediaType.ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; + } + + public static bool HasConfigMediaTypeContaining(this JObject manifest, string value) + { + var config = manifest[OciConstants.Manifest.Config.PropertyName]; + + return config is { Type: JTokenType.Object } + && config[OciConstants.Manifest.Config.MediaTypePropertyName] != null + && config[OciConstants.Manifest.Config.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; + } + + public static bool HasLayersMediaTypeContaining(this JObject manifest, string value) + { + var layers = manifest[OciConstants.Manifest.Layers.PropertyName]; + + if (layers is { Type: JTokenType.Array }) + { + foreach (var layer in layers) + { + if (layer[OciConstants.Manifest.Layers.MediaTypePropertyName] != null + && layer[OciConstants.Manifest.Layers.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + } + + return false; + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs b/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs deleted file mode 100644 index 72a77778f..000000000 --- a/source/Calamari.Shared/Integration/Packages/Download/OciArtifactManifestRetriever.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using System.Net; -using System.Net.Http; -using Octopus.Versioning; - -namespace Calamari.Integration.Packages.Download -{ - public class OciArtifactManifestRetriever - { - public OciArtifactTypes TryGetArtifactType(string packageId, - IVersion version, - Uri feedUri, - string? feedUsername, - string? feedPassword) - { - try - { - var versionString = Oci.FixVersion(version); - var apiUrl = Oci.GetApiUri(feedUri); - - var client = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); - - var jsonManifest = Oci.GetManifest(client, - apiUrl, - packageId, - versionString, - feedUsername, - feedPassword); - - // Check for Helm chart annotations - var isHelmChart = //Oci.HasAnnotationContaining(jsonManifest, Oci.Manifest.Annotations.HelmChartAnnotationKey, "helm") - Oci.HasConfigMediaTypeContaining(jsonManifest, Oci.Manifest.Config.OciImageMediaTypeValue) - || Oci.HasLayersMediaTypeContaining(jsonManifest, Oci.Manifest.Layers.HelmChartMediaTypeValue); - - if (isHelmChart) - return OciArtifactTypes.HelmChart; - - var isDockerImage = Oci.HasMediaTypeContaining(jsonManifest, Oci.Manifest.DockerImageMediaTypeValue) - || Oci.HasConfigMediaTypeContaining(jsonManifest, Oci.Manifest.Config.DockerImageMediaTypeValue) - || Oci.HasLayersMediaTypeContaining(jsonManifest, Oci.Manifest.Layers.DockerImageMediaTypeValue); - - return isDockerImage ? OciArtifactTypes.DockerImage : OciArtifactTypes.Unknown; - } - catch (Exception ex) - { - return OciArtifactTypes.Unknown; - } - } - } -} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs index 13afda7ac..53c390117 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs @@ -1,18 +1,13 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Text.RegularExpressions; -using System.Web; using Calamari.Common.Commands; using Calamari.Common.Features.Packages; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; -using Newtonsoft.Json; +using Calamari.Integration.Packages.Download.Oci; using Newtonsoft.Json.Linq; using Octopus.Versioning; @@ -31,20 +26,22 @@ public class OciPackageDownloader : IPackageDownloader readonly ICalamariFileSystem fileSystem; readonly ICombinedPackageExtractor combinedPackageExtractor; readonly ILog log; - readonly HttpClient client; + readonly OciClient ociClient; public OciPackageDownloader( ICalamariFileSystem fileSystem, ICombinedPackageExtractor combinedPackageExtractor, + OciClient ociClient, ILog log) { this.fileSystem = fileSystem; this.combinedPackageExtractor = combinedPackageExtractor; + this.ociClient = ociClient; this.log = log; - client = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); } - public PackagePhysicalFileMetadata DownloadPackage(string packageId, + public PackagePhysicalFileMetadata DownloadPackage( + string packageId, IVersion version, string feedId, Uri feedUri, @@ -83,23 +80,15 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, Directory.CreateDirectory(stagingDir); } - var versionString = Oci.FixVersion(version); - - var apiUrl = Oci.GetApiUri(feedUri); - var (digest, size, extension) = GetPackageDetails(apiUrl, packageId, versionString, feedUsername, feedPassword); - var hash = Oci.GetPackageHashFromDigest(digest); + var (digest, size, extension) = GetPackageDetails(feedUri, packageId, version, feedUsername, feedPassword); + var hash = OciClient.GetPackageHashFromDigest(digest); var cachedFileName = PackageName.ToCachedFileName(packageId, version, extension); var downloadPath = Path.Combine(Path.Combine(stagingDir, cachedFileName)); var retryStrategy = PackageDownloaderRetryUtils.CreateRetryStrategy(maxDownloadAttempts, downloadAttemptBackoff, log); - retryStrategy.Execute(() => DownloadPackage(apiUrl, - packageId, - digest, - feedUsername, - feedPassword, - downloadPath)); -; + retryStrategy.Execute( + () => DownloadPackage(feedUri, packageId, digest, feedUsername, feedPassword, downloadPath)); var localDownloadName = Path.Combine(cacheDirectory, cachedFileName); fileSystem.MoveFile(downloadPath, localDownloadName); @@ -110,23 +99,23 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, localDownloadName, hash, size) - : PackagePhysicalFileMetadata.Build(localDownloadName) + : PackagePhysicalFileMetadata.Build(localDownloadName) ?? throw new CommandException($"Unable to retrieve metadata for package {packageId}, version {version}"); } } (string digest, int size, string extension) GetPackageDetails( - Uri url, + Uri feedUri, string packageId, - string version, - string? feedUserName, + IVersion version, + string? feedUserName, string? feedPassword) { - var manifest = Oci.GetManifest(client, url, packageId, version, feedUserName, feedPassword); + var manifest = ociClient.GetManifest(feedUri, packageId, version, feedUserName, feedPassword); - var layer = manifest.Value(Oci.Manifest.Layers.PropertyName)[0]; - var digest = layer.Value(Oci.Manifest.Layers.DigestPropertyName); - var size = layer.Value(Oci.Manifest.Layers.SizePropertyName); + var layer = manifest.Value(OciConstants.Manifest.Layers.PropertyName)[0]; + var digest = layer.Value(OciConstants.Manifest.Layers.DigestPropertyName); + var size = layer.Value(OciConstants.Manifest.Layers.SizePropertyName); var extension = GetExtensionFromManifest(layer); return (digest, size, extension); @@ -134,17 +123,18 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, string GetExtensionFromManifest(JToken layer) { - var artifactTitle = layer.Value(Oci.Manifest.Layers.AnnotationsPropertyName)?[Oci.Manifest.Image.TitleAnnotationKey]?.Value() ?? ""; + var artifactTitle = layer.Value(OciConstants.Manifest.Layers.AnnotationsPropertyName)?[OciConstants.Manifest.Image.TitleAnnotationKey]?.Value() ?? ""; var extension = combinedPackageExtractor - .Extensions - .FirstOrDefault(ext => - Path.GetExtension(artifactTitle).Equals(ext, StringComparison.OrdinalIgnoreCase)); + .Extensions + .FirstOrDefault( + ext => + Path.GetExtension(artifactTitle).Equals(ext, StringComparison.OrdinalIgnoreCase)); - return extension ?? (layer.Value(Oci.Manifest.Layers.MediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); + return extension ?? (layer.Value(OciConstants.Manifest.Layers.MediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); } void DownloadPackage( - Uri url, + Uri feedUri, string packageId, string digest, string? feedUsername, @@ -152,12 +142,7 @@ void DownloadPackage( string downloadPath) { using var fileStream = fileSystem.OpenFile(downloadPath, FileAccess.Write); - using var response = Oci.Get(client, new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); - if (!response.IsSuccessStatusCode) - { - throw new CommandException( - $"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); - } + using var response = ociClient.GetPackage(feedUri, packageId, digest, feedUsername, feedPassword); response.Content.CopyToAsync(fileStream).GetAwaiter().GetResult(); } diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index 6c4e6edd7..290e049bc 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -5,6 +5,7 @@ using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; +using Calamari.Integration.Packages.Download.Oci; using Octopus.Versioning; namespace Calamari.Integration.Packages.Download @@ -63,14 +64,14 @@ public PackagePhysicalFileMetadata DownloadPackage( downloader = new HelmChartPackageDownloader(fileSystem, log); break; case FeedType.OciRegistry: - downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), log); + downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), new OciClient(), log); break; case FeedType.AwsElasticContainerRegistry: - var x = new OciArtifactManifestRetriever(); - if (x.TryGetArtifactType(packageId, version, feedUri, feedUsername, feedPassword) - == OciArtifactTypes.HelmChart) + var ociClient = new OciClient(); + var ociArtifactManifestRetriever = new OciArtifactManifestRetriever(ociClient, log); + if (ociArtifactManifestRetriever.TryGetArtifactType(packageId, version, feedUri, feedUsername, feedPassword) == OciArtifactTypes.HelmChart) { - downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), log); + downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), ociClient, log); } else { diff --git a/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs b/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs index 52dade37c..b4629f903 100644 --- a/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs +++ b/source/Calamari.Tests/Fixtures/PackageDownload/AwsEcrDownloadFixture.cs @@ -1,11 +1,9 @@ using System; using System.IO; using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Amazon; -using Amazon.ECR; -using Amazon.ECR.Model; using Calamari.Common.Features.Packages; using Calamari.Common.Features.Processes; using Calamari.Common.Features.Scripting; @@ -13,7 +11,7 @@ using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; using Calamari.Integration.Packages.Download; -using Calamari.Testing.Helpers; +using Calamari.Testing; using Calamari.Testing.Requirements; using Calamari.Tests.Helpers; using FluentAssertions; @@ -28,6 +26,7 @@ namespace Calamari.Tests.Fixtures.PackageDownload public class AwsEcrDownloadFixture : CalamariFixture { static readonly string Home = Path.GetTempPath(); + readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); [OneTimeSetUp] public void TestFixtureSetUp() @@ -36,20 +35,20 @@ public void TestFixtureSetUp() } [Test] - public void HelmChartIsSuccessfullyDownloaded() + public async Task HelmChartIsSuccessfullyDownloaded() { - var regionEndpoint = RegionEndpoint.USEast1; - const string repositoryName = "markc-test"; + var regionEndpoint = RegionEndpoint.USWest2; + const string repositoryName = "calamari-testing-helm-chart"; const string imageTag = "0.1.0"; - var packagePhysicalFileMetadata = DoDownload(regionEndpoint, repositoryName, imageTag); + var packagePhysicalFileMetadata = await DoDownload(regionEndpoint, repositoryName, imageTag, cancellationTokenSource.Token); packagePhysicalFileMetadata.Should().NotBeNull(); using (new AssertionScope()) { packagePhysicalFileMetadata.PackageId.Should().Be(repositoryName); - packagePhysicalFileMetadata.Version.Should().Be(imageTag); + packagePhysicalFileMetadata.Version.Should().Be(new SemanticVersion(imageTag)); packagePhysicalFileMetadata.Extension.Should().Be(".tgz"); packagePhysicalFileMetadata.FullFilePath.Should().Contain(repositoryName); } @@ -57,14 +56,14 @@ public void HelmChartIsSuccessfullyDownloaded() [Test] [RequiresDockerInstalled] - public void DockerImageIsSuccessfullyDownloaded() + public async Task DockerImageIsSuccessfullyDownloaded() { - const string repositoryName = "markc-test"; + const string repositoryName = "calamari-testing-container-image"; const string imageTag = "1.0.0"; - - var regionEndpoint = RegionEndpoint.USEast1; - var packagePhysicalFileMetadata = DoDownload(regionEndpoint, repositoryName, imageTag); + var regionEndpoint = RegionEndpoint.USWest2; + + var packagePhysicalFileMetadata = await DoDownload(regionEndpoint, repositoryName, imageTag, cancellationTokenSource.Token); packagePhysicalFileMetadata.Should().NotBeNull(); @@ -76,48 +75,41 @@ public void DockerImageIsSuccessfullyDownloaded() } } - static PackagePhysicalFileMetadata DoDownload(RegionEndpoint regionEndpoint, string repositoryName, string imageTag) + static async Task DoDownload(RegionEndpoint regionEndpoint, string repositoryName, string imageTag, CancellationToken cancellationToken) { var log = Substitute.For(); var runner = new CommandLineRunner(log, new CalamariVariables()); var engine = new ScriptEngine(Enumerable.Empty(), log); - var strategy = new PackageDownloaderStrategy(log, - engine, - CalamariPhysicalFileSystem.GetPhysicalFileSystem(), - runner, - new CalamariVariables()); - - var authDetails = GetEcrAuthDetails(regionEndpoint); - var registryUri = new Uri(authDetails.Registry); - - var packagePhysicalFileMetadata = strategy.DownloadPackage(repositoryName, - SemVerFactory.CreateVersion(imageTag), - "", - registryUri, - FeedType.AwsElasticContainerRegistry, - authDetails.Username, - authDetails.Password, - true, - 1, - TimeSpan.FromSeconds(30)); + var strategy = new PackageDownloaderStrategy( + log, + engine, + CalamariPhysicalFileSystem.GetPhysicalFileSystem(), + runner, + new CalamariVariables()); + + var authDetails = await GetEcrAuthDetails(regionEndpoint, cancellationToken); + var registryUri = new Uri(authDetails.RegistryUri); + + var packagePhysicalFileMetadata = strategy.DownloadPackage( + repositoryName, + SemVerFactory.CreateVersion(imageTag), + "", + registryUri, + FeedType.AwsElasticContainerRegistry, + authDetails.Username, + authDetails.Password, + true, + 1, + TimeSpan.FromSeconds(30)); return packagePhysicalFileMetadata; } - static (string Username, string Password, string Registry) GetEcrAuthDetails(RegionEndpoint regionEndpoint) + static async Task GetEcrAuthDetails(RegionEndpoint regionEndpoint, CancellationToken cancellationToken) { - var ecrClient = new AmazonECRClient(regionEndpoint); - var authTokenRequest = new GetAuthorizationTokenRequest(); - var authTokenResponse = ecrClient.GetAuthorizationTokenAsync(authTokenRequest).Result; - - var authorizationData = authTokenResponse.AuthorizationData[0]; - var token = authorizationData.AuthorizationToken; - var decodedToken = Encoding.UTF8.GetString(Convert.FromBase64String(token)); - var usernamePassword = decodedToken.Split(':'); - var username = usernamePassword[0]; - var password = usernamePassword[1]; - var registry = authorizationData.ProxyEndpoint; - - return (username, password, registry); + return new AwsElasticContainerRegistryCredentials().RetrieveTemporaryCredentials( + await ExternalVariables.Get(ExternalVariable.AwsCloudFormationAndS3AccessKey, cancellationToken), + await ExternalVariables.Get(ExternalVariable.AwsCloudFormationAndS3SecretKey, cancellationToken), + regionEndpoint.SystemName); } } } \ No newline at end of file diff --git a/source/Calamari.Tests/Fixtures/PackageDownload/AwsElasticContainerRegistryCredentials.cs b/source/Calamari.Tests/Fixtures/PackageDownload/AwsElasticContainerRegistryCredentials.cs new file mode 100644 index 000000000..1b5f2f73a --- /dev/null +++ b/source/Calamari.Tests/Fixtures/PackageDownload/AwsElasticContainerRegistryCredentials.cs @@ -0,0 +1,110 @@ +#pragma warning disable CS8601 // Possible null reference assignment +#pragma warning disable CS8618 // Non-nullable property {0} is uninitialized +using System; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Security.Authentication; +using System.Text; +using Amazon; +using Amazon.ECR; +using Amazon.ECR.Model; +using Amazon.Runtime; +using JetBrains.Annotations; +using SharpCompress; + +namespace Calamari.Tests.Fixtures.PackageDownload +{ + // Taken from Server + public class AwsElasticContainerRegistryCredentials + { + /// + /// AWS credentials expire after 12 hours: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html + /// Rather than hit the AWS service every time we want to use the registry, we can cache them locally based on the expiry time in the response + /// + static readonly ConcurrentDictionary<(string accessKey, string region), TemporaryCredentials> CachedCreds = new ConcurrentDictionary<(string accessKey, string region), TemporaryCredentials>(); + + public virtual TemporaryCredentials RetrieveTemporaryCredentials(string accessKey, string secretKey, string region, bool bypassCache = false) + { + if (!bypassCache) + { + if (TryUseCache(accessKey, region, out var cachedCreds)) return cachedCreds; + } + + var authToken = GetAuthorizationData(accessKey, secretKey, region); + var creds = DecodeCredentials(authToken); + var tempCreds = new TemporaryCredentials + { + Expiry = authToken.ExpiresAt, + RegistryUri = authToken.ProxyEndpoint, + Password = creds.Password, + Username = creds.Username + }; + + return CachedCreds.AddOrUpdate((accessKey, region), tempCreds, (tuple, credentials) => tempCreds); + } + + static bool TryUseCache(string accessKey, string region, out TemporaryCredentials temporaryCredentials) + { + // AWS Creds have 12 hour expiry + // to ensure it doesnt expire mid-deploy lets make sure we have at least 2 hours left + CachedCreds.ToArray() + .Where(kvp => kvp.Value.Expiry.AddHours(-2) < DateTime.UtcNow) + .ForEach(k => CachedCreds.TryRemove(k.Key, out var _)); + + return CachedCreds.TryGetValue((accessKey, region), out temporaryCredentials); + } + + (string Username, string Password) DecodeCredentials(AuthorizationData authToken) + { + try + { + var decodedToken = Encoding.UTF8.GetString(Convert.FromBase64String(authToken.AuthorizationToken)); + var parts = decodedToken.Split(':'); + if (parts.Length != 2) + { + throw new AuthenticationException("Token returned by AWS is in an unexpected format"); + } + + return (parts[0], parts[1]); + } + catch (Exception) + { + throw new AuthenticationException("Token returned by AWS is in an unexpected format"); + } + } + + static AuthorizationData GetAuthorizationData(string accessKey, string secretKey, string region) + { + var regionEndpoint = RegionEndpoint.GetBySystemName(region); + var credentials = new BasicAWSCredentials(accessKey, secretKey); + var client = new AmazonECRClient(credentials, regionEndpoint); + try + { + var token = client.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()).Result; + var authToken = token.AuthorizationData.FirstOrDefault(); + if (authToken == null) + { + throw new Exception("No AuthToken found"); + } + + return authToken; + } + catch (Exception ex) + { + throw new AuthenticationException($"Unable to retrieve AWS Authorization token:\r\n\t{ex.Message}"); + } + } + + public class TemporaryCredentials + { + public string Username { get; set; } + + [CanBeNull] + public string Password { get; set; } + + public string RegistryUri { get; set; } + public DateTime Expiry { get; set; } + } + } +} From 139f14246a8e6775a2014a17d9448142efa15836 Mon Sep 17 00:00:00 2001 From: Eddy Moulton Date: Tue, 17 Dec 2024 10:52:47 +1100 Subject: [PATCH 5/7] Add feature toggle --- .../FeatureToggles/OctopusFeatureToggle.cs | 2 + .../OciOrDockerImagePackageDownloader.cs | 64 +++++++++++++++++++ .../Packages/Download/OciPackageDownloader.cs | 2 - .../Download/PackageDownloaderStrategy.cs | 19 ++++-- 4 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs diff --git a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs index dcf856900..8db310dc7 100644 --- a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs +++ b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs @@ -9,12 +9,14 @@ public static class KnownSlugs public const string KubernetesObjectManifestInspection = "kubernetes-object-manifest-inspection"; public const string KOSForHelm = "kos-for-helm"; public const string ExecuteHelmUpgradeCommandViaShellScript = "execute-helm-upgrade-command-via-shell-script"; + public const string OciForContainerRegistries = "oci-for-container-registries"; }; public static readonly OctopusFeatureToggle NonPrimaryGitDependencySupportFeatureToggle = new OctopusFeatureToggle("non-primary-git-dependency-support"); public static readonly OctopusFeatureToggle KubernetesObjectManifestInspectionFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KubernetesObjectManifestInspection); public static readonly OctopusFeatureToggle KOSForHelmFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KOSForHelm); public static readonly OctopusFeatureToggle ExecuteHelmUpgradeCommandViaShellScriptFeatureToggle = new OctopusFeatureToggle(KnownSlugs.ExecuteHelmUpgradeCommandViaShellScript); + public static readonly OctopusFeatureToggle OciForContainerRegistriesFeatureToggle = new OctopusFeatureToggle(KnownSlugs.OciForContainerRegistries); public class OctopusFeatureToggle { diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs new file mode 100644 index 000000000..a9b026996 --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs @@ -0,0 +1,64 @@ +using System; +using Calamari.Common.Features.Packages; +using Calamari.Common.Plumbing.Logging; +using Calamari.Integration.Packages.Download.Oci; +using Octopus.Versioning; + +namespace Calamari.Integration.Packages.Download +{ + public class OciOrDockerImagePackageDownloader : IPackageDownloader + { + readonly OciPackageDownloader ociPackageDownloader; + readonly DockerImagePackageDownloader dockerImagePackageDownloader; + readonly OciClient ociClient; + readonly ILog log; + + public OciOrDockerImagePackageDownloader( + OciPackageDownloader ociPackageDownloader, + DockerImagePackageDownloader dockerImagePackageDownloader, + OciClient ociClient, + ILog log) + { + this.ociPackageDownloader = ociPackageDownloader; + this.dockerImagePackageDownloader = dockerImagePackageDownloader; + this.log = log; + this.ociClient = ociClient; + } + + public PackagePhysicalFileMetadata DownloadPackage( + string packageId, + IVersion version, + string feedId, + Uri feedUri, + string? feedUsername, + string? feedPassword, + bool forcePackageDownload, + int maxDownloadAttempts, + TimeSpan downloadAttemptBackoff) + { + var downloader = GetInnerDownloader(packageId, version, feedUri, feedUsername, feedPassword); + + return downloader.DownloadPackage( + packageId, + version, + feedId, + feedUri, + feedUsername, + feedPassword, + forcePackageDownload, + maxDownloadAttempts, + downloadAttemptBackoff); + } + + IPackageDownloader GetInnerDownloader(string packageId, IVersion version, Uri feedUri, string? feedUsername, string? feedPassword) + { + var ociArtifactManifestRetriever = new OciArtifactManifestRetriever(ociClient, log); + if (ociArtifactManifestRetriever.TryGetArtifactType(packageId, version, feedUri, feedUsername, feedPassword) == OciArtifactTypes.HelmChart) + { + return ociPackageDownloader; + } + + return dockerImagePackageDownloader; + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs index 53c390117..8f40ef214 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs @@ -1,8 +1,6 @@ using System; using System.IO; using System.Linq; -using System.Net; -using System.Net.Http; using Calamari.Common.Commands; using Calamari.Common.Features.Packages; using Calamari.Common.Plumbing.FileSystem; diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index 290e049bc..7f60b46a9 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -2,6 +2,7 @@ using Calamari.Common.Features.Packages; using Calamari.Common.Features.Processes; using Calamari.Common.Features.Scripting; +using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; @@ -64,25 +65,23 @@ public PackagePhysicalFileMetadata DownloadPackage( downloader = new HelmChartPackageDownloader(fileSystem, log); break; case FeedType.OciRegistry: - downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), new OciClient(), log); + downloader = OciPackageDownloader(); break; case FeedType.AwsElasticContainerRegistry: - var ociClient = new OciClient(); - var ociArtifactManifestRetriever = new OciArtifactManifestRetriever(ociClient, log); - if (ociArtifactManifestRetriever.TryGetArtifactType(packageId, version, feedUri, feedUsername, feedPassword) == OciArtifactTypes.HelmChart) + if (OctopusFeatureToggles.OciForContainerRegistriesFeatureToggle.IsEnabled(variables)) { - downloader = new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), ociClient, log); + downloader = new OciOrDockerImagePackageDownloader(OciPackageDownloader(), DockerImagePackageDownloader(), new OciClient(), log); } else { - downloader = new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); + downloader = DockerImagePackageDownloader(); } break; case FeedType.Docker: case FeedType.AzureContainerRegistry: case FeedType.GoogleContainerRegistry: - downloader = new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); + downloader = DockerImagePackageDownloader(); break; case FeedType.S3: downloader = new S3PackageDownloader(log, fileSystem); @@ -107,5 +106,11 @@ public PackagePhysicalFileMetadata DownloadPackage( maxDownloadAttempts, downloadAttemptBackoff); } + + OciPackageDownloader OciPackageDownloader() + => new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), new OciClient(), log); + + DockerImagePackageDownloader DockerImagePackageDownloader() + => new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log); } } \ No newline at end of file From 8822d19563fea029fff0ea559f9ca4c23ff52ca6 Mon Sep 17 00:00:00 2001 From: Eddy Moulton Date: Tue, 17 Dec 2024 15:14:37 +1100 Subject: [PATCH 6/7] Remove feature toggle --- .../FeatureToggles/OctopusFeatureToggle.cs | 2 -- .../Packages/Download/PackageDownloaderStrategy.cs | 10 +--------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs index 8db310dc7..dcf856900 100644 --- a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs +++ b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs @@ -9,14 +9,12 @@ public static class KnownSlugs public const string KubernetesObjectManifestInspection = "kubernetes-object-manifest-inspection"; public const string KOSForHelm = "kos-for-helm"; public const string ExecuteHelmUpgradeCommandViaShellScript = "execute-helm-upgrade-command-via-shell-script"; - public const string OciForContainerRegistries = "oci-for-container-registries"; }; public static readonly OctopusFeatureToggle NonPrimaryGitDependencySupportFeatureToggle = new OctopusFeatureToggle("non-primary-git-dependency-support"); public static readonly OctopusFeatureToggle KubernetesObjectManifestInspectionFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KubernetesObjectManifestInspection); public static readonly OctopusFeatureToggle KOSForHelmFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KOSForHelm); public static readonly OctopusFeatureToggle ExecuteHelmUpgradeCommandViaShellScriptFeatureToggle = new OctopusFeatureToggle(KnownSlugs.ExecuteHelmUpgradeCommandViaShellScript); - public static readonly OctopusFeatureToggle OciForContainerRegistriesFeatureToggle = new OctopusFeatureToggle(KnownSlugs.OciForContainerRegistries); public class OctopusFeatureToggle { diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index 7f60b46a9..f7b0f9547 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -68,15 +68,7 @@ public PackagePhysicalFileMetadata DownloadPackage( downloader = OciPackageDownloader(); break; case FeedType.AwsElasticContainerRegistry: - if (OctopusFeatureToggles.OciForContainerRegistriesFeatureToggle.IsEnabled(variables)) - { - downloader = new OciOrDockerImagePackageDownloader(OciPackageDownloader(), DockerImagePackageDownloader(), new OciClient(), log); - } - else - { - downloader = DockerImagePackageDownloader(); - } - + downloader = new OciOrDockerImagePackageDownloader(OciPackageDownloader(), DockerImagePackageDownloader(), new OciClient(), log); break; case FeedType.Docker: case FeedType.AzureContainerRegistry: From d2782881879923e16fb9b5e56ccecb364a9de353 Mon Sep 17 00:00:00 2001 From: Eddy Moulton Date: Thu, 19 Dec 2024 15:59:10 +1100 Subject: [PATCH 7/7] Rename to OciRegistryClient to align with server --- .../Oci/OciArtifactManifestRetriever.cs | 8 ++-- .../{OciClient.cs => OciRegistryClient.cs} | 38 +++++++++++-------- .../OciOrDockerImagePackageDownloader.cs | 8 ++-- .../Packages/Download/OciPackageDownloader.cs | 27 +++---------- .../Download/PackageDownloaderStrategy.cs | 4 +- 5 files changed, 38 insertions(+), 47 deletions(-) rename source/Calamari.Shared/Integration/Packages/Download/Oci/{OciClient.cs => OciRegistryClient.cs} (89%) diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs index 6d5be5b6b..5bad549ef 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs @@ -6,12 +6,12 @@ namespace Calamari.Integration.Packages.Download.Oci { public class OciArtifactManifestRetriever { - readonly OciClient ociClient; + readonly OciRegistryClient ociRegistryClient; readonly ILog log; - public OciArtifactManifestRetriever(OciClient ociClient, ILog log) + public OciArtifactManifestRetriever(OciRegistryClient ociRegistryClient, ILog log) { - this.ociClient = ociClient; + this.ociRegistryClient = ociRegistryClient; this.log = log; } @@ -24,7 +24,7 @@ public OciArtifactTypes TryGetArtifactType( { try { - var jsonManifest = ociClient.GetManifest(feedUri, packageId, version, feedUsername, feedPassword); + var jsonManifest = ociRegistryClient.GetManifest(feedUri, packageId, version, feedUsername, feedPassword); // Check for Helm chart annotations var isHelmChart = diff --git a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciClient.cs b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciRegistryClient.cs similarity index 89% rename from source/Calamari.Shared/Integration/Packages/Download/Oci/OciClient.cs rename to source/Calamari.Shared/Integration/Packages/Download/Oci/OciRegistryClient.cs index 234490c8d..816eee668 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/Oci/OciClient.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/Oci/OciRegistryClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; @@ -7,15 +8,22 @@ using System.Text; using System.Text.RegularExpressions; using Calamari.Common.Commands; +using Calamari.Common.Plumbing.FileSystem; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Octopus.Versioning; namespace Calamari.Integration.Packages.Download.Oci { - public class OciClient + public class OciRegistryClient { readonly HttpClient httpClient = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); + readonly ICalamariFileSystem fileSystem; + + public OciRegistryClient(ICalamariFileSystem fileSystem) + { + this.fileSystem = fileSystem; + } public JObject? GetManifest(Uri feedUri, string packageId, IVersion version, string? feedUsername, string? feedPassword) { @@ -27,26 +35,24 @@ public class OciClient void ApplyAcceptHeaderFunc(HttpRequestMessage request) => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciConstants.Manifest.Config.OciImageMediaTypeValue)); } - public HttpResponseMessage GetPackage(Uri feedUri, string packageId, string digest, string? feedUsername, string? feedPassword) + public void DownloadPackage( + Uri feedUri, + string packageId, + string digest, + string? feedUsername, + string? feedPassword, + string downloadPath) { var url = GetApiUri(feedUri); - HttpResponseMessage? response = null; - try - { - response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); - - if (!response.IsSuccessStatusCode) - { - throw new CommandException($"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); - } + using var fileStream = fileSystem.OpenFile(downloadPath, FileAccess.Write); + using var response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); - return response; - } - catch + if (!response.IsSuccessStatusCode) { - response?.Dispose(); - throw; + throw new CommandException($"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); } + + response.Content.CopyToAsync(fileStream).GetAwaiter().GetResult(); } HttpResponseMessage Get(Uri url, ICredentials credentials, Action? customAcceptHeader = null) diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs index a9b026996..cc4bf4246 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciOrDockerImagePackageDownloader.cs @@ -10,19 +10,19 @@ public class OciOrDockerImagePackageDownloader : IPackageDownloader { readonly OciPackageDownloader ociPackageDownloader; readonly DockerImagePackageDownloader dockerImagePackageDownloader; - readonly OciClient ociClient; + readonly OciRegistryClient ociRegistryClient; readonly ILog log; public OciOrDockerImagePackageDownloader( OciPackageDownloader ociPackageDownloader, DockerImagePackageDownloader dockerImagePackageDownloader, - OciClient ociClient, + OciRegistryClient ociRegistryClient, ILog log) { this.ociPackageDownloader = ociPackageDownloader; this.dockerImagePackageDownloader = dockerImagePackageDownloader; this.log = log; - this.ociClient = ociClient; + this.ociRegistryClient = ociRegistryClient; } public PackagePhysicalFileMetadata DownloadPackage( @@ -52,7 +52,7 @@ public PackagePhysicalFileMetadata DownloadPackage( IPackageDownloader GetInnerDownloader(string packageId, IVersion version, Uri feedUri, string? feedUsername, string? feedPassword) { - var ociArtifactManifestRetriever = new OciArtifactManifestRetriever(ociClient, log); + var ociArtifactManifestRetriever = new OciArtifactManifestRetriever(ociRegistryClient, log); if (ociArtifactManifestRetriever.TryGetArtifactType(packageId, version, feedUri, feedUsername, feedPassword) == OciArtifactTypes.HelmChart) { return ociPackageDownloader; diff --git a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs index 8f40ef214..cf7eb5677 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/OciPackageDownloader.cs @@ -24,17 +24,17 @@ public class OciPackageDownloader : IPackageDownloader readonly ICalamariFileSystem fileSystem; readonly ICombinedPackageExtractor combinedPackageExtractor; readonly ILog log; - readonly OciClient ociClient; + readonly OciRegistryClient ociRegistryClient; public OciPackageDownloader( ICalamariFileSystem fileSystem, ICombinedPackageExtractor combinedPackageExtractor, - OciClient ociClient, + OciRegistryClient ociRegistryClient, ILog log) { this.fileSystem = fileSystem; this.combinedPackageExtractor = combinedPackageExtractor; - this.ociClient = ociClient; + this.ociRegistryClient = ociRegistryClient; this.log = log; } @@ -79,14 +79,13 @@ public PackagePhysicalFileMetadata DownloadPackage( } var (digest, size, extension) = GetPackageDetails(feedUri, packageId, version, feedUsername, feedPassword); - var hash = OciClient.GetPackageHashFromDigest(digest); + var hash = OciRegistryClient.GetPackageHashFromDigest(digest); var cachedFileName = PackageName.ToCachedFileName(packageId, version, extension); var downloadPath = Path.Combine(Path.Combine(stagingDir, cachedFileName)); var retryStrategy = PackageDownloaderRetryUtils.CreateRetryStrategy(maxDownloadAttempts, downloadAttemptBackoff, log); - retryStrategy.Execute( - () => DownloadPackage(feedUri, packageId, digest, feedUsername, feedPassword, downloadPath)); + retryStrategy.Execute(() => ociRegistryClient.DownloadPackage(feedUri, packageId, digest, feedUsername, feedPassword, downloadPath)); var localDownloadName = Path.Combine(cacheDirectory, cachedFileName); fileSystem.MoveFile(downloadPath, localDownloadName); @@ -109,7 +108,7 @@ public PackagePhysicalFileMetadata DownloadPackage( string? feedUserName, string? feedPassword) { - var manifest = ociClient.GetManifest(feedUri, packageId, version, feedUserName, feedPassword); + var manifest = ociRegistryClient.GetManifest(feedUri, packageId, version, feedUserName, feedPassword); var layer = manifest.Value(OciConstants.Manifest.Layers.PropertyName)[0]; var digest = layer.Value(OciConstants.Manifest.Layers.DigestPropertyName); @@ -131,20 +130,6 @@ string GetExtensionFromManifest(JToken layer) return extension ?? (layer.Value(OciConstants.Manifest.Layers.MediaTypePropertyName).EndsWith("tar+gzip") ? ".tgz" : ".tar"); } - void DownloadPackage( - Uri feedUri, - string packageId, - string digest, - string? feedUsername, - string? feedPassword, - string downloadPath) - { - using var fileStream = fileSystem.OpenFile(downloadPath, FileAccess.Write); - using var response = ociClient.GetPackage(feedUri, packageId, digest, feedUsername, feedPassword); - - response.Content.CopyToAsync(fileStream).GetAwaiter().GetResult(); - } - PackagePhysicalFileMetadata? SourceFromCache(string packageId, IVersion version, string cacheDirectory) { log.VerboseFormat("Checking package cache for package {0} v{1}", packageId, version.ToString()); diff --git a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs index f7b0f9547..7a92f84aa 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs @@ -68,7 +68,7 @@ public PackagePhysicalFileMetadata DownloadPackage( downloader = OciPackageDownloader(); break; case FeedType.AwsElasticContainerRegistry: - downloader = new OciOrDockerImagePackageDownloader(OciPackageDownloader(), DockerImagePackageDownloader(), new OciClient(), log); + downloader = new OciOrDockerImagePackageDownloader(OciPackageDownloader(), DockerImagePackageDownloader(), new OciRegistryClient(fileSystem), log); break; case FeedType.Docker: case FeedType.AzureContainerRegistry: @@ -100,7 +100,7 @@ public PackagePhysicalFileMetadata DownloadPackage( } OciPackageDownloader OciPackageDownloader() - => new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), new OciClient(), log); + => new OciPackageDownloader(fileSystem, new CombinedPackageExtractor(log, fileSystem, variables, commandLineRunner), new OciRegistryClient(fileSystem), log); DockerImagePackageDownloader DockerImagePackageDownloader() => new DockerImagePackageDownloader(engine, fileSystem, commandLineRunner, variables, log);