Use ECR feed for OCI and docker

source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs

@@ -9,12 +9,14 @@ public static class KnownSlugs
             public const string KubernetesObjectManifestInspection = "kubernetes-object-manifest-inspection";
             public const string KOSForHelm = "kos-for-helm";
             public const string ExecuteHelmUpgradeCommandViaShellScript = "execute-helm-upgrade-command-via-shell-script";
+            public const string UseOciRegistryPackageFeedsFeatureToggle = "use-oci-registry-package-feeds";
         };
 
         public static readonly OctopusFeatureToggle NonPrimaryGitDependencySupportFeatureToggle = new OctopusFeatureToggle("non-primary-git-dependency-support");
         public static readonly OctopusFeatureToggle KubernetesObjectManifestInspectionFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KubernetesObjectManifestInspection);
         public static readonly OctopusFeatureToggle KOSForHelmFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KOSForHelm);
         public static readonly OctopusFeatureToggle ExecuteHelmUpgradeCommandViaShellScriptFeatureToggle = new OctopusFeatureToggle(KnownSlugs.ExecuteHelmUpgradeCommandViaShellScript);
+        public static readonly OctopusFeatureToggle UseOciRegistryPackageFeedsFeatureToggle = new OctopusFeatureToggle(KnownSlugs.UseOciRegistryPackageFeedsFeatureToggle);
 
         public class OctopusFeatureToggle
         {

source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs

@@ -0,0 +1,37 @@
+using System;
+
+namespace Calamari.Integration.Packages.Download.Oci
+{
+    public static class OciConstants
+    {
+        public class Manifest
+        {
+            internal const string MediaTypePropertyName = "mediaType";
+            internal const string DockerImageMediaTypeValue = "application/vnd.docker.distribution.manifest.v2+json";
+
+            public class Config
+            {
+                internal const string PropertyName = "config";
+                internal const string MediaTypePropertyName = "mediaType";
+                internal const string OciImageMediaTypeValue = "application/vnd.oci.image.config.v1+json";
+                internal const string DockerImageMediaTypeValue = "application/vnd.docker.container.image.v1+json";
+            }
+
+            public class Image
+            {
+                internal const string TitleAnnotationKey = "org.opencontainers.image.title";
+            }
+
+            public class Layers
+            {
+                internal const string PropertyName = "layers";
+                internal const string DigestPropertyName = "digest";
+                internal const string SizePropertyName = "size";
+                internal const string MediaTypePropertyName = "mediaType";
+                internal const string AnnotationsPropertyName = "annotations";
+                internal const string HelmChartMediaTypeValue = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v370
+                internal const string DockerImageMediaTypeValue = "application/vnd.docker.image.rootfs.diff.tar.gzip";
+            }
+        }
+    }
+}

source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs

@@ -0,0 +1,44 @@
+using System;
+using Newtonsoft.Json.Linq;
+
+namespace Calamari.Integration.Packages.Download.Oci
+{
+    static class OciJObjectExtensions
+    {
+        public static bool HasMediaTypeContaining(this JObject manifest, string value)
+        {
+            var mediaType = manifest[OciConstants.Manifest.MediaTypePropertyName];
+
+            return mediaType != null
+                   && mediaType.ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0;
+        }
+
+        public static bool HasConfigMediaTypeContaining(this JObject manifest, string value)
+        {
+            var config = manifest[OciConstants.Manifest.Config.PropertyName];
+
+            return config is { Type: JTokenType.Object }
+                   && config[OciConstants.Manifest.Config.MediaTypePropertyName] != null
+                   && config[OciConstants.Manifest.Config.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0;
+        }
+
+        public static bool HasLayersMediaTypeContaining(this JObject manifest, string value)
+        {
+            var layers = manifest[OciConstants.Manifest.Layers.PropertyName];
+
+            if (layers is { Type: JTokenType.Array })
+            {
+                foreach (var layer in layers)
+                {
+                    if (layer[OciConstants.Manifest.Layers.MediaTypePropertyName] != null
+                        && layer[OciConstants.Manifest.Layers.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0)
+                    {
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+    }
+}

source/Calamari.Shared/Integration/Packages/Download/Oci/OciRegistryClient.cs

@@ -0,0 +1,268 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.RegularExpressions;
+using Calamari.Common.Commands;
+using Calamari.Common.Plumbing.FileSystem;
+using Calamari.Common.Plumbing.Logging;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Octopus.Versioning;
+
+namespace Calamari.Integration.Packages.Download.Oci
+{
+    public class OciRegistryClient
+    {
+        readonly HttpClient httpClient = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None });
+        readonly ICalamariFileSystem fileSystem;
+        readonly ILog log;
+
+        public OciRegistryClient(ICalamariFileSystem fileSystem, ILog log)
+        {
+            this.fileSystem = fileSystem;
+            this.log = log;
+        }
+
+        public void DownloadPackage(
+            Uri feedUri,
+            string packageId,
+            string digest,
+            string? feedUsername,
+            string? feedPassword,
+            string downloadPath)
+        {
+            var url = GetApiUri(feedUri);
+            using var fileStream = fileSystem.OpenFile(downloadPath, FileAccess.Write);
+            using var response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword));
+
+            if (!response.IsSuccessStatusCode)
+            {
+                throw new CommandException($"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}");
+            }
+
+            response.Content.CopyToAsync(fileStream).GetAwaiter().GetResult();
+        }
+
+        public OciArtifactTypes TryGetArtifactType(
+            string packageId,
+            IVersion version,
+            Uri feedUri,
+            string? feedUsername,
+            string? feedPassword)
+        {
+            try
+            {
+                var jsonManifest = GetManifest(feedUri, packageId, version, feedUsername, feedPassword);
+
+                // Check for Helm chart annotations
+                var isHelmChart =
+                    jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.OciImageMediaTypeValue)
+                    || jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.HelmChartMediaTypeValue);
+
+                if (isHelmChart)
+                    return OciArtifactTypes.HelmChart;
+
+                var isDockerImage = jsonManifest.HasMediaTypeContaining(OciConstants.Manifest.DockerImageMediaTypeValue)
+                                    || jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.DockerImageMediaTypeValue)
+                                    || jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.DockerImageMediaTypeValue);
+
+                return isDockerImage ? OciArtifactTypes.DockerImage : OciArtifactTypes.Unknown;
+            }
+            catch (Exception ex)
+            {
+                log.ErrorFormat("Failed to get artifact type: {Message}", ex.Message);
+                return OciArtifactTypes.Unknown;
+            }
+        }
+
+        public JObject? GetManifest(Uri feedUri, string packageId, IVersion version, string? feedUsername, string? feedPassword)
+        {
+            var url = GetApiUri(feedUri);
+            var fixedVersion = FixVersion(version);
+            using var response = Get(new Uri($"{url}/{packageId}/manifests/{fixedVersion}"), new NetworkCredential(feedUsername, feedPassword), ApplyAcceptHeaderFunc);
+            return JsonConvert.DeserializeObject<JObject>(response.Content.ReadAsStringAsync().Result);
+
+            // void ApplyAcceptHeaderFunc(HttpRequestMessage request) => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciConstants.Manifest.AcceptHeader));
+            void ApplyAcceptHeaderFunc(HttpRequestMessage request) => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciConstants.Manifest.Config.OciImageMediaTypeValue));
+        }
+
+        HttpResponseMessage Get(Uri url, ICredentials credentials, Action<HttpRequestMessage>? customAcceptHeader = null)
+        {
+            var request = new HttpRequestMessage(HttpMethod.Get, url);
+            try
+            {
+                var networkCredential = credentials.GetCredential(url, "Basic");
+
+                if (!string.IsNullOrWhiteSpace(networkCredential?.UserName) || !string.IsNullOrWhiteSpace(networkCredential?.Password))
+                {
+                    request.Headers.Authorization = CreateAuthenticationHeader(networkCredential);
+                }
+
+                customAcceptHeader?.Invoke(request);
+                var response = SendRequest(httpClient, request);
+
+                if (response.StatusCode == HttpStatusCode.Unauthorized)
+                {
+                    var tokenFromAuthService = GetAuthRequestHeader(response, networkCredential);
+                    request = new HttpRequestMessage(HttpMethod.Get, url);
+                    request.Headers.Authorization = tokenFromAuthService;
+                    customAcceptHeader?.Invoke(request);
+                    response = SendRequest(httpClient, request);
+
+                    if (response.StatusCode == HttpStatusCode.Unauthorized)
+                    {
+                        throw new CommandException($"Authorization to `{url}` failed.");
+                    }
+                }
+
+                if (response.StatusCode == HttpStatusCode.NotFound)
+                {
+                    // Some registries do not support the Docker HTTP APIs
+                    // For example GitHub: https://github.community/t/ghcr-io-docker-http-api/130121
+                    throw new CommandException($"Docker registry located at `{url}` does not support this action.");
+                }
+
+                if (!response.IsSuccessStatusCode)
+                {
+                    var errorMessage = $"Request to Docker registry located at `{url}` failed with {response.StatusCode}:{response.ReasonPhrase}.";
+
+                    var responseBody = GetContent(response);
+                    if (!string.IsNullOrWhiteSpace(responseBody)) errorMessage += $" {responseBody}";
+
+                    throw new CommandException(errorMessage);
+                }
+
+                return response;
+            }
+            finally
+            {
+                request.Dispose();
+            }
+        }
+
+        AuthenticationHeaderValue GetAuthRequestHeader(HttpResponseMessage response, NetworkCredential credential)
+        {
+            var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer");
+            if (auth != null)
+            {
+                var authToken = RetrieveAuthenticationToken(auth, credential);
+                return new AuthenticationHeaderValue("Bearer", authToken);
+            }
+
+            if (response.Headers.WwwAuthenticate.Any(a => a.Scheme == "Basic"))
+            {
+                return CreateAuthenticationHeader(credential);
+            }
+
+            throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`");
+        }
+
+        string RetrieveAuthenticationToken(AuthenticationHeaderValue auth, NetworkCredential credential)
+        {
+            HttpResponseMessage? response = null;
+
+            try
+            {
+                var authUrl = GetOAuthServiceUrl(auth);
+                using (var msg = new HttpRequestMessage(HttpMethod.Get, authUrl))
+                {
+                    if (credential.UserName != null)
+                    {
+                        msg.Headers.Authorization = CreateAuthenticationHeader(credential);
+                    }
+
+                    response = SendRequest(httpClient, msg);
+                }
+
+                if (response.IsSuccessStatusCode)
+                {
+                    return ExtractTokenFromResponse(response);
+                }
+            }
+            finally
+            {
+                response?.Dispose();
+            }
+
+            throw new CommandException("Unable to retrieve authentication token required to perform operation.");
+        }
+
+        static string GetOAuthServiceUrl(AuthenticationHeaderValue auth)
+        {
+            var details = auth.Parameter.Split(',').ToDictionary(x => x.Substring(0, x.IndexOf('=')), y => y.Substring(y.IndexOf('=') + 1, y.Length - y.IndexOf('=') - 1).Trim('"'));
+            var oathUrl = new UriBuilder(details["realm"]);
+            var queryStringValues = new Dictionary<string, string>();
+            if (details.TryGetValue("service", out var service))
+            {
+                var encodedService = WebUtility.UrlEncode(service);
+                queryStringValues.Add("service", encodedService);
+            }
+
+            if (details.TryGetValue("scope", out var scope))
+            {
+                var encodedScope = WebUtility.UrlEncode(scope);
+                queryStringValues.Add("scope", encodedScope);
+            }
+
+            oathUrl.Query = "?" + string.Join("&", queryStringValues.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray());
+            return oathUrl.ToString();
+        }
+
+        static string ExtractTokenFromResponse(HttpResponseMessage response)
+        {
+            var token = GetContent(response);
+            return (string)JObject.Parse(token).SelectToken("token")
+                   ?? throw new CommandException("Unable to retrieve authentication token required to perform operation.");
+        }
+
+        static Uri GetApiUri(Uri feedUri)
+        {
+            var httpScheme = IsPlainHttp(feedUri) ? Uri.UriSchemeHttp : Uri.UriSchemeHttps;
+
+            var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/');
+            var uri = new Uri(r);
+
+            const string versionPath = "v2";
+            if (!r.EndsWith("/" + versionPath))
+            {
+                uri = new Uri(uri, versionPath);
+            }
+
+            return uri;
+
+            static bool IsPlainHttp(Uri uri)
+                => uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase);
+        }
+
+        // oci registries don't support the '+' tagging
+        // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380
+        static string FixVersion(IVersion version)
+            => version.ToString().Replace("+", "_");
+
+        static readonly Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?<hash>[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase);
+
+        internal static string? GetPackageHashFromDigest(string digest)
+            => PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value;
+
+        static AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential)
+        {
+            var byteArray = Encoding.ASCII.GetBytes($"{credential.UserName}:{credential.Password}");
+            return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
+        }
+
+        static HttpResponseMessage SendRequest(HttpClient client, HttpRequestMessage request)
+        {
+            return client.SendAsync(request).GetAwaiter().GetResult();
+        }
+
+        static string? GetContent(HttpResponseMessage response)
+        {
+            return response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+        }
+    }
+}