-
Notifications
You must be signed in to change notification settings - Fork 111
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use ECR feed for OCI and docker #1415
Draft
eddymoulton
wants to merge
11
commits into
main
Choose a base branch
from
em/use-ecr-feed-for-oci-and-docker
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Draft
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
684ff08
First go
MissedTheMark 0ba45c8
Added tests
MissedTheMark a5713da
Merge branch 'main' into MissedTheMark/Enh/CheckForOciArtifactType
eddymoulton 38431d3
Fix merge error
eddymoulton 98c1e45
Merge branch 'main' into MissedTheMark/Enh/CheckForOciArtifactType
eddymoulton bb188f0
Clean up and get tests working
eddymoulton 139f142
Add feature toggle
eddymoulton be32835
Merge branch 'main' into em/use-ecr-feed-for-oci-and-docker
eddymoulton 8822d19
Remove feature toggle
eddymoulton 3d5ff4d
Merge branch 'main' into em/use-ecr-feed-for-oci-and-docker
eddymoulton d278288
Rename to OciRegistryClient to align with server
eddymoulton File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
50 changes: 50 additions & 0 deletions
50
source/Calamari.Shared/Integration/Packages/Download/Oci/OciArtifactManifestRetriever.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
using System; | ||
using Calamari.Common.Plumbing.Logging; | ||
using Octopus.Versioning; | ||
|
||
namespace Calamari.Integration.Packages.Download.Oci | ||
{ | ||
public class OciArtifactManifestRetriever | ||
{ | ||
readonly OciClient ociClient; | ||
readonly ILog log; | ||
|
||
public OciArtifactManifestRetriever(OciClient ociClient, ILog log) | ||
{ | ||
this.ociClient = ociClient; | ||
this.log = log; | ||
} | ||
|
||
public OciArtifactTypes TryGetArtifactType( | ||
string packageId, | ||
IVersion version, | ||
Uri feedUri, | ||
string? feedUsername, | ||
string? feedPassword) | ||
{ | ||
try | ||
{ | ||
var jsonManifest = ociClient.GetManifest(feedUri, packageId, version, feedUsername, feedPassword); | ||
|
||
// Check for Helm chart annotations | ||
var isHelmChart = | ||
jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.OciImageMediaTypeValue) | ||
|| jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.HelmChartMediaTypeValue); | ||
|
||
if (isHelmChart) | ||
return OciArtifactTypes.HelmChart; | ||
|
||
var isDockerImage = jsonManifest.HasMediaTypeContaining(OciConstants.Manifest.DockerImageMediaTypeValue) | ||
|| jsonManifest.HasConfigMediaTypeContaining(OciConstants.Manifest.Config.DockerImageMediaTypeValue) | ||
|| jsonManifest.HasLayersMediaTypeContaining(OciConstants.Manifest.Layers.DockerImageMediaTypeValue); | ||
|
||
return isDockerImage ? OciArtifactTypes.DockerImage : OciArtifactTypes.Unknown; | ||
} | ||
catch (Exception ex) | ||
{ | ||
log.ErrorFormat("Failed to get artifact type: {Message}", ex.Message); | ||
return OciArtifactTypes.Unknown; | ||
} | ||
} | ||
} | ||
} |
226 changes: 226 additions & 0 deletions
226
source/Calamari.Shared/Integration/Packages/Download/Oci/OciClient.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,226 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Net; | ||
using System.Net.Http; | ||
using System.Net.Http.Headers; | ||
using System.Text; | ||
using System.Text.RegularExpressions; | ||
using Calamari.Common.Commands; | ||
using Newtonsoft.Json; | ||
using Newtonsoft.Json.Linq; | ||
using Octopus.Versioning; | ||
|
||
namespace Calamari.Integration.Packages.Download.Oci | ||
{ | ||
public class OciClient | ||
{ | ||
readonly HttpClient httpClient = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.None }); | ||
|
||
public JObject? GetManifest(Uri feedUri, string packageId, IVersion version, string? feedUsername, string? feedPassword) | ||
{ | ||
var url = GetApiUri(feedUri); | ||
var fixedVersion = FixVersion(version); | ||
using var response = Get(new Uri($"{url}/{packageId}/manifests/{fixedVersion}"), new NetworkCredential(feedUsername, feedPassword), ApplyAcceptHeaderFunc); | ||
return JsonConvert.DeserializeObject<JObject>(response.Content.ReadAsStringAsync().Result); | ||
|
||
void ApplyAcceptHeaderFunc(HttpRequestMessage request) => request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(OciConstants.Manifest.Config.OciImageMediaTypeValue)); | ||
} | ||
|
||
public HttpResponseMessage GetPackage(Uri feedUri, string packageId, string digest, string? feedUsername, string? feedPassword) | ||
{ | ||
var url = GetApiUri(feedUri); | ||
HttpResponseMessage? response = null; | ||
try | ||
{ | ||
response = Get(new Uri($"{url}/{packageId}/blobs/{digest}"), new NetworkCredential(feedUsername, feedPassword)); | ||
|
||
if (!response.IsSuccessStatusCode) | ||
{ | ||
throw new CommandException($"Failed to download artifact (Status Code {(int)response.StatusCode}). Reason: {response.ReasonPhrase}"); | ||
} | ||
|
||
return response; | ||
} | ||
catch | ||
{ | ||
response?.Dispose(); | ||
throw; | ||
} | ||
} | ||
|
||
HttpResponseMessage Get(Uri url, ICredentials credentials, Action<HttpRequestMessage>? customAcceptHeader = null) | ||
{ | ||
var request = new HttpRequestMessage(HttpMethod.Get, url); | ||
try | ||
{ | ||
var networkCredential = credentials.GetCredential(url, "Basic"); | ||
|
||
if (!string.IsNullOrWhiteSpace(networkCredential?.UserName) || !string.IsNullOrWhiteSpace(networkCredential?.Password)) | ||
{ | ||
request.Headers.Authorization = CreateAuthenticationHeader(networkCredential); | ||
} | ||
|
||
customAcceptHeader?.Invoke(request); | ||
var response = SendRequest(httpClient, request); | ||
|
||
if (response.StatusCode == HttpStatusCode.Unauthorized) | ||
{ | ||
var tokenFromAuthService = GetAuthRequestHeader(response, networkCredential); | ||
request = new HttpRequestMessage(HttpMethod.Get, url); | ||
request.Headers.Authorization = tokenFromAuthService; | ||
customAcceptHeader?.Invoke(request); | ||
response = SendRequest(httpClient, request); | ||
|
||
if (response.StatusCode == HttpStatusCode.Unauthorized) | ||
{ | ||
throw new CommandException($"Authorization to `{url}` failed."); | ||
} | ||
} | ||
|
||
if (response.StatusCode == HttpStatusCode.NotFound) | ||
{ | ||
// Some registries do not support the Docker HTTP APIs | ||
// For example GitHub: https://github.community/t/ghcr-io-docker-http-api/130121 | ||
throw new CommandException($"Docker registry located at `{url}` does not support this action."); | ||
} | ||
|
||
if (!response.IsSuccessStatusCode) | ||
{ | ||
var errorMessage = $"Request to Docker registry located at `{url}` failed with {response.StatusCode}:{response.ReasonPhrase}."; | ||
|
||
var responseBody = GetContent(response); | ||
if (!string.IsNullOrWhiteSpace(responseBody)) errorMessage += $" {responseBody}"; | ||
|
||
throw new CommandException(errorMessage); | ||
} | ||
|
||
return response; | ||
} | ||
finally | ||
{ | ||
request.Dispose(); | ||
} | ||
} | ||
|
||
AuthenticationHeaderValue GetAuthRequestHeader(HttpResponseMessage response, NetworkCredential credential) | ||
{ | ||
var auth = response.Headers.WwwAuthenticate.FirstOrDefault(a => a.Scheme == "Bearer"); | ||
if (auth != null) | ||
{ | ||
var authToken = RetrieveAuthenticationToken(auth, credential); | ||
return new AuthenticationHeaderValue("Bearer", authToken); | ||
} | ||
|
||
if (response.Headers.WwwAuthenticate.Any(a => a.Scheme == "Basic")) | ||
{ | ||
return CreateAuthenticationHeader(credential); | ||
} | ||
|
||
throw new CommandException($"Unknown Authentication scheme for Uri `{response.RequestMessage.RequestUri}`"); | ||
} | ||
|
||
string RetrieveAuthenticationToken(AuthenticationHeaderValue auth, NetworkCredential credential) | ||
{ | ||
HttpResponseMessage? response = null; | ||
|
||
try | ||
{ | ||
var authUrl = GetOAuthServiceUrl(auth); | ||
using (var msg = new HttpRequestMessage(HttpMethod.Get, authUrl)) | ||
{ | ||
if (credential.UserName != null) | ||
{ | ||
msg.Headers.Authorization = CreateAuthenticationHeader(credential); | ||
} | ||
|
||
response = SendRequest(httpClient, msg); | ||
} | ||
|
||
if (response.IsSuccessStatusCode) | ||
{ | ||
return ExtractTokenFromResponse(response); | ||
} | ||
} | ||
finally | ||
{ | ||
response?.Dispose(); | ||
} | ||
|
||
throw new CommandException("Unable to retrieve authentication token required to perform operation."); | ||
} | ||
|
||
static string GetOAuthServiceUrl(AuthenticationHeaderValue auth) | ||
{ | ||
var details = auth.Parameter.Split(',').ToDictionary(x => x.Substring(0, x.IndexOf('=')), y => y.Substring(y.IndexOf('=') + 1, y.Length - y.IndexOf('=') - 1).Trim('"')); | ||
var oathUrl = new UriBuilder(details["realm"]); | ||
var queryStringValues = new Dictionary<string, string>(); | ||
if (details.TryGetValue("service", out var service)) | ||
{ | ||
var encodedService = WebUtility.UrlEncode(service); | ||
queryStringValues.Add("service", encodedService); | ||
} | ||
|
||
if (details.TryGetValue("scope", out var scope)) | ||
{ | ||
var encodedScope = WebUtility.UrlEncode(scope); | ||
queryStringValues.Add("scope", encodedScope); | ||
} | ||
|
||
oathUrl.Query = "?" + string.Join("&", queryStringValues.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray()); | ||
return oathUrl.ToString(); | ||
} | ||
|
||
static string ExtractTokenFromResponse(HttpResponseMessage response) | ||
{ | ||
var token = GetContent(response); | ||
return (string)JObject.Parse(token).SelectToken("token") | ||
?? throw new CommandException("Unable to retrieve authentication token required to perform operation."); | ||
} | ||
|
||
static Uri GetApiUri(Uri feedUri) | ||
{ | ||
var httpScheme = IsPlainHttp(feedUri) ? Uri.UriSchemeHttp : Uri.UriSchemeHttps; | ||
|
||
var r = feedUri.ToString().Replace($"oci{Uri.SchemeDelimiter}", $"{httpScheme}{Uri.SchemeDelimiter}").TrimEnd('/'); | ||
var uri = new Uri(r); | ||
|
||
const string versionPath = "v2"; | ||
if (!r.EndsWith("/" + versionPath)) | ||
{ | ||
uri = new Uri(uri, versionPath); | ||
} | ||
|
||
return uri; | ||
|
||
static bool IsPlainHttp(Uri uri) | ||
=> uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase); | ||
} | ||
|
||
// oci registries don't support the '+' tagging | ||
// https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v380 | ||
static string FixVersion(IVersion version) | ||
=> version.ToString().Replace("+", "_"); | ||
|
||
static readonly Regex PackageDigestHashRegex = new Regex(@"[A-Za-z0-9_+.-]+:(?<hash>[A-Fa-f0-9]+)", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase); | ||
|
||
internal static string? GetPackageHashFromDigest(string digest) | ||
=> PackageDigestHashRegex.Match(digest).Groups["hash"]?.Value; | ||
|
||
static AuthenticationHeaderValue CreateAuthenticationHeader(NetworkCredential credential) | ||
{ | ||
var byteArray = Encoding.ASCII.GetBytes($"{credential.UserName}:{credential.Password}"); | ||
return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); | ||
} | ||
|
||
static HttpResponseMessage SendRequest(HttpClient client, HttpRequestMessage request) | ||
{ | ||
return client.SendAsync(request).GetAwaiter().GetResult(); | ||
} | ||
|
||
static string? GetContent(HttpResponseMessage response) | ||
{ | ||
return response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); | ||
} | ||
} | ||
} |
37 changes: 37 additions & 0 deletions
37
source/Calamari.Shared/Integration/Packages/Download/Oci/OciConstants.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
using System; | ||
|
||
namespace Calamari.Integration.Packages.Download.Oci | ||
{ | ||
public static class OciConstants | ||
{ | ||
public class Manifest | ||
{ | ||
internal const string MediaTypePropertyName = "mediaType"; | ||
internal const string DockerImageMediaTypeValue = "application/vnd.docker.distribution.manifest.v2+json"; | ||
|
||
public class Config | ||
{ | ||
internal const string PropertyName = "config"; | ||
internal const string MediaTypePropertyName = "mediaType"; | ||
internal const string OciImageMediaTypeValue = "application/vnd.oci.image.config.v1+json"; | ||
internal const string DockerImageMediaTypeValue = "application/vnd.docker.container.image.v1+json"; | ||
} | ||
|
||
public class Image | ||
{ | ||
internal const string TitleAnnotationKey = "org.opencontainers.image.title"; | ||
} | ||
|
||
public class Layers | ||
{ | ||
internal const string PropertyName = "layers"; | ||
internal const string DigestPropertyName = "digest"; | ||
internal const string SizePropertyName = "size"; | ||
internal const string MediaTypePropertyName = "mediaType"; | ||
internal const string AnnotationsPropertyName = "annotations"; | ||
internal const string HelmChartMediaTypeValue = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; // https://helm.sh/docs/topics/registries/#oci-feature-deprecation-and-behavior-changes-with-v370 | ||
internal const string DockerImageMediaTypeValue = "application/vnd.docker.image.rootfs.diff.tar.gzip"; | ||
} | ||
} | ||
} | ||
} |
44 changes: 44 additions & 0 deletions
44
source/Calamari.Shared/Integration/Packages/Download/Oci/OciJObjectExtensions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
using System; | ||
using Newtonsoft.Json.Linq; | ||
|
||
namespace Calamari.Integration.Packages.Download.Oci | ||
{ | ||
static class OciJObjectExtensions | ||
{ | ||
public static bool HasMediaTypeContaining(this JObject manifest, string value) | ||
{ | ||
var mediaType = manifest[OciConstants.Manifest.MediaTypePropertyName]; | ||
|
||
return mediaType != null | ||
&& mediaType.ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; | ||
} | ||
|
||
public static bool HasConfigMediaTypeContaining(this JObject manifest, string value) | ||
{ | ||
var config = manifest[OciConstants.Manifest.Config.PropertyName]; | ||
|
||
return config is { Type: JTokenType.Object } | ||
&& config[OciConstants.Manifest.Config.MediaTypePropertyName] != null | ||
&& config[OciConstants.Manifest.Config.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0; | ||
} | ||
|
||
public static bool HasLayersMediaTypeContaining(this JObject manifest, string value) | ||
{ | ||
var layers = manifest[OciConstants.Manifest.Layers.PropertyName]; | ||
|
||
if (layers is { Type: JTokenType.Array }) | ||
{ | ||
foreach (var layer in layers) | ||
{ | ||
if (layer[OciConstants.Manifest.Layers.MediaTypePropertyName] != null | ||
&& layer[OciConstants.Manifest.Layers.MediaTypePropertyName].ToString().IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0) | ||
{ | ||
return true; | ||
} | ||
} | ||
} | ||
|
||
return false; | ||
} | ||
} | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Note for myself before merging, I need to confirm these are all correct and expected.