From 41e71002fc19529e1a330a9022417005d2364420 Mon Sep 17 00:00:00 2001 From: xibz Date: Sun, 4 Feb 2024 21:40:54 -0600 Subject: [PATCH] fix(artifacts): Addresses a couple things regarding artifact store (#1145) * fix(artifacts): Updates Artifact Store README This commit updates the README refect the changes in https://github.com/spinnaker/kork/pull/1120 Also adds a Rosco/Helm section to describe the new `expandOverrides` field Signed-off-by: benjamin-j-powell * feat(artifacts): Move scheme to ArtifactReferenceURI This moves scheme from the artifact URI builders to the URI object instead. Also moves the `isArtifactReference` from artifact stores to the URI class instead. Signed-off-by: benjamin-j-powell * feat(artifacts): Add HelmConfig and ensure properties aren't null This commit adds the new HelmConfig which allows for users to turn on expanding overrides. This will inspect overrides values, and if it is an reference URI, it will expand and use that instead of the raw URI. This also updates the properties, `helmConfig` and `s3` to no longer be `null`. Signed-off-by: benjamin-j-powell --------- Signed-off-by: benjamin-j-powell Co-authored-by: benjamin-j-powell Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../spinnaker/kork/artifacts/README.md | 19 ++++++++++++--- .../artifactstore/ArtifactReferenceURI.java | 23 ++++++++++++------- .../artifactstore/ArtifactStore.java | 4 ---- .../ArtifactStoreConfigurationProperties.java | 9 +++++++- .../ArtifactStoreURIBuilder.java | 7 ------ .../ArtifactStoreURISHA256Builder.java | 4 ++-- .../s3/S3ArtifactStoreConfiguration.java | 2 +- .../ArtifactUriToReferenceConverter.java | 2 +- 8 files changed, 43 insertions(+), 27 deletions(-) diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md index 39433b123..0308f4093 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md @@ -81,12 +81,25 @@ To enable artifact storage, simple add this to your `spinnaker-local.yml` file ```yaml artifact-store: - enabled: true + type: s3 s3: enabled: true bucket: some-artifact-store-bucket ``` +### Rosco and Helm + +If any pipelines are passing artifact references to bake stages as a parameter, +enabling this field will allow those URIs to be expanded to the full +references: + +```yaml +artifact-store: + type: s3 + helm: + expandOverrides: true +``` + ## Storage Options ### S3 @@ -97,7 +110,7 @@ against AWS. ```yaml artifact-store: - enabled: true + type: s3 s3: enabled: true profile: dev # if you want to authenticate using a certain profile @@ -119,7 +132,7 @@ Next enable the configuration ```yaml artifact-store: - enabled: true + type: s3 s3: enabled: true url: http://localhost:8333 # this URL will be used to make S3 API requests to diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactReferenceURI.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactReferenceURI.java index 8f1cbb022..3fdfcfa0e 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactReferenceURI.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactReferenceURI.java @@ -29,24 +29,31 @@ @Builder @Getter public class ArtifactReferenceURI { - private final String scheme; + /** + * uriScheme is used as an HTTP scheme to let us further distinguish a String that is a URI to an + * artifact. This is helpful in determining what is an artifact since sometimes we are only given + * a string rather than a full artifact. + */ + private static final String uriScheme = "ref://"; + private final List uriPaths; public String uri() { - return String.format("%s://%s", scheme, paths()); + return uriScheme + paths(); } public String paths() { return Strings.join(uriPaths, '/'); } + /** Used to determine whether a String is in the artifact reference URI format. */ + public static boolean is(String reference) { + return reference.startsWith(uriScheme); + } + public static ArtifactReferenceURI parse(String reference) { - String noSchemeURI = - StringUtils.removeStart(reference, ArtifactStoreURIBuilder.uriScheme + "://"); + String noSchemeURI = StringUtils.removeStart(reference, uriScheme); String[] paths = StringUtils.split(noSchemeURI, '/'); - return ArtifactReferenceURI.builder() - .scheme(ArtifactStoreURIBuilder.uriScheme) - .uriPaths(Arrays.asList(paths)) - .build(); + return ArtifactReferenceURI.builder().uriPaths(Arrays.asList(paths)).build(); } } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java index 5e5ea591a..7e2d9566e 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java @@ -57,8 +57,4 @@ public static void setInstance(ArtifactStore storage) { log.warn("Multiple attempts in setting the singleton artifact store"); } } - - public boolean isArtifactURI(String value) { - return value.startsWith(ArtifactStoreURIBuilder.uriScheme + "://"); - } } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java index 019da7ce2..6f06ae786 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java @@ -44,5 +44,12 @@ public static class S3ClientConfig { private boolean forcePathStyle = true; } - private S3ClientConfig s3 = null; + @Data + public static class HelmConfig { + /** Enables Rosco to expand any artifact URIs passed as parameters for Helm. */ + private boolean expandOverrides = false; + } + + private S3ClientConfig s3 = new S3ClientConfig(); + private HelmConfig helm = new HelmConfig(); } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java index ba6712008..cef9904e4 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java @@ -18,13 +18,6 @@ import com.netflix.spinnaker.kork.artifacts.model.Artifact; public abstract class ArtifactStoreURIBuilder { - /** - * uriScheme is used as an HTTP scheme to let us further distinguish a String that is a URI to an - * artifact. This is helpful in determining what is an artifact since sometimes we are only given - * a string rather than a full artifact. - */ - public static final String uriScheme = "ref"; - /** * Returns the remote artifact URI that will be associated with some artifact. * diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java index b7744515b..ee739eaa0 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java @@ -38,7 +38,7 @@ public ArtifactReferenceURI buildArtifactURI(String context, Artifact artifact) Hashing.sha256() .hashBytes(artifact.getReference().getBytes(StandardCharsets.UTF_8)) .toString()); - return ArtifactReferenceURI.builder().scheme(uriScheme).uriPaths(uriPaths).build(); + return ArtifactReferenceURI.builder().uriPaths(uriPaths).build(); } @Override @@ -47,6 +47,6 @@ public ArtifactReferenceURI buildURIFromPaths(String context, String... paths) { uriPaths.add(context); uriPaths.addAll(List.of(paths)); - return ArtifactReferenceURI.builder().scheme(uriScheme).uriPaths(uriPaths).build(); + return ArtifactReferenceURI.builder().uriPaths(uriPaths).build(); } } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStoreConfiguration.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStoreConfiguration.java index 0b0fdaed1..e792d0c13 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStoreConfiguration.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStoreConfiguration.java @@ -66,7 +66,7 @@ public ArtifactStoreGetter artifactStoreGetter( "PermissionEvaluator is not present. This means anyone will be able to access any artifact in the store."); } - String bucket = properties.getS3() != null ? properties.getS3().getBucket() : null; + String bucket = properties.getS3().getBucket(); return new S3ArtifactStoreGetter(s3Client, permissionEvaluator.orElse(null), bucket); } diff --git a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java index bfe21ecb7..cacfbc635 100644 --- a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java +++ b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java @@ -55,7 +55,7 @@ public Object convertValue( return defaultTypeConverter.convertValue(value, sourceType, targetType); } - if (artifactStore == null || !artifactStore.isArtifactURI((String) value)) { + if (artifactStore == null || !ArtifactReferenceURI.is((String) value)) { return defaultTypeConverter.convertValue(value, sourceType, targetType); }