Merge pull request #32481 from vespa-engine/move-secret-impl-to-open-…

…source_pt2_try4

Move secret impl to open source pt2 try4

config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecrets.java

@@ -14,8 +14,8 @@
  */
 public class CloudSecrets extends SimpleComponent implements SecretsConfig.Producer {
 
-    private static final String CLASS = "ai.vespa.secret.cloud.SecretsImpl";
-    private static final String BUNDLE = "cloud-common";
+    private static final String CLASS = "ai.vespa.secret.aws.SecretsImpl";
+    private static final String BUNDLE = "jdisc-cloud-aws";
 
     private final List<SecretConfig> secrets = new ArrayList<>();
 

config-model/src/test/java/com/yahoo/vespa/model/container/xml/SecretsTest.java

@@ -23,7 +23,7 @@
  */
 public class SecretsTest extends ContainerModelBuilderTestBase {
 
-    private static String IMPL_ID = "ai.vespa.secret.cloud.SecretsImpl";
+    private static String IMPL_ID = "ai.vespa.secret.aws.SecretsImpl";
 
     @Test
     void testCloudSecretsNeedHosted() {

container-disc/src/main/java/ai/vespa/secret/internal/TypedSecretStore.java

@@ -0,0 +1,34 @@
+package ai.vespa.secret.internal;
+
+import ai.vespa.secret.model.Key;
+import ai.vespa.secret.model.Secret;
+import ai.vespa.secret.model.SecretVersionId;
+import com.yahoo.container.jdisc.secretstore.SecretStore;
+
+import java.util.List;
+
+public interface TypedSecretStore extends SecretStore {
+
+    enum Type {
+        PUBLIC,
+        TEST,
+        YAHOO
+    }
+
+    Secret getSecret(Key key);
+
+    Secret getSecret(Key key, SecretVersionId version);
+
+    /** Lists the existing versions of this secret */
+    default List<Secret> listSecretVersions(Key key) {
+        throw new UnsupportedOperationException("Secret store does not support listing versions");
+    }
+
+    Type type();
+
+    // Do not use! Only for legacy compatibility
+    default Secret getSecret(Key k, int i) {
+        return getSecret(k, SecretVersionId.of(String.valueOf(i)));
+    }
+
+}

container-disc/src/main/java/ai/vespa/secret/internal/package-info.java

@@ -0,0 +1,6 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+@ExportPackage
+package ai.vespa.secret.internal;
+
+import com.yahoo.osgi.annotation.ExportPackage;

container-disc/src/test/java/ai/vespa/secret/model/KeyTest.java

@@ -0,0 +1,24 @@
+package ai.vespa.secret.model;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author gjoranv
+ */
+public class KeyTest {
+
+    @Test
+    void string_can_be_converted_to_key() {
+        var vault = VaultName.of("vaultName");
+        var secret = SecretName.of("secretName");
+
+        var expected = new Key(vault, secret);
+        assertEquals(expected, Key.fromString("vaultName/secretName"));
+
+        assertThrows(IllegalArgumentException.class, () -> Key.fromString("vaultName"));
+        assertThrows(IllegalArgumentException.class, () -> Key.fromString("vaultName/secretName/extra"));
+    }
+}

container-disc/src/test/java/ai/vespa/secret/model/SecretNameTest.java

@@ -0,0 +1,27 @@
+package ai.vespa.secret.model;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author gjoranv
+ */
+public class SecretNameTest {
+
+    @Test
+    void testSecretName() {
+        SecretName.of("foo-bar");
+        SecretName.of("-");
+        SecretName.of("0");
+        SecretName.of("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcde");
+        assertThrows(IllegalArgumentException.class, () -> SecretName.of(""));
+
+        // TODO: enable when all secrets are < 64 characters
+        //assertThrows(IllegalArgumentException.class, () -> SecretName.of("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"));
+
+        for (char c : "+/$ {}[]()!\"@#?\\'".toCharArray())
+            assertThrows(IllegalArgumentException.class, () -> SecretName.of("foo" + c + "bar"));
+    }
+
+}

container-disc/src/test/java/ai/vespa/secret/model/SecretTest.java

@@ -0,0 +1,58 @@
+package ai.vespa.secret.model;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static ai.vespa.secret.model.SecretVersionState.CURRENT;
+import static ai.vespa.secret.model.SecretVersionState.DEPRECATED;
+import static ai.vespa.secret.model.SecretVersionState.PENDING;
+import static ai.vespa.secret.model.SecretVersionState.PREVIOUS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author gjoranv
+ */
+public class SecretTest {
+
+    @Test
+    void secrets_are_sorted_on_vault_then_name_then_state() {
+
+        var s11pe = secret("vault1", "name1", PENDING);
+        var s11cu = secret("vault1", "name1", CURRENT);
+        var s12cu = secret("vault1", "name2", CURRENT);
+        var s21pe = secret("vault2", "name1", PENDING);
+        var s21cu = secret("vault2", "name1", CURRENT);
+        var s21pr = secret("vault2", "name1", PREVIOUS);
+        var s21de = secret("vault2", "name1", DEPRECATED);
+
+        var secrets = List.of( s21pe, s11cu, s12cu, s11pe, s21de, s21pr, s21cu );
+
+        var expected = List.of( s11pe, s11cu, s12cu, s21pe, s21cu, s21pr, s21de );
+
+        assertEquals(expected, secrets.stream().sorted().toList());
+    }
+
+    // This is relevant for secrets from CKMS, which don't use state, but ascending version numbers.
+    @Test
+    void secrets_with_same_state_are_sorted_by_version_descending() {
+        var v1 = secretWithIntVersion(1);
+        var v2 = secretWithIntVersion(2);
+        var v3 = secretWithIntVersion(3);
+
+        var secrets = List.of(v3, v1, v2);
+        var expected = List.of(v3, v2, v1);
+        assertEquals(expected, secrets.stream().sorted().toList());
+    }
+
+    private static Secret secretWithIntVersion(Integer version) {
+        return new Secret(new Key(VaultName.of("foo"), SecretName.of("bar")), new byte[0],
+                          SecretVersionId.of(version.toString()));
+    }
+
+    private static Secret secret(String vault, String name, SecretVersionState state) {
+        return new Secret(new Key(VaultName.of(vault), SecretName.of(name)), new byte[0],
+                          SecretVersionId.of("0"), state);
+    }
+
+}

dependency-versions/pom.xml

@@ -78,6 +78,7 @@
                  find zkfacade/src/main/java/org/apache/curator -name package-info.java | \
                      xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g'
         -->
+        <aws-sdk2.vespa.version>2.27.4</aws-sdk2.vespa.version>
         <bouncycastle.vespa.version>1.78.1</bouncycastle.vespa.version>
         <byte-buddy.vespa.version>1.14.18</byte-buddy.vespa.version>
         <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version>

jdisc-cloud-aws/CMakeLists.txt

@@ -1,3 +1,3 @@
 # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-install_jar(jdisc-cloud-aws-jar-with-dependencies.jar)
+install_configserver_component(jdisc-cloud-aws)
 install_config_definitions()

jdisc-cloud-aws/pom.xml

@@ -14,7 +14,6 @@
     <artifactId>jdisc-cloud-aws</artifactId>
     <version>8-SNAPSHOT</version>
     <packaging>container-plugin</packaging>
-    <name>${project.artifactId}</name>
 
     <dependencies>
         <dependency>
@@ -29,6 +28,12 @@
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespa-athenz</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-core</artifactId>
@@ -51,6 +56,37 @@
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-ssm</artifactId>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>secretsmanager</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <!-- Used by SecretsManagerClient.builder() to load a http client  -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>apache-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
@@ -71,6 +107,17 @@
     </dependencies>
 
     <build>
+        <resources>
+            <!-- Exclude config defs - they are already installed in the configserver's serverdb  -->
+            <resource>
+                <directory>src/main/resources</directory>
+                <excludes>
+                    <exclude>configdefinitions/*</exclude>
+                </excludes>
+                <filtering>false</filtering>
+            </resource>
+        </resources>
+
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>