Aws bulk loading

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Next Release
+
+### Features Added
+- Aws bulk loading for secp256k1 keys in eth1 mode [#889](https://github.com/Consensys/web3signer/pull/889)
+
 ## 23.9.0
 
 ### Features Added

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java

@@ -16,7 +16,7 @@
 import tech.pegasys.web3signer.core.config.client.ClientTlsOptions;
 import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider;
 import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
-import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
+import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
 import tech.pegasys.web3signer.signing.config.KeystoresParameters;
 
@@ -41,7 +41,7 @@ public class SignerConfiguration {
   private final List<String> metricsCategories;
   private final boolean metricsEnabled;
   private final Optional<AzureKeyVaultParameters> azureKeyVaultParameters;
-  private final Optional<AwsSecretsManagerParameters> awsSecretsManagerParameters;
+  private final Optional<AwsVaultParameters> awsSecretsManagerParameters;
   private final Optional<KeystoresParameters> keystoresParameters;
   private final Optional<TlsOptions> serverTlsOptions;
   private final Optional<TlsCertificateDefinition> overriddenCaTrustStore;
@@ -89,7 +89,7 @@ public SignerConfiguration(
       final List<String> metricsCategories,
       final boolean metricsEnabled,
       final Optional<AzureKeyVaultParameters> azureKeyVaultParameters,
-      final Optional<AwsSecretsManagerParameters> awsSecretsManagerParameters,
+      final Optional<AwsVaultParameters> awsSecretsManagerParameters,
       final Optional<KeystoresParameters> keystoresParameters,
       final Optional<TlsOptions> serverTlsOptions,
       final Optional<TlsCertificateDefinition> overriddenCaTrustStore,
@@ -221,7 +221,7 @@ public Optional<AzureKeyVaultParameters> getAzureKeyVaultParameters() {
     return azureKeyVaultParameters;
   }
 
-  public Optional<AwsSecretsManagerParameters> getAwsSecretsManagerParameters() {
+  public Optional<AwsVaultParameters> getAwsParameters() {
     return awsSecretsManagerParameters;
   }
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java

@@ -20,7 +20,7 @@
 import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider;
 import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ConfigurationChainId;
 import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
-import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
+import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
 import tech.pegasys.web3signer.signing.config.KeystoresParameters;
 
@@ -50,7 +50,7 @@ public class SignerConfigurationBuilder {
   private Path slashingProtectionDbPoolConfigurationFile = null;
   private String mode;
   private AzureKeyVaultParameters azureKeyVaultParameters;
-  private AwsSecretsManagerParameters awsSecretsManagerParameters;
+  private AwsVaultParameters awsVaultParameters;
   private Map<String, String> web3SignerEnvironment;
   private Duration startupTimeout =
       Boolean.getBoolean("debugSubProcess") ? Duration.ofHours(1) : Duration.ofSeconds(30);
@@ -143,9 +143,8 @@ public SignerConfigurationBuilder withAzureKeyVaultParameters(
     return this;
   }
 
-  public SignerConfigurationBuilder withAwsSecretsManagerParameters(
-      final AwsSecretsManagerParameters awsSecretsManagerParameters) {
-    this.awsSecretsManagerParameters = awsSecretsManagerParameters;
+  public SignerConfigurationBuilder withAwsParameters(final AwsVaultParameters awsVaultParameters) {
+    this.awsVaultParameters = awsVaultParameters;
     return this;
   }
 
@@ -332,7 +331,7 @@ public SignerConfiguration build() {
         metricsCategories,
         metricsEnabled,
         Optional.ofNullable(azureKeyVaultParameters),
-        Optional.ofNullable(awsSecretsManagerParameters),
+        Optional.ofNullable(awsVaultParameters),
         Optional.ofNullable(keystoresParameters),
         Optional.ofNullable(serverTlsOptions),
         Optional.ofNullable(overriddenCaTrustStore),

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java

@@ -12,6 +12,13 @@
  */
 package tech.pegasys.web3signer.dsl.signer.runner;
 
+import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ACCESS_KEY_ID_OPTION;
+import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_AUTH_MODE_OPTION;
+import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ENABLED_OPTION;
+import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_REGION_OPTION;
+import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_SECRET_ACCESS_KEY_OPTION;
+import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_NAMES_FILTER_OPTION;
+import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_VALUES_FILTER_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
@@ -31,7 +38,7 @@
 import tech.pegasys.web3signer.dsl.signer.SignerConfiguration;
 import tech.pegasys.web3signer.dsl.signer.WatermarkRepairParameters;
 import tech.pegasys.web3signer.dsl.utils.DatabaseUtil;
-import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
+import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
 import tech.pegasys.web3signer.signing.config.KeystoresParameters;
 
@@ -143,8 +150,9 @@ public List<String> createCmdLineParams() {
       }
 
       signerConfig
-          .getAwsSecretsManagerParameters()
-          .ifPresent(awsParams -> yamlConfig.append(awsBulkLoadingOptions(awsParams)));
+          .getAwsParameters()
+          .ifPresent(
+              awsParams -> yamlConfig.append(awsSecretsManagerBulkLoadingOptions(awsParams)));
 
       final CommandArgs subCommandArgs = createSubCommandArgs();
       params.addAll(subCommandArgs.params);
@@ -160,6 +168,10 @@ public List<String> createCmdLineParams() {
       signerConfig
           .getV3KeystoresBulkloadParameters()
           .ifPresent(setV3KeystoresBulkloadParameters(yamlConfig));
+
+      signerConfig
+          .getAwsParameters()
+          .ifPresent(awsParams -> yamlConfig.append(awsKmsBulkLoadingOptions(awsParams)));
     }
 
     signerConfig
@@ -475,71 +487,70 @@ private String createEth2SlashingProtectionArgs() {
     return yamlConfig.toString();
   }
 
-  private String awsBulkLoadingOptions(
-      final AwsSecretsManagerParameters awsSecretsManagerParameters) {
+  private String awsSecretsManagerBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) {
     final StringBuilder yamlConfig = new StringBuilder();
 
     yamlConfig.append(
         String.format(
             YAML_BOOLEAN_FMT,
             "eth2." + AWS_SECRETS_ENABLED_OPTION.substring(2),
-            awsSecretsManagerParameters.isEnabled()));
+            awsVaultParameters.isEnabled()));
 
     yamlConfig.append(
         String.format(
             YAML_STRING_FMT,
             "eth2." + AWS_SECRETS_AUTH_MODE_OPTION.substring(2),
-            awsSecretsManagerParameters.getAuthenticationMode().name()));
+            awsVaultParameters.getAuthenticationMode().name()));
 
-    if (awsSecretsManagerParameters.getAccessKeyId() != null) {
+    if (awsVaultParameters.getAccessKeyId() != null) {
       yamlConfig.append(
           String.format(
               YAML_STRING_FMT,
               "eth2." + AWS_SECRETS_ACCESS_KEY_ID_OPTION.substring(2),
-              awsSecretsManagerParameters.getAccessKeyId()));
+              awsVaultParameters.getAccessKeyId()));
     }
 
-    if (awsSecretsManagerParameters.getSecretAccessKey() != null) {
+    if (awsVaultParameters.getSecretAccessKey() != null) {
       yamlConfig.append(
           String.format(
               YAML_STRING_FMT,
               "eth2." + AWS_SECRETS_SECRET_ACCESS_KEY_OPTION.substring(2),
-              awsSecretsManagerParameters.getSecretAccessKey()));
+              awsVaultParameters.getSecretAccessKey()));
     }
 
-    if (awsSecretsManagerParameters.getRegion() != null) {
+    if (awsVaultParameters.getRegion() != null) {
       yamlConfig.append(
           String.format(
               YAML_STRING_FMT,
               "eth2." + AWS_SECRETS_REGION_OPTION.substring(2),
-              awsSecretsManagerParameters.getRegion()));
+              awsVaultParameters.getRegion()));
     }
 
-    if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) {
+    if (!awsVaultParameters.getPrefixesFilter().isEmpty()) {
       yamlConfig.append(
           String.format(
               YAML_STRING_FMT,
               "eth2." + AWS_SECRETS_PREFIXES_FILTER_OPTION.substring(2),
-              String.join(",", awsSecretsManagerParameters.getPrefixesFilter())));
+              String.join(",", awsVaultParameters.getPrefixesFilter())));
     }
 
-    if (!awsSecretsManagerParameters.getTagNamesFilter().isEmpty()) {
+    if (!awsVaultParameters.getTagNamesFilter().isEmpty()) {
       yamlConfig.append(
           String.format(
               YAML_STRING_FMT,
               "eth2." + AWS_SECRETS_TAG_NAMES_FILTER_OPTION.substring(2),
-              String.join(",", awsSecretsManagerParameters.getTagNamesFilter())));
+              String.join(",", awsVaultParameters.getTagNamesFilter())));
     }
 
-    if (!awsSecretsManagerParameters.getTagValuesFilter().isEmpty()) {
+    if (!awsVaultParameters.getTagValuesFilter().isEmpty()) {
       yamlConfig.append(
           String.format(
               YAML_STRING_FMT,
               "eth2." + AWS_SECRETS_TAG_VALUES_FILTER_OPTION.substring(2),
-              String.join(",", awsSecretsManagerParameters.getTagValuesFilter())));
+              String.join(",", awsVaultParameters.getTagValuesFilter())));
     }
 
-    awsSecretsManagerParameters
+    awsVaultParameters
         .getEndpointOverride()
         .ifPresent(
             uri ->
@@ -552,6 +563,74 @@ private String awsBulkLoadingOptions(
     return yamlConfig.toString();
   }
 
+  private String awsKmsBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) {
+    final StringBuilder yamlConfig = new StringBuilder();
+
+    yamlConfig.append(
+        String.format(
+            YAML_BOOLEAN_FMT,
+            "eth1." + AWS_KMS_ENABLED_OPTION.substring(2),
+            awsVaultParameters.isEnabled()));
+
+    yamlConfig.append(
+        String.format(
+            YAML_STRING_FMT,
+            "eth1." + AWS_KMS_AUTH_MODE_OPTION.substring(2),
+            awsVaultParameters.getAuthenticationMode().name()));
+
+    if (awsVaultParameters.getAccessKeyId() != null) {
+      yamlConfig.append(
+          String.format(
+              YAML_STRING_FMT,
+              "eth1." + AWS_KMS_ACCESS_KEY_ID_OPTION.substring(2),
+              awsVaultParameters.getAccessKeyId()));
+    }
+
+    if (awsVaultParameters.getSecretAccessKey() != null) {
+      yamlConfig.append(
+          String.format(
+              YAML_STRING_FMT,
+              "eth1." + AWS_KMS_SECRET_ACCESS_KEY_OPTION.substring(2),
+              awsVaultParameters.getSecretAccessKey()));
+    }
+
+    if (awsVaultParameters.getRegion() != null) {
+      yamlConfig.append(
+          String.format(
+              YAML_STRING_FMT,
+              "eth1." + AWS_KMS_REGION_OPTION.substring(2),
+              awsVaultParameters.getRegion()));
+    }
+
+    if (!awsVaultParameters.getTagNamesFilter().isEmpty()) {
+      yamlConfig.append(
+          String.format(
+              YAML_STRING_FMT,
+              "eth1." + AWS_KMS_TAG_NAMES_FILTER_OPTION.substring(2),
+              String.join(",", awsVaultParameters.getTagNamesFilter())));
+    }
+
+    if (!awsVaultParameters.getTagValuesFilter().isEmpty()) {
+      yamlConfig.append(
+          String.format(
+              YAML_STRING_FMT,
+              "eth1." + AWS_KMS_TAG_VALUES_FILTER_OPTION.substring(2),
+              String.join(",", awsVaultParameters.getTagValuesFilter())));
+    }
+
+    awsVaultParameters
+        .getEndpointOverride()
+        .ifPresent(
+            uri ->
+                yamlConfig.append(
+                    String.format(
+                        YAML_STRING_FMT,
+                        "eth1." + AWS_ENDPOINT_OVERRIDE_OPTION.substring(2),
+                        uri)));
+
+    return yamlConfig.toString();
+  }
+
   private String formatStringList(final String key, final List<String> stringList) {
     return stringList.isEmpty()
         ? String.format("%s: []%n", key)