Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Aws bulk loading #889

Merged
merged 24 commits into from
Sep 12, 2023
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
ff917d5
Add Aws params to Eth1Config
jframe Aug 21, 2023
fc4eb12
Rename AWSBulkLoadingArtifactSignerProvider to BlsAwsBulkLoader to ma…
jframe Aug 21, 2023
ad2d0ff
SecpAwsBulkLoader
jframe Aug 22, 2023
6b2960b
AwsKmsClient tests
jframe Aug 23, 2023
c3a3fc4
Add support for tags
jframe Aug 24, 2023
46fe53c
Change existing tests to use AwsKmsUtil
jframe Aug 24, 2023
93514aa
Wire in Eth1 AWS bulk loading
jframe Aug 29, 2023
c63fde9
acceptance tests for Aws Kms
jframe Aug 31, 2023
614851a
change the CachedAwsKmsClientFactory to use the kms properties and bu…
jframe Aug 31, 2023
2129c87
Bulk loading kms with values specified using env variables
jframe Aug 31, 2023
5b80cfe
cleanup
jframe Sep 1, 2023
313989a
remove unnecessary renames
jframe Sep 1, 2023
10e7516
additional AwsKmsClient tests for being enabled and secp256k1
jframe Sep 1, 2023
77343bb
filter on enabled keys before checking tags as we might not have perm…
jframe Sep 4, 2023
f9748a4
Use AwsRegionProvider if auth mode is not specified
jframe Sep 4, 2023
ea12373
catch any exceptions that occur filtering keys
jframe Sep 4, 2023
b8953a9
after pr review
jframe Sep 4, 2023
23062d6
Merge remote-tracking branch 'upstream/master' into aws_bulk_loading
jframe Sep 4, 2023
3775f9f
changelog
jframe Sep 5, 2023
4bd6d95
rename AwsParameters to AwsVaultParameters
jframe Sep 5, 2023
cec9f06
Merge remote-tracking branch 'upstream/master' into aws_bulk_loading
jframe Sep 5, 2023
03867ee
spotless
jframe Sep 5, 2023
5bd260a
Fix intg test broken validation of aws and azure vault params and cha…
jframe Sep 7, 2023
0e7cf33
Merge remote-tracking branch 'upstream/master' into aws_bulk_loading
jframe Sep 12, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import tech.pegasys.web3signer.core.config.client.ClientTlsOptions;
import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider;
import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsParameters;
import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
import tech.pegasys.web3signer.signing.config.KeystoresParameters;

Expand All @@ -41,7 +41,7 @@ public class SignerConfiguration {
private final List<String> metricsCategories;
private final boolean metricsEnabled;
private final Optional<AzureKeyVaultParameters> azureKeyVaultParameters;
private final Optional<AwsSecretsManagerParameters> awsSecretsManagerParameters;
private final Optional<AwsParameters> awsSecretsManagerParameters;
private final Optional<KeystoresParameters> keystoresParameters;
private final Optional<TlsOptions> serverTlsOptions;
private final Optional<TlsCertificateDefinition> overriddenCaTrustStore;
Expand Down Expand Up @@ -88,7 +88,7 @@ public SignerConfiguration(
final List<String> metricsCategories,
final boolean metricsEnabled,
final Optional<AzureKeyVaultParameters> azureKeyVaultParameters,
final Optional<AwsSecretsManagerParameters> awsSecretsManagerParameters,
final Optional<AwsParameters> awsSecretsManagerParameters,
final Optional<KeystoresParameters> keystoresParameters,
final Optional<TlsOptions> serverTlsOptions,
final Optional<TlsCertificateDefinition> overriddenCaTrustStore,
Expand Down Expand Up @@ -218,7 +218,7 @@ public Optional<AzureKeyVaultParameters> getAzureKeyVaultParameters() {
return azureKeyVaultParameters;
}

public Optional<AwsSecretsManagerParameters> getAwsSecretsManagerParameters() {
public Optional<AwsParameters> getAwsSecretsManagerParameters() {
return awsSecretsManagerParameters;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider;
import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ConfigurationChainId;
import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsParameters;
import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
import tech.pegasys.web3signer.signing.config.KeystoresParameters;

Expand Down Expand Up @@ -50,7 +50,7 @@ public class SignerConfigurationBuilder {
private Path slashingProtectionDbPoolConfigurationFile = null;
private String mode;
private AzureKeyVaultParameters azureKeyVaultParameters;
private AwsSecretsManagerParameters awsSecretsManagerParameters;
private AwsParameters awsParameters;
private Map<String, String> web3SignerEnvironment;
private Duration startupTimeout =
Boolean.getBoolean("debugSubProcess") ? Duration.ofHours(1) : Duration.ofSeconds(30);
Expand Down Expand Up @@ -142,8 +142,8 @@ public SignerConfigurationBuilder withAzureKeyVaultParameters(
}

public SignerConfigurationBuilder withAwsSecretsManagerParameters(
final AwsSecretsManagerParameters awsSecretsManagerParameters) {
this.awsSecretsManagerParameters = awsSecretsManagerParameters;
final AwsParameters awsParameters) {
this.awsParameters = awsParameters;
return this;
}

Expand Down Expand Up @@ -324,7 +324,7 @@ public SignerConfiguration build() {
metricsCategories,
metricsEnabled,
Optional.ofNullable(azureKeyVaultParameters),
Optional.ofNullable(awsSecretsManagerParameters),
Optional.ofNullable(awsParameters),
Optional.ofNullable(keystoresParameters),
Optional.ofNullable(serverTlsOptions),
Optional.ofNullable(overriddenCaTrustStore),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,23 @@
*/
package tech.pegasys.web3signer.dsl.signer.runner;

import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ENABLED_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_PREFIXES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_REGION_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_SECRET_ACCESS_KEY_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_TAG_NAMES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_TAG_VALUES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_AUTH_MODE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_ENABLED_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_PREFIXES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_REGION_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_SECRET_ACCESS_KEY_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_TAG_NAMES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_TAG_VALUES_FILTER_OPTION;

import tech.pegasys.web3signer.core.config.ClientAuthConstraints;
import tech.pegasys.web3signer.core.config.TlsOptions;
import tech.pegasys.web3signer.core.config.client.ClientTlsOptions;
import tech.pegasys.web3signer.dsl.signer.SignerConfiguration;
import tech.pegasys.web3signer.dsl.signer.WatermarkRepairParameters;
import tech.pegasys.web3signer.dsl.utils.DatabaseUtil;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsParameters;
import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
import tech.pegasys.web3signer.signing.config.KeystoresParameters;

Expand Down Expand Up @@ -441,71 +441,70 @@ private String createEth2SlashingProtectionArgs() {
return yamlConfig.toString();
}

private String awsBulkLoadingOptions(
final AwsSecretsManagerParameters awsSecretsManagerParameters) {
private String awsBulkLoadingOptions(final AwsParameters awsParameters) {
final StringBuilder yamlConfig = new StringBuilder();

yamlConfig.append(
String.format(
YAML_BOOLEAN_FMT,
"eth2." + AWS_SECRETS_ENABLED_OPTION.substring(2),
awsSecretsManagerParameters.isEnabled()));
awsParameters.isEnabled()));

yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_AUTH_MODE_OPTION.substring(2),
awsSecretsManagerParameters.getAuthenticationMode().name()));
awsParameters.getAuthenticationMode().name()));

if (awsSecretsManagerParameters.getAccessKeyId() != null) {
if (awsParameters.getAccessKeyId() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_ACCESS_KEY_ID_OPTION.substring(2),
awsSecretsManagerParameters.getAccessKeyId()));
awsParameters.getAccessKeyId()));
}

if (awsSecretsManagerParameters.getSecretAccessKey() != null) {
if (awsParameters.getSecretAccessKey() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_SECRET_ACCESS_KEY_OPTION.substring(2),
awsSecretsManagerParameters.getSecretAccessKey()));
awsParameters.getSecretAccessKey()));
}

if (awsSecretsManagerParameters.getRegion() != null) {
if (awsParameters.getRegion() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_REGION_OPTION.substring(2),
awsSecretsManagerParameters.getRegion()));
awsParameters.getRegion()));
}

if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) {
if (!awsParameters.getPrefixesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_PREFIXES_FILTER_OPTION.substring(2),
String.join(",", awsSecretsManagerParameters.getPrefixesFilter())));
String.join(",", awsParameters.getPrefixesFilter())));
}

if (!awsSecretsManagerParameters.getTagNamesFilter().isEmpty()) {
if (!awsParameters.getTagNamesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_TAG_NAMES_FILTER_OPTION.substring(2),
String.join(",", awsSecretsManagerParameters.getTagNamesFilter())));
String.join(",", awsParameters.getTagNamesFilter())));
}

if (!awsSecretsManagerParameters.getTagValuesFilter().isEmpty()) {
if (!awsParameters.getTagValuesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_TAG_VALUES_FILTER_OPTION.substring(2),
String.join(",", awsSecretsManagerParameters.getTagValuesFilter())));
String.join(",", awsParameters.getTagValuesFilter())));
}

awsSecretsManagerParameters
awsParameters
.getEndpointOverride()
.ifPresent(
uri ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,23 @@
*/
package tech.pegasys.web3signer.dsl.signer.runner;

import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ENABLED_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_PREFIXES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_REGION_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_SECRET_ACCESS_KEY_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_TAG_NAMES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_TAG_VALUES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_AUTH_MODE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_ENABLED_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_PREFIXES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_REGION_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_SECRET_ACCESS_KEY_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_TAG_NAMES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsParameters.AWS_SECRETS_TAG_VALUES_FILTER_OPTION;

import tech.pegasys.web3signer.core.config.ClientAuthConstraints;
import tech.pegasys.web3signer.core.config.TlsOptions;
import tech.pegasys.web3signer.core.config.client.ClientTlsOptions;
import tech.pegasys.web3signer.dsl.signer.SignerConfiguration;
import tech.pegasys.web3signer.dsl.signer.WatermarkRepairParameters;
import tech.pegasys.web3signer.dsl.utils.DatabaseUtil;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsParameters;
import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
import tech.pegasys.web3signer.signing.config.KeystoresParameters;

Expand Down Expand Up @@ -276,51 +276,50 @@ private Collection<String> createEth2Args() {
return params;
}

private Collection<String> awsBulkLoadingOptions(
final AwsSecretsManagerParameters awsSecretsManagerParameters) {
private Collection<String> awsBulkLoadingOptions(final AwsParameters awsParameters) {
final List<String> params = new ArrayList<>();

params.add(AWS_SECRETS_ENABLED_OPTION + "=" + awsSecretsManagerParameters.isEnabled());
params.add(AWS_SECRETS_ENABLED_OPTION + "=" + awsParameters.isEnabled());

params.add(AWS_SECRETS_AUTH_MODE_OPTION);
params.add(awsSecretsManagerParameters.getAuthenticationMode().name());
params.add(awsParameters.getAuthenticationMode().name());

if (awsSecretsManagerParameters.getAccessKeyId() != null) {
if (awsParameters.getAccessKeyId() != null) {
params.add(AWS_SECRETS_ACCESS_KEY_ID_OPTION);
params.add(awsSecretsManagerParameters.getAccessKeyId());
params.add(awsParameters.getAccessKeyId());
}

if (awsSecretsManagerParameters.getSecretAccessKey() != null) {
if (awsParameters.getSecretAccessKey() != null) {
params.add(AWS_SECRETS_SECRET_ACCESS_KEY_OPTION);
params.add(awsSecretsManagerParameters.getSecretAccessKey());
params.add(awsParameters.getSecretAccessKey());
}

if (awsSecretsManagerParameters.getRegion() != null) {
if (awsParameters.getRegion() != null) {
params.add(AWS_SECRETS_REGION_OPTION);
params.add(awsSecretsManagerParameters.getRegion());
params.add(awsParameters.getRegion());
}

awsSecretsManagerParameters
awsParameters
.getEndpointOverride()
.ifPresent(
uri -> {
params.add(AWS_ENDPOINT_OVERRIDE_OPTION);
params.add(uri.toString());
});

if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) {
if (!awsParameters.getPrefixesFilter().isEmpty()) {
params.add(AWS_SECRETS_PREFIXES_FILTER_OPTION);
params.add(String.join(",", awsSecretsManagerParameters.getPrefixesFilter()));
params.add(String.join(",", awsParameters.getPrefixesFilter()));
}

if (!awsSecretsManagerParameters.getTagNamesFilter().isEmpty()) {
if (!awsParameters.getTagNamesFilter().isEmpty()) {
params.add(AWS_SECRETS_TAG_NAMES_FILTER_OPTION);
params.add(String.join(",", awsSecretsManagerParameters.getTagNamesFilter()));
params.add(String.join(",", awsParameters.getTagNamesFilter()));
}

if (!awsSecretsManagerParameters.getTagValuesFilter().isEmpty()) {
if (!awsParameters.getTagValuesFilter().isEmpty()) {
params.add(AWS_SECRETS_TAG_VALUES_FILTER_OPTION);
params.add(String.join(",", awsSecretsManagerParameters.getTagValuesFilter()));
params.add(String.join(",", awsParameters.getTagValuesFilter()));
}

return params;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
import tech.pegasys.web3signer.common.config.AwsAuthenticationMode;
import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder;
import tech.pegasys.web3signer.signing.KeyType;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder;
import tech.pegasys.web3signer.signing.config.AwsParameters;
import tech.pegasys.web3signer.signing.config.AwsParametersBuilder;
import tech.pegasys.web3signer.tests.AcceptanceTestBase;

import java.net.URI;
Expand Down Expand Up @@ -102,8 +102,8 @@ void setupAwsResources() {
@ParameterizedTest(name = "{index} - Using config file: {0}")
@ValueSource(booleans = {true, false})
void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean useConfigFile) {
final AwsSecretsManagerParameters awsSecretsManagerParameters =
AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters()
final AwsParameters awsParameters =
AwsParametersBuilder.anAwsSecretsManagerParameters()
.withAuthenticationMode(AwsAuthenticationMode.SPECIFIED)
.withRegion(AWS_REGION)
.withAccessKeyId(RO_AWS_ACCESS_KEY_ID)
Expand All @@ -118,7 +118,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
new SignerConfigurationBuilder()
.withUseConfigFile(useConfigFile)
.withMode("eth2")
.withAwsSecretsManagerParameters(awsSecretsManagerParameters);
.withAwsSecretsManagerParameters(awsParameters);

startSigner(configBuilder.build());

Expand All @@ -144,8 +144,8 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
@Test
void healthCheckErrorCountWhenInvalidCredentialsAreUsed() {
final boolean useConfigFile = false;
final AwsSecretsManagerParameters invalidCredsParams =
AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters()
final AwsParameters invalidCredsParams =
AwsParametersBuilder.anAwsSecretsManagerParameters()
.withAuthenticationMode(AwsAuthenticationMode.SPECIFIED)
.withRegion("us-east-2")
.withAccessKeyId("invalid")
Expand Down Expand Up @@ -189,8 +189,8 @@ private static int getAwsBulkLoadingData(String healthCheckJsonBody, String data
@ValueSource(booleans = {true, false})
void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPublicApi(
final boolean useConfigFile) {
final AwsSecretsManagerParameters awsSecretsManagerParameters =
AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters()
final AwsParameters awsParameters =
AwsParametersBuilder.anAwsSecretsManagerParameters()
.withAuthenticationMode(AwsAuthenticationMode.ENVIRONMENT)
.withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
.withTagNamesFilter(List.of("TagName2", "TagName3"))
Expand All @@ -202,7 +202,7 @@ void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPu
new SignerConfigurationBuilder()
.withUseConfigFile(useConfigFile)
.withMode("eth2")
.withAwsSecretsManagerParameters(awsSecretsManagerParameters);
.withAwsSecretsManagerParameters(awsParameters);

startSigner(configBuilder.build());

Expand Down
Loading