Fix IMDSv2 token handling and docker permissions

README.md

@@ -85,10 +85,9 @@ The following table provides a sample cost breakdown for deploying this Guidance
 * An [AWS Identity and Access Management](http://aws.amazon.com/iam) (IAM) user with administrator access
 * [Configured AWS credentials](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html#getting_started_prerequisites)
 * [Docker](https://docs.docker.com/get-docker/), [Node.js](https://nodejs.org/en/download/)
-  , [Python 3.9](https://www.python.org/downloads/release/python-3916), [pip](https://pip.pypa.io/en/stable/installing/),
+  , [>=Python 3.11](https://www.python.org/downloads/release/python-3110/), [pip](https://pip.pypa.io/en/stable/installing/),
   and [jq](https://stedolan.github.io/jq/) installed on the workstation that you plan to deploy the guidance from.
 
-Note that the guidance is **only** compatible with Python 3.9.
 
 ### Deploy with AWS CDK
 
@@ -177,6 +176,27 @@ cdk deploy prodNitroSigner -O output.json
 
 Follow all subsequent steps from the dev deployment pointed out above.
 
+## Troubleshooting
+
+**Docker Image Push/Pull Error**
+* On `building` instance during `cdk deploy` step:
+```shell
+devNitroWalletEth: fail: docker push 012345678910.dkr.ecr.us-east-1.amazonaws.com/cdk-hnb659fds-container-assets-012345678910-us-east-1:ab3fe... exited with error code 1: failed commit on ref "manifest-sha256:7141...": unexpected status from PUT request to https://012345678910.dkr.ecr.us-east-1.amazonaws.com/v2/cdk-hnb659fds-container-assets-012345678910-us-east-1/manifests/ab3fe...: 400 Bad Request
+Failed to publish asset ab3fe...:012345678910-us-east-1
+```
+
+* On EC2 instance pulling docker container
+```shell
+ab3fe...: Pulling from cdk-hnb659fds-container-assets-012345678910-us-east-1
+unsupported media type application/vnd.in-toto+json
+```
+
+**Solution**
+* Issue might be related building and publishing docker containers from an `arm` based instances such as Apple Silicon, requiring docker `buildx` [issue](https://github.com/aws/aws-cdk/issues/30258)
+* Cleanup images from local docker repository (`docker rmi ...`) and from Amazon Elastic Container Registry (ECR) e.g. via AWS console
+* Set environment variable in terminal session (`export BUILDX_NO_DEFAULT_ATTESTATIONS=1`) or specify it during cdk deployment  (`BUILDX_NO_DEFAULT_ATTESTATIONS=1 cdk deploy`)
+
+
 ## Security
 
 See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information.

application/eth2/lambda/layer/requirements.txt

@@ -1,3 +1,3 @@
-cryptography==39.0.2 ; python_version >= "3.9" and python_version < "4"
-requests>=2.28.1 ; python_version >= "3.9" and python_version < "4"
-urllib3<2; python_version >= "3.9" and python_version < "4"
+cryptography==41.0.7
+requests>=2.31.0
+urllib3==2.1.0

application/eth2/watchdog/watchdog.py

@@ -28,29 +28,51 @@
 _logger.addHandler(handler)
 
 
-def get_aws_session_token() -> dict:
+def get_imds_token():
     http_ec2_client = client.HTTPConnection("169.254.169.254")
-    http_ec2_client.request("GET", "/latest/meta-data/iam/security-credentials/")
-    r = http_ec2_client.getresponse()
+    headers = {
+        "X-aws-ec2-metadata-token-ttl-seconds": "21600"  # Token valid for 6 hours
+    }
+    http_ec2_client.request("PUT", "/latest/api/token", headers=headers)
+    token_response = http_ec2_client.getresponse()
+    return token_response.read().decode()
 
-    instance_profile_name = r.read().decode()
 
-    http_ec2_client = client.HTTPConnection("169.254.169.254")
-    http_ec2_client.request(
-        "GET",
-        f"/latest/meta-data/iam/security-credentials/{instance_profile_name}",
-    )
-    r = http_ec2_client.getresponse()
+def get_aws_session_token():
+    try:
+        token = get_imds_token()
 
-    response = json.loads(r.read())
+        http_ec2_client = client.HTTPConnection("169.254.169.254")
+        headers = {"X-aws-ec2-metadata-token": token}
 
-    credential = {
-        "access_key_id": response["AccessKeyId"],
-        "secret_access_key": response["SecretAccessKey"],
-        "token": response["Token"],
-    }
+        # Get instance profile name
+        http_ec2_client.request(
+            "GET",
+            "/latest/meta-data/iam/security-credentials/",
+            headers=headers
+        )
+        r = http_ec2_client.getresponse()
+        instance_profile_name = r.read().decode()
+
+        # Get credentials
+        http_ec2_client.request(
+            "GET",
+            f"/latest/meta-data/iam/security-credentials/{instance_profile_name}",
+            headers=headers
+        )
+        r = http_ec2_client.getresponse()
+        response = json.loads(r.read())
+        return {
+            "access_key_id": response["AccessKeyId"],
+            "secret_access_key": response["SecretAccessKey"],
+            "token": response["Token"],
+        }
 
-    return credential
+    except Exception as e:
+        raise Exception(f"Failed to retrieve instance credentials: {str(e)}")
+    finally:
+        if 'http_ec2_client' in locals():
+            http_ec2_client.close()
 
 
 def get_cloudformation_stack_id(cf_stack_name: str) -> str:
@@ -92,8 +114,8 @@ def nitro_cli_describe_call(name: str = None) -> bool:
             return False
 
         if (
-            response[0].get("EnclaveName") != name
-            and response[0].get("State") != "Running"
+                response[0].get("EnclaveName") != name
+                and response[0].get("State") != "Running"
         ):
             return False
 
@@ -230,19 +252,19 @@ def get_encrypted_tls_key(tls_keys_table_name: str, key_id=1) -> str:
 
 
 def init_web3signer_call(
-    tls_keys_table_name: str, cf_stack_name: str, validator_keys_table_name: str
+        tls_keys_table_name: str, cf_stack_name: str, validator_keys_table_name: str
 ) -> None:
     uuid = get_cloudformation_stack_id(cf_stack_name)
     encrypted_validator_keys = get_encrypted_validator_keys(
         validator_keys_table_name, uuid
     )
     encrypted_tls_key = get_encrypted_tls_key(tls_keys_table_name=tls_keys_table_name)
 
-    credential = get_aws_session_token()
+    credentials = get_aws_session_token()
 
     payload = {
         "operation": "init",
-        "credential": credential,
+        "credential": credentials,
         "encrypted_tls_key": encrypted_tls_key,
         "encrypted_validator_keys": encrypted_validator_keys,
     }

nitro_wallet/nitro_wallet_stack.py

@@ -94,13 +94,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             "EthereumSigningServerImage",
             directory="./application/{}/server".format(application_type),
             build_args={"REGION_ARG": self.region, "LOG_LEVEL_ARG": log_level},
+            platform=ecr_assets.Platform.LINUX_AMD64,
+            asset_name="EthereumSigningServerImage"
+
         )
 
         signing_enclave_image = ecr_assets.DockerImageAsset(
             self,
             "EthereumSigningEnclaveImage",
             directory="./application/{}/enclave".format(application_type),
             build_args={"REGION_ARG": self.region, "LOG_LEVEL_ARG": log_level},
+            platform=ecr_assets.Platform.LINUX_AMD64,
+            asset_name="EthereumSigningEnclaveImage"
         )
 
         watchdog = s3_assets.Asset(
@@ -152,7 +157,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
         nitro_instance_sg.add_ingress_rule(signer_client_sg, ec2.Port.tcp(8443))
 
         # AMI
-        amzn_linux = ec2.MachineImage.latest_amazon_linux(generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2)
+        amzn_linux = ec2.MachineImage.latest_amazon_linux2()
 
         # Instance Role and SSM Managed Policy
         role = iam.Role(
@@ -227,6 +232,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             block_devices=[block_device],
             role=role,
             security_group=nitro_instance_sg,
+            http_put_response_hop_limit=3
         )
 
         nitro_nlb = elbv2.NetworkLoadBalancer(
@@ -292,7 +298,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             self,
             "NitroInvokeLambdaLayer",
             entry="application/{}/lambda/layer".format(params["application_type"]),
-            compatible_runtimes=[lambda_.Runtime.PYTHON_3_9],
+            compatible_runtimes=[lambda_.Runtime.PYTHON_3_11],
         )
 
         invoke_lambda = lambda_python.PythonFunction(
@@ -301,7 +307,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             entry="application/{}/lambda/NitroInvoke".format(params["application_type"]),
             handler="lambda_handler",
             index="lambda_function.py",
-            runtime=lambda_.Runtime.PYTHON_3_9,
+            runtime=lambda_.Runtime.PYTHON_3_11,
             timeout=Duration.minutes(2),
             memory_size=256,
             environment={
@@ -314,6 +320,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             vpc=vpc,
             vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
             security_groups=[signer_client_sg],
+            architecture=lambda_.Architecture.X86_64
         )
 
         encryption_key.grant_encrypt(invoke_lambda)

requirements-dev.txt

@@ -1,5 +1,5 @@
-pytest==6.2.5 ; python_version >= "3.9" and python_version < "3.10"
-black>=22.10.0 ; python_version >= "3.9" and python_version < "3.10"
-pre-commit>=2.20.0 ; python_version >= "3.9" and python_version < "3.10"
-bandit>=1.7.4 ; python_version >= "3.9" and python_version < "3.10"
-flake8==7.0.0 ; python_version >= "3.9" and python_version < "3.10"
+pytest==7.4.4
+black>=23.12.1
+pre-commit>=3.6.0
+bandit>=1.7.6
+flake8==7.0.0

requirements.txt

@@ -1,4 +1,4 @@
-aws-cdk-lib==2.51.1 ; python_version >= "3.9" and python_version < "3.10"
-constructs>=10.0.0,<11.0.0 ; python_version >= "3.9" and python_version < "3.10"
-aws-cdk.aws-lambda-python-alpha==2.51.0a0 ; python_version >= "3.9" and python_version < "3.10"
-cdk-nag>=2.21.11 ; python_version >= "3.9" and python_version < "3.10"
+aws-cdk-lib==2.98.0
+constructs==10.1.271
+aws-cdk.aws-lambda-python-alpha==2.51.0a0
+cdk-nag==2.27.88

scripts/generate_key_policy.sh

@@ -3,28 +3,30 @@
 #  SPDX-License-Identifier: MIT-0
 set -e
 set +x
+set -o pipefail
 
 output=${1}
 secure_keygen_stack_name=${2}
 
 # instance id
-stack_name=$(jq -r '. |= keys | .[0]' ${output})
-asg_name=$(jq -r '."'${stack_name}'".ASGGroupName' ${output})
-instance_id=$(./scripts/get_asg_instances.sh ${asg_name} | head -n 1)
+stack_name=$(jq -r '. |= keys | .[0]' "${output}")
+asg_name=$(jq -r '."'${stack_name}'".ASGGroupName' "${output}")
+instance_id=$(./scripts/get_asg_instances.sh "${asg_name}" | head -n 1)
 
 # pcr_0
 # 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 for debug
-pcr_0=$(./scripts/get_pcr0.sh ${instance_id})
+pcr_0=$(./scripts/get_pcr0.sh "${instance_id}")
 
 # ec2 role
-ec2_role_arn=$(jq -r ".${stack_name}.EC2InstanceRoleARN" ${output})
+ec2_role_arn=$(jq -r ".${stack_name}.EC2InstanceRoleARN" "${output}")
 # lambda role
-lambda_execution_arn=$(jq -r ".${stack_name}.LambdaExecutionArn" ${output})
+lambda_execution_arn=$(jq -r ".${stack_name}.LambdaExecutionArn" "${output}")
 
 if [[ -n "${secure_keygen_stack_name}" ]]; then
   echo "Retrieving ValidatorKeyGenFunction Lambda Role of $secure_keygen_stack_name deployed in https://github.com/aws-samples/eth-keygen-lambda-sam"
   imported_lambda_execution_arn=$(aws cloudformation describe-stacks \
-      --stack-name $secure_keygen_stack_name \
+      --region "${CDK_DEPLOY_REGION}" \
+      --stack-name "${secure_keygen_stack_name}" \
       --query "Stacks[0].Outputs[?OutputKey=='ValidatorKeyGenFunctionIamRole'].OutputValue | [0]" \
       --output text)
 fi

scripts/get_asg_instances.sh

@@ -5,4 +5,7 @@ set +x
 set -e
 
 # avoid old terminated instances
-aws autoscaling describe-auto-scaling-groups --region "${CDK_DEPLOY_REGION}" --auto-scaling-group-name "${1}" | jq -r '.AutoScalingGroups[0].Instances[] | select ( .LifecycleState | contains("InService")) | .InstanceId '
+aws autoscaling describe-auto-scaling-groups \
+--region "${CDK_DEPLOY_REGION}" \
+--auto-scaling-group-name "${1}" \
+| jq -r '.AutoScalingGroups[0].Instances[] | select ( .LifecycleState | contains("InService")) | .InstanceId '

scripts/get_pcr0.sh

@@ -4,16 +4,51 @@
 set +x
 set -e
 
-flag_id=$(aws ssm send-command --region "${CDK_DEPLOY_REGION}" --document-name "AWS-RunShellScript" --instance-ids "${1}" --parameters 'commands=["sudo cat /etc/environment | head -n 1 | tr \"=\" \"\n\" | tail -n 1"]' | jq -r '.Command.CommandId')
-flags=$(aws ssm list-command-invocations --region "${CDK_DEPLOY_REGION}" --instance-id "${1}" --command-id "${flag_id}" --details | jq -r '.CommandInvocations[0].CommandPlugins[0].Output')
+flag_id=$(aws ssm send-command \
+ --region "${CDK_DEPLOY_REGION}" \
+ --document-name "AWS-RunShellScript" \
+ --instance-ids "${1}" \
+ --parameters 'commands=["sudo cat /etc/environment | head -n 1 | tr \"=\" \"\n\" | tail -n 1"]' \
+ | jq -r '.Command.CommandId')
 
+flags=$(aws ssm list-command-invocations \
+ --region "${CDK_DEPLOY_REGION}" \
+ --instance-id "${1}" \
+ --command-id "${flag_id}" \
+ --details \
+ | jq -r '.CommandInvocations[0].CommandPlugins[0].Output')
+
+# validate that flags value has been read correctly from ec2 instance - it should be either true or false
+if [[ "${flags}" != "TRUE" && "${flags}" != "FALSE" ]]; then
+  echo "flags is not true or false"
+  exit 1
+fi
+
+# if debug flag is true, provide 000 string in key policy, otherwise get PCR value from eif file running on EC2 instance
 if [[ "${flags}" == "TRUE" ]]; then
   pcr_0="000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
 else
-  command_id=$(aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids "${1}" --parameters 'commands=["sudo nitro-cli describe-eif --eif-path /home/ec2-user/app/server/signing_server.eif | jq -r '.Measurements.PCR0'"]' | jq -r '.Command.CommandId')
+  command_id=$(aws ssm send-command \
+  --region "${CDK_DEPLOY_REGION}" \
+  --document-name "AWS-RunShellScript" \
+  --instance-ids "${1}" \
+  --parameters 'commands=["sudo nitro-cli describe-eif --eif-path /home/ec2-user/app/server/signing_server.eif | jq -r '.Measurements.PCR0'"]' \
+  | jq -r '.Command.CommandId')
+
   # takes about 5sec to return the pcr0 value from a non running enclave
-  sleep 7
-  pcr_0=$(aws ssm list-command-invocations --instance-id "${1}" --command-id "${command_id}" --details | jq -r '.CommandInvocations[0].CommandPlugins[0].Output')
+  sleep 10
+  pcr_0=$(aws ssm list-command-invocations \
+  --region "${CDK_DEPLOY_REGION}" \
+  --instance-id "${1}" \
+  --command-id "${command_id}" \
+  --details \
+  | jq -r '.CommandInvocations[0].CommandPlugins[0].Output')
+fi
+
+# ensure that pcr0 is not empty
+if [[ "${pcr_0}" == "" ]]; then
+  echo "pcr_0 is empty"
+  exit 1
 fi
 
 echo "${pcr_0}"

scripts/load_validator_keys/load_validator_keys.py

@@ -37,20 +37,25 @@
 logger.addHandler(handler)
 logger.propagate = False
 
+region = os.getenv("CDK_DEPLOY_REGION", "us-east-1")
+
 kms_key_arn = os.getenv("KMS_KEY_ARN")
 table_name = os.getenv("DDB_TABLE_NAME")
 cf_stack_name = os.getenv("CF_STACK_NAME")
 
-client_kms = boto3.client("kms")
-dynamodb = boto3.resource("dynamodb")
+client_kms = boto3.client(service_name="kms",
+                          region_name=region)
+dynamodb = boto3.resource(service_name="dynamodb",
+                          region_name=region)
 
 words_list_path = "word_lists"
 
 
 def get_cloudformation_stack_id(cf_stack_name):
     """Get CF Stack ID"""
 
-    client = boto3.client(service_name="cloudformation")
+    client = boto3.client(service_name="cloudformation",
+                          region_name=region)
 
     try:
         response = client.describe_stacks(
@@ -78,12 +83,11 @@ def verify_keystore(credential: Credential, keystore: Keystore, password: str) -
 
 
 def main(
-    num_validators=5,
-    mnemonic_language="english",
-    chain="goerli",
-    eth1_withdrawal_address="0x6F4b46423fc6181a0cF34e6716c220BD4d6C2471",
+        num_validators=5,
+        mnemonic_language="english",
+        chain="sepolia",
+        eth1_withdrawal_address="0x6F4b46423fc6181a0cF34e6716c220BD4d6C2471",
 ) -> list:
-
     if kms_key_arn is None:
         raise ValueError("Specify KMS_KEY_ARN environment variable")