diff --git a/tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml b/tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml index b1ce0f1f5e..b5288d9370 100644 --- a/tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml +++ b/tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml @@ -12,9 +12,8 @@ mainSteps: runCommand: - mkdir docker-images - cd docker-images - # Install choco and git - Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null - - choco install git --version 2.23.0 -y + - Start-Service -Name docker - $env:path+='C:\Program Files\Git\cmd' # Git clone aws-lc repo. - git clone https://github.com/GITHUB_OWNER_PLACEHOLDER/aws-lc.git diff --git a/tests/ci/cdk/cdk/windows_docker_image_build_stack.py b/tests/ci/cdk/cdk/windows_docker_image_build_stack.py index 97009dfdd6..fa1079c7ed 100644 --- a/tests/ci/cdk/cdk/windows_docker_image_build_stack.py +++ b/tests/ci/cdk/cdk/windows_docker_image_build_stack.py @@ -1,66 +1,124 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 OR ISC -from aws_cdk import Stack, Tags, aws_ec2 as ec2, aws_s3 as s3, aws_iam as iam, aws_ssm as ssm +from aws_cdk import ( + Stack, + Tags, + aws_ec2 as ec2, + aws_s3 as s3, + aws_iam as iam, + aws_ssm as ssm, +) from constructs import Construct -from util.iam_policies import ecr_power_user_policy_in_json, s3_read_write_policy_in_json -from util.metadata import AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO, S3_BUCKET_NAME, GITHUB_REPO_OWNER, WIN_EC2_TAG_KEY, \ - WIN_EC2_TAG_VALUE, SSM_DOCUMENT_NAME, GITHUB_SOURCE_VERSION +from util.iam_policies import ( + ecr_power_user_policy_in_json, + s3_read_write_policy_in_json, +) +from util.metadata import ( + AWS_ACCOUNT, + AWS_REGION, + WINDOWS_X86_ECR_REPO, + S3_BUCKET_NAME, + GITHUB_REPO_OWNER, + WIN_EC2_TAG_KEY, + WIN_EC2_TAG_VALUE, + SSM_DOCUMENT_NAME, + GITHUB_SOURCE_VERSION, +) from util.yml_loader import YmlLoader class WindowsDockerImageBuildStack(Stack): """Define a temporary stack used to build Windows Docker images. After build, this stack will be destroyed.""" - def __init__(self, - scope: Construct, - id: str, - **kwargs) -> None: + def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Define SSM command document. - ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO) - placeholder_map = {"ECR_PLACEHOLDER": ecr_repo, "GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER, - "REGION_PLACEHOLDER": AWS_REGION, "GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION} - content = YmlLoader.load("./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map) - ssm.CfnDocument(scope=self, - id="{}-ssm-document".format(id), - name=SSM_DOCUMENT_NAME, - content=content, - document_type="Command") + ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format( + AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO + ) + placeholder_map = { + "ECR_PLACEHOLDER": ecr_repo, + "GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER, + "REGION_PLACEHOLDER": AWS_REGION, + "GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION, + } + content = YmlLoader.load( + "./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map + ) + ssm.CfnDocument( + scope=self, + id="{}-ssm-document".format(id), + name=SSM_DOCUMENT_NAME, + content=content, + document_type="Command", + ) # Define a S3 bucket to store windows docker files and build scripts. - s3.Bucket(scope=self, - id="{}-s3".format(id), - bucket_name=S3_BUCKET_NAME, - block_public_access=s3.BlockPublicAccess.BLOCK_ALL) + s3.Bucket( + scope=self, + id="{}-s3".format(id), + bucket_name=S3_BUCKET_NAME, + block_public_access=s3.BlockPublicAccess.BLOCK_ALL, + ) # Define a role for EC2. - ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json([WINDOWS_X86_ECR_REPO])) - s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json(S3_BUCKET_NAME)) - inline_policies = {"ecr_power_user_policy": ecr_power_user_policy, "s3_read_write_policy": s3_read_write_policy} - role = iam.Role(scope=self, id="{}-role".format(id), - assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), - inline_policies=inline_policies, - managed_policies=[ - iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMManagedInstanceCore") - ]) + ecr_power_user_policy = iam.PolicyDocument.from_json( + ecr_power_user_policy_in_json([WINDOWS_X86_ECR_REPO]) + ) + s3_read_write_policy = iam.PolicyDocument.from_json( + s3_read_write_policy_in_json(S3_BUCKET_NAME) + ) + inline_policies = { + "ecr_power_user_policy": ecr_power_user_policy, + "s3_read_write_policy": s3_read_write_policy, + } + role = iam.Role( + scope=self, + id="{}-role".format(id), + assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), + inline_policies=inline_policies, + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name( + "AmazonSSMManagedInstanceCore" + ) + ], + ) # Define Windows EC2 instance, where the SSM document will be executed. - # TODO: This AMI does not have docker installed by default anymore. Find another Windows machine - # that has docker by default or update the ssm document to properly install docker. machine_image = ec2.MachineImage.latest_windows( - ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE) + ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE + ) vpc = ec2.Vpc(scope=self, id="{}-vpc".format(id)) - block_device_volume = ec2.BlockDeviceVolume.ebs(volume_size=200, delete_on_termination=True) - block_device = ec2.BlockDevice(device_name="/dev/sda1", volume=block_device_volume) - instance = ec2.Instance(scope=self, - id="{}-instance".format(id), - instance_type=ec2.InstanceType(instance_type_identifier="m5d.xlarge"), - vpc=vpc, - role=role, - block_devices=[block_device], - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), - machine_image=machine_image) + block_device_volume = ec2.BlockDeviceVolume.ebs( + volume_size=200, delete_on_termination=True + ) + block_device = ec2.BlockDevice( + device_name="/dev/sda1", volume=block_device_volume + ) + + setup_user_data = ec2.UserData.for_windows() + setup_user_data.add_commands( + "Install-WindowsFeature -Name Containers -IncludeAllSubFeature -IncludeManagementTools", + "Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null", + "choco install docker-cli -y", + "choco install docker-engine -y", + "choco install git --version 2.23.0 -y", + "Set-Service -Name docker -StartupType Automatic", + "Restart-Computer -Force", + ) + + instance = ec2.Instance( + scope=self, + id="{}-instance".format(id), + instance_type=ec2.InstanceType(instance_type_identifier="m5d.xlarge"), + vpc=vpc, + role=role, + block_devices=[block_device], + vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), + machine_image=machine_image, + user_data=setup_user_data, + ) Tags.of(instance).add(WIN_EC2_TAG_KEY, WIN_EC2_TAG_VALUE) diff --git a/tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh b/tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh index 69335a248f..9f33459468 100755 --- a/tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh +++ b/tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh @@ -15,9 +15,11 @@ export CXXFLAGS="-fsanitize=address,fuzzer-no-link -D_GLIBCXX_DEBUG -O2 -g" # Setup base of Cryptofuzz cd "$FUZZ_ROOT" MODULES_ROOT="${FUZZ_ROOT}/modules" -git clone --depth 1 https://github.com/guidovranken/cryptofuzz.git +# TODO this is not the latest (which is cryptofuzz-9461c91.tar.gz, but newer boton is not compiling so pinning) +curl -OL https://d2yr98kym3baw0.cloudfront.net/cryptofuzz-508c384.tar.gz +tar xvzf cryptofuzz-*.tar.gz +rm cryptofuzz-*.tar.gz cd cryptofuzz -git rev-parse HEAD CRYPTOFUZZ_SRC=$(pwd) python3 gen_repository.py @@ -26,14 +28,17 @@ cd "$MODULES_ROOT" # Setup the other crypto libraries for differential fuzzing # Botan https://github.com/guidovranken/cryptofuzz/blob/master/docs/botan.md -git clone --depth 1 https://github.com/randombit/botan.git +git clone https://github.com/randombit/botan.git cd botan +# TODO: Current tip of botan is not compiling for us (maybe C++20 related?) +# reverting to the version of botan we built with cryptofuzz@508c384 +git checkout 51b06ca93d1998d19927699f78b8d67539148dde git rev-parse HEAD -python3 configure.py --cc-bin=$CXX --cc-abi-flags="$CXXFLAGS" --disable-shared --disable-modules=locking_allocator,x509,tls --build-targets=static --without-documentation +python3 configure.py --cc-bin=$CXX --cc-abi-flags="${CXXFLAGS}" --disable-shared --disable-modules=locking_allocator,x509,tls --build-targets=static --without-documentation make -j$(nproc) -env LIBBOTAN_A_PATH `realpath libbotan-3.a` -env BOTAN_INCLUDE_PATH `realpath build/include/public` -export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_BOTAN -I $(realpath build/include/internal)" +env LIBBOTAN_A_PATH "$(realpath libbotan-3.a)" +env BOTAN_INCLUDE_PATH "$(realpath build/include)" +export CXXFLAGS="${CXXFLAGS} -DCRYPTOFUZZ_BOTAN" cd "${CRYPTOFUZZ_SRC}/modules/botan/" make -j$(nproc) @@ -43,9 +48,9 @@ git clone --depth 1 https://github.com/weidai11/cryptopp.git cd cryptopp/ git rev-parse HEAD make libcryptopp.a -j$(nproc) -export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_CRYPTOPP" -env LIBCRYPTOPP_A_PATH `realpath libcryptopp.a` -env CRYPTOPP_INCLUDE_PATH `realpath .` +export CXXFLAGS="${CXXFLAGS} -DCRYPTOFUZZ_CRYPTOPP" +env LIBCRYPTOPP_A_PATH "$(realpath libcryptopp.a)" +env CRYPTOPP_INCLUDE_PATH "$(realpath .)" cd "${CRYPTOFUZZ_SRC}/modules/cryptopp/" make -j$(nproc) @@ -53,8 +58,8 @@ make -j$(nproc) cd "$FUZZ_ROOT" unzip cryptofuzz_data.zip rm cryptofuzz_data.zip -env CRYPTOFUZZ_SEED_CORPUS `realpath cryptofuzz_seed_corpus` -env CRYPTOFUZZ_DICT `realpath cryptofuzz-dict.txt` +env CRYPTOFUZZ_SEED_CORPUS "$(realpath cryptofuzz_seed_corpus)" +env CRYPTOFUZZ_DICT "$(realpath cryptofuzz-dict.txt)" # Save final common flags env CFLAGS "$CFLAGS" diff --git a/tests/ci/docker_images/linux-x86/amazonlinux-2023_base/Dockerfile b/tests/ci/docker_images/linux-x86/amazonlinux-2023_base/Dockerfile index 01b129e48e..8e09dc3402 100644 --- a/tests/ci/docker_images/linux-x86/amazonlinux-2023_base/Dockerfile +++ b/tests/ci/docker_images/linux-x86/amazonlinux-2023_base/Dockerfile @@ -34,7 +34,7 @@ RUN set -ex && \ # valgrind/memcheck.h is provided by the valgrind-devel package on AL2. see P63119011. valgrind-devel \ unzip && \ - wget https://raw.githubusercontent.com/aws/aws-codebuild-docker-images/master/al2/x86_64/standard/4.0/amazon-ssm-agent.json -P /etc/amazon/ssm/ && \ + wget https://raw.githubusercontent.com/aws/aws-codebuild-docker-images/master/al/x86_64/standard/5.0/amazon-ssm-agent.json -P /etc/amazon/ssm/ && \ # Based on https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && \ diff --git a/tests/ci/docker_images/linux-x86/amazonlinux-2_gcc-7x-intel-sde/Dockerfile b/tests/ci/docker_images/linux-x86/amazonlinux-2_gcc-7x-intel-sde/Dockerfile index 5bbcb8e2e5..a90cbc6cae 100644 --- a/tests/ci/docker_images/linux-x86/amazonlinux-2_gcc-7x-intel-sde/Dockerfile +++ b/tests/ci/docker_images/linux-x86/amazonlinux-2_gcc-7x-intel-sde/Dockerfile @@ -5,8 +5,8 @@ FROM amazonlinux-2:gcc-7x SHELL ["/bin/bash", "-c"] -ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-lin -ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-lin.tar.xz" +ENV SDE_VERSION_TAG=sde-external-9.44.0-2024-08-22-lin +ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/831748/sde-external-9.44.0-2024-08-22-lin.tar.xz" # Enable the EPEL repository on Amazon Linux 2 before installing packages # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/add-repositories.html diff --git a/tests/ci/docker_images/linux-x86/centos-7_gcc-4x/Dockerfile b/tests/ci/docker_images/linux-x86/centos-7_gcc-4x/Dockerfile index e319b22ad3..8cf34c8203 100644 --- a/tests/ci/docker_images/linux-x86/centos-7_gcc-4x/Dockerfile +++ b/tests/ci/docker_images/linux-x86/centos-7_gcc-4x/Dockerfile @@ -8,6 +8,8 @@ ENV GOROOT=/usr/local/go ENV PATH="$GOROOT/bin:$PATH" RUN set -ex && \ + sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && \ yum -y update && \ yum --enablerepo=extras install epel-release -y && \ yum -y install \ diff --git a/tests/ci/docker_images/linux-x86/ubuntu-22.04_clang-14x-sde/Dockerfile b/tests/ci/docker_images/linux-x86/ubuntu-22.04_clang-14x-sde/Dockerfile index a7fb95a64e..401d281bc1 100644 --- a/tests/ci/docker_images/linux-x86/ubuntu-22.04_clang-14x-sde/Dockerfile +++ b/tests/ci/docker_images/linux-x86/ubuntu-22.04_clang-14x-sde/Dockerfile @@ -5,8 +5,8 @@ FROM ubuntu-22.04:base SHELL ["/bin/bash", "-c"] -ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-lin -ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-lin.tar.xz" +ENV SDE_VERSION_TAG=sde-external-9.44.0-2024-08-22-lin +ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/831748/sde-external-9.44.0-2024-08-22-lin.tar.xz" RUN set -ex && \ apt-get update && \ diff --git a/tests/ci/docker_images/windows/vs2017/Dockerfile b/tests/ci/docker_images/windows/vs2017/Dockerfile index 21befce32d..449aa6a6dd 100644 --- a/tests/ci/docker_images/windows/vs2017/Dockerfile +++ b/tests/ci/docker_images/windows/vs2017/Dockerfile @@ -6,8 +6,8 @@ # Keep parity with the upstream tags at https://hub.docker.com/_/microsoft-windows-servercore FROM aws-lc/windows_base:2019 -ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-win -ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-win.tar.xz" +ENV SDE_VERSION_TAG=sde-external-9.44.0-2024-08-22-win +ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/831748/sde-external-9.44.0-2024-08-22-win.tar.xz" SHELL ["cmd", "/S", "/C"] RUN ` diff --git a/tests/ci/setup.py b/tests/ci/setup.py index 90a72de56e..0e6356accf 100644 --- a/tests/ci/setup.py +++ b/tests/ci/setup.py @@ -20,18 +20,18 @@ install_requires=[ # CDK dependencies. - "aws-cdk-lib==2.74.0", - "constructs==10.1.314", + "aws-cdk-lib==2.173.1", + "constructs==10.4.2", # PyYAML is a YAML parser and emitter for Python. Used to read build_spec.yaml. - "pyyaml==6.0.1", + "pyyaml==6.0.2", # A formatter for Python code. - "yapf==0.30.0", + "yapf==0.43.0", # Introduced by benchmark framework. - "boto3==1.26.126", + "boto3==1.35.82", # Introduced by Android Device Farm CI. "requests", "arnparse==0.0.2", - "urllib3==1.25.10" + "urllib3==2.2.3" ], python_requires=">=3.6",