Skip to content

Commit

Permalink
Merge branch 'main' into sha3_absorb_squeeze
Browse files Browse the repository at this point in the history
  • Loading branch information
manastasova authored Jan 8, 2025
2 parents b2228b6 + 3cea179 commit 077ef78
Show file tree
Hide file tree
Showing 9 changed files with 134 additions and 70 deletions.
3 changes: 1 addition & 2 deletions tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,8 @@ mainSteps:
runCommand:
- mkdir docker-images
- cd docker-images
# Install choco and git
- Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null
- choco install git --version 2.23.0 -y
- Start-Service -Name docker
- $env:path+='C:\Program Files\Git\cmd'
# Git clone aws-lc repo.
- git clone https://github.com/GITHUB_OWNER_PLACEHOLDER/aws-lc.git
Expand Down
144 changes: 101 additions & 43 deletions tests/ci/cdk/cdk/windows_docker_image_build_stack.py
Original file line number Diff line number Diff line change
@@ -1,66 +1,124 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0 OR ISC

from aws_cdk import Stack, Tags, aws_ec2 as ec2, aws_s3 as s3, aws_iam as iam, aws_ssm as ssm
from aws_cdk import (
Stack,
Tags,
aws_ec2 as ec2,
aws_s3 as s3,
aws_iam as iam,
aws_ssm as ssm,
)
from constructs import Construct
from util.iam_policies import ecr_power_user_policy_in_json, s3_read_write_policy_in_json
from util.metadata import AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO, S3_BUCKET_NAME, GITHUB_REPO_OWNER, WIN_EC2_TAG_KEY, \
WIN_EC2_TAG_VALUE, SSM_DOCUMENT_NAME, GITHUB_SOURCE_VERSION
from util.iam_policies import (
ecr_power_user_policy_in_json,
s3_read_write_policy_in_json,
)
from util.metadata import (
AWS_ACCOUNT,
AWS_REGION,
WINDOWS_X86_ECR_REPO,
S3_BUCKET_NAME,
GITHUB_REPO_OWNER,
WIN_EC2_TAG_KEY,
WIN_EC2_TAG_VALUE,
SSM_DOCUMENT_NAME,
GITHUB_SOURCE_VERSION,
)
from util.yml_loader import YmlLoader


class WindowsDockerImageBuildStack(Stack):
"""Define a temporary stack used to build Windows Docker images. After build, this stack will be destroyed."""

def __init__(self,
scope: Construct,
id: str,
**kwargs) -> None:
def __init__(self, scope: Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)

# Define SSM command document.
ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO)
placeholder_map = {"ECR_PLACEHOLDER": ecr_repo, "GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER,
"REGION_PLACEHOLDER": AWS_REGION, "GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION}
content = YmlLoader.load("./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map)
ssm.CfnDocument(scope=self,
id="{}-ssm-document".format(id),
name=SSM_DOCUMENT_NAME,
content=content,
document_type="Command")
ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(
AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO
)
placeholder_map = {
"ECR_PLACEHOLDER": ecr_repo,
"GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER,
"REGION_PLACEHOLDER": AWS_REGION,
"GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION,
}
content = YmlLoader.load(
"./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map
)
ssm.CfnDocument(
scope=self,
id="{}-ssm-document".format(id),
name=SSM_DOCUMENT_NAME,
content=content,
document_type="Command",
)

# Define a S3 bucket to store windows docker files and build scripts.
s3.Bucket(scope=self,
id="{}-s3".format(id),
bucket_name=S3_BUCKET_NAME,
block_public_access=s3.BlockPublicAccess.BLOCK_ALL)
s3.Bucket(
scope=self,
id="{}-s3".format(id),
bucket_name=S3_BUCKET_NAME,
block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
)

# Define a role for EC2.
ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json([WINDOWS_X86_ECR_REPO]))
s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json(S3_BUCKET_NAME))
inline_policies = {"ecr_power_user_policy": ecr_power_user_policy, "s3_read_write_policy": s3_read_write_policy}
role = iam.Role(scope=self, id="{}-role".format(id),
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
inline_policies=inline_policies,
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMManagedInstanceCore")
])
ecr_power_user_policy = iam.PolicyDocument.from_json(
ecr_power_user_policy_in_json([WINDOWS_X86_ECR_REPO])
)
s3_read_write_policy = iam.PolicyDocument.from_json(
s3_read_write_policy_in_json(S3_BUCKET_NAME)
)
inline_policies = {
"ecr_power_user_policy": ecr_power_user_policy,
"s3_read_write_policy": s3_read_write_policy,
}
role = iam.Role(
scope=self,
id="{}-role".format(id),
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
inline_policies=inline_policies,
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSSMManagedInstanceCore"
)
],
)

# Define Windows EC2 instance, where the SSM document will be executed.
# TODO: This AMI does not have docker installed by default anymore. Find another Windows machine
# that has docker by default or update the ssm document to properly install docker.
machine_image = ec2.MachineImage.latest_windows(
ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE)
ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE
)
vpc = ec2.Vpc(scope=self, id="{}-vpc".format(id))
block_device_volume = ec2.BlockDeviceVolume.ebs(volume_size=200, delete_on_termination=True)
block_device = ec2.BlockDevice(device_name="/dev/sda1", volume=block_device_volume)
instance = ec2.Instance(scope=self,
id="{}-instance".format(id),
instance_type=ec2.InstanceType(instance_type_identifier="m5d.xlarge"),
vpc=vpc,
role=role,
block_devices=[block_device],
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
machine_image=machine_image)
block_device_volume = ec2.BlockDeviceVolume.ebs(
volume_size=200, delete_on_termination=True
)
block_device = ec2.BlockDevice(
device_name="/dev/sda1", volume=block_device_volume
)

setup_user_data = ec2.UserData.for_windows()
setup_user_data.add_commands(
"Install-WindowsFeature -Name Containers -IncludeAllSubFeature -IncludeManagementTools",
"Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null",
"choco install docker-cli -y",
"choco install docker-engine -y",
"choco install git --version 2.23.0 -y",
"Set-Service -Name docker -StartupType Automatic",
"Restart-Computer -Force",
)

instance = ec2.Instance(
scope=self,
id="{}-instance".format(id),
instance_type=ec2.InstanceType(instance_type_identifier="m5d.xlarge"),
vpc=vpc,
role=role,
block_devices=[block_device],
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
machine_image=machine_image,
user_data=setup_user_data,
)

Tags.of(instance).add(WIN_EC2_TAG_KEY, WIN_EC2_TAG_VALUE)
29 changes: 17 additions & 12 deletions tests/ci/docker_images/dependencies/build_cryptofuzz_modules.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@ export CXXFLAGS="-fsanitize=address,fuzzer-no-link -D_GLIBCXX_DEBUG -O2 -g"
# Setup base of Cryptofuzz
cd "$FUZZ_ROOT"
MODULES_ROOT="${FUZZ_ROOT}/modules"
git clone --depth 1 https://github.com/guidovranken/cryptofuzz.git
# TODO this is not the latest (which is cryptofuzz-9461c91.tar.gz, but newer boton is not compiling so pinning)
curl -OL https://d2yr98kym3baw0.cloudfront.net/cryptofuzz-508c384.tar.gz
tar xvzf cryptofuzz-*.tar.gz
rm cryptofuzz-*.tar.gz
cd cryptofuzz
git rev-parse HEAD
CRYPTOFUZZ_SRC=$(pwd)
python3 gen_repository.py

Expand All @@ -26,14 +28,17 @@ cd "$MODULES_ROOT"

# Setup the other crypto libraries for differential fuzzing
# Botan https://github.com/guidovranken/cryptofuzz/blob/master/docs/botan.md
git clone --depth 1 https://github.com/randombit/botan.git
git clone https://github.com/randombit/botan.git
cd botan
# TODO: Current tip of botan is not compiling for us (maybe C++20 related?)
# reverting to the version of botan we built with cryptofuzz@508c384
git checkout 51b06ca93d1998d19927699f78b8d67539148dde
git rev-parse HEAD
python3 configure.py --cc-bin=$CXX --cc-abi-flags="$CXXFLAGS" --disable-shared --disable-modules=locking_allocator,x509,tls --build-targets=static --without-documentation
python3 configure.py --cc-bin=$CXX --cc-abi-flags="${CXXFLAGS}" --disable-shared --disable-modules=locking_allocator,x509,tls --build-targets=static --without-documentation
make -j$(nproc)
env LIBBOTAN_A_PATH `realpath libbotan-3.a`
env BOTAN_INCLUDE_PATH `realpath build/include/public`
export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_BOTAN -I $(realpath build/include/internal)"
env LIBBOTAN_A_PATH "$(realpath libbotan-3.a)"
env BOTAN_INCLUDE_PATH "$(realpath build/include)"
export CXXFLAGS="${CXXFLAGS} -DCRYPTOFUZZ_BOTAN"
cd "${CRYPTOFUZZ_SRC}/modules/botan/"
make -j$(nproc)

Expand All @@ -43,18 +48,18 @@ git clone --depth 1 https://github.com/weidai11/cryptopp.git
cd cryptopp/
git rev-parse HEAD
make libcryptopp.a -j$(nproc)
export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_CRYPTOPP"
env LIBCRYPTOPP_A_PATH `realpath libcryptopp.a`
env CRYPTOPP_INCLUDE_PATH `realpath .`
export CXXFLAGS="${CXXFLAGS} -DCRYPTOFUZZ_CRYPTOPP"
env LIBCRYPTOPP_A_PATH "$(realpath libcryptopp.a)"
env CRYPTOPP_INCLUDE_PATH "$(realpath .)"
cd "${CRYPTOFUZZ_SRC}/modules/cryptopp/"
make -j$(nproc)

# Extract the seed corpus, docker layers are already compressed so this won't use any more space and save time when running
cd "$FUZZ_ROOT"
unzip cryptofuzz_data.zip
rm cryptofuzz_data.zip
env CRYPTOFUZZ_SEED_CORPUS `realpath cryptofuzz_seed_corpus`
env CRYPTOFUZZ_DICT `realpath cryptofuzz-dict.txt`
env CRYPTOFUZZ_SEED_CORPUS "$(realpath cryptofuzz_seed_corpus)"
env CRYPTOFUZZ_DICT "$(realpath cryptofuzz-dict.txt)"

# Save final common flags
env CFLAGS "$CFLAGS"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ RUN set -ex && \
# valgrind/memcheck.h is provided by the valgrind-devel package on AL2. see P63119011.
valgrind-devel \
unzip && \
wget https://raw.githubusercontent.com/aws/aws-codebuild-docker-images/master/al2/x86_64/standard/4.0/amazon-ssm-agent.json -P /etc/amazon/ssm/ && \
wget https://raw.githubusercontent.com/aws/aws-codebuild-docker-images/master/al/x86_64/standard/5.0/amazon-ssm-agent.json -P /etc/amazon/ssm/ && \
# Based on https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ FROM amazonlinux-2:gcc-7x

SHELL ["/bin/bash", "-c"]

ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-lin
ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-lin.tar.xz"
ENV SDE_VERSION_TAG=sde-external-9.44.0-2024-08-22-lin
ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/831748/sde-external-9.44.0-2024-08-22-lin.tar.xz"

# Enable the EPEL repository on Amazon Linux 2 before installing packages
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/add-repositories.html
Expand Down
2 changes: 2 additions & 0 deletions tests/ci/docker_images/linux-x86/centos-7_gcc-4x/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ ENV GOROOT=/usr/local/go
ENV PATH="$GOROOT/bin:$PATH"

RUN set -ex && \
sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && \
yum -y update && \
yum --enablerepo=extras install epel-release -y && \
yum -y install \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ FROM ubuntu-22.04:base

SHELL ["/bin/bash", "-c"]

ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-lin
ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-lin.tar.xz"
ENV SDE_VERSION_TAG=sde-external-9.44.0-2024-08-22-lin
ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/831748/sde-external-9.44.0-2024-08-22-lin.tar.xz"

RUN set -ex && \
apt-get update && \
Expand Down
4 changes: 2 additions & 2 deletions tests/ci/docker_images/windows/vs2017/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
# Keep parity with the upstream tags at https://hub.docker.com/_/microsoft-windows-servercore
FROM aws-lc/windows_base:2019

ENV SDE_VERSION_TAG=sde-external-9.21.1-2023-04-24-win
ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-win.tar.xz"
ENV SDE_VERSION_TAG=sde-external-9.44.0-2024-08-22-win
ENV SDE_MIRROR_URL="https://downloadmirror.intel.com/831748/sde-external-9.44.0-2024-08-22-win.tar.xz"

SHELL ["cmd", "/S", "/C"]
RUN `
Expand Down
12 changes: 6 additions & 6 deletions tests/ci/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,18 @@

install_requires=[
# CDK dependencies.
"aws-cdk-lib==2.74.0",
"constructs==10.1.314",
"aws-cdk-lib==2.173.1",
"constructs==10.4.2",
# PyYAML is a YAML parser and emitter for Python. Used to read build_spec.yaml.
"pyyaml==6.0.1",
"pyyaml==6.0.2",
# A formatter for Python code.
"yapf==0.30.0",
"yapf==0.43.0",
# Introduced by benchmark framework.
"boto3==1.26.126",
"boto3==1.35.82",
# Introduced by Android Device Farm CI.
"requests",
"arnparse==0.0.2",
"urllib3==1.25.10"
"urllib3==2.2.3"
],

python_requires=">=3.6",
Expand Down

0 comments on commit 077ef78

Please sign in to comment.