Skip to content

Commit

Permalink
Merge pull request #634 from amazonlinux/codebuild-infra-containers
Browse files Browse the repository at this point in the history
Add CodeBuild Infra containers supporting resources
  • Loading branch information
jahkeup authored Jan 13, 2020
2 parents 65bda1e + 88127e0 commit b47dad9
Show file tree
Hide file tree
Showing 9 changed files with 331 additions and 59 deletions.
2 changes: 1 addition & 1 deletion tools/infra/buildspec/infra-pr-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ env:
phases:
build:
commands:
- make -C "$INFRA_DIR/stacks" --keep-going validate
- make -C "$INFRA_DIR/stacks" --keep-going validate check
2 changes: 2 additions & 0 deletions tools/infra/buildspec/thar-pr-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ artifacts:
base-directory: 'build/'
files:
- '*.img*'
- '*.ext4*'
- '*.verity*'
secondary-artifacts:
meta:
base-directory: 'build/meta'
Expand Down
5 changes: 4 additions & 1 deletion tools/infra/container/Dockerfile.builder
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,12 @@ FROM amazonlinux:2 as base
RUN yum update -y \
&& yum groupinstall -y 'Development Tools' \
&& yum install -y socat procps-ng awscli jq openssh rsync \
&& amazon-linux-extras install -y docker \
&& amazon-linux-extras enable docker \
&& yum install -y docker amazon-ecr-credential-helper \
&& yum clean all \
&& rm -rf /var/cache/yum /var/cache/amzn2extras
RUN install -D /dev/null /root/.docker/config.json \
&& echo '{ "credsStore": "ecr-login" }' >> /root/.docker/config.json

FROM base
ENV PATH="$PATH:/build/runtime/bin:/build/scripts:/build/.cargo/bin"
Expand Down
44 changes: 37 additions & 7 deletions tools/infra/container/Makefile
Original file line number Diff line number Diff line change
@@ -1,20 +1,50 @@
# SHELL is bash, silly sh.
SHELL = bash
# DOCKERFILES are the detected container images that are being worked
# with. It is expected that NAME be part of the file name, as in
# Dockerfile.NAME, which is used throughout the infrastructure.
DOCKERFILES = $(filter-out %~,$(wildcard Dockerfile.*))
# NAMES are the detected NAMES given the provided Dockerfiles.
NAMES = $(DOCKERFILES:Dockerfile.%=%)

# IMAGE_REPO_PREFIX is prepended to the image's tag. In the case of
# `push', the IMAGE_REPO_PREFIX provides the ECR repository URI prefix
# for each image.
IMAGE_REPO_PREFIX ?= infra/
# IMAGE_TAG provides the registry/image-name:IMAGE_TAG portion of the
# URI tagged to images.
IMAGE_TAG ?= develop
# IMAGE_NAME is the name that the container image is tagged with.
IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(IMAGE_TAG)
# ECR_URI_PREFIX is the ECR URI prefix based on the resolved builder
# image URI which, like other container images, is discoverable under
# its in-region SSM parameter - so we can lob off the builder part and
# use it as our model for the pushed repository name.
ECR_URI_PREFIX = $(shell aws ssm get-parameter --name /infra/container/infra/builder --query Parameter.Value --output text | sed 's/builder$$//')
# ECR_NAME_PREFIX provides a prefix to derive the ECR repository-name
# (the attribute) from the images' NAME - the infra/ prefix is
# conventional across automations' consumed images.
ECR_NAME_PREFIX ?= infra/

.PHONY: force all release $(NAMES)
.DEFAULT: all
.PHONY: force all release $(NAMES)
force:

all : IMAGE_TAG ?= develop
all: $(NAMES)

release : IMAGE_TAG ?= latest
release: $(if $(NAME),$(NAME),$(NAMES))
all: $(if $(NAME),$(NAME),$(NAMES))

$(NAMES) : NAME = $@
$(NAMES): force
@echo "Building container image for '$(NAME)'"
docker build -t $(IMAGE_NAME) -f Dockerfile.$(NAME) .

# Push images (must explicitly provide IMAGE_TAG=release to be pulled
# by consumers).
push: IMAGE_REPO_PREFIX = $(ECR_URI_PREFIX)
push: IMAGE_TAG = staging
push: all
@echo "Pushing container images with tag '$(IMAGE_TAG)'"
@echo "Images: $(foreach NAME,$(NAMES),$(IMAGE_NAME))"
@$(foreach NAME,$(NAMES),\
echo "Pushing '$(NAME)' to '$(IMAGE_NAME)'" && \
aws ecr describe-repositories --repository-names $(ECR_NAME_PREFIX)$(NAME) &> /dev/null \
&& docker push $(IMAGE_NAME) \
|| echo "Could not push $(NAME) to ECR repository as $(IMAGE_NAME)";)
28 changes: 25 additions & 3 deletions tools/infra/container/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,38 @@ make all

Each `Dockerfile.<name>` can be built individually with `make $name` as needed.

## Release Images
## Release Images (using a tag)

As with the development images, all images may be built at once:

```bash
make release
make all IMAGE_TAG=release
```

To build a specific image, for instance named `builder`, `make` may be provided this name to build its release image:

```bash
make release NAME=builder
make all NAME=builder IMAGE_TAG=release
```

# Releasing

The `push` target is provided to build & push release container images for use, at least in the context of build and release automation.

The default target will prepare to push the images using the environment's AWS profile to confirm that the ECR repositories line up and subsequently pushing with a default of `IMAGE_TAG=staging`.
This invocation **will** push to the ECR repository, but with the image tagged as "staging".
Doing a push this way will stage the layers in the ECR repository so that subsequent pushes update lightweight references only (pushing a tag that refers to the same layers).

``` bash
make push
```

To push a container image tagged as a release image, which is required for the CodeBuild project to use, the `IMAGE_TAG` must be set explicitly to the same tag that's configured to be pulled by projects.
If the release tag is `release`, then the call to `push` these images would be:

``` bash
make push IMAGE_TAG=release
```

The `Makefile` target would then match the images to their respective ECR repositories, as before, and `docker push` to the images' respective repositories.
If the `make push IMAGE_TAG=release` followed an earlier `make push` then this the `make push IMAGE_TAG=release` call will simply update the references in the remote ECR repository to point to the same layers.
8 changes: 8 additions & 0 deletions tools/infra/stacks/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,12 @@ validate/%:
$(info validating stack: $*)
aws cloudformation validate-template --template-body "$$(< $*)"

check: check-readme

check-readme:
@$(foreach stack,$(stacks:.yml=),\
grep -Fw -q -e "$(stack)" README.md ||\
{ MISSING=1; echo "Missing README section mentioning $(stack)"; }; )\
$${MISSING:+exit 1}

.PHONY: test list
163 changes: 163 additions & 0 deletions tools/infra/stacks/infra-container.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
# stack: infra-container

AWSTemplateFormatVersion: "2010-09-09"

Description: >-
Infra's container images ECR repositories used in release automation.
Parameters:
SSMPathNamespace:
Type: String
Default: /infra/container
AllowedPattern: '^/.*[^/]$'
Description: >
Namespace under which SSM Parameters will be created for container images (should start but *not* end with '/')
Resources:
SDKx8664Repo:
Type: AWS::ECR::Repository
Metadata:
Source: extras/sdk-container
Properties:
RepositoryName: thar/sdk-x86_64

SDKx8664Parameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub "${SSMPathNamespace}/${SDKx8664Repo}"
Type: String
Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SDKx8664Repo}"

SDKaarch64Repo:
Type: AWS::ECR::Repository
Metadata:
Source: extras/sdk-container
Properties:
RepositoryName: thar/sdk-aarch64

SDKaarch64Parameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub "${SSMPathNamespace}/${SDKaarch64Repo}"
Type: String
Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SDKaarch64Repo}"

BuilderRepo:
Type: AWS::ECR::Repository
Metadata:
Source: tools/infra/container/Dockerfile.builder
Properties:
RepositoryName: infra/builder
RepositoryPolicyText:
Version: "2012-10-17"
Statement:
- Sid: "codeBuildPull"
Effect: Allow
Principal:
Service: "codebuild.amazonaws.com"
Action:
- "ecr:GetDownloadUrlForLayer"
- "ecr:BatchGetImage"
- "ecr:BatchCheckLayerAvailability"

BuilderParameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub "${SSMPathNamespace}/${BuilderRepo}"
Type: String
Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${BuilderRepo}"

SigningRepo:
Type: AWS::ECR::Repository
Metadata:
Source: tools/infra/container/Dockerfile.signing
Properties:
RepositoryName: infra/signing
RepositoryPolicyText:
Version: "2012-10-17"
Statement:
- Sid: "codeBuildPull"
Effect: Allow
Principal:
Service: "codebuild.amazonaws.com"
Action:
- "ecr:GetDownloadUrlForLayer"
- "ecr:BatchGetImage"
- "ecr:BatchCheckLayerAvailability"

SigningParameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub "${SSMPathNamespace}/${SigningRepo}"
Type: String
Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SigningRepo}"

PullPolicy:
Type: AWS::IAM::ManagedPolicy
Properties:
ManagedPolicyName: pull
Path: !Sub "/${AWS::StackName}/"
PolicyDocument:
Version: '2012-10-17'
Statement:
- Sid: "imagePull"
Effect: Allow
Resource:
- !GetAtt BuilderRepo.Arn
- !GetAtt SigningRepo.Arn
- !GetAtt SDKx8664Repo.Arn
- !GetAtt SDKaarch64Repo.Arn
Action:
- "ecr:GetAuthorizationToken"
- "ecr:BatchCheckLayerAvailability"
- "ecr:GetDownloadUrlForLayer"
- "ecr:ListImages"
- "ecr:DescribeImages"
- "ecr:BatchGetImage"
- "ecr:ListTagsForResource"
- Sid: "imageResolve"
Effect: Allow
Resource:
- !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*"
Action:
- "ssm:GetParameter"
- "ssm:GetParameters"
- "ssm:GetParametersByPath"
- "ssm:DescribeParameters"

PushPolicy:
Type: AWS::IAM::ManagedPolicy
Properties:
ManagedPolicyName: push
Path: !Sub "/${AWS::StackName}/"
PolicyDocument:
Version: '2012-10-17'
Statement:
- Sid: "imagePush"
Effect: Allow
Resource:
- !GetAtt BuilderRepo.Arn
- !GetAtt SigningRepo.Arn
- !GetAtt SDKx8664Repo.Arn
- !GetAtt SDKaarch64Repo.Arn
Action:
- "ecr:GetAuthorizationToken"
- "ecr:BatchCheckLayerAvailability"
- "ecr:GetDownloadUrlForLayer"
- "ecr:DescribeImages"
- "ecr:BatchGetImage"
- "ecr:ListTagsForResource"
- "ecr:InitiateLayerUpload"
- "ecr:UploadLayerPart"
- "ecr:CompleteLayerUpload"
- "ecr:PutImage"

Outputs:
PullPolicy:
Export:
Name: !Sub "${AWS::StackName}-pull-policy"
Value: !Ref PullPolicy
PushPolicy:
Export:
Name: !Sub "${AWS::StackName}-push-policy"
Value: !Ref PushPolicy
Loading

0 comments on commit b47dad9

Please sign in to comment.