diff --git a/.env.defaults b/.env.defaults deleted file mode 100644 index 4e5a213b8..000000000 --- a/.env.defaults +++ /dev/null @@ -1,11 +0,0 @@ -# These credential information is no secret, it's just for easy -# development. Don't use these values for production deployment! -JWTSECRET=super-secret-string -JWTAUDIENCE=api.dev -AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXXXX -AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxx -AWS_BUCKET=aws-bucket -HARBOR_REGISTRY_STORAGE_AMAZON_BUCKET=bucket-name -HARBOR_REGISTRY_STORAGE_AMAZON_REGION=bucket-region -REGISTRY_STORAGE_S3_ACCESSKEY=AWS-ID -REGISTRY_STORAGE_S3_SECRETKEY=AWS-Secret \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 75478c5d8..9e2f714f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,169 +1,154 @@ node ('lagoon-images') { - withEnv(['AWS_BUCKET=jobs.amazeeio.services', 'AWS_DEFAULT_REGION=us-east-2']) { - withCredentials([ - usernamePassword(credentialsId: 'aws-s3-lagoon', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY'), - string(credentialsId: 'SKIP_IMAGE_PUBLISH', variable: 'SKIP_IMAGE_PUBLISH') - ]) { - try { - env.CI_BUILD_TAG = env.BUILD_TAG.replaceAll('%2f','').replaceAll("[^A-Za-z0-9]+", "").toLowerCase() - env.SAFEBRANCH_NAME = env.BRANCH_NAME.replaceAll('%2f','-').replaceAll("[^A-Za-z0-9]+", "-").toLowerCase() - env.SYNC_MAKE_OUTPUT = 'target' - // make/tests will synchronise (buffer) output by default to avoid interspersed - // lines from multiple jobs run in parallel. However this means that output for - // each make target is not written until the command completes. - // - // See `man -P 'less +/-O' make` for more information about this option. - // - // Uncomment the line below to disable output synchronisation. - // env.SYNC_MAKE_OUTPUT = 'none' - - stage ('env') { - sh "env" - } + withCredentials([ + string(credentialsId: 'SKIP_IMAGE_PUBLISH', variable: 'SKIP_IMAGE_PUBLISH') + ]) { + try { + env.CI_BUILD_TAG = env.BUILD_TAG.replaceAll('%2f','').replaceAll("[^A-Za-z0-9]+", "").toLowerCase() + env.SAFEBRANCH_NAME = env.BRANCH_NAME.replaceAll('%2f','-').replaceAll("[^A-Za-z0-9]+", "-").toLowerCase() + env.SYNC_MAKE_OUTPUT = 'target' + // make/tests will synchronise (buffer) output by default to avoid interspersed + // lines from multiple jobs run in parallel. However this means that output for + // each make target is not written until the command completes. + // + // See `man -P 'less +/-O' make` for more information about this option. + // + // Uncomment the line below to disable output synchronisation. + // env.SYNC_MAKE_OUTPUT = 'none' + + stage ('env') { + sh "env" + } - deleteDir() + deleteDir() - stage ('Checkout') { - def checkout = checkout scm - env.GIT_COMMIT = checkout["GIT_COMMIT"] - } + stage ('Checkout') { + def checkout = checkout scm + env.GIT_COMMIT = checkout["GIT_COMMIT"] + } - // in order to have the newest images from upstream (with all the security updates) we clean our local docker cache on tag deployments - // we don't do this all the time to still profit from image layer caching - // but we want this on tag deployments in order to ensure that we publish images always with the newest possible images. - if (env.TAG_NAME || env.SAFEBRANCH_NAME == 'main') { - stage ('clean docker image cache') { - sh script: "make docker-buildx-remove", label: "removing leftover buildx" - sh script: "docker image prune -af", label: "Pruning images" - sh script: "docker buildx prune -af", label: "Pruning builder cache" - } + // in order to have the newest images from upstream (with all the security updates) we clean our local docker cache on tag deployments + // we don't do this all the time to still profit from image layer caching + // but we want this on tag deployments in order to ensure that we publish images always with the newest possible images. + if (env.TAG_NAME || env.SAFEBRANCH_NAME == 'main') { + stage ('clean docker image cache') { + sh script: "make docker-buildx-remove", label: "removing leftover buildx" + sh script: "docker image prune -af", label: "Pruning images" + sh script: "docker buildx prune -af", label: "Pruning builder cache" } + } - stage ('build images') { - sh script: "docker run --privileged --rm tonistiigi/binfmt --install all", label: "setting binfmt correctly" - sh script: "make docker-buildx-configure", label: "Configuring buildx for multi-platform build" - env.SCAN_IMAGES = 'true' - sh script: "make docker_pull", label: "Ensuring fresh upstream images" - sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build", label: "Building images" - } + stage ('build images') { + sh script: "docker run --privileged --rm tonistiigi/binfmt --install all", label: "setting binfmt correctly" + sh script: "make docker-buildx-configure", label: "Configuring buildx for multi-platform build" + env.SCAN_IMAGES = 'true' + sh script: "make docker_pull", label: "Ensuring fresh upstream images" + sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build", label: "Building images" + } - stage ('show built images') { - sh 'cat build.txt' - sh 'docker image ls | grep ${CI_BUILD_TAG} | sort -u' - } + stage ('show built images') { + sh 'cat build.txt' + sh 'docker image ls | grep ${CI_BUILD_TAG} | sort -u' + } - stage ('Copy examples down') { - sh script: "git clone https://github.com/uselagoon/lagoon-examples.git tests" - dir ('tests') { - sh script: "git submodule sync && git submodule update --init" - sh script: "mkdir -p ./all-images && cp ../helpers/*docker-compose.yml ./all-images/ && cp ../helpers/TESTING_*_dockercompose.md ./all-images/" - sh script: "sed -i '/image: uselagoon/ s/uselagoon/${CI_BUILD_TAG}/' ./all-images/*-docker-compose.yml" - sh script: "yarn install" - sh script: "docker network inspect amazeeio-network >/dev/null || docker network create amazeeio-network" - } + stage ('Copy examples down') { + sh script: "git clone https://github.com/uselagoon/lagoon-examples.git tests" + dir ('tests') { + sh script: "git submodule sync && git submodule update --init" + sh script: "mkdir -p ./all-images && cp ../helpers/*docker-compose.yml ./all-images/ && cp ../helpers/TESTING_*_dockercompose.md ./all-images/" + sh script: "sed -i '/image: uselagoon/ s/uselagoon/${CI_BUILD_TAG}/' ./all-images/*-docker-compose.yml" + sh script: "yarn install" + sh script: "docker network inspect amazeeio-network >/dev/null || docker network create amazeeio-network" } + } - parallel ( - 'build and push images to testlagoon dockerhub': { - stage ('push branch images to testlagoon/*') { - withCredentials([string(credentialsId: 'amazeeiojenkins-dockerhub-password', variable: 'PASSWORD')]) { - try { - if (env.SKIP_IMAGE_PUBLISH != 'true') { - sh script: 'docker login -u amazeeiojenkins -p $PASSWORD', label: "Docker login" - sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 publish-testlagoon-baseimages BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Publishing built images to testlagoon" - if (env.SAFEBRANCH_NAME == 'main') { - sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build PUBLISH_IMAGES=true REGISTRY_ONE=testlagoon TAG_ONE=${SAFEBRANCH_NAME} REGISTRY_TWO=testlagoon TAG_TWO=latest", label: "Publishing built images to testlagoon main&latest images" - } else if (env.SAFEBRANCH_NAME == 'arm64-images') { - sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build PUBLISH_IMAGES=true REGISTRY_ONE=testlagoon TAG_ONE=${SAFEBRANCH_NAME} REGISTRY_TWO=testlagoon TAG_TWO=multiarch", label: "Publishing built images to testlagoon arm images" - } else { - sh script: 'echo "No multi-arch images required for this build"', label: "Skipping image publishing" - } + parallel ( + 'build and push images to testlagoon dockerhub': { + stage ('push branch images to testlagoon/*') { + withCredentials([string(credentialsId: 'amazeeiojenkins-dockerhub-password', variable: 'PASSWORD')]) { + try { + if (env.SKIP_IMAGE_PUBLISH != 'true') { + sh script: 'docker login -u amazeeiojenkins -p $PASSWORD', label: "Docker login" + sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 publish-testlagoon-baseimages BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Publishing built images to testlagoon" + if (env.SAFEBRANCH_NAME == 'main') { + sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build PUBLISH_IMAGES=true REGISTRY_ONE=testlagoon TAG_ONE=${SAFEBRANCH_NAME} REGISTRY_TWO=testlagoon TAG_TWO=latest", label: "Publishing built images to testlagoon main&latest images" + } else if (env.SAFEBRANCH_NAME == 'arm64-images') { + sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build PUBLISH_IMAGES=true REGISTRY_ONE=testlagoon TAG_ONE=${SAFEBRANCH_NAME} REGISTRY_TWO=testlagoon TAG_TWO=multiarch", label: "Publishing built images to testlagoon arm images" } else { - sh script: 'echo "skipped because of SKIP_IMAGE_PUBLISH env variable"', label: "Skipping image publishing" + sh script: 'echo "No multi-arch images required for this build"', label: "Skipping image publishing" } - } catch (e) { - echo "Something went wrong, trying to cleanup" - cleanup() - throw e + } else { + sh script: 'echo "skipped because of SKIP_IMAGE_PUBLISH env variable"', label: "Skipping image publishing" } + } catch (e) { + echo "Something went wrong, trying to cleanup" + cleanup() + throw e } } - }, - 'Run all the tests on the local images': { - stage ('running test suite') { - dir ('tests') { - sh script: "docker buildx use default", label: "Ensure to use default builder" - sh script: "grep -rl uselagoon . | xargs sed -i '/^FROM/ s/uselagoon/${CI_BUILD_TAG}/'" - sh script: "grep -rl uselagoon . | xargs sed -i '/image: uselagoon/ s/uselagoon/${CI_BUILD_TAG}/'" - sh script: "find . -maxdepth 2 -name docker-compose.yml | xargs sed -i -e '/###/d'" - sh script: "TEST=./all-images/TESTING_base_images* yarn test", label: "Run base-images tests" - sh script: "TEST=./all-images/TESTING_service_images* yarn test", label: "Run service-images tests" - sh script: "yarn test:simple", label: "Run simple Drupal tests" - sh script: "yarn test:advanced", label: "Run advanced Drupal tests" - } + } + }, + 'Run all the tests on the local images': { + stage ('running test suite') { + dir ('tests') { + sh script: "docker buildx use default", label: "Ensure to use default builder" + sh script: "grep -rl uselagoon . | xargs sed -i '/^FROM/ s/uselagoon/${CI_BUILD_TAG}/'" + sh script: "grep -rl uselagoon . | xargs sed -i '/image: uselagoon/ s/uselagoon/${CI_BUILD_TAG}/'" + sh script: "find . -maxdepth 2 -name docker-compose.yml | xargs sed -i -e '/###/d'" + sh script: "TEST=./all-images/TESTING_base_images* yarn test", label: "Run base-images tests" + sh script: "TEST=./all-images/TESTING_service_images* yarn test", label: "Run service-images tests" + sh script: "yarn test:simple", label: "Run simple Drupal tests" + sh script: "yarn test:advanced", label: "Run advanced Drupal tests" } } - ) + } + ) - stage ('publish experimental image tags to testlagoon') { - if (env.SAFEBRANCH_NAME == 'main' || env.CHANGE_ID && pullRequest.labels.contains("experimental")) { - sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 publish-testlagoon-experimental-baseimages BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Publishing experimental images to testlagoon" - } else { - sh script: 'echo "not a PR or main branch push"', label: "Skipping experimantal image publishing" - } + stage ('publish experimental image tags to testlagoon') { + if (env.SAFEBRANCH_NAME == 'main' || env.CHANGE_ID && pullRequest.labels.contains("experimental")) { + sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 publish-testlagoon-experimental-baseimages BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Publishing experimental images to testlagoon" + } else { + sh script: 'echo "not a PR or main branch push"', label: "Skipping experimantal image publishing" } + } - if (env.TAG_NAME && env.SKIP_IMAGE_PUBLISH != 'true') { - parallel ( - 'build and push images to uselagoon dockerhub': { - stage ('push branch images to uselagoon/*') { - withCredentials([string(credentialsId: 'amazeeiojenkins-dockerhub-password', variable: 'PASSWORD')]) { - try { - if (env.SKIP_IMAGE_PUBLISH != 'true') { - sh script: 'docker login -u amazeeiojenkins -p $PASSWORD', label: "Docker login" - sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build PUBLISH_IMAGES=true REGISTRY_ONE=uselagoon TAG_ONE=${TAG_NAME} REGISTRY_TWO=uselagoon TAG_TWO=latest", label: "Publishing built images to uselagoon" - } else { - sh script: 'echo "skipped because of SKIP_IMAGE_PUBLISH env variable"', label: "Skipping image publishing" - } - } catch (e) { - echo "Something went wrong, trying to cleanup" - cleanup() - throw e - } - } - } - }, - 'push legacy images to amazeeio dockerhub': { - stage ('publish-amazeeio') { - withCredentials([string(credentialsId: 'amazeeiojenkins-dockerhub-password', variable: 'PASSWORD')]) { - sh script: 'docker login -u amazeeiojenkins -p $PASSWORD', label: "Docker login" - sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 publish-amazeeio-baseimages", label: "Publishing legacy images to amazeeio" - } + if (env.TAG_NAME && env.SKIP_IMAGE_PUBLISH != 'true') { + stage ('push branch images to uselagoon/*') { + withCredentials([string(credentialsId: 'amazeeiojenkins-dockerhub-password', variable: 'PASSWORD')]) { + try { + if (env.SKIP_IMAGE_PUBLISH != 'true') { + sh script: 'docker login -u amazeeiojenkins -p $PASSWORD', label: "Docker login" + sh script: "make -O${SYNC_MAKE_OUTPUT} -j8 build PUBLISH_IMAGES=true REGISTRY_ONE=uselagoon TAG_ONE=${TAG_NAME} REGISTRY_TWO=uselagoon TAG_TWO=latest", label: "Publishing built images to uselagoon" + } else { + sh script: 'echo "skipped because of SKIP_IMAGE_PUBLISH env variable"', label: "Skipping image publishing" } + } catch (e) { + echo "Something went wrong, trying to cleanup" + cleanup() + throw e } - ) - } - - if (env.TAG_NAME || env.SAFEBRANCH_NAME == 'main' || env.SAFEBRANCH_NAME == 'testing-scans' ) { - stage ('scan built images') { - sh script: 'make scan-images', label: "perform scan routines" - sh script: 'find ./scans/*trivy* -type f | xargs tail -n +1', label: "Show Trivy vulnerability scan results" - sh script: 'find ./scans/*grype* -type f | xargs tail -n +1', label: "Show Grype vulnerability scan results" - sh script: 'find ./scans/*syft* -type f | xargs tail -n +1', label: "Show Syft SBOM results" } } + } - } catch (e) { - currentBuild.result = 'FAILURE' - echo "Something went wrong, trying to cleanup" - throw e - } finally { - cleanup() - notifySlack(currentBuild.result) + if (env.TAG_NAME || env.SAFEBRANCH_NAME == 'main' || env.SAFEBRANCH_NAME == 'testing-scans' ) { + stage ('scan built images') { + sh script: 'make scan-images', label: "perform scan routines" + sh script: 'find ./scans/*trivy* -type f | xargs tail -n +1', label: "Show Trivy vulnerability scan results" + sh script: 'find ./scans/*grype* -type f | xargs tail -n +1', label: "Show Grype vulnerability scan results" + sh script: 'find ./scans/*syft* -type f | xargs tail -n +1', label: "Show Syft SBOM results" + } } - + + } catch (e) { + currentBuild.result = 'FAILURE' + echo "Something went wrong, trying to cleanup" + throw e + } finally { cleanup() + notifySlack(currentBuild.result) } + + cleanup() } } diff --git a/Makefile b/Makefile index ff652aadb..a66aaf469 100644 --- a/Makefile +++ b/Makefile @@ -125,9 +125,6 @@ docker_publish_testlagoon = docker tag $(CI_BUILD_TAG)/$(1) testlagoon/$(2) && d # Tags an image with the `uselagoon` repository and pushes it docker_publish_uselagoon = docker tag $(CI_BUILD_TAG)/$(1) uselagoon/$(2) && docker push uselagoon/$(2) | cat -# Tags an image with the `amazeeio` repository and pushes it -docker_publish_amazeeio = docker tag $(CI_BUILD_TAG)/$(1) amazeeio/$(2) && docker push amazeeio/$(2) | cat - .PHONY: docker_pull docker_pull: grep -Eh 'FROM' $$(find . -type f -name *Dockerfile) | grep -Ev 'IMAGE_REPO' | sed 's/\-\-platform\=linux\/amd64//g' | awk '{print $$2}' | sort --unique | xargs -tn1 -P8 docker pull -q @@ -145,7 +142,6 @@ unversioned-images := commons \ # base-images is a variable that will be constantly filled with all base image there are base-images += $(unversioned-images) -s3-images += $(unversioned-images) # List with all images prefixed with `build/`. Which are the commands to actually build images build-images = $(foreach image,$(unversioned-images),build/$(image)) @@ -286,8 +282,6 @@ $(build-versioned-images): base-images-with-versions += $(versioned-images) base-images-with-versions += $(default-versioned-images) base-images-with-versions += $(experimental-images) -s3-images += $(versioned-images) -s3-images += $(experimental-images) build/php-8.1-fpm build/php-8.2-fpm build/php-8.3-fpm build/php-8.4-fpm: build/commons build/php-8.1-cli: build/php-8.1-fpm @@ -483,96 +477,6 @@ $(publish-uselagoon-baseimages-with-versions): $(call docker_publish_uselagoon,$(image),$(image):$(LAGOON_VERSION)) -####### -####### All tagged releases are also pushed to amazeeio repository with legacy tags -####### - -# Publish command to amazeeio docker hub, this should probably only be done during a master deployments -publish-amazeeio-baseimages = $(foreach image,$(base-images),[publish-amazeeio-baseimages]-$(image)) -publish-amazeeio-baseimages-with-versions = $(foreach image,$(base-images-with-versions),[publish-amazeeio-baseimages-with-versions]-$(image)) -# Special handler for the previously unversioned images that now have versions -publish-amazeeio-baseimages-without-versions = $(foreach image,$(default-versioned-images),[publish-amazeeio-baseimages-without-versions]-$(image)) - -# tag and push all images -.PHONY: publish-amazeeio-baseimages -publish-amazeeio-baseimages: $(publish-amazeeio-baseimages) $(publish-amazeeio-baseimages-with-versions) $(publish-amazeeio-baseimages-without-versions) - -# tag and push of each image -.PHONY: $(publish-amazeeio-baseimages) -$(publish-amazeeio-baseimages): -# Calling docker_publish for image, but remove the prefix '[publish-amazeeio-baseimages]-' first - $(eval image = $(subst [publish-amazeeio-baseimages]-,,$@)) -# Publish images as :latest - $(call docker_publish_amazeeio,$(image),$(image):latest) -# Publish images with version tag - $(call docker_publish_amazeeio,$(image),$(image):$(LAGOON_VERSION)) - -# tag and push of base image with version -.PHONY: $(publish-amazeeio-baseimages-with-versions) -$(publish-amazeeio-baseimages-with-versions): -# Calling docker_publish for image, but remove the prefix '[publish-amazeeio-baseimages-with-versions]-' first - $(eval image = $(subst [publish-amazeeio-baseimages-with-versions]-,,$@)) - $(eval variant = $(word 1,$(subst -, ,$(image)))) - $(eval version = $(word 2,$(subst -, ,$(image)))) - $(eval type = $(word 3,$(subst -, ,$(image)))) - $(eval subtype = $(word 4,$(subst -, ,$(image)))) -# Construct a "legacy" tag of the form `amazeeio/variant:version-type-subtype` e.g. `amazeeio/php:8.2-cli-drupal` - $(eval legacytag = $(shell echo $(variant)$(if $(version),:$(version))$(if $(type),-$(type))$(if $(subtype),-$(subtype)))) -# These images already use a tag to differentiate between different versions of the service itself (like node:9 and node:10) -# We push a version without the `-latest` suffix - $(call docker_publish_amazeeio,$(image),$(legacytag)) -# Plus a version with the `-latest` suffix, this makes it easier for people with automated testing - $(call docker_publish_amazeeio,$(image),$(legacytag)-latest) -# We add the Lagoon Version just as a dash - $(call docker_publish_amazeeio,$(image),$(legacytag)-$(LAGOON_VERSION)) - -# tag and push of unversioned base images -.PHONY: $(publish-amazeeio-baseimages-without-versions) -$(publish-amazeeio-baseimages-without-versions): -# Calling docker_publish for image, but remove the prefix '[publish-amazeeio-baseimages-with-versions]-' first - $(eval image = $(subst [publish-amazeeio-baseimages-without-versions]-,,$@)) - $(eval variant = $(word 1,$(subst -, ,$(image)))) - $(eval version = $(word 2,$(subst -, ,$(image)))) - $(eval type = $(word 3,$(subst -, ,$(image)))) - $(eval subtype = $(word 4,$(subst -, ,$(image)))) -# Construct a "legacy" tag of the form `amazeeio/variant-type-subtype` e.g. `amazeeio/postgres-ckan` - $(eval legacytag = $(shell echo $(variant)$(if $(type),-$(type))$(if $(subtype),-$(subtype)))) -# These images previously had no version tracking, publish them for legacy compatibility only - $(call docker_publish_amazeeio,$(image),$(legacytag):latest) - $(call docker_publish_uselagoon,$(image),$(legacytag):latest) -# These images previously had no version tracking, publish them for legacy compatibility only - $(call docker_publish_amazeeio,$(image),$(legacytag):$(LAGOON_VERSION)) - $(call docker_publish_uselagoon,$(image),$(legacytag):$(LAGOON_VERSION)) - - -####### -####### Transferring Images to S3 -####### - -s3-save = $(foreach image,$(s3-images),[s3-save]-$(image)) -# save all images to s3 -.PHONY: s3-save -s3-save: $(s3-save) -# tag and push of each image -.PHONY: $(s3-save) -$(s3-save): -# remove the prefix '[s3-save]-' first - $(eval image = $(subst [s3-save]-,,$@)) - $(eval image = $(subst __,:,$(image))) - docker save $(CI_BUILD_TAG)/$(image) $$(docker history -q $(CI_BUILD_TAG)/$(image) | grep -v missing) | gzip -9 | aws s3 cp - s3://lagoon-images/$(image).tar.gz - -s3-load = $(foreach image,$(s3-images),[s3-load]-$(image)) -# save all images to s3 -.PHONY: s3-load -s3-load: $(s3-load) -# tag and push of each image -.PHONY: $(s3-load) -$(s3-load): -# remove the prefix '[s3-load]-' first - $(eval image = $(subst [s3-load]-,,$@)) - $(eval image = $(subst __,:,$(image))) - curl -s https://s3.us-east-2.amazonaws.com/lagoon-images/$(image).tar.gz | gunzip -c | docker load - # Clean all build touches, which will case make to rebuild the Docker Images (Layer caching is # still active, so this is a very safe command)