From 2b632b9063656c04f3e7fe9fa97ea8a878194c30 Mon Sep 17 00:00:00 2001 From: Kamil Kaczynski Date: Tue, 4 Jun 2024 12:22:59 +0200 Subject: [PATCH 1/3] [RELENG-284] Replace Coveralls with Codacy Remove coverage-reports bump coverage-reporter version Add coverage generation Force Go coverage parser Replace orb with command Update codact badge --- .circleci/config.yml | 13 ++++++++++++- README.MD | 4 ++-- devops/make/common-go.mk | 11 +---------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90483a3..d7004d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,6 +50,17 @@ jobs: - run: name: lint and test command: make test-circle + - run: + name: Upload Coverage Results to Codacy + command: | + if [ -x "$(which curl)" ]; then + curl -Ls https://coverage.codacy.com/get.sh > get.sh + elif [ -x "$(which wget)" ] ; then + wget -qO - https://coverage.codacy.com/get.sh > get.sh + else + printf "Could not find curl or wget, please install one." + fi + source get.sh report --force-coverage-parser go -r coverage.out # Tag for release release: executor: go-build @@ -91,4 +102,4 @@ workflows: filters: branches: only: - - main \ No newline at end of file + - main diff --git a/README.MD b/README.MD index cb90147..9057015 100644 --- a/README.MD +++ b/README.MD @@ -2,6 +2,8 @@ [![Early Access](https://img.shields.io/badge/Pantheon-Early_Access-yellow?logo=pantheon&color=FFDC28)](https://docs.pantheon.io/oss-support-levels#early-access) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/5a16739ac87e496c80e9578e64ab79ca)](https://app.codacy.com/gh/pantheon-systems/sites-yml-validator/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) + A utility for validating a `sites.yml` file on a Pantheon site during a WordPress multisite's search and replace tasks. For more information, see [Pantheon's Multisite Docs](https://docs.pantheon.io/guides/multisite/search-replace/). ## Usage @@ -73,8 +75,6 @@ curl -LO https://github.com/pantheon-systems/sites-yml-validator/releases/latest ## Testing -[![Coverage Status](https://coveralls.io/repos/github/pantheon-systems/sites-yml-validator/badge.svg?t=PGhafd)](https://coveralls.io/github/pantheon-systems/sites-yml-validator) - `make test` runs linting and testing. ## Releases diff --git a/devops/make/common-go.mk b/devops/make/common-go.mk index a46fd74..067387a 100644 --- a/devops/make/common-go.mk +++ b/devops/make/common-go.mk @@ -3,7 +3,6 @@ # INPUT VARIABLES # - GOLINT_ARGS: Override the options passed to golangci-lint for linting (-v --timeout 3m by default) # - GOTEST_ARGS: Override the options passed by default ot go test (--race by default) -# - COVERALLS_TOKEN: Token to use when pushing coverage to coveralls. # # - FETCH_CA_CERT: The presence of this variable will cause the root CA certs # to be downloaded to the file ca-certificates.crt before building. @@ -16,7 +15,7 @@ deps:: deps-go deps-circle:: deps-circle-go deps lint:: lint-go test:: lint-go test-go-tparse -test-circle:: test test-coveralls +test-circle:: test-coverage-go test test-coverage:: test-coverage-go build:: $(APP) build-go:: $(APP) @@ -131,14 +130,6 @@ test-coverage-go:: ## run coverage report $(call INFO, "running go coverage tests with $(GO_TEST_COVERAGE_ARGS)") @$(GO_TEST_COVERAGE_CMD) > /dev/null -test-coveralls:: deps-coveralls-go test-coverage-go ## run coverage and report to coveralls -ifdef COVERALLS_TOKEN - $(call INFO, "reporting coverage to coveralls") - @goveralls -repotoken $$COVERALLS_TOKEN -service=circleci -coverprofile=coverage.out > /dev/null -else - $(call WARN, "You asked to use Coveralls but neglected to set the COVERALLS_TOKEN environment variable") -endif - test-coverage-html:: test-coverage ## output html coverage file $(call INFO, "generating html coverage report") @go tool cover -html=coverage.out > /dev/null From 0efd4fb94bd88cfee74b8354aa395426ab8208b0 Mon Sep 17 00:00:00 2001 From: Kamil Kaczynski Date: Tue, 4 Jun 2024 20:04:30 +0200 Subject: [PATCH 2/3] Update circle CI config and common-go.mk to align with common_makefiles --- .circleci/config.yml | 2 +- devops/make/common-go.mk | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7004d5..633ce35 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: - restore-go-mod-cache - run: name: lint and test - command: make test-circle + command: make test-circle && make test-coverage - run: name: Upload Coverage Results to Codacy command: | diff --git a/devops/make/common-go.mk b/devops/make/common-go.mk index 067387a..7b58a17 100644 --- a/devops/make/common-go.mk +++ b/devops/make/common-go.mk @@ -15,11 +15,12 @@ deps:: deps-go deps-circle:: deps-circle-go deps lint:: lint-go test:: lint-go test-go-tparse -test-circle:: test-coverage-go test +test-circle:: test test-coverage:: test-coverage-go build:: $(APP) build-go:: $(APP) clean:: clean-go +format:: format-go ifndef GOLINT_ARGS GOLINT_ARGS := -v --timeout 3m @@ -69,9 +70,17 @@ deps-go:: deps-lint ## install dependencies for project assumes you have go bina ifneq (,$(wildcard vendor)) @find ./vendor/* -maxdepth 0 -type d -exec rm -rf "{}" \; || true endif - $(call INFO, "restoring dependencies using modules via: go get $(GO_GET_ARGS)") - @GO111MODULE=on go get $(GO_GET_ARGS) - + $(call INFO, "restoring dependencies using modules via: go mod download $(GO_GET_ARGS)") + @GO111MODULE=on go mod download $(GO_GET_ARGS) + +format-go: + $(call INFO, "cleaning up go.mod") + @go mod tidy + $(call INFO, "formatting go-code") + @go fmt + $(call INFO, "running golangci-lint with fixes") + @# TODO: call lint-go + @golangci-lint run -E goimports --fix $(GOLINT_ARGS) lint-go:: deps-go deps-lint $(call INFO, "scanning source with golangci-lint") @@ -108,12 +117,6 @@ else endif endif -deps-coveralls-go:: ## install goveralls for sending go test coverage reports to Coveralls.io -ifeq (, $(shell command -v goveralls;)) - $(call INFO, "installing goveralls") - @GO111MODULE=off go get github.com/mattn/goveralls > /dev/null -endif - deps-status:: ## check status of deps with gostatus ifeq (, $(shell command -v gostatus;)) $(call INFO, "installing gostatus") @@ -144,5 +147,5 @@ ifdef FETCH_CA_CERT @curl -s -L https://curl.haxx.se/ca/cacert.pem -o ca-certificates.crt > /dev/null endif -.PHONY:: _fetch-cert test-coverage-html test-coveralls deps-status deps-coveralls-go deps-circle deps-go deps-lint \ +.PHONY:: _fetch-cert test-coverage-html deps-status deps-circle deps-go deps-lint \ lint-go test-circle test-go build-circle build-linux build-go clean-go From 9f5b3655ed9531a292035365e4c888b4a2f1d417 Mon Sep 17 00:00:00 2001 From: Phil Tyler Date: Mon, 17 Jun 2024 11:21:15 -0700 Subject: [PATCH 3/3] Squashed 'devops/make/' changes from 4641812..1997ff8 1997ff8 Retire Coveralls (#394) 9d9de49 Allow unzip to overwrite files. (#397) d7c18aa [RELENG-284] Retire Coveralls (#398) 612b754 Persist COMMIT_NO length (#395) 056497e RELENG-280: Add a new image tag strategy (#393) 7888620 Fix BUILD_NUM (#392) d6b4eb8 Remove duplicate `make help` lines (#383) cd8960a Trim unnecessarily duplicated lines in _docker.mk 91a5d83 Merge pull request #391 from pantheon-systems/donebox 9f1e08f Fixed 15 minute timeout. d3928fd Added timeout of 15 minutes. 00cc75e Removed onebox code that breaks CI now that onebox is gone. b4abc03 Merge pull request #385 from pantheon-systems/rover-graph-variant-param 0994cb4 Codeformat for readme keys 4a625a3 Update common-apollo.mk a9967a9 Merge pull request #387 from pantheon-systems/apple-si ae7c9a2 Changed docker run to specify platform for M1/M2 workstations. cf64473 Update var names, add to readme 62aa949 Support parameterized args for check-apollo-schema and update-apollo-schema 1ffb675 add timeout to format-go (#384) 3d268bf Add Visual Studio Code devcontainer support (#381) 76539c8 Improve error handling when installing pants 82165da Allow override of docker command (podman) da7537d Bump Go builders to 1.21 a1907cb Use local markdown-toc when available e372990 Squelch some unset variable warnings 966c357 Fix some README lint warnings 70c7db0 shellcheck fix for setup circle ar docker (#380) a024296 Use `go mod download` instead of `go get` (#379) 5efb805 add format and check-format targets (#377) 4dfe66f remove common_makefiles remote after updating (#378) 996b5c5 Merge pull request #376 from pantheon-systems/update-timeout-to-match-ci 42865ed Update verify deployment timeout 2f37121 Adjusting jq filter so we return the latest deb format pvault package (#375) git-subtree-dir: devops/make git-subtree-split: 1997ff88e0c47192477c5188c6f9f64d52bad696 --- .circleci/config.yml | 17 +- .devcontainer/Dockerfile | 50 +++ .devcontainer/devcontainer.json | 48 ++ .devcontainer/on-create.sh | 16 + CODEOWNERS | 3 + README.md | 768 ++++++++++++++++---------------- _docker.mk | 73 ++- common-apollo.mk | 12 +- common-conda.mk | 4 +- common-docker-ar.mk | 12 +- common-docker-quay.mk | 13 +- common-docs.mk | 30 +- common-go.mk | 34 +- common-kube.mk | 2 +- common-pants.mk | 3 +- common-python.mk | 29 +- common-python3.mk | 6 +- common.mk | 30 +- go.mod | 2 +- sh/build-docker.sh | 6 +- sh/create-tls-cert.sh | 14 +- sh/install-pants.sh | 8 +- sh/setup-circle-ar-docker.sh | 4 +- sh/setup-circle-vault.sh | 6 +- sh/update-gcloud.sh | 2 +- test/make/docker-ar.mk | 4 + test/make/docker.mk | 4 + 27 files changed, 661 insertions(+), 539 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/on-create.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index e7ebee5..1928d0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,10 +8,10 @@ version: 2.1 executors: go-build: docker: - - image: quay.io/getpantheon/go-build:latest + - image: us-docker.pkg.dev/pantheon-artifacts/internal/go-build:1.21 auth: - username: $QUAY_USER - password: $QUAY_PASSWD + username: _json_key + password: $CIRCLE_CI_COMMON_KEY deploy-toolbox: docker: @@ -43,6 +43,7 @@ jobs: - run: echo 'export COMMON_MAKE_DIR="."' >> $BASH_ENV - run: make test-deps-build - run: make test-common-build + - run: NEW_TAG_STRATEGY=true make test-common-build test-deploy: executor: deploy-toolbox @@ -50,11 +51,12 @@ jobs: - checkout - run: echo 'export COMMON_MAKE_DIR="."' >> $BASH_ENV - run: make test-deps-deploy - - run: make test-common-deploy + - run: + command: make test-common-deploy + no_output_timeout: 30m test-artifact-setup: - docker: - - image: cimg/go:1.17 + executor: go-build steps: - checkout - run: echo 'export COMMON_MAKE_DIR="."' >> $BASH_ENV @@ -78,6 +80,7 @@ workflows: context: - sig-go-release - docker-executor-auth + - gcp-credentials-ar-ci - test-deploy: context: - sig-go-release @@ -87,10 +90,10 @@ workflows: context: - sig-go-release - docker-executor-auth + - gcp-credentials-ar-ci - autotag-release: context: - sig-go-release - - docker-executor-auth - gcp-credentials-ar-ci filters: branches: diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..f2c84f2 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,50 @@ +FROM mcr.microsoft.com/devcontainers/go:1.21-bookworm + +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] + +# hadolint ignore=DL3008 +RUN \ + echo "deb [signed-by=/etc/apt/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main" \ + | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ + && curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \ + > /etc/apt/keyrings/cloud.google.asc \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + dh-autoreconf \ + fuse \ + google-cloud-cli \ + google-cloud-sdk-gke-gcloud-auth-plugin \ + kubernetes-client \ + libfuse-dev \ + libglib2.0-dev \ + libjemalloc-dev \ + libleveldb-dev \ + libsystemd-dev \ + liburiparser-dev \ + npm \ + podman \ + shellcheck \ + sudo \ + && apt-get autoremove --yes \ + && rm -rf /var/lib/apt/lists/* + +RUN \ + rm -f /usr/local/bin/hadolint \ + && curl -fsSL https://api.github.com/repos/hadolint/hadolint/releases/latest \ + | jq '.assets[] | select(.name=="hadolint-Linux-x86_64") | .browser_download_url' \ + | xargs curl -fsS -o /usr/local/bin/hadolint -L \ + && chmod 0755 /usr/local/bin/hadolint + +# hadolint ignore=DL3016 +RUN \ + npm install --save markdown-toc --global + +# uid/gid may be changed at runtime. +USER vscode + +RUN \ + mkdir -p "${HOME}/.kube" \ + && go install github.com/mfridman/tparse@latest diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..d9194ca --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,48 @@ +{ + "name": "docker", + "build": { + "dockerfile": "Dockerfile" + }, + "mounts": [ + "type=bind,source=${localEnv:HOME}/.config/gcloud,target=/home/vscode/.config/gcloud", + "type=bind,source=${localEnv:HOME}/.netrc,target=/home/vscode/.netrc,readonly", + "type=bind,source=${localEnv:HOME}/.pants.yaml,target=/home/vscode/.pants.yaml,readonly" + ], + "onCreateCommand": [ + "/bin/bash", + ".devcontainer/on-create.sh" + ], + "containerEnv": { + "COMMON_MAKE_DIR": "/workspaces/common_makefiles" + }, + "customizations": { + "codespaces": { + "extensions": [ + "davidanson.vscode-markdownlint", + "golang.go", + "ms-azuretools.vscode-docker", + "ms-vscode.makefile-tools", + "redhat.vscode-yaml" + ], + "repositories": { + "pantheon-systems/pants": { + "permissions": { + "contents": "read" + } + } + } + }, + "vscode": { + "settings": {}, + "extensions": [ + "davidanson.vscode-markdownlint", + "golang.go", + "ms-azuretools.vscode-docker", + "ms-vscode.makefile-tools", + "redhat.vscode-yaml" + ] + } + }, + "containerUser": "vscode", + "updateRemoteUserUID": true +} diff --git a/.devcontainer/on-create.sh b/.devcontainer/on-create.sh new file mode 100644 index 0000000..3101d2f --- /dev/null +++ b/.devcontainer/on-create.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail + +# If GITHUB_TOKEN isn't set, try to extract from ~/.netrc +if [[ -z "${GITHUB_TOKEN:-}" && -r "${HOME}/.netrc" ]]; then + GITHUB_TOKEN=$(awk '/machine github.com / {print $6}' "${HOME}/.netrc") + export GITHUB_TOKEN +fi + +# If GITHUB_TOKEN is still not set, continue anyway and let install-pants.sh complain. + +CIRCLECI=true sudo -E sh/install-pants.sh || echo "Installing pants failed; continuing anyway" + +echo +echo "NEXT: You may want to run 'pants gke pull-creds pantheon-sandbox' to configure Kubernetes credentials." +echo diff --git a/CODEOWNERS b/CODEOWNERS index 8dae14a..7511984 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,3 +5,6 @@ # owned by #sig-go *.go @pantheon-systems/sig-go common-go.mk @pantheon-systems/sig-go + +# Doug is temporary devcontainer SME until a suitable group is found/formed +.devcontainers/ @djschaap @pantheon-systems/engops @pantheon-systems/sig-go diff --git a/README.md b/README.md index 7b85dad..1976d57 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -Common make tasks -================= +# Common make tasks @@ -13,7 +12,7 @@ Common make tasks + [Initial import](#initial-import) + [Future updates](#future-updates) * [Usage with more complicated projects](#usage-with-more-complicated-projects) - + [recursive make](#recursive-make) + + [Recursive make](#recursive-make) - [Tasks](#tasks) * [common.mk](#commonmk) + [help](#help) @@ -29,119 +28,126 @@ Common make tasks + [update-readme-toc](#update-readme-toc) + [test-readme-toc](#test-readme-toc) * [common-docker.mk](#common-dockermk) - + [push-circle::](#push-circle) + + [push-circle](#push-circle) * [_docker.mk](#_dockermk) - + [Input Environment Variables:](#input-environment-variables) - + [Exported Environment Variables:](#exported-environment-variables) - + [build-docker::](#build-docker) - + [lint-hadolint::](#lint-hadolint) - * [common-docker-ar.mk](#common-docker-armk) + [Input Environment Variables](#input-environment-variables) + + [Exported Environment Variables](#exported-environment-variables) + + [build-docker](#build-docker) + + [lint-hadolint](#lint-hadolint) + * [common-docker-ar.mk](#common-docker-armk) + + [Input Environment Variables](#input-environment-variables-1) + [Export Environment Variables](#export-environment-variables) - + [push::](#push) - + [push-ar::](#push-ar) - + [setup-ar::](#setup-ar) + + [push](#push) + + [push-ar](#push-ar) + + [setup-ar](#setup-ar) * [common-docker-quay.mk](#common-docker-quaymk) + [circleci 2.0](#circleci-20-1) - + [push::](#push-1) - + [push-quay::](#push-quay) - + [Input Environment Variables](#input-environment-variables-1) + + [push](#push-1) + + [push-quay](#push-quay) + + [Input Environment Variables](#input-environment-variables-2) + [Export Environment Variables](#export-environment-variables-1) * [common-shell.mk](#common-shellmk) - + [Input Environment Variables:](#input-environment-variables-1) + + [Input Environment Variables](#input-environment-variables-3) + [test-shell](#test-shell) * [common-pants.mk](#common-pantsmk) * [install-circle-pants](#install-circle-pants) * [delete-circle-pants](#delete-circle-pants) * [init-circle-pants](#init-circle-pants) * [common-conda.mk](#common-condamk) - + [Notes:](#notes) - + [Inheritable Input Environment Variables from common-python.mk:](#inheritable-input-environment-variables-from-common-pythonmk) - + [Input Environment Variables:](#input-environment-variables-2) - + [deps-conda::](#deps-conda) - + [setup-conda::](#setup-conda) - + [clean-conda::](#clean-conda) - + [reset-conda-environment::](#reset-conda-environment) - + [build-conda::](#build-conda) - + [build-conda-deployment-environment::](#build-conda-deployment-environment) - + [deploy-conda::](#deploy-conda) - + [deploy-conda-pypi::](#deploy-conda-pypi) - + [regenerate-anaconda-cloud-repo-token::](#regenerate-anaconda-cloud-repo-token) - + [add-conda-private-channel::](#add-conda-private-channel) - + [generate-conda-requirements::](#generate-conda-requirements) - + [reset-conda-environment::](#reset-conda-environment-1) + + [Notes](#notes) + + [Inheritable Input Environment Variables from common-python.mk](#inheritable-input-environment-variables-from-common-pythonmk) + + [Input Environment Variables](#input-environment-variables-4) + + [deps-conda](#deps-conda) + + [setup-conda](#setup-conda) + + [clean-conda](#clean-conda) + + [reset-conda-environment](#reset-conda-environment) + + [build-conda](#build-conda) + + [build-conda-deployment-environment](#build-conda-deployment-environment) + + [deploy-conda](#deploy-conda) + + [deploy-conda-pypi](#deploy-conda-pypi) + + [regenerate-anaconda-cloud-repo-token](#regenerate-anaconda-cloud-repo-token) + + [add-conda-private-channel](#add-conda-private-channel) + + [generate-conda-requirements](#generate-conda-requirements) + + [reset-conda-environment](#reset-conda-environment-1) * [common-python.mk](#common-pythonmk) - + [Input Environment Variables:](#input-environment-variables-3) - + [build-python::](#build-python) - + [test-python::](#test-python) - + [test-circle-python::](#test-circle-python) - + [deps-python::](#deps-python) - + [deps-circle::](#deps-circle) - + [deps-coverage::](#deps-coverage) - + [test-coverage-python::](#test-coverage-python) - + [test-coveralls::](#test-coveralls) - + [coverage-report::](#coverage-report) - + [lint-python::](#lint-python) - + [lint-pylint::](#lint-pylint) - + [lint-flake8::](#lint-flake8) + + [Input Environment Variables](#input-environment-variables-5) + + [build-python](#build-python) + + [test-python](#test-python) + + [test-circle-python](#test-circle-python) + + [deps-python](#deps-python) + + [deps-circle](#deps-circle) + + [deps-coverage](#deps-coverage) + + [test-coverage-python](#test-coverage-python) + + [coverage-report](#coverage-report) + + [lint-python](#lint-python) + + [lint-pylint](#lint-pylint) + + [lint-flake8](#lint-flake8) * [common-go.mk](#common-gomk) + [circleci 2.0](#circleci-20-2) - + [Input Environment Variables:](#input-environment-variables-4) - + [build-go::](#build-go) - + [build-linux::](#build-linux) - + [build-circle::](#build-circle) - + [test-go::](#test-go) - + [test-go-tparse::](#test-go-tparse) - + [test-no-race::](#test-no-race) - + [test-circle::](#test-circle) - + [deps-go::](#deps-go) - + [deps-circle::](#deps-circle-1) - + [deps-coverage::](#deps-coverage-1) - + [deps-status::](#deps-status) - + [clean-go::](#clean-go) - + [test-coverage-go::](#test-coverage-go) - + [test-coveralls::](#test-coveralls-1) - + [test-coverage-html::](#test-coverage-html) + + [Input Environment Variables](#input-environment-variables-6) + + [build-go](#build-go) + + [build-linux](#build-linux) + + [build-circle](#build-circle) + + [test-go](#test-go) + + [test-go-tparse](#test-go-tparse) + + [test-no-race](#test-no-race) + + [test-circle](#test-circle) + + [deps-go](#deps-go) + + [deps-circle](#deps-circle-1) + + [deps-coverage](#deps-coverage-1) + + [deps-status](#deps-status) + + [clean-go](#clean-go) + + [test-coverage-go](#test-coverage-go) + + [test-coverage-html](#test-coverage-html) * [common-kube.mk](#common-kubemk) + [circleci 2.0](#circleci-20-3) - + [Input Environment Variables:](#input-environment-variables-5) - + [Exported Environment Variables:](#exported-environment-variables-1) + + [Input Environment Variables](#input-environment-variables-7) + + [Exported Environment Variables](#exported-environment-variables-1) + [Multi-cluster deployments](#multi-cluster-deployments) - [Default Behavior](#default-behavior) - [Deloying to Other Clusters (not `general-01`)](#deloying-to-other-clusters-not-general-01) - [Deploying to Many Clusters](#deploying-to-many-clusters) - + [lint-kubeval::](#lint-kubeval) + + [lint-kubeval](#lint-kubeval) - [customizing](#customizing) - + [force-pod-restart::](#force-pod-restart) - + [update-secrets::](#update-secrets) - + [clean-secrets::](#clean-secrets) - + [update-configmaps::](#update-configmaps) - + [verify-deployment-rollout::](#verify-deployment-rollout) + + [force-pod-restart](#force-pod-restart) + + [update-secrets](#update-secrets) + - [How It Works](#how-it-works) + - [Directory Example](#directory-example) + - [Literal File Example](#literal-file-example) + - [Labels](#labels) + + [clean-secrets](#clean-secrets) + + [update-configmaps](#update-configmaps) + - [How It Works](#how-it-works-1) + - [Directory Example](#directory-example-1) + - [Literal File Example](#literal-file-example-1) + - [Labels](#labels-1) + + [verify-deployment-rollout](#verify-deployment-rollout) * [common-python3.mk](#common-python3mk) - + [Input Environment Variables:](#input-environment-variables-6) - + [check-format-python3::](#check-format-python3) - + [lint-python3::](#lint-python3) - + [test-python3-docker::](#test-python3-docker) - + [deps-python3::](#deps-python3) - + [deps-python3-docker::](#deps-python3-docker) + + [Input Environment Variables](#input-environment-variables-8) + + [check-format-python3](#check-format-python3) + + [lint-python3](#lint-python3) + + [test-python3-docker](#test-python3-docker) + + [deps-python3](#deps-python3) + + [deps-python3-docker](#deps-python3-docker) * [common-kustomize.mk](#common-kustomizemk) - + [Input Environment Variables:](#input-environment-variables-7) - + [build-kustomize::](#build-kustomize) - + [diff-kustomize::](#diff-kustomize) - + [deploy-kustomize::](#deploy-kustomize) + + [Input Environment Variables](#input-environment-variables-9) + - [Required](#required) + - [Optional](#optional) + + [build-kustomize](#build-kustomize) + + [diff-kustomize](#diff-kustomize) + + [deploy-kustomize](#deploy-kustomize) - [Contributing](#contributing) - [Versioning](#versioning) - * [Logging](#logging) - * [Pathfinding](#pathfinding) - * [Common Patterns for adding to the repo](#common-patterns-for-adding-to-the-repo) - * [Adding support for a new language](#adding-support-for-a-new-language) - * [README.md updates](#readmemd-updates) +- [Logging](#logging) +- [Pathfinding](#pathfinding) +- [Common Patterns for Adding to the Repo](#common-patterns-for-adding-to-the-repo) +- [Adding Support for a New Language](#adding-support-for-a-new-language) +- [README.md updates](#readmemd-updates) - [Handy Make stuff](#handy-make-stuff) -Introduction -============ +## Introduction This repo contains a library of Makefile tasks and shell scripts that provide common functions for working with a variety of systems at Pantheon. The @@ -151,26 +157,25 @@ the task is defined in a project's `Makefile` and you find yourself copying it into another repository, that is a good sign that it belongs here. Here are some good examples of common tasks that belong here: -* tasks for building go projects -* tasks for building docker containers -* tasks for managing resources in Kubernetes -* tasks for installing and deploying to a sandbox environment -This repository is **NOT** a good place to define every handy task. Please be +- tasks for building go projects +- tasks for building docker containers +- tasks for managing resources in Kubernetes +- tasks for installing and deploying to a sandbox environment + +This repository is __NOT__ a good place to define every handy task. Please be selective. -Usage -===== +## Usage -Setting Up the common makefiles ------------------------------- +### Setting Up the common makefiles Add these common tasks to your project by using git subtree from the root of your project. First add the remote. -``` +```sh git remote add common_makefiles git@github.com:pantheon-systems/common_makefiles.git --no-tags ``` @@ -179,12 +184,11 @@ Now add the subtree **note:** it is important that you keep the import path set to `devops/make` as the makefiles assume this structure. -``` +```sh git subtree add --prefix devops/make common_makefiles master --squash ``` -Using in your Makefile ----------------------- +### Using in your Makefile you simply need to include the common makefiles you want in your projects root Makefile: @@ -198,14 +202,13 @@ include devops/make/common-kube.mk include devops/make/common-go.mk ``` -Updating common makefiles -------------------------- +### Updating common makefiles The `common.mk` file includes a task named `update-makefiles` which you can invoke to pull and squash the latest versions of the common tasks into your project. Put the changes on a branch. -``` +```sh git checkout -b update-make make update-makefiles @@ -214,13 +217,12 @@ make update-makefiles If this is a new clone you may need to re-run these commands to register the common-make git repo before running `make update-makefiles`: -``` +```sh git remote add common_makefiles git@github.com:pantheon-systems/common_makefiles.git --no-tags git subtree add --prefix devops/make common_makefiles master --squash ``` -Extending Tasks ---------------- +### Extending Tasks All the common makefile tasks can be extended in your top level Makefile by defining them again. Each common task that can be extended has a `::` target. @@ -250,8 +252,7 @@ or set variables to modify its behavior (in this case directing it to compile build:: CMD=foo ``` -Usage with public repos ------------------------ +### Usage with public repos Ideally we should never add anything sensitive or secret to common-makefiles. Nonetheless, it is safest to prune out anything not needed by your project if @@ -259,9 +260,9 @@ it is going to be a public Github repo. Here is a method for pruning out unused files: -### Initial import +#### Initial import -- Create /Makefile with an extended `update-makefiles` task and a new +- Create PROJECT/Makefile with an extended `update-makefiles` task and a new `prune-common-make` task. The prune task should be customized to include only the files your project will need. Everything else will be removed locally and from git. Example project that only needs `common.mk` and `common-docker.mk`: @@ -284,25 +285,24 @@ prune-common-make: - Follow the standard procedures for adding the common_makefiles to your project: -``` +```sh git remote add common_makefiles git@github.com:pantheon-systems/common_makefiles.git --no-tags git subtree add --prefix devops/make common_makefiles master --squash ``` - And then execute the prune task created in the first step: -``` +```sh make prune-common-make ``` -### Future updates +#### Future updates After the initial import of common_makefiles the project can be updated in the standard way: `make update-makefiles`. The `prune-common-make` task will be executed after the updates are pulled in. -Usage with more complicated projects ----- +### Usage with more complicated projects This section describes approaches to building a Makefile for a project that builds and/or deploys more than one thing. @@ -311,7 +311,7 @@ Rather than have a makefile that uses the default targets directly, some indirection is necessary to accomplish multiple different invocations of those targets. -### recursive make +#### Recursive make To use this strategy, define your own targets with the same name as the default targets you plan to reuse, but implement them by calling make again, using @@ -320,7 +320,7 @@ that the default target could normally handle. For example, consider this directory layout: -``` +```text devops/make make/ - foo.mk @@ -335,21 +335,21 @@ Makefile your `Makefile` would have targets like this: -``` +```make build: $(MAKE) -C . -f make/$(TARGET).mk ``` and if you invoked it like this: -``` +```sh make build TARGET=foo ``` then you would use `make/foo` as your makefile, and it would look like a normal Makefile that consumes common-make: -``` +```make include $(COMMON_MAKE_DIR)/common-go.mk ``` @@ -358,7 +358,7 @@ We can further reuse default targets in our submake with another submake. For example, here we create a separate dev instance of `build-docker` and create a separate image for use with our `test` target: -``` +```make build-docker-dev:: export DOCKER_BUILD_ARGS := $(DOCKER_BUILD_ARGS) --build-arg=dev=true build-docker-dev:: export IMAGE := $(IMAGE)-dev build-docker-dev:: @@ -367,13 +367,12 @@ build-docker-dev:: test:: build-docker-dev docker run $(IMAGE)-dev $(TEST_CMD) ``` -Tasks -===== -common.mk ---------- +## Tasks + +### common.mk -### help +#### help `make help` prints out a list of tasks and descriptions. @@ -386,7 +385,7 @@ foo: ## this help message will be display by `make help` Example: -``` +```text $ make help foo this help message will be display by `make help` ``` @@ -398,7 +397,7 @@ your Makefile, eg: HIDE_TASKS = deps deps-circle test-circle ``` -### update-makefiles +#### update-makefiles Use this to pull the latest `master` branch from `common_makefiles` into the local project. (Assumes `common_makefiles` were imported to `./devops/make`) @@ -406,41 +405,42 @@ local project. (Assumes `common_makefiles` were imported to `./devops/make`) If you get an error such as `fatal: 'common_makefiles' does not appear to be a git repository` you may need to add the remote repo before running this task: -``` +```sh git remote add common_makefiles git@github.com:pantheon-systems/common_makefiles.git --no-tags ``` -#### output variables +##### output variables - `ROOT_DIR` -- the full path to the root of your repository. Useful for supporting execution of `make` in subdirectories of your repo, and some scenarios where you find you need multiple includes and/or recursive make invocation. -## common-apollo.mk +### common-apollo.mk Common tasks for working with Apollo Studio for GraphQL Services. -### Input +#### Input - `APP`: (required) The name of the app. - `GQL_SCHEMA_PATH`: (required) path to the schema.graphqls file +- `APOLLO_GRAPH_TARGET`: (optional) the name of the graph to check or update against. Defaults to "pantheon". +- `APOLLO_VARIANT_TARGET`: (optional) the name of the variant to check or update. Defaults to "current". -### Tasks +#### Tasks -#### check-apollo-schema +##### check-apollo-schema Checks schema changes against production to ensure any changes are compatable -#### update-apollo-schema +##### update-apollo-schema Updates schema for your app on Apollo Studio -common-docs.mk --------------- +### common-docs.mk Common tasks for managing documentation. -### circleci 2.0 +#### circleci 2.0 If using a docker-based circleci 2.0 build environment ensure that a remote docker is available in your `.circleci/config.yml`: @@ -450,13 +450,12 @@ is available in your `.circleci/config.yml`: - setup_remote_docker ``` -### update-readme-toc +#### update-readme-toc Run `make update-readme-toc` to update the TOC in `./README.md`. Uses [markdown-toc](https://github.com/jonschlinkert/markdown-toc#cli) to edit in place. - -### test-readme-toc +#### test-readme-toc This task executes `markdoc-toc` via Docker and compares the output to the current TOC in the on-disk `./README.md` file. If they differ, a diff output @@ -469,33 +468,35 @@ This task requires Docker to be running. This task is added to the global `test` task. -common-docker.mk ----------------- +### common-docker.mk + We push our images to quay.io by default. This file includes `common-docker-quay.mk` to maintain compatbility for existing projects that use `common-docker.mk`. -### push-circle:: +#### push-circle Runs the `build-docker` and `push` tasks. DEPRECATED. Run these commands separately instead. -_docker.mk ----------------- -### Input Environment Variables: +### _docker.mk -None +#### Input Environment Variables -### Exported Environment Variables: +- `NEW_TAG_STRATEGY`: (optional) Indicating if the new tag strategy should be used. -- `BUILD_NUM`: The build number for this build. Will use `$(DEFAULT_SANDBOX_NAME)-$(COMMIT)` if not building - on circleCI, will use `$(CIRCLE_BUILD_NUM)-$(CIRCLE_BRANCH)"` otherwise. +#### Exported Environment Variables +- `BUILD_NUM`: The build number for this build. + - `$(CIRCLE_BUILD_NUM)-$(CIRCLE_BRANCH)"` if in CircleCI and `NEW_TAG_STRATEGY` is not defined. (`123-master`) + - `$(PANTS_DEFAULT_SANDBOX_NAME)-$(COMMIT[8])` if `pants` return a value. (`sandbox-pantheor-12345678`) + - `dev-$(COMMIT[8])"` if no result from `pants` and `BRANCH` is not defined. (`dev-12345678`) + - `$(BRANCH)-$(COMMIT[8])"` if no result from `pants` and `BRANCH` is defined. (`feature-branch-12345678`) - `DOCKER_BYPASS_DEFAULT_PUSH`: If you need to provide custom logic for tagging and pushing images to an artifact repository, add `DOCKER_BYPASS_DEFAULT_PUSH=true` to your `Makefile` and the default push step will be skipped. -### build-docker:: +#### build-docker Runs `build-docker.sh` -### lint-hadolint:: +#### lint-hadolint Runs `hadolint` on all Dockerfiles found in the repository. @@ -504,45 +505,51 @@ You can set `REQUIRE_DOCKER_LINT := no` in your `Makefile` to make it pass silen This task is added to the global `lint` task. -common-docker-ar.mk --------------- -### Input Environment Variables +### common-docker-ar.mk + +#### Input Environment Variables + - `AR_IMAGE`: the docker image to use. Will be computed if it doesn't exist - `AR_REGISTRY`: The docker registry to use. Set to Google Artifact Registry -### Export Environment Variables +#### Export Environment Variables + - `AR_IMAGE`: The image to use for the build. - `AR_REGISTRY`: The registry to use for the build - `AR_IMAGE_BASENAME`: The image without the tag field on it.. i.e. foo:1.0.0 would have image basename of 'foo' - `AR_REGISTRY_PATH`: Registry url and repo name -### push:: +#### push + Invokes `push-ar` -### push-ar:: +#### push-ar + Invokes `setup-ar` then: Runs `docker push $(IMAGE)` to push the docker image and tag to artifact registry. -### setup-ar:: +#### setup-ar + When invoked from Circle CI this task will setup the gsa for the common circle GSA to be used with our common AR repositories. Invoked as a dependency of `push-ar` Runs `setup-circle-vault.sh`. This script does the following + - Installs vault - Installs pvault - Runs `gcloud auth activate-service-account` which authenticates the account into the Google Cloud CLI by reading and decoding production vault GSA for `circleci-common@pantheon-internal.iam.gserviceaccount.com` +### common-docker-quay.mk -common-docker-quay.mk --------------- -### circleci 2.0 +#### circleci 2.0 To push a container to quay.io upon a successful master build: + - On Circle-CI, navigate to *Project Settings > Environment Variables*. - Add the following environment vars. Ask `@infra` on Slack if you need assistance. -``` +```text QUAY_USER: getpantheon+circleci QUAY_PASSWD: ``` @@ -558,26 +565,30 @@ is available in your `.circleci/config.yml`: Note that some functionality is not available with remote docker such as volume mounts. If you need to use volume mounts on circleci 2.0 you will need to use the VM-based build environment instead of docker. -### push:: + +#### push Invokes `push-quay` -### push-quay:: + +#### push-quay + Runs `docker push $(IMAGE)` to push the docker image and tag to quay. -### Input Environment Variables +#### Input Environment Variables + - `QUAY_USER`: The quay.io user to use (usually set in CI) - `QUAY_PASSWD`: The quay passwd to use (usually set in CI) - `IMAGE`: the docker image to use. will be computed if it doesn't exist. - `REGISTRY`: The docker registry to use. Defaults to quay. -### Export Environment Variables +#### Export Environment Variables + - `IMAGE`: The image to use for the build. - `REGISTRY`: The registry to use for the build. - `IMAGE_BASENAME`: The image without the tag field on it.. i.e. foo:1.0.0 would have image basename of 'foo' - `REGISTRY_PATH`: Registry url and repo name -common-shell.mk --------------- +### common-shell.mk Common tasks for shell scripts, such as [shellcheck](https://www.shellcheck.net/) for linting shell scripts. @@ -589,9 +600,9 @@ when writing shell scripts. > Shellcheck will error with "Segmentation Fault" on debian/ubuntu based images > which includes our deploy-toolbox and most circleci/* images. > -> Workarounds: Alpine or Fedora/Redhat images should work. Or this workaround: https://github.com/koalaman/shellcheck/issues/1053 +> Workarounds: Alpine or Fedora/Redhat images should work or this [workaround](https://github.com/koalaman/shellcheck/issues/1053). -### Input Environment Variables: +#### Input Environment Variables - `SHELL_SOURCES`: (optional) A list of shell scripts that should be tested by the `test-shell` and `test` tasks. If none is provided, `find . -name \*.sh` @@ -600,14 +611,13 @@ when writing shell scripts. - `SHELLCHECK_VERSION`: (optional) The version of shellcheck to be installed by the `deps-circle` task. -### test-shell +#### test-shell Run shell script tests such as `shellcheck`. This task is added to the global `test` task. -common-pants.mk ---------------- +### common-pants.mk Installs pants version greater than `0.1.3` unless overridden with `PANTS_VERSION` so that a sandbox integration environment can be created on @@ -619,14 +629,14 @@ circle for integration testing and acceptance criteria review. - `PANTS_INCLUDE`: (optional) The services for pants to install or update. E.g `make init-pants PANTS_INCLUDE=notification-service,ticket-management` -## install-circle-pants +### install-circle-pants -Installs the `pants` utility on Circle-CI from https://github.com/pantheon-systems/pants +Installs the `pants` utility on Circle-CI from the [pants git repo](https://github.com/pantheon-systems/pants). This task is added to the global `deps-circle` task. If `make deps-circle` is already in your circle-ci config file then you only need to `include common-pants.mk` in your Makefile. -## delete-circle-pants +### delete-circle-pants Deletes the sandbox environment based on the branch if one exists to prepare for the deployment. @@ -634,16 +644,15 @@ for the deployment. This task is added to the global `deps-circle` task. If `make deps-circle` is already in your circle-ci config file then you only need to `include common-pants.mk` in your Makefile. -## init-circle-pants +### init-circle-pants Creates a kube environment against the `testing.onebox.panth.io` as the ygg-api. The kube environment is set with the `KUBE_NAMESPACE` following the convention `sandbox-REPO_NAME-BRANCH_NAME`. -common-conda.mk ---------------- +### common-conda.mk -### Notes: +#### Notes Conda is an open source package management system and environment management system for installing multiple versions of software packages and their @@ -663,14 +672,12 @@ To prevent mistakes, some targets are protected from being run inside a conda environment by using the `_assert-conda-env-active` and `_assert-conda-env-not-active` targets respectively. -### Inheritable Input Environment Variables from common-python.mk: +#### Inheritable Input Environment Variables from common-python.mk - `PYTHON_PACKAGE_NAME`: (required) The name of the python package. - `TEST_RUNNER`: (optional) The name of the python test runner to execute. Defaults to `trial` -- `COVERALLS_TOKEN`: (optional) Token to use when pushing coverage to coveralls. Required if using - the `test-coveralls` task -### Input Environment Variables: +#### Input Environment Variables - `TEST_RUNNER`: (required) The name of the test runner to execute. Inherited from common-python.mk @@ -692,7 +699,7 @@ targets respectively. - `CONDA_PACKAGE_LABEL`: (optional) The label that will be applied to the conda package on deployment. Defaults to `main` -### deps-conda:: +#### deps-conda Downloads the miniconda installation script. The target uses `uname -s` to determine which installation script to download. Currently only `Darwin` (OSX) @@ -701,7 +708,7 @@ and `Linux` are supported. Runs the installation script and adds the path to to disable automatic upload to anaconda cloud after a build. This target is added to the global `deps` target. -### setup-conda:: +#### setup-conda Setup the conda virtual environment for this project. Looks for an environment.yml file in the project root. @@ -709,7 +716,7 @@ file in the project root. Runs `conda env create || conda env update` This target is added to the global `setup` target. -### clean-conda:: +#### clean-conda Removes index cache, lock files, tarballs, unused cache packages, and source cache. @@ -717,19 +724,19 @@ Runs `conda clean --all -y`. This target is added to the global `clean` target. -### reset-conda-environment:: +#### reset-conda-environment Reset a conda environment by removing and reinstalling all of its packages. Runs `conda remove --name $(CONDA_PACKAGE_NAME) --all -y` and `conda env update` -### build-conda:: +#### build-conda Build conda package for project with current arch. A no arch package can be built by configuring the conda recipe. Runs `conda build recipe --no-anaconda-upload`. -### build-conda-deployment-environment:: +#### build-conda-deployment-environment Clones the project conda environment into the project directory `./local`. This environment can be copied directly into the Docker container as the deployment @@ -737,14 +744,14 @@ artifact. Runs `conda create --clone $(CONDA_PACKAGE_NAME) -y --prefix ./local --copy`. -### deploy-conda:: +#### deploy-conda Requires ANACONDA_CLOUD_DEPLOY_TOKEN to be set. Deploys the built conda package to Anaconda Cloud. Runs `conda build -q --user $(ANACONDA_CLOUD_ORGANIZATION) --token $(ANACONDA_CLOUD_DEPLOY_TOKEN) recipe` -### deploy-conda-pypi:: +#### deploy-conda-pypi Requires ANACONDA_CLOUD_DEPLOY_TOKEN to be set. Deploys the latest built pypi package to Anaconda Cloud. Distributing private pypi packages is a paid feature @@ -753,7 +760,7 @@ be downloaded on the dashboard or using the API. Runs `anaconda --token $(ANACONDA_CLOUD_DEPLOY_TOKEN) upload -u $(ANACONDA_CLOUD_ORGANIZATION) --label $(CONDA_PACKAGE_LABEL) --no-register --force dist/$(CONDA_PACKAGE_NAME)-$(CONDA_PACKAGE_VERSION).tar.gz`. -### regenerate-anaconda-cloud-repo-token:: +#### regenerate-anaconda-cloud-repo-token A helper to generate a personal read-only token for downloading private conda packages suitable for local development. If not logged into anaconda client this @@ -765,50 +772,45 @@ The output of this target is an ANACONDA_CLOUD_REPO_TOKEN which should be export Run `make admin-regenerate-anaconda-cloud-repo-token` Copy the token then run `export ANACONDA_CLOUD_REPO_TOKEN=_TOKEN_GOES_HERE_` -### add-conda-private-channel:: +#### add-conda-private-channel Adds the pantheon private channel to your conda config for downloading conda packages from Anaconda Cloud. Requires ANACONDA_CLOUD_REPO_TOKEN to be set. -### generate-conda-requirements:: +#### generate-conda-requirements Helper to generate a full dependency tree of this conda environment into a requirements_full.txt -### reset-conda-environment:: +#### reset-conda-environment Helper to reset a conda environment by removing and reinstalling all of its packages. -common-python.mk ----------------- +### common-python.mk -### Input Environment Variables: +#### Input Environment Variables - `PYTHON_PACKAGE_NAME`: (required) The name of the python package. - `TEST_RUNNER`: (optional) The name of the python test runner to execute. Defaults to `trial` -- `COVERALLS_TOKEN`: (optional) Token to use when pushing coverage to coveralls. Required if using - the `test-coveralls` task -### build-python:: +#### build-python Run `python setup.py sdist` in the current project directory. This task is added to the global `build` task. -### test-python:: +#### test-python Runs targets `test-coverage-python` and target the global `lint` target. This task is added to the global `test` task. -### test-circle-python:: +#### test-circle-python -Intended for use in circle-ci config to run tests under the Circle-CI context. This -target additionally calls target test-coveralls-python which runs `coveralls` -to report coverage metrics. +Intended for use in circle-ci config to run tests under the CircleCI context. -### deps-python:: +#### deps-python Install this projects' Python dependencies which includes the targets deps-testrunner-python, deps-lint-python and deps-coverage-python @@ -817,38 +819,33 @@ NOTE: Currently assumes this project is using `pip` for dependency management. This task is added to the global `deps` task. -### deps-circle:: +#### deps-circle -Install dependencies on Circle-CI which includes the targets deps-coveralls-python +Install dependencies on CircleCI. -### deps-coverage:: +#### deps-coverage -Install dependencies necessary for running the test coverage utilities like -coveralls. +Install dependencies necessary for running the test coverage utilities. -### test-coverage-python:: +#### test-coverage-python Run `coverage run --branch --source $(PYTHON_PACKAGE_NAME) $(shell which $(TEST_RUNNER)) $(PYTHON_PACKAGE_NAME)` which creates the coverage report. This task is added to the global `test-coverage` task. -### test-coveralls:: - -Run `coveralls` which sends the coverage report to coveralls. - -Requires `COVERALLS_TOKEN` environment variable. - -### coverage-report:: +#### coverage-report Run `coverage report` on the last generated coverage source. -### lint-python:: +#### lint-python + Run targets `lint-pylint` and `lint-flake8` This task is added to the global `lint` task. -### lint-pylint:: +#### lint-pylint + Run `pylint $(PYTHON_PACKAGE_NAME)` Pylint is a Python source code analyzer which looks for programming errors, @@ -858,7 +855,8 @@ python package which is useful for catching misconfigured setup.py files. This task is added to `lint-python` task. -### lint-flake8:: +#### lint-flake8 + Run `flake8 --show-source --statistics --benchmark $(PYTHON_PACKAGE_NAME)` Flake8 is a combination of three tools (Pyflakes, pep8 and mccabe). Flake8 @@ -870,22 +868,20 @@ and can be run on an installed python package. This preserves flexibility. This task is added to `lint-python` task. -common-go.mk ------------- +### common-go.mk -### circleci 2.0 +#### circleci 2.0 When using circleci 2.0 it is recommended to use one of the circleci provided Go primary containers if possible. The latest version of Go will be available. Updating to a new version involves bumping the container tag. -### Input Environment Variables: +#### Input Environment Variables -- `COVERALLS_TOKEN`: Token to use when pushing coverage to coveralls. - `FETCH_CA_CERT:` The presence of this variable will add a Pull root ca certs - to ca-certificats.crt before build. + to ca-certificats.crt before build. -### build-go:: +#### build-go Run `go build` in the current project directory if any go code files have been updated and the binary (at `./$(APP)` by default) isn't already present. @@ -897,94 +893,86 @@ images, ie one for each binary. This task is added to the global `build` task. -### build-linux:: +#### build-linux Build a static Linux binary. (Works on any platform.) -### build-circle:: +#### build-circle Intended for use in circle-ci config files to run a build under the Circle-CI context. -### test-go:: +#### test-go Run `go test` against all go packages in the project. Does not run tests for packages in directories `devops/` and `vendor/` or with `e2e_tests` in their name. -### test-go-tparse:: +#### test-go-tparse -Run `go test` against all go packages in the project and formats the output using tparse (https://github.com/mfridman/tparse). +Run `go test` against all go packages in the project and formats the output using [tparse](https://github.com/mfridman/tparse). Does not run tests for packages in directories `devops/` and `vendor/` or with `e2e_tests` in their name. This task is added to the global `test` task. -### test-no-race:: +#### test-no-race Run `go test` without the race dectector. -### test-circle:: +#### test-circle Intended for use in circle-ci config to run tests under the Circle-CI context. -### deps-go:: +#### deps-go Install this projects' Go dependencies and tools. -If you are using go modules, then the target will use `go get`. +If you are using Go modules, then the target will use `go mod download`. -Pass arguments to `go get` by setting the `GO_GET_ARGS` variable in your +Pass arguments to `go mod download` by setting the `GO_GET_ARGS` variable in your Makefile: ```make GO_GET_ARGS := -d ``` -If there is a `./vendor` directory, then `go get` is not called, +If there is a `./vendor` directory, then `go mod download` is not called, and the `./vendor` directory is deleted. This task is added to the global `deps` task. -### deps-circle:: +#### deps-circle Install dependencies on Circle-CI -### deps-coverage:: +#### deps-coverage -Install dependencies necessary for running the test coverage utilities like -coveralls. +Install dependencies necessary for running the test coverage utilities. -### deps-status:: +#### deps-status Check status of dependencies with gostatus. -### clean-go:: +#### clean-go Delete all build artifacts. Executed by default with common target `clean::`. -### test-coverage-go:: +#### test-coverage-go Run `go cov` test coverage report. This task is added to the global `test-coverage` task. -### test-coveralls:: - -Run test coverage report and send it to coveralls. - -Requires `COVERALLS_TOKEN` environment variable. - -### test-coverage-html:: +#### test-coverage-html Run go test coverage report and output to `./coverage.html`. -common-kube.mk --------------- +### common-kube.mk -### circleci 2.0 +#### circleci 2.0 - On Circle-CI, navigate to *Project Settings > Environment Variables*. - Ask `@infra` on Slack to assist in adding the `GCLOUD_EMAIL` and `GCLOUD_KEY` env vars. @@ -1001,7 +989,7 @@ jobs: steps: ``` -### Input Environment Variables: +#### Input Environment Variables - `APP`: should be defined in your topmost Makefile - `SECRET_FILES`: list of files that should exist in secrets/* used by @@ -1014,7 +1002,7 @@ jobs: - `KUBE_NAMESPACE`: set this to the namespace you wish to deploy to, ie `template-sandbox` if you're trying to update template-sandbox -### Exported Environment Variables: +#### Exported Environment Variables NOTE: variables that appear hear and above are determined automatically if not set @@ -1037,11 +1025,11 @@ NOTE: variables that appear hear and above are determined automatically if not s If no namespace or context is defined, common-kube will use a different set of defaults depending on the current environment and branch: -| CircleCI | Branch | Default context | Default namespace | -|----------|-------------------|-----------------|--------------------------| -| Yes | `master` / `main` | `general-01` | `production` | -| Yes | Other branch | `sandbox-01` | `sandbox-[APP]-[BRANCH]` | -| No | Any branch | _pants default_ | _pants default_ | +| CircleCI | Branch | Default context | Default namespace | +|----------|-------------------|-------------------|--------------------------| +| Yes | `master` / `main` | `general-01` | `production` | +| Yes | Other branch | `sandbox-01` | `sandbox-[APP]-[BRANCH]` | +| No | Any branch | __pants default__ | __pants default__ | - If `CIRCLECI` env var is not defined (ie: not running on circle), the `pants` utility will be invoked to retrieve your default sandbox name. If this succeeds the value @@ -1057,9 +1045,10 @@ To override default context and/or namespace: - Abbreviated context: `CLUSTER_DEFAULT=general-04` - Namespace: `KUBE_NAMESPACE=namespace` - Template sandbox: `KUBE_NAMESPACE=template-sandbox` - - is equivalent to: `KUBE_NAMESPACE=template-sandbox CLUSTER_DEFAULT=sandbox-02` + - equivalent to: `KUBE_NAMESPACE=template-sandbox CLUSTER_DEFAULT=sandbox-02` + +#### Multi-cluster deployments -### Multi-cluster deployments Common make creates all of the connection information for main kubernetes clusters that we interact with. These are setup to be used as contexts for your `kubectl` commands for switching between clusters. If the context is not specified then commands will default to the cluster specified in CLUSTER_DEFAULT @@ -1071,17 +1060,20 @@ Example: This will get the pods on the cluster and namespace specified. This should be the long cluster name. -#### Default Behavior +##### Default Behavior + By default, all common make tools will use the default cluster (as set for `kubectl` tooling), in addition it will create connections to each of the other clusters. -#### Deloying to Other Clusters (not `general-01`) +##### Deloying to Other Clusters (not `general-01`) + To uniformly deploy to a different cluster than the default you can specify `CLUSTER_DEFAULT=` inside your make file to point everything to that deployment cluster. -#### Deploying to Many Clusters +##### Deploying to Many Clusters + To do this you must specify the --context argument to your `kubectl` calls inside your common make primary makefile. The context will specify for that command what cluster to deploy to. For example: @@ -1093,7 +1085,7 @@ Using this for resources deploying to general-01 Using this for CronJob, deploying to general-02 `kubectl --context general-02 create ...` -### lint-kubeval:: +#### lint-kubeval Runs `kubeval` by default on all YAMLS found in `KUBE_YAMLS_PATH` (defaults to `./devops/k8s/`) in the repository, but it will skip files in `KUBE_YAMLS_PATH/configmaps` and any path with substring `template.`. @@ -1102,15 +1094,16 @@ If `KUBE_YAMLS_PATH` is not present, `lint-kubeval` is skipped. If `KUBE_YAMLS_PATH` is present, but the command `kubeval` is not installed, `lint-kubevals` fails with an error message: -``` +```text devops/make/common-kube.mk:186: *** "kubeval is not installed! please install it.". Stop. ``` This task is added to the global `lint` task. -#### customizing +##### customizing Set these variables to change behavior + - `KUBE_YAMLS_PATH` -- Defaults to `./devops/k8s`. Extend it so it references a subdirectory, eg `KUBE_YAMLS_PATH=./devops/k8s/manifests/database` so you can 1. deploy a separate database component without deploying anything else @@ -1123,15 +1116,16 @@ Set these variables to change behavior - `SKIP_KUBEVAL` -- if defined, the call to lint-kubeval from common-kube.mk will be skipped. This provides the opportunity to render configuration files of the project before calling lint-kubeval. -### force-pod-restart:: +#### force-pod-restart Nuke the pod in the current `KUBE_NAMESPACE`. -### update-secrets:: +#### update-secrets Requires `$APP` variable to be set. Requires `$KUBE_NAMESPACE` variable to be set. -Requires **one** of these directories to have files meant to be applied: +Requires __one__ of these directories to have files meant to be applied: + - ./devops/k8s/secrets/[KUBE_CONTEXT]/production/ - ./devops/k8s/secrets/[KUBE_CONTEXT]/[NAMESPACE]/ - ./devops/k8s/secrets/[KUBE_CONTEXT]/non-prod/ @@ -1140,13 +1134,14 @@ Requires **one** of these directories to have files meant to be applied: - ./devops/k8s/secrets/non-prod/ There secrets can be created two ways: + 1. From a set of files in a directory named after the secret. Each file will use its name as a key name for the secret and the data in the file as the value. 2. From a 'literal' map. Make a file that has a set of k=v pairs in it one per line. Each line will have its data split into secrets keys and values. -_How it works:_ +##### How It Works Put secrets into files in a directory such as `./devops/k8s/secrets/non-prod/[namespace]`, @@ -1164,27 +1159,26 @@ NOTE: The `$APP` variable will be prepended to the volume name. eg: A directory path of `./devops/k8s/secrets/template-sandbox/certs` and `APP=foo` will create a secret volume named `foo-certs` in the template-sandbox namespace. -_Directory Example:_ +##### Directory Example The Directory method can be used for anthing that you want translated directly into a kube secret from a file. This could be an RSA key or an entire JSON file from GCE. The contents of the file will be encoded using base64 and added to the kube secret. Move the file to a directory under the last level of the path used for literal files. In the example below, the extra directory after 'production' for 'api-keys' will cause the `make update-secrets` command below to follow the path of creating the kube object using a file with a directory instead of from a literal file. Additionally, naming the directory 'api-keys' will append that name to the end of the 'app' name, making the final secret name in Kube "app-api-keys". -``` +```sh # for production: +mkdir -p ./devops/k8s/secrets/production/api-keys +echo -n "secret-API-key!" >./devops/k8s/secrets/production/api-keys/key1.txt +echo -n "another-secret-API-key!" >./devops/k8s/secrets/production/api-keys/key2.txt +make update-secrets KUBE_NAMESPACE=production APP=foo -$ mkdir -p ./devops/k8s/secrets/production/api-keys -$ echo -n "secret-API-key!" >./devops/k8s/secrets/production/api-keys/key1.txt -$ echo -n "another-secret-API-key!" >./devops/k8s/secrets/production/api-keys/key2.txt -$ make update-secrets KUBE_NAMESPACE=production APP=foo - -# cleanup secrets, do not check them into git! -$ rm -rf -- ./devops/k8s/secrets/* +# clean up secrets, do not check them into git! +rm -rf -- ./devops/k8s/secrets/* ``` Verify the volume was created and contains the expected files: -``` +```text $ kubectl describe secret foo-api-keys --namespace=production Name: foo-api-keys Namespace: production @@ -1198,10 +1192,11 @@ key1.txt: 15 bytes key2.txt: 22 bytes ``` -_Literal File Example_ +##### Literal File Example Make a file with k=value pairs, and name it what you want the secret to be called. -``` + +```text $ cat ./devops/k8s/secrets/non-prod/foo-secrets secret1=foo secret2=bar @@ -1209,26 +1204,28 @@ secret3=baz ``` Apply the secrets -``` -$ make update-secrets KUBE_NAMESPACE=template-sandbox + +```sh +make update-secrets KUBE_NAMESPACE=template-sandbox ``` Verify the secrets contents -``` + +```text $ kubectl describe secrets myapp-foo-secrets --namespace=template-sandbox -Name: myapp-foo-secrets -Namespace: template-sandbox -Labels: app=myapp -Annotations: +Name: myapp-foo-secrets +Namespace: template-sandbox +Labels: app=myapp +Annotations: Data ==== -secret1: 3 bytes -secret2: 3 bytes -secret3: 3 bytes +secret1: 3 bytes +secret2: 3 bytes +secret3: 3 bytes ``` -_Labels_ +##### Labels By default a `app=$APP` label will be applied to the configmaps. This can be overridden by setting the `LABELS` environment variable. You should almost always @@ -1242,18 +1239,19 @@ Example `Makefile` task to set custom `LABELS`: update-configmaps:: LABELS="app=$(APP),cos-system-service=true" ``` -### clean-secrets:: +#### clean-secrets Delete all uncommitted files and directories in ./devops/k8s/secrets, including the directory itself. Executed by default with common target `clean::`. -### update-configmaps:: +#### update-configmaps Requires `$APP` variable to be set. Requires `$KUBE_NAMESPACE` variable to be set. Requires one of these directories to have files meant to be applied: + - ./devops/k8s/configmaps/[KUBE_CONTEXT]/production/ - ./devops/k8s/configmaps/[KUBE_CONTEXT]/[NAMESPACE]/ - ./devops/k8s/configmaps/[KUBE_CONTEXT]/non-prod/ @@ -1263,14 +1261,14 @@ Requires one of these directories to have files meant to be applied: Use this task to upload Kubernetes configmaps. -_How it works:_ +##### How It Works There are 2 types of configmaps that can be used. A configmap complied from a set of files in a directory or a 'literal' map. Directory of files is what it sounds like; make a directory and put files in it. Each file will use its name as a key name for the configmap, and the data in the file as the value. A Literal map is a file that has a set of k=v pairs in it one per line. Each line will have it's -data split into configmap keys and values. _BEWARE_ that the value should not be +data split into configmap keys and values. *BEWARE* that the value should not be quoted. Due to how the shell interpolation happens when passing these k=v pairs to kubectl quote strings will be literal quoted strings in the kube config map. @@ -1293,21 +1291,24 @@ A directory path of `./devops/k8s/configmaps/template-sandbox/config-files` and `APP=foo` will create a configmap named `foo-config-files` in the `template-sandbox` namespace. -_Directory Example:_ +##### Directory Example Make the map directory. Given the app named foo this will become a configmap named foo-nginx-config -``` -$ mkdir -p ./devops/k8s/configmaps/non-prod/nginx-config + +```sh +mkdir -p ./devops/k8s/configmaps/non-prod/nginx-config ``` Put your app config in the directory you just created -``` + +```text $ ls ./devops/k8s/configmaps/non-prod/nginx-config common-location.conf common-proxy.conf common-server.conf nginx.conf verify-client-ssl.conf websocket-proxy.conf ``` Apply the map with the make task -``` + +```text $ make update-configmaps KUBE_NAMESPACE=sandbox-foo # this error is fine, it would say deleted if it existed Error from server: configmaps "foo-nginx-config" not found @@ -1317,30 +1318,31 @@ configmap "foo-nginx-config" labeled Verify the volume was created and contains the expected files: -``` +```text $ kubectl describe configmap foo-nginx-config --namespace=sandbox-foo kubectl describe configmap foo-nginx-config --namespace=sandbox-foo -Name: foo-nginx-config -Namespace: sandbox-foo -Labels: app=foo -Annotations: +Name: foo-nginx-config +Namespace: sandbox-foo +Labels: app=foo +Annotations: Data ==== -verify-client-ssl.conf: 214 bytes -websocket-proxy.conf: 227 bytes -common-location.conf: 561 bytes -common-proxy.conf: 95 bytes -common-server.conf: 928 bytes -nginx.conf: 2357 bytes +verify-client-ssl.conf: 214 bytes +websocket-proxy.conf: 227 bytes +common-location.conf: 561 bytes +common-proxy.conf: 95 bytes +common-server.conf: 928 bytes +nginx.conf: 2357 bytes ``` -_Literal File Example_ +##### Literal File Example Make a file with k=value pairs, and name it what you want the map to be called. Given I am in 'myapp' using commonmake and I run these configs the resultant map will be 'myapp-foo-config' -``` + +```text $ cat ./devops/k8s/configmaps/non-prod/foo-conf setting1=foo setting2=bar @@ -1348,26 +1350,28 @@ setting3=baz ``` Apply the map -``` -$ make update-configmaps KUBE_NAMESPACE=sandbox-foo + +```sh +make update-configmaps KUBE_NAMESPACE=sandbox-foo ``` Verify the map contents -``` + +```text $ kubectl describe configmap myapp-foo-config --namespace=sandbox-foo -Name: myapp-foo-config -Namespace: sandbox-foo -Labels: app=myapp -Annotations: +Name: myapp-foo-config +Namespace: sandbox-foo +Labels: app=myapp +Annotations: Data ==== -setting1: 3 bytes -setting2: 3 bytes -setting3: 3 bytes +setting1: 3 bytes +setting2: 3 bytes +setting3: 3 bytes ``` -_Labels_ +##### Labels By default a `app=$APP` label will be applied to the configmaps. This can be overridden by setting the `LABELS` environment variable. You should almost always @@ -1381,40 +1385,40 @@ Example `Makefile` task to set custom `LABELS`: update-configmaps:: LABELS="app=$(APP),cos-system-service=true" ``` -### verify-deployment-rollout:: +#### verify-deployment-rollout Checks for a successful rollout of a Kubernetes Deployment. This would typically be called in your CI/CD pipeline's `deploy` step. If the rollout fails an attempt will be made to rollback to the previous Deployment. The rollback may also fail, however, and you are responsible for confirming the status of your service. Note that this is only intended for use with Kubernetes Deployment resources (eg: not StatefulSets, CronJobs) -common-python3.mk ----------------- +### common-python3.mk -### Input Environment Variables: +#### Input Environment Variables - `PYTHON_PACKAGE_NAME`: The name of the python package. - `IMAGE`: The image to use for the build. If also using `common-docker.mk`, this should already be defined. -### check-format-python3:: +#### check-format-python3 Run `pipenv run black --check --skip-string-normalization --line-length 120 .` in the current project directory. This task is added to the global `lint` task. -### lint-python3:: +#### lint-python3 Runs `pipenv run pylint $(PYTHON_PACKAGE_NAME)`. This task is added to the global `lint` task. -### test-python3-docker:: +#### test-python3-docker Runs tests in a Docker image with the following command: -``` -docker run $(IMAGE) .venv/bin/python setup.py test + +```sh +docker run $IMAGE .venv/bin/python setup.py test ``` -### deps-python3:: +#### deps-python3 Install this project's Python runtime and dev dependencies. @@ -1422,15 +1426,13 @@ NOTE: Currently assumes this project is using `pipenv` for dependency management This task is added to the global `deps` task. -### deps-python3-docker:: +#### deps-python3-docker Install this project's Python runtime dependencies. NOTE: Currently assumes this project is using `pipenv` for dependency management. - -common-kustomize.mk ----------------- +### common-kustomize.mk Provides targets for deploying apps to kube using [kustomize](https://kustomize.io/). @@ -1441,16 +1443,16 @@ The instance path (relative to this directory) is then provided (via `INSTANCE`) For example: `make deploy-kustomize INSTANCE=prod/cluster1` will build `devops/kustomize/instances/prod/cluster1` and apply it to the current kube cluster/namespace. -### Input Environment Variables: +#### Input Environment Variables -**Required:** +##### Required Only one of the following is required: - `INSTANCE`: The kustomize instance (a path relative to `devops/kustomize/instances`) to deploy/diff/build. - `KUSTOMIZATION`: The full kustomization path. May be provided if your project structure is difference than the default. -**Optional:** +##### Optional - `AR_IMAGE`: The image to deploy. If also using `common-docker-ar.mk`, this should already be defined. - `IMAGE`: The image to deploy. If also using `common-docker.mk` or `common-docker-quay.mk`, this should already be defined. This will take precedence over `AR_IMAGE`. @@ -1459,45 +1461,43 @@ Only one of the following is required: - `KUBECTL_CMD`: Path to the `kubectl` binary. If also using `common-kube.mk`, this should already be defined. - `KUSTOMIZE_CMD`: Path to the `kustomize` binary. (Automatically determined from `$PATH` if not provided). -### build-kustomize:: +#### build-kustomize Builds an instance kustomization and displays it by printing it to stdout. -### diff-kustomize:: +#### diff-kustomize Builds an instance kustomization and diffs it's content against the content in the kube server. -### deploy-kustomize:: +#### deploy-kustomize Builds an instance kustomization and applies it's content to the kube server. -Contributing -============ +## Contributing -make edits here and open a PR against this repo. Please do not push from your +Make edits here and open a PR against this repo. Please do not push from your subtree on your project. -1. Have an idea -2. Get feedback from the beautiful people around you -3. Document your new or modified task in this README +1. Have an idea. +2. Get feedback from the beautiful people around you. +3. Document your new or modified task in this README. 4. Not everyone reads markdown files in a web browser, please try to wrap lines in this README at or near 80 characters. 5. Paragraphs should be separated by blank lines to facilitate proper rendering in both web and text formats. 6. Commit on a branch (please squash closely related commits into contextually - single commits) -7. Send PR + single commits). +7. Send PR. + +## Versioning -Versioning -============ -This repository uses SemVer style versioning. We use [autotag](https://github.com/pantheon-systems/autotag) +This repository uses SemVer style versioning and [autotag](https://github.com/pantheon-systems/autotag) to automatically increment version tags based on commit messages on changes to the `master` branch. Refer to the [autotag README](https://github.com/pantheon-systems/autotag#scheme-autotag-default) to learn how to trigger major, minor, and patch tags. If no keywords are -specified a Patch bump is applied. +specified a patch bump is applied. -Logging -------- +## Logging There are 3 logging functions defined in `common.mk` INFO, WARN, and ERROR. If you want to have clean output for make tasks you should redirect STDOUT to @@ -1511,14 +1511,13 @@ footask: When dostuff errors the error will still be reported, and the task will fail. -Pathfinding --------------- +## Pathfinding Have to do something in a real project where you're using common-make, and it's tedious, but you don't yet know what the solution is? Try setting `COMMON_MAKE_DIR` to your local checkout of common_makefiles outside your project, eg: -``` +```sh COMMON_MAKE_DIR=$HOME/pantheon/common_makefiles ``` @@ -1528,13 +1527,12 @@ Additionally, you can proceed with no danger of ending up in the ninth circle of merge hell due to mistakenly trying to do almost anything to the subtree checkout in your consuming project. -Common Patterns for adding to the repo --------------------------------------- +## Common Patterns for Adding to the Repo Tasks should follow the form of `--` for example ifs I have a build task and you want to add windows support you would add as -`build-windows` or if you wanted to add a build for onebox you might dos -`build-onebox-linux` or simply `build-onebox`. +`build-windows` or if you wanted to add a build for sandbox you might do +`build-sandbox-linux` or simply `build-sandbox`. There is the expectation that if you are doing a context specific task you add the context to your tasks. I.E. `test-circle`. @@ -1543,8 +1541,7 @@ This isn't written in stone, but I think it is a reasonable expectation that any engineer should be able to checkout any project and run: `make deps && make build && make test` to get things running / testing. -Adding support for a new language ---------------------------------- +## Adding Support for a New Language Programming languages tend to share a similar set of common tasks like `test`, `build`, `deps`. Commmon-make tries to handle this situation by setting a list @@ -1573,8 +1570,7 @@ The reason for this pattern is: not be completely accurate if the project includes multiple common-LANG files. - Supports running all of a project's tests and builds as the default case. -README.md updates ------------------ +## README.md updates When updating this README, run `make update-readme-toc` before committing to update the table of contents. @@ -1582,14 +1578,8 @@ the table of contents. Ensure your text wraps at 80 columns. Exceptions made for code and command line examples as well as URLs. -Handy Make stuff -================ - -Introduction to make Slide deck -http://martinvseticka.eu/temp/make/presentation.html - -The make cheet sheet -https://github.com/mxenoph/cheat_sheets/blob/master/make_cheatsheet.pdf +## Handy Make stuff -The make Manual -https://www.gnu.org/software/make/manual/make.html +- [Introduction to GNU Make slide deck](http://martinvseticka.eu/temp/make/presentation.html) +- [GNU Make Cheat Sheet](https://github.com/mxenoph/cheat_sheets/blob/master/make_cheatsheet.pdf) +- [GNU Make Manual](https://www.gnu.org/software/make/manual/make.html) diff --git a/_docker.mk b/_docker.mk index a936d87..5c63a66 100644 --- a/_docker.mk +++ b/_docker.mk @@ -20,29 +20,41 @@ _DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) include $(_DIR)/_base.mk +# DOCKER_PATH can be used to override the path to the docker command (i.e., "podman --remote"). +DOCKER_PATH ?= docker + ## Append tasks to the global tasks lint:: lint-hadolint # use pants if it exists outside of circle to get the default namespace and use it for the build -ifndef CIRCLECI - BUILD_NUM := $(shell pants config get default-sandbox-name 2> /dev/null || echo dev)-$(COMMIT) +PANTS_SANDBOX := $(shell pants config get default-sandbox-name 2> /dev/null) +ifeq ($(strip $(PANTS_SANDBOX)),) + ifdef BRANCH + BUILD_NUM_PREFIX := $(shell echo "${BRANCH}" | tr -cd '[:alnum:]_-') + endif +else + BUILD_NUM_PREFIX := $(PANTS_SANDBOX) endif -BUILD_NUM ?= dev + +BUILD_NUM_PREFIX ?= dev +BUILD_NUM := $(BUILD_NUM_PREFIX)-$(COMMIT) # TODO: the docker login -e email flag logic can be removed when all projects stop using circleci 1.0 or # if circleci 1.0 build container upgrades its docker > 1.14 -ifdef CIRCLE_BUILD_NUM - BUILD_NUM := $(CIRCLE_BUILD_NUM) - ifeq (email-required, $(shell docker login --help | grep -q Email && echo email-required)) - QUAY := docker login -p "$$QUAY_PASSWD" -u "$$QUAY_USER" -e "unused@unused" quay.io - else - QUAY := docker login -p "$$QUAY_PASSWD" -u "$$QUAY_USER" quay.io - endif -endif - -# If we have a circle branch, tag the image with it -ifdef CIRCLE_BRANCH - BUILD_NUM := $(BUILD_NUM)-$(shell echo "${CIRCLE_BRANCH}" | tr -cd '[:alnum:]_-') +ifndef NEW_TAG_STRATEGY + ifdef CIRCLE_BUILD_NUM + BUILD_NUM := $(CIRCLE_BUILD_NUM) + ifeq (email-required, $(shell $(DOCKER_PATH) login --help | grep -q Email && echo email-required)) + QUAY := $(DOCKER_PATH) login -p "$$QUAY_PASSWD" -u "$$QUAY_USER" -e "unused@unused" quay.io + else + QUAY := $(DOCKER_PATH) login -p "$$QUAY_PASSWD" -u "$$QUAY_USER" quay.io + endif + endif + + # If we have a circle branch, tag the image with it + ifdef CIRCLE_BRANCH + BUILD_NUM := $(BUILD_NUM)-$(shell echo "${CIRCLE_BRANCH}" | tr -cd '[:alnum:]_-') + endif endif DOCKER_TRY_PULL ?= false @@ -54,28 +66,6 @@ DOCKER_BUILD_ARGS ?= "" # Overriding this flag in your makefile is useful for custom push logic. DOCKER_BYPASS_DEFAULT_PUSH ?= false -# use pants if it exists outside of circle to get the default namespace and use it for the build -ifndef CIRCLECI - BUILD_NUM := $(shell pants config get default-sandbox-name 2> /dev/null || echo dev)-$(COMMIT) -endif -BUILD_NUM ?= dev - -# TODO: the docker login -e email flag logic can be removed when all projects stop using circleci 1.0 or -# if circleci 1.0 build container upgrades its docker > 1.14 -ifdef CIRCLE_BUILD_NUM - BUILD_NUM := $(CIRCLE_BUILD_NUM) - ifeq (email-required, $(shell docker login --help | grep -q Email && echo email-required)) - QUAY := docker login -p "$$QUAY_PASSWD" -u "$$QUAY_USER" -e "unused@unused" quay.io - else - QUAY := docker login -p "$$QUAY_PASSWD" -u "$$QUAY_USER" quay.io - endif -endif - -# If we have a circle branch, tag the image with it -ifdef CIRCLE_BRANCH - BUILD_NUM := $(BUILD_NUM)-$(shell echo "${CIRCLE_BRANCH}" | tr -cd '[:alnum:]_-') -endif - # if there is a docker file then set the docker variable so things can trigger off it ifneq ("$(wildcard Dockerfile))","") # file is there @@ -84,12 +74,11 @@ endif DOCKER_BUILD_CONTEXT ?= . +build-docker:: ## build the Docker image + # stub build-linux std target build-linux:: -## Append tasks to the global tasks -lint:: lint-hadolint - DOCKERFILES := $(shell find . -name 'Dockerfile*' -not -path "./devops/make*") lint-hadolint:: ## lint Dockerfiles ifdef DOCKERFILES @@ -103,6 +92,8 @@ ifdef DOCKERFILES endif endif -.PHONY:: build-docker lint-hadolint +push:: ## push the image to the registry + +.PHONY:: build-docker lint-hadolint push endif # ifndef COMMON_MAKE_DOCKER_INCLUDED diff --git a/common-apollo.mk b/common-apollo.mk index 1abdd98..97b8716 100644 --- a/common-apollo.mk +++ b/common-apollo.mk @@ -2,21 +2,25 @@ # # INPUT VARIABLES # - APP: (required) The name of the app. -# - GQL_SCHEMA_PATH: (required) path to the schema.graphqls file +# - GQL_SCHEMA_PATH: (required) path to the schema.graphqls file. +# - APOLLO_GRAPH_TARGET: (optional) the name of the graph to check or update against. Defaults to "pantheon". +# - APOLLO_VARIANT_TARGET: (optional) the name of the variant to check or update. Defaults to "current". # #------------------------------------------------------------------------------- +APOLLO_GRAPH_TARGET ?= pantheon +APOLLO_VARIANT_TARGET ?= current update-apollo-schema: ## Updates schema for your app on Apollo Studio - rover subgraph publish pantheon@current \ + rover subgraph publish $(APOLLO_GRAPH_TARGET)@$(APOLLO_VARIANT_TARGET) \ --schema $(GQL_SCHEMA_PATH) \ --name $(APP) \ --routing-url https://$(APP)/graphql/ check-apollo-schema: ## Checks schema changes against production to ensure any changes are compatable - @if rover subgraph list pantheon@current | grep -wq $(APP); then \ + @if rover subgraph list $(APOLLO_GRAPH_TARGET)@$(APOLLO_VARIANT_TARGET) | grep -wq $(APP); then \ echo "'$(APP)' found in the current Graph, running schema check"; \ - rover subgraph check pantheon@current --schema $(GQL_SCHEMA_PATH) --name $(APP); \ + rover subgraph check $(APOLLO_GRAPH_TARGET)@$(APOLLO_VARIANT_TARGET) --schema $(GQL_SCHEMA_PATH) --name $(APP); \ else \ echo "'$(APP)' not found in the current Graph, skipping schema check"; \ fi diff --git a/common-conda.mk b/common-conda.mk index 9e84566..f54f2d3 100644 --- a/common-conda.mk +++ b/common-conda.mk @@ -31,8 +31,8 @@ MINICONDA_VERSION ?= latest ANACONDA_CLOUD_ORGANIZATION:=pantheon # Default to exported environment variables if they are set and exist. # This approach is used in circle and local development -ANACONDA_CLOUD_DEPLOY_TOKEN:=$(shell echo $${ANACONDA_CLOUD_DEPLOY_TOKEN}) -ANACONDA_CLOUD_REPO_TOKEN:=$(shell echo $${ANACONDA_CLOUD_REPO_TOKEN}) +ANACONDA_CLOUD_DEPLOY_TOKEN:=$(shell echo $${ANACONDA_CLOUD_DEPLOY_TOKEN:-}) +ANACONDA_CLOUD_REPO_TOKEN:=$(shell echo $${ANACONDA_CLOUD_REPO_TOKEN:-}) # FQDN for the anaconda cloud api ANACONDA_CLOUD_API_FQDN:=api.anaconda.org # FQDN for conda packages in anaconda cloud diff --git a/common-docker-ar.mk b/common-docker-ar.mk index 320b29b..dad24d6 100644 --- a/common-docker-ar.mk +++ b/common-docker-ar.mk @@ -21,32 +21,36 @@ AR_IMAGE_BASENAME := $(firstword $(subst :, ,$(AR_IMAGE))) # used for testing in Makefile AR_REGISTRY_PATH := $(AR_REGISTRY)/$(APP) +# TODO Wipe out ifdef CIRCLE_BUILD_NUM VAULT_TOKEN = $(shell $(COMMON_MAKE_DIR)/sh/setup-circle-vault.sh 2>&1 > /dev/null; . $$BASH_ENV; echo $$VAULT_TOKEN) export VAULT_TOKEN endif -build-docker:: build-docker-ar ## build the docker container +build-docker:: build-docker-ar build-docker-ar:: setup-ar build-linux @FORCE_BUILD=$(DOCKER_FORCE_BUILD) TRY_PULL=$(DOCKER_TRY_PULL) \ + DOCKER_PATH=$(DOCKER_PATH) \ $(COMMON_MAKE_DIR)/sh/build-docker.sh \ $(AR_IMAGE) $(DOCKER_BUILD_CONTEXT) $(DOCKER_BUILD_ARGS) ifeq ("$(DOCKER_BYPASS_DEFAULT_PUSH)", "false") -push:: push-ar ## push the container to the registry +push:: push-ar else push:: endif setup-ar:: +# TODO Wipe out ifdef CIRCLE_BUILD_NUM + DOCKER_PATH=$(DOCKER_PATH) \ $(COMMON_MAKE_DIR)/sh/setup-circle-ar-docker.sh; endif push-ar:: setup-ar push-ar:: $(call INFO,"Pushing image $(AR_IMAGE)") - docker push $(AR_IMAGE); + $(DOCKER_PATH) push $(AR_IMAGE); -.PHONY:: push-ar build-docker-ar +.PHONY:: build-docker-ar push-ar setup-ar diff --git a/common-docker-quay.mk b/common-docker-quay.mk index 4ac35ad..bc5c5b9 100644 --- a/common-docker-quay.mk +++ b/common-docker-quay.mk @@ -26,22 +26,25 @@ QUAY_IMAGE_BASENAME := $(IMAGE_BASENAME) REGISTRY_PATH := $(REGISTRY)/$(APP) -build-docker:: build-docker-quay ## build the docker container +build-docker:: build-docker-quay build-docker-quay:: setup-quay build-linux @FORCE_BUILD=$(DOCKER_FORCE_BUILD) TRY_PULL=$(DOCKER_TRY_PULL) \ + DOCKER_PATH=$(DOCKER_PATH) \ $(COMMON_MAKE_DIR)/sh/build-docker.sh \ $(IMAGE) $(DOCKER_BUILD_CONTEXT) $(DOCKER_BUILD_ARGS) ifeq ("$(DOCKER_BYPASS_DEFAULT_PUSH)", "false") -push:: push-quay ## push the container to the registry +push:: push-quay else push:: endif push-quay:: setup-quay - $(call INFO,"pushing image $(IMAGE)") - @docker push $(IMAGE) + $(call INFO, "pushing image $(IMAGE)") + $(call WARN, "Quay is deprecated. Please migrate to Google Artifact Registry.") + @$(DOCKER_PATH) push $(IMAGE) + $(call WARN, "Quay is deprecated. Please migrate to Google Artifact Registry.") setup-quay:: # setup docker login for quay.io @@ -59,4 +62,4 @@ else $(call INFO, "We will fail if the docker config.json does not have the quay credentials.") endif -.PHONY:: setup-quay build-docker-quay push-quay lint-hadolint +.PHONY:: build-docker-quay push-quay setup-quay diff --git a/common-docs.mk b/common-docs.mk index ca58041..77b75c4 100644 --- a/common-docs.mk +++ b/common-docs.mk @@ -1,3 +1,9 @@ +# When run in dev container, markdown-toc is run from the local install (to avoid complications due to nested docker/podman volume mounts). +# When run by CircleCI, markdown-toc is run as a container. +# +# DOCKER_PATH can be used to override the path to the docker command (i.e., "podman --remote" or "/usr/local/bin/podman-devcontainer.sh"). +DOCKER_PATH ?= "docker" + ## Append tasks to the global tasks test:: test-readme-toc @@ -5,15 +11,27 @@ test:: test-readme-toc test-readme-toc: ## test if table of contents in README.md needs to be updated $(call INFO, "validating documentation TOC") @if grep -q '' ./README.md; then \ - bash -c "diff -c --ignore-blank-lines --strip-trailing-cr \ - <(cat ./README.md | docker run --rm -i quay.io/getpantheon/markdown-toc -; echo) \ - <(cat ./README.md | awk '//{flag=1;next}//{flag=0}flag' | sed '1d;\$$d')\ - " > /dev/null 2>&1 \ - || { echo "ERROR: README.md table of contents needs updating. Run 'make update-readme-toc', commit and push changes to your branch."; exit 1; } \ + if [ -x /usr/local/bin/markdown-toc ]; then \ + bash -c "diff -c --ignore-blank-lines --strip-trailing-cr \ + <(cat ./README.md | /usr/local/bin/markdown-toc -; echo) \ + <(cat ./README.md | awk '//{flag=1;next}//{flag=0}flag' | sed '1d;\$$d')\ + " > /dev/null 2>&1 \ + || { echo "ERROR: README.md table of contents needs updating. Run 'make update-readme-toc', commit and push changes to your branch."; exit 1; }; \ + else \ + bash -c "diff -c --ignore-blank-lines --strip-trailing-cr \ + <(cat ./README.md | $(DOCKER_PATH) run --rm -i quay.io/getpantheon/markdown-toc -; echo) \ + <(cat ./README.md | awk '//{flag=1;next}//{flag=0}flag' | sed '1d;\$$d')\ + " > /dev/null 2>&1 \ + || { echo "ERROR: README.md table of contents needs updating. Run 'make update-readme-toc', commit and push changes to your branch."; exit 1; }; \ + fi \ fi update-readme-toc: ## update the Table of Contents in ./README.md (replaces tag) $(call INFO, "updating documentation TOC") - @docker run --rm -v `pwd`:/src quay.io/getpantheon/markdown-toc -i /src/README.md > /dev/null + @if [ -x /usr/local/bin/markdown-toc ]; then \ + /usr/local/bin/markdown-toc -i README.md; \ + else \ + $(DOCKER_PATH) run --platform linux/amd64 --rm -v `pwd`:/src quay.io/getpantheon/markdown-toc -i /src/README.md > /dev/null; \ + fi .PHONY:: test-readme-toc update-readme-toc diff --git a/common-go.mk b/common-go.mk index a46fd74..7b58a17 100644 --- a/common-go.mk +++ b/common-go.mk @@ -3,7 +3,6 @@ # INPUT VARIABLES # - GOLINT_ARGS: Override the options passed to golangci-lint for linting (-v --timeout 3m by default) # - GOTEST_ARGS: Override the options passed by default ot go test (--race by default) -# - COVERALLS_TOKEN: Token to use when pushing coverage to coveralls. # # - FETCH_CA_CERT: The presence of this variable will cause the root CA certs # to be downloaded to the file ca-certificates.crt before building. @@ -16,11 +15,12 @@ deps:: deps-go deps-circle:: deps-circle-go deps lint:: lint-go test:: lint-go test-go-tparse -test-circle:: test test-coveralls +test-circle:: test test-coverage:: test-coverage-go build:: $(APP) build-go:: $(APP) clean:: clean-go +format:: format-go ifndef GOLINT_ARGS GOLINT_ARGS := -v --timeout 3m @@ -70,9 +70,17 @@ deps-go:: deps-lint ## install dependencies for project assumes you have go bina ifneq (,$(wildcard vendor)) @find ./vendor/* -maxdepth 0 -type d -exec rm -rf "{}" \; || true endif - $(call INFO, "restoring dependencies using modules via: go get $(GO_GET_ARGS)") - @GO111MODULE=on go get $(GO_GET_ARGS) - + $(call INFO, "restoring dependencies using modules via: go mod download $(GO_GET_ARGS)") + @GO111MODULE=on go mod download $(GO_GET_ARGS) + +format-go: + $(call INFO, "cleaning up go.mod") + @go mod tidy + $(call INFO, "formatting go-code") + @go fmt + $(call INFO, "running golangci-lint with fixes") + @# TODO: call lint-go + @golangci-lint run -E goimports --fix $(GOLINT_ARGS) lint-go:: deps-go deps-lint $(call INFO, "scanning source with golangci-lint") @@ -109,12 +117,6 @@ else endif endif -deps-coveralls-go:: ## install goveralls for sending go test coverage reports to Coveralls.io -ifeq (, $(shell command -v goveralls;)) - $(call INFO, "installing goveralls") - @GO111MODULE=off go get github.com/mattn/goveralls > /dev/null -endif - deps-status:: ## check status of deps with gostatus ifeq (, $(shell command -v gostatus;)) $(call INFO, "installing gostatus") @@ -131,14 +133,6 @@ test-coverage-go:: ## run coverage report $(call INFO, "running go coverage tests with $(GO_TEST_COVERAGE_ARGS)") @$(GO_TEST_COVERAGE_CMD) > /dev/null -test-coveralls:: deps-coveralls-go test-coverage-go ## run coverage and report to coveralls -ifdef COVERALLS_TOKEN - $(call INFO, "reporting coverage to coveralls") - @goveralls -repotoken $$COVERALLS_TOKEN -service=circleci -coverprofile=coverage.out > /dev/null -else - $(call WARN, "You asked to use Coveralls but neglected to set the COVERALLS_TOKEN environment variable") -endif - test-coverage-html:: test-coverage ## output html coverage file $(call INFO, "generating html coverage report") @go tool cover -html=coverage.out > /dev/null @@ -153,5 +147,5 @@ ifdef FETCH_CA_CERT @curl -s -L https://curl.haxx.se/ca/cacert.pem -o ca-certificates.crt > /dev/null endif -.PHONY:: _fetch-cert test-coverage-html test-coveralls deps-status deps-coveralls-go deps-circle deps-go deps-lint \ +.PHONY:: _fetch-cert test-coverage-html deps-status deps-circle deps-go deps-lint \ lint-go test-circle test-go build-circle build-linux build-go clean-go diff --git a/common-kube.mk b/common-kube.mk index 9eb35bd..81cd600 100644 --- a/common-kube.mk +++ b/common-kube.mk @@ -145,7 +145,7 @@ clean-secrets:: ## delete local secrets clean-kube:: clean-secrets verify-deployment-rollout:: ## validate that deployment to kube was successful and rollback if not - @$(KUBECTL_CMD) rollout status deployment/$(APP) --timeout=10m \ + @$(KUBECTL_CMD) rollout status deployment/$(APP) --timeout=15m \ | grep 'successfully' && echo 'Deploy succeeded.' && exit 0 \ || echo 'Deploy unsuccessful. Rolling back. Investigate!' \ && $(KUBECTL_CMD) rollout undo deployment/$(APP) && exit 1 diff --git a/common-pants.mk b/common-pants.mk index 2fbcde1..8de2113 100644 --- a/common-pants.mk +++ b/common-pants.mk @@ -11,11 +11,10 @@ # PANTS_SANDBOX_NAME # Name of sandbox. Default is ${APP}-${BUILD_NUM}. # PANTS_VERSION_CONSTRAINT ?= latest -PANTS_UPDATE_ONEBOX ?= false PANTS_SANDBOX_NAME ?= $(shell echo $(KUBE_NAMESPACE) | tr A-Z a-z) # lowercased for compatibility PANTS_SANDBOX_CLUSTER ?= $(KUBE_CONTEXT) PANTS_DEBUG ?= false -PANTS_FLAGS ?= -s $(PANTS_SANDBOX_NAME) --update-onebox=$(PANTS_UPDATE_ONEBOX) --target-cluster=$(PANTS_SANDBOX_CLUSTER) --skip-machines +PANTS_FLAGS ?= -s $(PANTS_SANDBOX_NAME) --target-cluster=$(PANTS_SANDBOX_CLUSTER) --skip-machines ifdef PANTS_INCLUDE PANTS_FLAGS += -i $(PANTS_INCLUDE) diff --git a/common-python.mk b/common-python.mk index 0d67acc..e06cc88 100644 --- a/common-python.mk +++ b/common-python.mk @@ -4,7 +4,6 @@ # - PYTHON_PACKAGE_NAME: (required) The name of the python package. # - TEST_RUNNER: (optional) The name of the python test runner to execute. Defaults to `unittest` # - TEST_RUNNER_ARGS: (optional) Extra arguements to pass to the test runner. Defaults to `discover` -# - COVERALLS_TOKEN: (optional) Token to use when pushing coverage to coveralls (`test-coveralls`). # #------------------------------------------------------------------------------- @@ -16,16 +15,8 @@ TEST_RUNNER_ARGS ?= discover FLAKE8_BIN := $(shell command -v flake8;) PYLINT_BIN := $(shell command -v pylint;) COVERAGE_BIN := $(shell command -v coverage;) -COVERALLS_BIN := $(shell command -v coveralls;) BUMPVERSION_BIN := $(shell command -v bumpversion;) -# TODO(joe): remove after we confirm any github repos using common-python.mk have switched their circleCI config to use COVERALLS_TOKEN -# Note to future selves: query all pantheon github repos with a `/Makefile` containing "common-python-mk": https://github.com/search?utf8=%E2%9C%93&q=org%3Apantheon-systems+common-python.mk+filename%3AMakefile+path%3A%2F&type=Code&ref=advsearch&l=&l= -ifdef COVERALLS_REPO_TOKEN -COVERALLS_TOKEN = $(COVERALLS_REPO_TOKEN) -$(warning "Environment variable COVERALLS_REPO_TOKEN is deprecated, please switch this project's CI config to use COVERALLS_TOKEN instead) -endif - ## Append tasks to the global tasks deps:: deps-python deps-circle:: deps-circle-python @@ -73,14 +64,7 @@ ifndef COVERAGE_BIN pip install coverage endif -deps-circle-python:: deps-coveralls-python ## Install python dependencies for circle - -deps-coveralls-python:: -ifdef COVERALLS_REPO_TOKEN -ifndef COVERALLS_BIN - pip install coveralls -endif -endif +deps-circle-python:: ## Install python dependencies for circle deps-bumpversion-python: ifndef BUMPVERSION_BIN @@ -107,7 +91,7 @@ endif test-python:: test-coverage-python -test-circle-python:: test-coveralls-python +test-circle-python:: test-coverage-python:: deps-testrunner-python deps-coverage-python ## Run tests and generate code coverage. Configuration file '.coveragerc' coverage run --branch --source $(PYTHON_PACKAGE_NAME) -m $(TEST_RUNNER) $(TEST_RUNNER_ARGS) $(PYTHON_PACKAGE_NAME) @@ -115,13 +99,6 @@ test-coverage-python:: deps-testrunner-python deps-coverage-python ## Run tests coverage-report: ## Display the coverage report. Requires that make test has been run. coverage report -test-coveralls-python:: deps-coveralls-python ## run coverage and report to coveralls -ifdef COVERALLS_TOKEN - coveralls -else - $(call ERROR, "COVERALLS_TOKEN is not set. Skipping coveralls reporting") -endif - bumpmicro: bumppatch ## Bump the micro (patch) version of the python package. Configuration file '.bumpversion.cfg' bumppatch: deps-bumpversion ## Alias for bumpmicro @@ -133,4 +110,4 @@ bumpminor: deps-bumpversion ## Bump the minor version of the python package. Con bumpmajor: deps-bumpversion ## Bump the major version of the python package. Configuration file '.bumpversion.cfg' bumpversion major -.PHONY:: deps-coverage-python deps-circle-python deps-lint-python deps-coveralls-python deps-pylint deps-flake8 test-python test-circle-python test-coveralls-python build-python test-coverage-python coverage-report test-circle test-circle-python bumpmicro bumpminor bumpmajor bumppatch +.PHONY:: deps-coverage-python deps-circle-python deps-lint-python deps-pylint deps-flake8 test-python test-circle-python build-python test-coverage-python coverage-report test-circle test-circle-python bumpmicro bumpminor bumpmajor bumppatch diff --git a/common-python3.mk b/common-python3.mk index de13aff..018ac4d 100644 --- a/common-python3.mk +++ b/common-python3.mk @@ -5,6 +5,10 @@ # - IMAGE: (required) The docker image to use. #------------------------------------------------------------------------------- + +# DOCKER_PATH can be used to override the path to the docker command (i.e., "podman --remote"). +DOCKER_PATH ?= docker + export PATH := $(PATH):$(HOME)/.local/bin deps:: deps-python3 @@ -17,7 +21,7 @@ lint-python3: pipenv run pylint $(PYTHON_PACKAGE_NAME) test-python3-docker: - docker run $(IMAGE) .venv/bin/python setup.py test + $(DOCKER_PATH) run $(IMAGE) .venv/bin/python setup.py test deps-python3: pip3 install pipenv --user diff --git a/common.mk b/common.mk index e79b650..a860eb3 100644 --- a/common.mk +++ b/common.mk @@ -19,8 +19,11 @@ ROOT_DIR ?= $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) # override to use eg your own checkout for making pr's to upstream COMMON_MAKE_DIR ?= $(ROOT_DIR)/devops/make +# the branch or ref to get common-make updates from +COMMON_MAKE_UPDATE_TARGET ?= master + help: ## print list of tasks and descriptions - @grep --no-filename -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) $(filter_tasks_cmd) | sort | awk 'BEGIN {FS = ":.*?##"}; { printf "\033[36m%-30s\033[0m %s \n", $$1, $$2}' + @grep --no-filename -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) $(filter_tasks_cmd) | sort | uniq | awk 'BEGIN {FS = ":.*?##"}; { printf "\033[36m%-30s\033[0m %s \n", $$1, $$2}' .DEFAULT_GOAL := help # Invoke this with $(call INFO,) @@ -50,7 +53,7 @@ ifndef BRANCH endif ifndef COMMIT_NO - COMMIT_NO := $(shell git rev-parse --short HEAD) + COMMIT_NO := $(shell git rev-parse --short=8 HEAD) endif ifndef COMMIT @@ -68,6 +71,19 @@ test-circle:: ## invoke test tasks for CI test-coverage:: ## run test coverage reports build:: ## run all build clean:: ## clean up artifacts from test and build +format:: ## attempts to properly format or cleanup files, writing changes to disk + +# runs format and errors if anything has changed +check-format: format + @s=$$(git status --porcelain); if [ -z "$$s" ]; then \ + echo "workdir is clean"; \ + else \ + echo "The following files have changed on disk"; \ + echo "$$s"; \ + git diff; \ + exit 1; \ + fi + update-makefiles:: ## update the make subtree, assumes the subtree is in devops/make ifneq (, $(wildcard scripts/make)) @@ -79,12 +95,12 @@ update-makefiles:: ## update the make subtree, assumes the subtree is in devops/ @echo "git commit -am \"Move common_makefiles to new prefix\"" @exit 1 endif - # best effort attempt to do one time setup - @set -x; if ! git remote show common_makefiles &> /dev/null; then \ - echo "adding common_makefiles as a remote"; \ + @if ! git remote show common_makefiles &> /dev/null; then \ + echo "temporarily adding common_makefiles as a remote"; \ git remote add common_makefiles git@github.com:pantheon-systems/common_makefiles.git --no-tags; \ - git subtree add --prefix devops/make common_makefiles master --squash &>/dev/null || true; \ + git subtree add --prefix devops/make common_makefiles $(COMMON_MAKE_UPDATE_TARGET) --squash &>/dev/null || true; \ fi - @git subtree pull --prefix devops/make common_makefiles master --squash + @git subtree pull --prefix devops/make common_makefiles $(COMMON_MAKE_UPDATE_TARGET) --squash -m 'update common make' + @git remote remove common_makefiles .PHONY:: all help update-makefiles diff --git a/go.mod b/go.mod index 5a9059a..63c6b9c 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/pantheon-systems/common_makefiles -go 1.16 +go 1.21 diff --git a/sh/build-docker.sh b/sh/build-docker.sh index 7dbb8bb..c904579 100755 --- a/sh/build-docker.sh +++ b/sh/build-docker.sh @@ -18,7 +18,7 @@ function build() { docker_build_args=("$@") log "building docker image $image" # shellcheck disable=SC2068 - docker build --pull ${docker_build_args[@]} -t "$image" "$context" + $DOCKER_PATH build --pull ${docker_build_args[@]} -t "$image" "$context" } IMAGE="$1" @@ -38,13 +38,13 @@ if [[ $FORCE_BUILD == true ]]; then build "$IMAGE" "$CONTEXT" "$DOCKER_BUILD_ARGS" exit fi -if [[ $(docker images -q "$IMAGE" | wc -l) -gt 0 ]]; then +if [[ $($DOCKER_PATH images -q "$IMAGE" | wc -l) -gt 0 ]]; then log "found $IMAGE locally, not building" exit fi if [[ $TRY_PULL == true ]]; then log "attempting to pull docker image $IMAGE" - docker pull "$IMAGE" &>/dev/null && exit; + $DOCKER_PATH pull "$IMAGE" &>/dev/null && exit; log "unable to pull image, proceeding to build" fi build "$IMAGE" "$CONTEXT" "$DOCKER_BUILD_ARGS" diff --git a/sh/create-tls-cert.sh b/sh/create-tls-cert.sh index db254bc..7851c74 100755 --- a/sh/create-tls-cert.sh +++ b/sh/create-tls-cert.sh @@ -8,7 +8,7 @@ # Optional environment vars: # DIRECTORY: Directory to store downloaded certs in. Defaults to current dir ('.') -# CA_HOST: Defaults to 'cimt.getpantheon.com' (production CA). Use your onebox address to create a development/sandbox cert. +# CA_HOST: Defaults to 'cimt.getpantheon.com' (production CA) # SANS: SubjectAltNames to add to the cert. The CN will automatically be added to the SAN list so you don't need to add it. # The format is OpenSSL's SAN format documented here: https://www.openssl.org/docs/man1.0.2/apps/x509v3_config.html (Subject Alt Name section) # Example - 1 DNS name, 1 ipv4 address, 1 ipv6 address: @@ -19,7 +19,6 @@ # CASSANDRA_CA_KEY_FILE: Full path to the key file, used to generate the certs, when CASSANDRA=yes # # Notes on Cassandra flags: -# - If USE_ONEBOX_CA=true and CASSANDRA=true, then the CA `onebox_cassandra_ca.{crt,key} will be used. # - Currently Valhalla Cassandra shares a CA, in future we would like to have each one use a separate CA. # # Usage examples: @@ -32,10 +31,6 @@ # # CN=foo OU=bar FILENAME=mycert SANS="DNS:foobar.com;IP:10.0.0.1" bash ./create-tls-cert.sh # -# - Issue a development certificate from a onebox (any onebox can be used, so use yours if you have one): -# -# CA_HOST=onebox CN=foo OU=bar FILENAME=mycert bash ./create-tls-cert.sh -# # - Generate certs for use by Cassandra, using the Ygg CA (Valhalla make commands use this) # # CN=valhalla OU=valhalla \ @@ -54,19 +49,12 @@ FILENAME="${FILENAME:-}" OU="${OU:-}" CN="${CN:-}" SANS="${SANS:-}" -USE_ONEBOX_CA="${USE_ONEBOX_CA:-false}" CA_CERT_FILE="/etc/pantheon/ca.crt" CA_KEY_FILE="/etc/pantheon/ca.key" CASSANDRA="${CASSANDRA:-}" CASSANDRA_CA_CERT_FILE="${CASSANDRA_CA_CERT_FILE:-}" CASSANDRA_CA_KEY_FILE="${CASSANDRA_CA_KEY_FILE:-}" -if [[ "$USE_ONEBOX_CA" == "true" ]]; then - CA_CERT_FILE="/etc/pantheon/onebox_ca.crt" - CA_KEY_FILE="/etc/pantheon/onebox_ca.key" - CASSANDRA_CA_CERT_FILE="/etc/pantheon/onebox_cassandra_ca.crt" - CASSANDRA_CA_KEY_FILE="/etc/pantheon/onebox_cassandra_ca.key" -fi if [[ "$CASSANDRA" == "true" ]]; then CA_CERT_FILE="${CASSANDRA_CA_CERT_FILE}" CA_KEY_FILE="${CASSANDRA_CA_KEY_FILE}" diff --git a/sh/install-pants.sh b/sh/install-pants.sh index bedac6e..173a81a 100755 --- a/sh/install-pants.sh +++ b/sh/install-pants.sh @@ -26,12 +26,18 @@ if ! command -v pants >/dev/null; then PANTS_URL="https://$GITHUB_TOKEN:@api.github.com/repos/pantheon-systems/pants/releases" + PANTS_JSON=$(curl -fsS "$PANTS_URL" || true) + if [[ -z "${PANTS_JSON}" ]] ; then + echo "Unable to retrieve pants release catalog" + exit 1 + fi + JQ_FILTER=".[0].assets | map(select(.name|test(\"pants_.*_linux_amd64\")))[0].id" if [[ $PANTS_VERSION_CONSTRAINT != "latest" ]]; then JQ_FILTER=". | map(select(.tag_name == \"v$PANTS_VERSION_CONSTRAINT\"))[0].assets | map(select(.name|test(\"pants_.*_linux_amd64\")))[0].id" fi - ASSET=$(curl -s "$PANTS_URL" | jq -r "$JQ_FILTER") + ASSET=$(echo "${PANTS_JSON}" | jq -r "$JQ_FILTER") if [[ "$ASSET" == "null" ]]; then echo "Asset Not Found" exit 1 diff --git a/sh/setup-circle-ar-docker.sh b/sh/setup-circle-ar-docker.sh index 3209332..9d5cb75 100755 --- a/sh/setup-circle-ar-docker.sh +++ b/sh/setup-circle-ar-docker.sh @@ -30,7 +30,7 @@ setup_ar_docker_repos() { # setup all registries to use this identity registries=(us-docker.pkg.dev gcr.io us.gcr.io eu.gcr.io asia.gcr.io staging-k8s.gcr.io marketplace.gcr.io) for i in "${registries[@]}" ; do - echo "$gsa" | docker login -u _json_key --password-stdin "https://$i" + echo "$gsa" | $DOCKER_PATH login -u _json_key --password-stdin "https://$i" done } @@ -86,7 +86,7 @@ main() { "shared") VAULT_PATH="secret/circleci/gsa" ;; esac - setup_ar_docker_repos $VAULT_PATH + setup_ar_docker_repos "$VAULT_PATH" } main "$@" diff --git a/sh/setup-circle-vault.sh b/sh/setup-circle-vault.sh index a33f191..f3b3324 100755 --- a/sh/setup-circle-vault.sh +++ b/sh/setup-circle-vault.sh @@ -72,7 +72,7 @@ install_vault() { verify_vault "${VERSION}" "${ARCH}" "${PLATFORM}" fi - unzip "${FILENAME}" + unzip "${FILENAME}" vault rm "${FILENAME}" ${SUDO} mv ./vault /usr/local/bin/vault vault version @@ -95,9 +95,9 @@ install_pvault() { fi local PVAULT_URL="https://${GITHUB_TOKEN}:@api.github.com/repos/pantheon-systems/pvault/releases" - local JQ_FILTER=".[0].assets | map(select(.name|test(\"pvault.*_linux_amd64\")))[0].id" + local JQ_FILTER=".[0].assets | map(select(.name|test(\"pvault.*amd64.deb\")))[0].id" if [[ "${PVAULT_VERSION}" != "latest" ]] ; then - JQ_FILTER=". | map(select(.tag_name == \"v${PVAULT_VERSION}\"))[0].assets | map(select(.name|test(\"pvault_.*_linux_amd64\")))[0].id" + JQ_FILTER=". | map(select(.tag_name == \"v${PVAULT_VERSION}\"))[0].assets | map(select(.name|test(\"pvault.*amd64.deb\")))[0].id" fi local ASSET diff --git a/sh/update-gcloud.sh b/sh/update-gcloud.sh index abc48bf..fd4010c 100755 --- a/sh/update-gcloud.sh +++ b/sh/update-gcloud.sh @@ -7,7 +7,7 @@ # GCLOUD_KEY # base64 encoded key set -eou pipefail -if [ "$CIRCLECI" != "true" ]; then +if [ "${CIRCLECI:-}" != "true" ]; then echo "This script is only intended to run on Circle-CI." exit 1 fi diff --git a/test/make/docker-ar.mk b/test/make/docker-ar.mk index 127fd67..79bc354 100644 --- a/test/make/docker-ar.mk +++ b/test/make/docker-ar.mk @@ -12,6 +12,10 @@ include common-docker-ar.mk test-common-docker: build-docker push $(call INFO, "testing common docker") @test "$(AR_REGISTRY_PATH)" = "$(AR_IMAGE_BASENAME)" +ifdef NEW_TAG_STRATEGY + @test "$(shell echo "${BRANCH}" | tr -cd '[:alnum:]_-')-$(COMMIT)" = "$(BUILD_NUM)" +else ifdef CIRCLE_BUILD_NUM @test "$(CIRCLE_BUILD_NUM)-$(CIRCLE_BRANCH)" = "$(BUILD_NUM)" endif +endif diff --git a/test/make/docker.mk b/test/make/docker.mk index ef1deb5..c1c9256 100644 --- a/test/make/docker.mk +++ b/test/make/docker.mk @@ -12,6 +12,10 @@ include common-docker.mk test-common-docker: build-docker push $(call INFO, "testing common docker") @test "$(REGISTRY_PATH)" = "$(IMAGE_BASENAME)" +ifdef NEW_TAG_STRATEGY + @test "$(shell echo "${BRANCH}" | tr -cd '[:alnum:]_-')-$(COMMIT)" = "$(BUILD_NUM)" +else ifdef CIRCLE_BUILD_NUM @test "$(CIRCLE_BUILD_NUM)-$(CIRCLE_BRANCH)" = "$(BUILD_NUM)" endif +endif