Promote Helm Charts and Docker Images Across Repositories (#85)

## Breaking Changes
* `REPO_GATEWAY_ENDPOINT` renamed to `TARGET_CHART_REPO_ENDPOINT` (related to  `helm/build/*` targets)
* `LATEST_TAG` __depricated__ in`semver` module. Now it always `0.0.0`

## Change log
* Reinvent helm build and promote tasks
* Added Codefresh pipeline logic

.travis.yml

@@ -1,4 +1,3 @@
-sudo: required
 language: go
 go:
   - 1.9.x

modules/chamber/Makefile

@@ -17,7 +17,7 @@ chamber/install: packages/install/chamber
 ## Start a chamber shell with secrets exported to the environment
 chamber/shell: chamber/check-shell
 	$(call assert-unset,IN_CHAMBER_MODE)
-	$(WITH_CHAMBER) bash --rcfile $(BUILD_HARNESS_PATH)/modules/chamber/chamber.bash.rc
+	@$(WITH_CHAMBER) bash --rcfile $(BUILD_HARNESS_PATH)/modules/chamber/chamber.bash.rc
 
 # Ensure that variables required for chamber are set
 chamber/check-shell: aws/check-shell

modules/codefresh/Makefile

@@ -12,4 +12,7 @@ codefresh/trigger/webhook:
 	 --data-binary \
 		'{"serviceId":"$(CF_TRIGGER_SERVICE_ID)","type":"$(CF_TRIGGER_TYPE)","repoOwner":"$(CF_TRIGGER_REPO_OWNER)","branch":"$(CF_TRIGGER_BRANCH)","repoName":"$(CF_TRIGGER_REPO_NAME)"}'
 
-
+## Export codefresh additional envvars
+codefresh/export:
+	$(call assert-set,CF_BUILD_TIMESTAMP)
+	@echo "CF_BUILD_UNIX_TIMESTAMP=$(shell expr $(CF_BUILD_TIMESTAMP) / 1000)"

modules/codefresh/Makefile.pipeline

@@ -0,0 +1,61 @@
+## To get GIT_* env variables reference https://github.com/cloudposse/build-harness/blob/master/modules/git/bootstrap.Makefile
+
+# Extract ticket number from branch name (e.g. feature/ch37684/add-codefresh)
+FEATURE ?= $(shell echo "$(GIT_BRANCH)" | cut -s -d'/' -f2 | tr '[:upper:]' '[:lower:]')
+
+PIPELINE_ACTION = pass
+
+## Pre-production deploys only released versions
+ifeq ($(PIPELINE_ENV),pre-production)
+  STAGE ?= pre-prod
+  ifeq ($(GIT_IS_TAG),1)
+  	PIPELINE_ACTION = deploy
+  endif
+endif
+
+## Staging deploys all commits (working branch should be set on codefresh pipeline trigger config)
+ifeq ($(PIPELINE_ENV),staging)
+  STAGE ?= stage
+  PIPELINE_ACTION = deploy
+endif
+
+## Integration deploys on each PR create/update. On PR close - cleanup
+ifeq ($(PIPELINE_ENV),integration)
+  STAGE ?= $(FEATURE)
+
+  ifeq ($(CF_PULL_REQUEST_ACTION),opened)
+    PIPELINE_ACTION = deploy
+  endif
+
+  ifeq ($(CF_PULL_REQUEST_ACTION),synchronize)
+    PIPELINE_ACTION = deploy
+  endif
+
+  ifeq ($(CF_PULL_REQUEST_ACTION),closed)
+    PIPELINE_ACTION = destroy
+  endif
+
+endif
+
+ifneq ($(PIPELINE_ENV),)
+  NAMESPACE ?= $(PROJECT)-$(STAGE)
+  APP_HOST ?= $(APP_NAME).$(STAGE).$(BASE_HOST)
+  RELEASE_NAME ?= $(NAMESPACE)-$(APP_NAME)
+endif
+
+## Export pipeline vars
+codefresh/pipeline/export:
+	$(call assert-set,PROJECT)
+	$(call assert-set,PIPELINE_ACTION)
+	$(call assert-set,NAMESPACE)
+	$(call assert-set,APP_HOST)
+	$(call assert-set,RELEASE_NAME)
+	$(call assert-set,BASE_HOST)
+	$(call assert-set,APP_NAME)
+	@echo "PROJECT=$(PROJECT)"
+	@echo "NAMESPACE=$(NAMESPACE)"
+	@echo "PIPELINE_ACTION=$(PIPELINE_ACTION)"
+	@echo "APP_HOST=$(APP_HOST)"
+	@echo "RELEASE_NAME=$(RELEASE_NAME)"
+
+

modules/docker/Makefile.build

@@ -16,3 +16,31 @@ endif
 	done`; \
 	echo "Building $(DOCKER_IMAGE_NAME) from $(DOCKER_FILE) with [$(DOCKER_BUILD_FLAGS)] build args..."; \
 	$(DOCKER) build $(DOCKER_BUILD_FLAGS) $$BUILD_ARGS -t $(DOCKER_IMAGE_NAME) -f $(DOCKER_FILE) $(DOCKER_BUILD_PATH)
+
+
+## Promote $SOURCE_DOCKER_REGISTRY/$IMAGE_NAME:$SOURCE_VERSION to $TARGET_DOCKER_REGISTRY/$IMAGE_NAME:$TARGET_VERSION
+docker/image/promote/local:
+	$(call assert-set,DOCKER)
+	$(call assert-set,IMAGE_NAME)
+	$(call assert-set,SOURCE_DOCKER_REGISTRY)
+	$(call assert-set,SOURCE_VERSION)
+	$(call assert-set,TARGET_DOCKER_REGISTRY)
+	$(call assert-set,TARGET_VERSION)
+	@$(DOCKER) tag $(SOURCE_DOCKER_REGISTRY)/$(IMAGE_NAME):$(SOURCE_VERSION) $(TARGET_DOCKER_REGISTRY)/$(IMAGE_NAME):$(TARGET_VERSION)
+
+## Pull $SOURCE_DOCKER_REGISTRY/$IMAGE_NAME:$SOURCE_VERSION and promote to $TARGET_DOCKER_REGISTRY/$IMAGE_NAME:$TARGET_VERSION
+docker/image/promote/remote:
+	$(call assert-set,DOCKER)
+	$(call assert-set,IMAGE_NAME)
+	$(call assert-set,SOURCE_DOCKER_REGISTRY)
+	$(call assert-set,SOURCE_VERSION)
+	@$(DOCKER) pull $(SOURCE_DOCKER_REGISTRY)/$(IMAGE_NAME):$(SOURCE_VERSION)
+	@$(SELF) -s docker/image/promote/local
+
+## Push $TARGET_DOCKER_REGISTRY/$IMAGE_NAME:$TARGET_VERSION
+docker/image/push:
+	$(call assert-set,DOCKER)
+	$(call assert-set,IMAGE_NAME)
+	$(call assert-set,TARGET_DOCKER_REGISTRY)
+	$(call assert-set,TARGET_VERSION)
+	$(DOCKER) push $(TARGET_DOCKER_REGISTRY)/$(IMAGE_NAME):$(TARGET_VERSION)

modules/git/Makefile

@@ -9,22 +9,12 @@ git/submodules-update:
 git/aliases-update:
 	@$(GIT) config --global --add alias.permission-reset '!git diff -p -R --no-color | grep -E "^(diff|(old|new) mode)" --color=never | git apply'
 
-## Show vars
-git/show:
-	@echo "commit - $(GIT_COMMIT)"
-	@echo "commit short - $(GIT_COMMIT_SHORT)"
-	@echo "commit branch - $(GIT_BRANCH)"
-	@echo "latest tag - $(GIT_LATEST_TAG)"
-	@echo "tag - $(GIT_IS_TAG)"
-	@echo "branch - $(GIT_IS_BRANCH)"
-	@echo "timestamp - $(GIT_TIMESTAMP)"
-
 ## Export git vars
 git/export:
 	@echo "GIT_COMMIT=$(GIT_COMMIT)"
 	@echo "GIT_COMMIT_SHORT=$(GIT_COMMIT_SHORT)"
 	@echo "GIT_BRANCH=$(GIT_BRANCH)"
-	@echo "GIT_LATEST_TAG=$(GIT_LATEST_TAG)"
+	@echo "GIT_TAG=$(GIT_TAG)"
 	@echo "GIT_IS_BRANCH=$(GIT_IS_BRANCH)"
 	@echo "GIT_IS_TAG=$(GIT_IS_TAG)"
 	@echo "GIT_TIMESTAMP=$(GIT_TIMESTAMP)"

modules/git/bootstrap.Makefile

@@ -5,17 +5,20 @@ ifeq ($(wildcard .git),)
     $(warning disabling git bootstrapping)
   endif
 else
+
 GIT ?= $(shell which git)
 
 export GIT_COMMIT ?= $(shell $(GIT) rev-parse --verify HEAD)
 export GIT_COMMIT_SHORT ?= $(shell $(GIT) rev-parse --verify --short HEAD)
 export GIT_BRANCH ?= $(shell $(GIT) rev-parse --abbrev-ref HEAD)
-export GIT_LATEST_TAG ?= $(shell $(GIT)  describe --tags --abbrev=0 2>/dev/null)
+export GIT_TAG ?= $(shell $(GIT) tag -l --points-at HEAD)
 
 export GIT_TIMESTAMP ?= $(shell $(GIT) log -1 --format=%ct 2>/dev/null)
 
-ifeq ($(GIT_LATEST_TAG),)
-  export GIT_LATEST_TAG="0.0.0"
+ifeq ($(GIT_TAG),)
+  export GIT_IS_TAG := 0
+else
+  export GIT_IS_TAG := 1
 endif
 
 ifeq ($(GIT_BRANCH),)
@@ -24,10 +27,4 @@ else
   export GIT_IS_BRANCH := 1
 endif
 
-ifeq ($(shell $(GIT) describe --exact-match --tags 2>/dev/null),)
-  export GIT_IS_TAG := 0
-else
-  export GIT_IS_TAG := 1
-endif
-
 endif

modules/git/template/module.sh

@@ -12,7 +12,7 @@ cat << EOF > $GIT_FILE
   commit: ${GIT_COMMIT}
   commit_short: ${GIT_COMMIT_SHORT}
   branch: ${GIT_BRANCH}
-  latest_tag: ${LATEST_TAG}
+  tag: ${GIT_TAG}
   is_tag: ${GIT_IS_TAG}
   is_branch: ${GIT_IS_BRANCH}
 EOF

modules/helm/Makefile

@@ -25,9 +25,15 @@ helm/toolbox/upsert:
 
 ## Delete all releases in a `NAMEPSACE` as well as the namespace
 helm/delete/namespace:
-	@helm list --namespace $(NAMESPACE) --short | xargs -r helm delete --purge
+	$(call assert-set,NAMESPACE)
+	@$(HELM) list --namespace $(NAMESPACE) --short | xargs -r $(HELM) delete --purge
 	@kubectl delete namespace $(NAMESPACE) --ignore-not-found
 
+## Delete `NAMESPACE` if there are no releases in it
+helm/delete/namespace/empty:
+	$(call assert-set,NAMESPACE)
+	@[ -n "$$($(HELM) list --namespace $(NAMESPACE) --short --all)" ] || kubectl delete namespace $(NAMESPACE) --ignore-not-found
+
 ## Delete all failed releases in a `NAMESPACE` subject to `FILTER`
 helm/delete/failed:
-	@helm list --namespace $(NAMESPACE) --failed --short $(FILTER) 2>/dev/null | xargs -r helm delete --purge
+	@$(HELM) list --namespace $(NAMESPACE) --failed --short $(FILTER) 2>/dev/null | xargs -r $(HELM) delete --purge

modules/helm/Makefile.build

@@ -0,0 +1,110 @@
+## Build chart $CHART_NAME from $SOURCE_CHART_TPL
+helm/chart/build: helm/chart/build/prepare helm/chart/build/deps helm/chart/build/package
+	@$(SELF) -s helm/chart/build/clean
+
+## Build chart $CHART_NAME from $SOURCE_CHART_TPL for all available $SEMVERSIONS
+helm/chart/build/all: helm/chart/build/prepare helm/chart/build/deps
+	@for version in $(SEMVERSIONS) ; do \
+		echo "Build chart $(CHART_NAME) version $$version"; \
+		$(SELF) -s helm/chart/build/package TARGET_VERSION=$$version; \
+	done
+	@$(SELF) -s helm/chart/build/clean
+
+## Alias for helm/chart/build/all. Depricated.
+helm/chart/build-all: helm/chart/build/all
+	@exit 0
+
+## Promote $SOURCE_CHART_FILE to $TARGET_VERSION
+helm/chart/promote/local: helm/chart/promote/local/prepare helm/chart/build/package
+	@$(SELF) -s helm/chart/build/clean
+
+## Promote $CHART_NAME from $SOURCE_VERSION to $TARGET_VERSION. ($SOURCE_CHART_REPO_ENDPOINT required)
+helm/chart/promote/remote: helm/chart/promote/remote/prepare helm/chart/build/package
+	@$(SELF) -s helm/chart/build/clean
+
+## Alias for helm/chart/publish/all. WARNING: Eventually will became functional equal to helm/chart/publish/one
+helm/chart/publish: helm/chart/publish/all
+
+## Publish chart $CHART_NAME to $TARGET_CHART_REPO_ENDPOINT
+helm/chart/publish/all:
+	$(call assert-set,CHART_NAME)
+	$(call assert-set,TARGET_CHART_REPO_ENDPOINT)
+	@for package in $(shell find ${HELM_PACKAGE_PATH} -maxdepth 1 -mindepth 1 -type f -name '${CHART_NAME}*.tgz') ; do \
+		$(SELF) -s helm/chart/publish/package SOURCE_CHART_FILE=$$package; \
+	done
+
+## Publish chart $SOURCE_CHART_FILE to $REPO_GATEWAY_ENDPOINT
+helm/chart/publish/package:
+	$(call assert-set,SOURCE_CHART_FILE)
+	$(call assert-set,TARGET_CHART_REPO_ENDPOINT)
+	echo "Publish package $(SOURCE_CHART_FILE)"
+	curl --insecure --data-binary "@$(SOURCE_CHART_FILE)" $(TARGET_CHART_REPO_ENDPOINT)/api/charts
+
+## Clean chart packages
+helm/chart/clean:
+	@rm -rf $(HELM_PACKAGE_PATH)
+	@echo "Remove packages from $(HELM_PACKAGE_PATH)"
+
+#### Helpers --------------------------------------------------------------------------------------------
+
+# Copy $SOURCE_CHART_TPL directory to $HELM_PACKAGE_PATH and rename it $CHART_NAME
+helm/chart/build/prepare: helm/chart/build/clean
+	$(call assert-set,CHART_NAME)
+	$(call assert-set,SOURCE_CHART_TPL)
+	$(call assert-set,HELM_PACKAGE_PATH)
+	@echo "Create $(HELM_PACKAGE_PATH)$(CHART_NAME) from $(SOURCE_CHART_TPL)"
+	@mkdir -p $(HELM_PACKAGE_PATH)
+	@cp -R $(SOURCE_CHART_TPL) $(HELM_PACKAGE_PATH)$(CHART_NAME)
+
+
+# Clean temporary chart $CHART_NAME directory in $HELM_PACKAGE_PATH
+helm/chart/build/clean:
+	$(call assert-set,CHART_NAME)
+	$(call assert-set,HELM_PACKAGE_PATH)
+	@echo "Cleaning up $(HELM_PACKAGE_PATH)$(CHART_NAME)"
+	@rm -rf $(HELM_PACKAGE_PATH)$(CHART_NAME)
+
+# Get dependencies for chart named CHART_NAME that stored in HELM_PACKAGE_PATH
+helm/chart/build/deps:
+	$(call assert-set,CHART_NAME)
+	$(call assert-set,HELM_PACKAGE_PATH)
+	@echo "Fetch dependencies for $(HELM_PACKAGE_PATH)$(CHART_NAME)"
+	@$(HELM) dependency build --debug $(HELM_PACKAGE_PATH)$(CHART_NAME)
+
+# Create a helm package called CHART_NAME and store it in HELM_PACKAGE_PATH
+helm/chart/build/package:
+	$(call assert-set,CHART_NAME)
+	$(call assert-set,TARGET_VERSION)
+	$(call assert-set,HELM_PACKAGE_PATH)
+	@echo "For back compatibility pinning image.tag to $(TARGET_VERSION)"
+	@set -o pipefail; yq write --inplace $(HELM_PACKAGE_PATH)$(CHART_NAME)/values.yaml image.tag $(TARGET_VERSION) | head -1
+	@echo "Pack $(CHART_NAME) with version $(TARGET_VERSION)"
+	@$(HELM) package  \
+		--version $(TARGET_VERSION) \
+		--app-version $(TARGET_VERSION) \
+		--save=false \
+		--debug \
+		--destination $(HELM_PACKAGE_PATH) \
+		$(HELM_PACKAGE_PATH)$(CHART_NAME)
+
+# Unpack $SOURCE_CHART_FILE tarball to $HELM_PACKAGE_PATH
+helm/chart/promote/local/prepare: helm/chart/build/clean
+	$(call assert-set,SOURCE_CHART_FILE)
+	$(call assert-set,HELM_PACKAGE_PATH)
+	@echo "Create chart from $(SOURCE_CHART_FILE)"
+	@mkdir -p $(HELM_PACKAGE_PATH)
+	@tar -zxvf $(SOURCE_CHART_FILE) -C $(HELM_PACKAGE_PATH)
+
+# Unpack $SOURCE_CHART_REPO_ENDPOINT/$CHART_NAME-$SOURCE_VERSION tarball to $HELM_PACKAGE_PATH
+helm/chart/promote/remote/prepare: helm/chart/build/clean
+	$(call assert-set,CHART_NAME)
+	$(call assert-set,SOURCE_VERSION)
+	$(call assert-set,HELM_PACKAGE_PATH)
+	$(call assert-set,SOURCE_CHART_REPO_ENDPOINT)
+	@echo "Create $(HELM_PACKAGE_PATH)$(CHART_NAME) from {SOURCE_CHART_REPO_ENDPOINT}/$(CHART_NAME)-$(SOURCE_VERSION)"
+	@mkdir -p $(HELM_PACKAGE_PATH)
+	@$(HELM) fetch --untar --untardir $(HELM_PACKAGE_PATH) \
+		--repo $(SOURCE_CHART_REPO_ENDPOINT)\
+		--version $(SOURCE_VERSION) \
+		$(CHART_NAME)
+

modules/helm/Makefile.chart

@@ -3,7 +3,7 @@ CLOUDPOSSE_HELM_STARTER_REPO ?= https://github.com/cloudposse/helm-chart-scaffol
 CLOUDPOSSE_HELM_STARTER_REF ?= master
 CLOUDPOSSE_HELM_STARTER_NAME ?= cloudposse
 
-CHART_TPL ?= chart
+SOURCE_CHART_TPL ?= chart
 
 ## Fetch starter
 helm/chart/starter/fetch:
@@ -23,51 +23,9 @@ helm/chart/starter/update: helm/chart/starter/remove helm/chart/starter/fetch
 
 ## Create chart $CHART from starter scaffold
 helm/chart/create:
-	$(call assert-set,CHART)
-	@$(HELM) create $(CHART) -p $(CLOUDPOSSE_HELM_STARTER_NAME)
-	@chmod -R 777 $(CHART)
-
-## Build chart $CHART_NAME from $CHART_TPL
-helm/chart/build:
-	$(call assert-set,CHART_NAME)
-	$(call assert-set,CHART_TPL)
-	@rm -rf $(HELM_PACKAGE_PATH)$(CHART_NAME)
-	@mkdir -p $(HELM_PACKAGE_PATH)$(CHART_NAME)
-	@cp -R $(CHART_TPL)/* $(HELM_PACKAGE_PATH)$(CHART_NAME)
-	@echo "Pinning chart version and image.tag to $(SEMVERSION)"
-	@set -o pipefail; yq write --inplace $(HELM_PACKAGE_PATH)$(CHART_NAME)/Chart.yaml version $(SEMVERSION) | head -1
-	@set -o pipefail; yq write --inplace $(HELM_PACKAGE_PATH)$(CHART_NAME)/values.yaml image.tag $(SEMVERSION) | head -1
-	@echo "Fetch dependencies"
-	@$(HELM) dependency build --debug $(HELM_PACKAGE_PATH)$(CHART_NAME)
-	@echo "Create package"
-	@$(HELM) package --save=false --debug $(HELM_PACKAGE_PATH)$(CHART_NAME)
-	@find . -maxdepth 1 -mindepth 1 -type f -name '$(CHART_NAME)*.tgz' | xargs -I '{}' mv {} $(HELM_PACKAGE_PATH)
-	@echo "Cleaning up"
-	@rm -rf $(HELM_PACKAGE_PATH)$(CHART_NAME)
-
-## Build chart $CHART_NAME from $CHART_TPL for all available $SEMVERSIONS
-helm/chart/build-all:
-	$(call assert-set,CHART_NAME)
-	$(call assert-set,CHART_TPL)
-	@for version in $(SEMVERSIONS) ; do \
-		echo "Build chart $(CHART_NAME) version $$version"; \
-		$(SELF) -s helm/chart/build \
-			CHART_NAME=$(CHART_NAME) \
-			CHART_TPL=$(CHART_TPL) \
-			SEMVERSION=$$version; \
-	done
-
-## Publish chart $CHART_NAME to $REPO_GATEWAY_ENDPOINT
-helm/chart/publish:
 	$(call assert-set,CHART_NAME)
-	$(call assert-set,REPO_GATEWAY_ENDPOINT)
-	@for package in $(shell find ${HELM_PACKAGE_PATH} -maxdepth 1 -mindepth 1 -type f -name '${CHART_NAME}*.tgz') ; do \
-		echo "Publish package $$package"; \
-		curl --insecure --data-binary "@$$package" $(REPO_GATEWAY_ENDPOINT)/api/charts; \
-		echo "" ; \
-	done
+	$(call assert-set,SOURCE_CHART_TPL)
+	@$(HELM) create $(CHART_NAME) -p $(CLOUDPOSSE_HELM_STARTER_NAME)
+	@chmod -R 777 $(CHART_NAME)
+	@mv $(CHART_NAME) $(SOURCE_CHART_TPL)
 
-## Clean chart packages
-helm/chart/clean:
-	@rm -rf $(HELM_PACKAGE_PATH)
-	@echo "Remove packages from $(HELM_PACKAGE_PATH)"