From c9c8870cf89780aae9b3a2c108ad1a7d2a8f0783 Mon Sep 17 00:00:00 2001 From: TK Kiatkungwanglai Date: Mon, 5 Nov 2018 10:12:04 -0800 Subject: [PATCH] Remove MARKETPLACE_TOOLS_PATH references. (#258) * Add latest tagging to cloudbuild-tag.yaml. * Remove MARKETPLACE_TOOLS_PATH and simplify MARKETPLACE_TOOLS_TAG derivation. * Tune cloudbuild.yaml. * Revert "Add latest tagging to cloudbuild-tag.yaml." This reverts commit 433a87abf8fe66e93b27f7bfc83b7916509e4af1. We will publish latest tags explicitly. --- Makefile | 22 +--------------------- cloudbuild.yaml | 34 ++++------------------------------ common.Makefile | 25 +++---------------------- crd.Makefile | 8 +++----- marketplace.Makefile | 40 +++++++++++++++++----------------------- scripts/derive_tag.sh | 30 ------------------------------ testing.Makefile | 7 ++++--- testrunner.Makefile | 11 ++++------- var.Makefile | 2 +- 9 files changed, 37 insertions(+), 142 deletions(-) delete mode 100755 scripts/derive_tag.sh diff --git a/Makefile b/Makefile index d7cf4ff7..adba67af 100644 --- a/Makefile +++ b/Makefile @@ -1,27 +1,7 @@ -include common.Makefile - - -### Submodule ### - - -.PHONY: submodule/init -submodule/init: - git submodule init - git submodule sync --recursive - git submodule update --init --recursive - - -.PHONY: submodule/init-force -submodule/init-force: - git submodule init - git submodule sync --recursive - git submodule update --init --recursive --force - - ### Images publishing ### - include marketplace.Makefile +include testing.Makefile ### Testing ### diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 6af6b904..abd9ce60 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -126,39 +126,17 @@ steps: - marketplace/deployer_helm_tiller_base/onbuild/Dockerfile - . -- id: &GetKubectlCredentials Get Kubernetes Credentials +- id: &InitializeCredentials Initialize Credentials name: gcr.io/cloud-builders/gcloud waitFor: - '-' - args: - - container - - clusters - - get-credentials - - 'limani-integ' - - --region - - 'us-central1' - - --project - - '$PROJECT_ID' - -- id: &CopyKubectlCredentials Copy kubectl Credentials - name: gcr.io/google-appengine/debian9 - waitFor: - - Get Kubernetes Credentials entrypoint: bash args: - -exc - | + gcloud container clusters get-credentials 'limani-integ' --region 'us-central1' --project '$PROJECT_ID' mkdir -p /workspace/.kube/ cp -r $$HOME/.kube/ /workspace/ - -- id: &CopyGcloudCredentials Copy gcloud Credentials - name: gcr.io/google-appengine/debian9 - waitFor: - - Get Kubernetes Credentials - entrypoint: bash - args: - - -exc - - | mkdir -p /workspace/.config/gcloud/ cp -r $$HOME/.config/gcloud/ /workspace/.config/ @@ -166,20 +144,16 @@ steps: name: gcr.io/$PROJECT_ID/k8s/dev:sha_$COMMIT_SHA waitFor: - *BuildDev - - *BuildEnvsubst - - *BuildHelm - - *BuildHelmTiller - - *CopyGcloudCredentials - - *CopyKubectlCredentials + - *InitializeCredentials env: - 'KUBE_CONFIG=/workspace/.kube' - 'GCLOUD_CONFIG=/workspace/.config/gcloud' # Use local Docker network named cloudbuild as described here: # https://cloud.google.com/cloud-build/docs/overview#build_configuration_and_build_steps - 'EXTRA_DOCKER_PARAMS=--net cloudbuild' + - MARKETPLACE_TOOLS_TAG=testing-sha_$COMMIT_SHA args: - make - - --file=testing.Makefile - -j4 - testing/all diff --git a/common.Makefile b/common.Makefile index 020eca45..d03e071f 100644 --- a/common.Makefile +++ b/common.Makefile @@ -14,30 +14,19 @@ define print_error printf "\n\033[93m\033[1m$(1)\033[0m\n" endef -makefile_dir := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) -include $(makefile_dir)/var.Makefile +include var.Makefile -# MARKETPLACE_TOOLS_PATH is the path to the root of the -# marketplace-k8s-app-tools submodule. We derive it here by taking the -# full path of the directory containing this common.Makefile. patsubst -# operation removes the trailing slash. -MARKETPLACE_TOOLS_PATH ?= $(patsubst %/,%,$(dir $(realpath $(lastword $(MAKEFILE_LIST))))) - -$(info ---- MARKETPLACE_TOOLS_PATH = $(MARKETPLACE_TOOLS_PATH)) - # MARKETPLACE_TOOLS_TAG is the tag of the container images published # by marketplace-k8s-app-tools. -MARKETPLACE_TOOLS_TAG ?= $(shell cd $(MARKETPLACE_TOOLS_PATH) && $(MARKETPLACE_TOOLS_PATH)/scripts/derive_tag.sh) +MARKETPLACE_TOOLS_TAG ?= $(shell echo local-$(USER)) export MARKETPLACE_TOOLS_TAG $(info ---- MARKETPLACE_TOOLS_TAG = $(MARKETPLACE_TOOLS_TAG)) - -.build: | tools_path_exists +.build: mkdir -p "$@" - .build/tmp: | .build mkdir -p "$@" @@ -46,12 +35,4 @@ $(info ---- MARKETPLACE_TOOLS_TAG = $(MARKETPLACE_TOOLS_TAG)) clean:: rm -Rf .build - -.PHONY: tools_path_exists -tools_path_exists: -ifeq ($(shell test -d "$(MARKETPLACE_TOOLS_PATH)" && echo "OK"),) - $(error '$(MARKETPLACE_TOOLS_PATH)' directory does not exist. Must set proper path for marketplace tools) -endif - - endif diff --git a/crd.Makefile b/crd.Makefile index f3ebd65e..574af1b3 100644 --- a/crd.Makefile +++ b/crd.Makefile @@ -2,19 +2,17 @@ ifndef __CRD_MAKEFILE__ __CRD_MAKEFILE__ := included - -makefile_dir := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) -include $(makefile_dir)/common.Makefile +include common.Makefile # Installs the application CRD on the cluster. .PHONY: crd/install crd/install: - kubectl apply -f "$(MARKETPLACE_TOOLS_PATH)/crd/app-crd.yaml" + kubectl apply -f "crd/app-crd.yaml" # Uninstalls the application CRD from the cluster. .PHONY: crd/uninstall crd/uninstall: - kubectl delete -f "$(MARKETPLACE_TOOLS_PATH)/crd/app-crd.yaml" + kubectl delete -f "crd/app-crd.yaml" endif diff --git a/marketplace.Makefile b/marketplace.Makefile index 1585c7e4..59314eb2 100644 --- a/marketplace.Makefile +++ b/marketplace.Makefile @@ -4,23 +4,21 @@ __MARKETPLACE_MAKEFILE__ := included COMMIT ?= $(shell git rev-parse HEAD | fold -w 12 | head -n 1) -makefile_dir := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) -include $(makefile_dir)/common.Makefile -include $(makefile_dir)/var.Makefile +include common.Makefile +include var.Makefile .build/marketplace: | .build mkdir -p "$@" -.build/marketplace/dev: $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_util/* \ - $(MARKETPLACE_TOOLS_PATH)/marketplace/dev/* \ - $(MARKETPLACE_TOOLS_PATH)/scripts/* \ - $(MARKETPLACE_TOOLS_PATH)/marketplace.Makefile \ +.build/marketplace/dev: marketplace/deployer_util/* \ + marketplace/dev/* \ + scripts/* \ + marketplace.Makefile \ .build/var/MARKETPLACE_TOOLS_TAG \ | .build/marketplace $(call print_target) - cd "$(MARKETPLACE_TOOLS_PATH)" ; \ docker build \ --tag "gcr.io/cloud-marketplace-tools/k8s/dev:$(MARKETPLACE_TOOLS_TAG)" \ -f marketplace/dev/Dockerfile \ @@ -32,13 +30,12 @@ include $(makefile_dir)/var.Makefile mkdir -p "$@" -.build/marketplace/deployer/envsubst: $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_util/* \ - $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_envsubst_base/* \ +.build/marketplace/deployer/envsubst: marketplace/deployer_util/* \ + marketplace/deployer_envsubst_base/* \ .build/marketplace/delete_deprecated \ .build/var/MARKETPLACE_TOOLS_TAG \ | .build/marketplace/deployer $(call print_target) - cd "$(MARKETPLACE_TOOLS_PATH)"; \ docker build \ --tag "gcr.io/cloud-marketplace-tools/k8s/deployer_envsubst:$(MARKETPLACE_TOOLS_TAG)" \ -f marketplace/deployer_envsubst_base/Dockerfile \ @@ -46,15 +43,14 @@ include $(makefile_dir)/var.Makefile @touch "$@" -.build/marketplace/deployer/helm: $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_util/* \ - $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_helm_base/* \ +.build/marketplace/deployer/helm: marketplace/deployer_util/* \ + marketplace/deployer_helm_base/* \ .build/marketplace/delete_deprecated \ .build/var/COMMIT \ .build/var/MARKETPLACE_TOOLS_TAG \ | .build/marketplace/deployer $(call print_target) - cd $(MARKETPLACE_TOOLS_PATH) \ - && docker build \ + docker build \ --build-arg VERSION="$(COMMIT)" \ --tag "gcr.io/cloud-marketplace-tools/k8s/deployer_helm:$(MARKETPLACE_TOOLS_TAG)" \ -f marketplace/deployer_helm_base/Dockerfile \ @@ -64,12 +60,11 @@ include $(makefile_dir)/var.Makefile .build/marketplace/deployer/helm_tiller: \ .build/var/MARKETPLACE_TOOLS_TAG \ - $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_helm_tiller_base/* \ - $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_util/* \ + marketplace/deployer_helm_tiller_base/* \ + marketplace/deployer_util/* \ | .build/marketplace/deployer $(call print_target) - cd $(MARKETPLACE_TOOLS_PATH) \ - && docker build \ + docker build \ --tag "gcr.io/cloud-marketplace-tools/k8s/deployer_helm_tiller:$(MARKETPLACE_TOOLS_TAG)" \ -f marketplace/deployer_helm_tiller_base/Dockerfile \ . @@ -79,12 +74,11 @@ include $(makefile_dir)/var.Makefile .build/marketplace/deployer/helm_tiller_onbuild: \ .build/marketplace/deployer/helm_tiller \ .build/var/MARKETPLACE_TOOLS_TAG \ - $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_helm_tiller_base/onbuild/* \ - $(MARKETPLACE_TOOLS_PATH)/marketplace/deployer_util/* \ + marketplace/deployer_helm_tiller_base/onbuild/* \ + marketplace/deployer_util/* \ | .build/marketplace/deployer $(call print_target) - cd $(MARKETPLACE_TOOLS_PATH) \ - && docker build \ + docker build \ --build-arg FROM="gcr.io/cloud-marketplace-tools/k8s/deployer_helm_tiller:$(MARKETPLACE_TOOLS_TAG)" \ --tag "gcr.io/cloud-marketplace-tools/k8s/deployer_helm_tiller/onbuild:$(MARKETPLACE_TOOLS_TAG)" \ -f marketplace/deployer_helm_tiller_base/onbuild/Dockerfile \ diff --git a/scripts/derive_tag.sh b/scripts/derive_tag.sh deleted file mode 100755 index b6d14196..00000000 --- a/scripts/derive_tag.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -if [[ ! -z "$(git status --porcelain)" ]]; then - >&2 echo " -----------------------------------------------------------" - >&2 echo " / /" - >&2 echo " / The marketplace-k8s-app-tools submodule is not clean. /" - >&2 echo " / Please clean the marketplace-k8s-app-tools submodule. /" - >&2 echo " / /" - >&2 echo "-----------------------------------------------------------" - echo -n '' - exit 1 -fi - -echo -n "sha_$(git rev-parse HEAD)" diff --git a/testing.Makefile b/testing.Makefile index d6513c5b..36d39374 100644 --- a/testing.Makefile +++ b/testing.Makefile @@ -2,17 +2,18 @@ ifndef __TESTING_MAKEFILE__ __TESTING_MAKEFILE__ := included +include common.Makefile + # TODO: Move testing targets in top-level Makefile to here and include # this file in top-level Makefile. -include common.Makefile include gcloud.Makefile include marketplace.Makefile TEST_ID := $(shell cat /dev/urandom | tr -dc 'a-z0-9' | head -c 8) .testing/marketplace/deployer/helm_tiller_onbuild: - mkdir -p $@ + mkdir -p "$@" .testing/marketplace/deployer/helm_tiller_onbuild/helm-dependency-build: \ .build/marketplace/deployer/helm_tiller_onbuild \ @@ -53,7 +54,7 @@ testing/marketplace/deployer/helm_tiller_onbuild: \ .testing/marketplace/deployer/envsubst: - mkdir -p $@ + mkdir -p "$@" .testing/marketplace/deployer/envsubst/standard: \ .build/marketplace/deployer/envsubst \ diff --git a/testrunner.Makefile b/testrunner.Makefile index bdacb5c7..674c8f16 100644 --- a/testrunner.Makefile +++ b/testrunner.Makefile @@ -2,19 +2,16 @@ ifndef __TESTRUNNER_MAKEFILE__ __TESTRUNNER_MAKEFILE__ := included - -makefile_dir := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) -include $(makefile_dir)/common.Makefile - +include common.Makefile .build/testrunner: | .build/tmp $(call print_target) - cd $(MARKETPLACE_TOOLS_PATH)/testrunner \ + cd testrunner \ && bazel build //runner:main rm -rf .build/tmp/testrunner mkdir -p .build/tmp/testrunner - cp $(MARKETPLACE_TOOLS_PATH)/testrunner/bazel-bin/runner/main .build/tmp/testrunner/testrunner - docker build --tag=testrunner --file=$(MARKETPLACE_TOOLS_PATH)/testrunner/runner/Dockerfile .build/tmp/testrunner + cp testrunner/bazel-bin/runner/main .build/tmp/testrunner/testrunner + docker build --tag=testrunner --file=testrunner/runner/Dockerfile .build/tmp/testrunner @touch "$@" endif diff --git a/var.Makefile b/var.Makefile index 9149af6d..36032aef 100644 --- a/var.Makefile +++ b/var.Makefile @@ -23,7 +23,7 @@ __VAR_MAKEFILE__ := included .build/var: - mkdir -p .build/var + mkdir -p "$@" # Since we can't make pattern targets phony, we make them