From 330edfd316011400ac728d3ade26d8358bf76d4b Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Thu, 20 Sep 2018 15:06:18 -0700 Subject: [PATCH] Squashed 'hack/libbuild/' changes from 70ef1128..1de36b18 1de36b18 Use docker build --pull xref: https://github.com/appscodelabs/issues/issues/90 43d3c958 Use forked client-go d2f59a3f Add k8s.io/metrics to dep list 91a37ca4 Update commit message c23c8744 Use go-1.11 66580929 Fix depfixer 8f5aa600 Add revendor() helper (#27) dcff0532 Update gke version (#26) 8aab0b38 Allow REPO_ROOT independent of github.com (#25) 636f43cc fix uninstall (#24) git-subtree-dir: hack/libbuild git-subtree-split: 1de36b1897e683bf603d3604c6fd9f237a86dbbf --- common/lib.sh | 2 +- concourse/cluster.sh | 19 ++--- concourse/init.sh | 9 +- libbuild.py | 199 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 209 insertions(+), 20 deletions(-) diff --git a/common/lib.sh b/common/lib.sh index 7c7410731..a559f2111 100755 --- a/common/lib.sh +++ b/common/lib.sh @@ -110,7 +110,7 @@ EOL } build() { - local cmd="docker build -t $DOCKER_REGISTRY/$IMG:$TAG ." + local cmd="docker build --pull -t $DOCKER_REGISTRY/$IMG:$TAG ." echo $cmd; $cmd } diff --git a/concourse/cluster.sh b/concourse/cluster.sh index fe4d2b618..a73f9e6ef 100755 --- a/concourse/cluster.sh +++ b/concourse/cluster.sh @@ -24,8 +24,12 @@ function cleanup_test_stuff() { kubectl get nodes kubectl describe nodes - pushd $GOPATH/src/github.com/$ORG_NAME/$REPO_NAME - ./hack/deploy/setup.sh --uninstall --purge + if [ -d "$BASE_DIR/creds" ]; then + rm -rf $BASE_DIR/creds + fi + + pushd $REPO_ROOT + ./hack/concourse/uninstall.sh popd # delete cluster on exit @@ -44,15 +48,6 @@ function cleanup_test_stuff() { pharmer apply "$NAME" pharmer get cluster fi - - pushd $GOPATH/src/github.com/$ORG_NAME/$REPO_NAME - ./hack/deploy/setup.sh --uninstall --purge - popd - - # delete docker image on exit - curl -LO https://raw.githubusercontent.com/appscodelabs/libbuild/master/docker.py - chmod +x docker.py - ./docker.py del_tag $DOCKER_REGISTRY $OPERATOR_NAME $TAG } trap cleanup_test_stuff EXIT @@ -241,7 +236,7 @@ if [ "${ClusterProvider}" = "gke" ]; then CredProvider=GoogleCloud ZONE=us-central1-f NODE=n1-standard-2 - K8S_VERSION=${K8S_VERSION:-"1.10.4-gke.2"} + K8S_VERSION=${K8S_VERSION:-"1.10.6-gke.2"} pharmer_common elif [ "${ClusterProvider}" = "aws" ]; then diff --git a/concourse/init.sh b/concourse/init.sh index 39caa28b1..a6d66af6e 100755 --- a/concourse/init.sh +++ b/concourse/init.sh @@ -1,20 +1,21 @@ #!/usr/bin/env bash +BASE_DIR=$(pwd) GOPATH=$(go env GOPATH) -REPO_ROOT="$GOPATH/src/github.com/$ORG_NAME/$REPO_NAME" +REPO_ROOT=${REPO_ROOT:-"$GOPATH/src/github.com/$ORG_NAME/$REPO_NAME"} PHARMER_VERSION="0.1.0-rc.5" ONESSL_VERSION="0.7.0" ClusterProvider=$ClusterProvider # copy $REPO_ROOT to $GOPATH -mkdir -p "$GOPATH"/src/github.com/$ORG_NAME -cp -r $REPO_NAME "$GOPATH"/src/github.com/$ORG_NAME +mkdir -p $REPO_ROOT +cp -r $REPO_NAME/. $REPO_ROOT # install all the dependencies and prepeare cluster source "$REPO_ROOT/hack/libbuild/concourse/dependencies.sh" source "$REPO_ROOT/hack/libbuild/concourse/cluster.sh" -pushd "$GOPATH"/src/github.com/$ORG_NAME/$REPO_NAME +pushd $REPO_ROOT # changed name of branch # this is necessary because operator image tag is based on branch name diff --git a/libbuild.py b/libbuild.py index 79bba0e23..10b123f99 100644 --- a/libbuild.py +++ b/libbuild.py @@ -10,14 +10,16 @@ import json import os import os.path +import random import re import socket +import string import subprocess import sys -from collections import OrderedDict +import yaml +from collections import Counter, OrderedDict from os.path import expandvars - REPO_ROOT = '' BIN_MATRIX = None BUCKET_MATRIX = None @@ -182,7 +184,7 @@ def go_build(name, goos, goarch, main, compress=False, upx=False): if goos == 'alpine': repo_dir = REPO_ROOT[len(GOPATH):] uid = check_output('id -u').strip() - cmd = "docker run --rm -u {uid} -v /tmp:/.cache -v {repo_root}:/go{repo_dir} -w /go{repo_dir} -e {cgo_env} golang:1.9-alpine {goc} build -o {bindir}/{name}-{goos}-{goarch}{ext} {cgo} {ldflags} {tags} {main}".format( + cmd = "docker run --rm -u {uid} -v /tmp:/.cache -v {repo_root}:/go{repo_dir} -w /go{repo_dir} -e {cgo_env} golang:1.11-alpine {goc} build -o {bindir}/{name}-{goos}-{goarch}{ext} {cgo} {ldflags} {tags} {main}".format( repo_root=REPO_ROOT, repo_dir=repo_dir, uid=uid, @@ -338,3 +340,194 @@ def _ungroup_go_imports(fname): f.seek(0) f.writelines(out) f.truncate() + +def git_branch_exists(branch): + return call('git show-ref --quiet refs/heads/{0}'.format(branch), cwd=REPO_ROOT) == 0 + + +def git_checkout(branch): + call('git fetch --all --prune', cwd=REPO_ROOT) + call('git fetch --tags', cwd=REPO_ROOT) + if git_branch_exists(branch): + call('git checkout {0}'.format(branch), cwd=REPO_ROOT) + else: + call('git checkout -b {0}'.format(branch), cwd=REPO_ROOT) + + +def git_requires_commit(): + changed_files = check_output('git diff --name-only', cwd=REPO_ROOT).strip().split('\n') + return Counter(changed_files) != Counter(['glide.lock']) + + +def glide_mod(glide_config): + for x in DEP_LIST: + for idx, dep in enumerate(glide_config['import']): + if dep['package'] == x['package']: + glide_config['import'][idx] = x + break + + +def glide_write(f, glide_config): + f.seek(0) + pkg = glide_config.pop('package') + out = 'package: ' + pkg + '\n' + yaml.dump(glide_config, default_flow_style=False) + f.write(out) + f.truncate() + glide_config['package'] = pkg + + +DEP_LIST = [ + { + 'package': 'github.com/cpuguy83/go-md2man', + 'version': 'v1.0.8', + }, + { + 'package': 'github.com/json-iterator/go', + 'version': '1.1.5', + }, + { + 'package': 'github.com/coreos/prometheus-operator', + 'version': 'v0.23.2', + }, + { + "package": "k8s.io/api", + "version": "kubernetes-1.11.3" + }, + { + "package": "k8s.io/apiextensions-apiserver", + "version": "kubernetes-1.11.3" + }, + { + "package": "k8s.io/apimachinery", + "repo": "https://github.com/pharmer/apimachinery.git", + "vcs": "git", + "version": "release-1.11.3" + }, + { + "package": "k8s.io/apiserver", + "repo": "https://github.com/pharmer/apiserver.git", + "vcs": "git", + "version": "release-1.11.3" + }, + { + "package": "k8s.io/client-go", + "repo": "https://github.com/pharmer/client-go.git", + "vcs": "git", + "version": "release-1.11.3" + }, + { + "package": "k8s.io/kubernetes", + "version": "v1.11.3" + }, + { + "package": "k8s.io/kube-aggregator", + "version": "kubernetes-1.11.3" + }, + { + "package": "k8s.io/metrics", + "version": "kubernetes-1.11.3" + }, + { + "package": "k8s.io/kube-openapi", + "version": "master" + }, + { + "package": "github.com/appscode/kutil", + "version": "release-8.0" + }, + { + "package": "github.com/appscode/kubernetes-webhook-util", + "version": "release-8.0" + }, + { + "package": "kmodules.xyz/monitoring-agent-api", + "repo": "https://github.com/kmodules/monitoring-agent-api.git", + "vcs": "git", + "version": "release-8.0" + }, + { + "package": "kmodules.xyz/objectstore-api", + "repo": "https://github.com/kmodules/objectstore-api.git", + "vcs": "git", + "version": "release-8.0" + }, + { + "package": "kmodules.xyz/offshoot-api", + "repo": "https://github.com/kmodules/offshoot-api.git", + "vcs": "git", + "version": "release-8.0" + }, + { + "package": "github.com/appscode/kubernetes-webhook-util", + "version": "release-8.0" + }, + { + "package": "github.com/openshift/api", + "version": "31a7bbd2266d178da3c12bb83f5274d387f775e6" + }, + { + "package": "github.com/openshift/client-go", + "version": "4688ad28de2e88110c0ea30179c51b9b205f99be" + }, + { + "package": "github.com/openshift/origin", + "version": "fecffb2fce100260088a1b9f268c0901a778cf2b" + }, + { + "package": "github.com/spf13/cobra", + "version": "v0.0.3" + }, + { + "package": "github.com/spf13/pflag", + "version": "v1.0.1" + }, + { + "package": "github.com/graymeta/stow", + "repo": "https://github.com/appscode/stow.git", + "vcs": "git", + "version": "master" + }, + { + "package": "github.com/Azure/azure-sdk-for-go", + "version": "v14.6.0" + }, + { + "package": "github.com/Azure/go-autorest", + "version": "v10.6.2" + }, + { + "package": "github.com/aws/aws-sdk-go", + "version": "v1.12.7" + }, + { + "package": "google.golang.org/api/storage/v1", + "version": "master" + }, + { + "package": "cloud.google.com/go", + "version": "v0.2.0" + }, +] + + +def revendor(): + seed = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + revendor_branch = 'depfixer-{0}'.format(seed) + print(REPO_ROOT) + + call('git reset HEAD --hard', cwd=REPO_ROOT) + call('git clean -xfd', cwd=REPO_ROOT) + git_checkout('master') + call('git pull --rebase origin master', cwd=REPO_ROOT) + git_checkout(revendor_branch) + with open(REPO_ROOT + '/glide.yaml', 'r+') as glide_file: + glide_config = yaml.load(glide_file) + glide_mod(glide_config) + glide_write(glide_file, glide_config) + call('glide slow', cwd=REPO_ROOT) + if git_requires_commit(): + call('git add --all', cwd=REPO_ROOT) + call('git commit -s -a -m "Use kubernetes-1.11.3"', cwd=REPO_ROOT) + call('git push origin {0}'.format(revendor_branch), cwd=REPO_ROOT) + else: + call('git reset HEAD --hard', cwd=REPO_ROOT)