From 6dfeb126e1ea131b3df2cfcecc9d572b03785109 Mon Sep 17 00:00:00 2001 From: Liran Yogev <23477645+lyogev@users.noreply.github.com> Date: Sun, 24 May 2020 14:22:25 +0300 Subject: [PATCH] fix(travis): parallelise build (#296) * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix --- .travis.yml | 86 ++++++++++++++++++++++--------- scripts/before_cache.sh | 9 ++-- scripts/build.sh | 2 +- scripts/clear_from_cache.sh | 7 +++ scripts/docker-without-cache.sh | 7 --- scripts/docker_load_dev.sh | 14 +++++ scripts/docker_publish_dev.sh | 2 + scripts/load_docker_from_cache.sh | 11 ++++ scripts/load_from_cache.sh | 5 ++ scripts/save_docker_to_cache.sh | 6 +++ scripts/test.sh | 2 + scripts/travis_build.sh | 48 ----------------- 12 files changed, 115 insertions(+), 84 deletions(-) create mode 100755 scripts/clear_from_cache.sh delete mode 100755 scripts/docker-without-cache.sh create mode 100755 scripts/docker_load_dev.sh create mode 100755 scripts/load_docker_from_cache.sh create mode 100755 scripts/load_from_cache.sh create mode 100755 scripts/save_docker_to_cache.sh diff --git a/.travis.yml b/.travis.yml index 617f6c4e6..de6ba7842 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ cache: directories: - $HOME/.ivy2/cache - $HOME/.sbt + - $TARGET_CACHE branches: only: - master @@ -22,32 +23,69 @@ env: - HUDI_HIVE1_VERSION=0.4.7 - HIVE_VERSION=2.3.3 - HUDI_VERSION=0.5.2-incubating + - TARGET_CACHE=$HOME/target-cache/${TRAVIS_COMMIT} before_script: - export -f travis_fold - export -f travis_time_start - export -f travis_time_finish - export -f travis_nanoseconds -script: scripts/travis_build.sh -before_cache: scripts/before_cache.sh -deploy: - - provider: script - skip_cleanup: true - script: scripts/publish.sh - on: - all_branches: true - condition: ($TRAVIS_BRANCH = master) || ($TRAVIS_TAG =~ ^v.*) - - provider: script - skip_cleanup: true - script: scripts/docker_publish.sh - on: - all_branches: true - condition: $TRAVIS_TAG =~ ^v.* - - provider: releases - skip_cleanup: true - token: - secure: i+W1C7xBogQ5XUxctAV92+YaraEGrHscjnuIkXHesfGEvF5rz9AdwGvpLzRIdLOHH0S4tR5uPFVUF+w86IWQOHLyZhNDybbkIpoQqRRuB0xiOwsqlcjz5IwNkhraTRF+8sPl9FBI5OuGfOKh8jy91xLpYvcMt+EKsai+5rugDxyb8/L3G5nQcH+/5c40dflDmDChkJF/qoNB7iLofMoW9uG72QHKeO87JoVJKg6o73anpuqQ9qZA7W4GYZBmU5QrHNMfdQqt/8XgrPvxq4DW8EnH1Tzg4nXuBREvY92rBxBYpAuff0uxlRJmd8n19Nk4hNRNNpwBIBlHwmbzCAY4AnRJmZT1G/A3E5wznfUt1JekIdwCfaVBbIaAbeFwrTV2DjJYSWdmeB79HF82CYRHuWY4tkxUzUMuBUK2KKFovfMvNG2Lx33cQRwC1YrCb+9W587rWx3dNfIY4vj9aWbCcfZnfrM0N00rlQKZH16dkdYIEZlg396h3v9M41MmvNgez7O/ftUILF3cIc6AG2X8xzcpbXB0lGCFwNdMXKtEpUf04Nw5Lu84Ov662OWfPAM2G/edSnXF1d56XxLnax8dQNcoqc25wcn+1VmpguzLr530rI9dNREZC2sQ7C0/cLnk6SAP12PKXk9nmigyWIbbGHe2FBC4WBSssb8xMg+bgLs= - file_glob: true - file: "$TRAVIS_BUILD_DIR/target/scala-2.11/*.jar" - on: - all_branches: true - condition: $TRAVIS_TAG =~ ^v.* +jobs: + include: + - stage: "Build" + script: scripts/travis_build.sh + before_cache: scripts/before_cache.sh + - stage: "Tests" + name: "Unit & integration tests" + script: scripts/test.sh + before_script: scripts/load_from_cache.sh + before_cache: scripts/clear_from_cache.sh + - name: "influxdb" + script: (cd e2e/influxdb && ./test.sh) + before_script: scripts/docker_load_dev.sh + - name: "kafka" + script: (cd e2e/kafka && ./test.sh) + before_script: scripts/docker_load_dev.sh + - name: "elasticsearch" + script: (cd e2e/elasticsearch && ./test.sh) + before_script: scripts/docker_load_dev.sh + - name: "hive1" + script: (cd e2e/hive1 && ./test.sh) + before_script: scripts/docker_load_dev.sh + - name: "hive" + script: (cd e2e/hive && ./test.sh) + before_script: scripts/docker_load_dev.sh + - name: "hudi" + script: (cd e2e/hudi && ./test.sh) + before_script: scripts/docker_load_dev.sh + - name: "cdc" + script: (cd e2e/cdc && ./test.sh) + before_script: scripts/docker_load_dev.sh + - stage: "Deploy" + if: type != pull_request and (branch = master or tag =~ ^v) + before_script: + - scripts/load_from_cache.sh + - scripts/docker_load_dev.sh + script: skip + deploy: + - provider: script + skip_cleanup: true + script: scripts/publish.sh + on: + all_branches: true + condition: ($TRAVIS_BRANCH = master) || ($TRAVIS_TAG =~ ^v.*) + - provider: script + skip_cleanup: true + script: scripts/docker_publish.sh + on: + all_branches: true + condition: $TRAVIS_TAG =~ ^v.* + - provider: releases + skip_cleanup: true + token: + secure: i+W1C7xBogQ5XUxctAV92+YaraEGrHscjnuIkXHesfGEvF5rz9AdwGvpLzRIdLOHH0S4tR5uPFVUF+w86IWQOHLyZhNDybbkIpoQqRRuB0xiOwsqlcjz5IwNkhraTRF+8sPl9FBI5OuGfOKh8jy91xLpYvcMt+EKsai+5rugDxyb8/L3G5nQcH+/5c40dflDmDChkJF/qoNB7iLofMoW9uG72QHKeO87JoVJKg6o73anpuqQ9qZA7W4GYZBmU5QrHNMfdQqt/8XgrPvxq4DW8EnH1Tzg4nXuBREvY92rBxBYpAuff0uxlRJmd8n19Nk4hNRNNpwBIBlHwmbzCAY4AnRJmZT1G/A3E5wznfUt1JekIdwCfaVBbIaAbeFwrTV2DjJYSWdmeB79HF82CYRHuWY4tkxUzUMuBUK2KKFovfMvNG2Lx33cQRwC1YrCb+9W587rWx3dNfIY4vj9aWbCcfZnfrM0N00rlQKZH16dkdYIEZlg396h3v9M41MmvNgez7O/ftUILF3cIc6AG2X8xzcpbXB0lGCFwNdMXKtEpUf04Nw5Lu84Ov662OWfPAM2G/edSnXF1d56XxLnax8dQNcoqc25wcn+1VmpguzLr530rI9dNREZC2sQ7C0/cLnk6SAP12PKXk9nmigyWIbbGHe2FBC4WBSssb8xMg+bgLs= + file_glob: true + file: "$TRAVIS_BUILD_DIR/target/scala-2.11/*.jar" + on: + all_branches: true + condition: $TRAVIS_TAG =~ ^v.* + diff --git a/scripts/before_cache.sh b/scripts/before_cache.sh index 608294b68..d020a3dd8 100755 --- a/scripts/before_cache.sh +++ b/scripts/before_cache.sh @@ -1,7 +1,8 @@ #!/bin/bash set -e -find $HOME/.sbt -name "*.lock" | xargs rm -find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm -rm -f $HOME/.ivy2/.sbt.ivy.lock -rm -f $HOME/.ivy2/cache/org.apache.hbase/hbase-server/ivy-1.1.1.xml +./scripts/clear_from_cache.sh + +echo "Saving metorikku JAR to cache" +rm -rf target/streams/\$global/assembly target/streams/\$global/assemblyOption target/streams/test/assemblyOption target/streams/test/assembly +cp -r target $TARGET_CACHE diff --git a/scripts/build.sh b/scripts/build.sh index 4a93d4a1d..a1fbd7411 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,4 +1,4 @@ #!/bin/bash set -e -sbt -DsparkVersion=$SPARK_VERSION clean scalastyle assembly "set test in (Test, assembly) := {}" test:assembly +sbt -DsparkVersion=$SPARK_VERSION clean scalastyle "set test in (assembly) := {}" assembly "set test in (Test, assembly) := {}" test:assembly diff --git a/scripts/clear_from_cache.sh b/scripts/clear_from_cache.sh new file mode 100755 index 000000000..608294b68 --- /dev/null +++ b/scripts/clear_from_cache.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +find $HOME/.sbt -name "*.lock" | xargs rm +find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm +rm -f $HOME/.ivy2/.sbt.ivy.lock +rm -f $HOME/.ivy2/cache/org.apache.hbase/hbase-server/ivy-1.1.1.xml diff --git a/scripts/docker-without-cache.sh b/scripts/docker-without-cache.sh deleted file mode 100755 index 1a16eea4a..000000000 --- a/scripts/docker-without-cache.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -docker build -t metorikku/spark --build-arg SPARK_VERSION=$SPARK_VERSION --build-arg HADOOP_VERSION=$HADOOP_VERSION -f docker/spark/Dockerfile docker/spark -docker build -t metorikku/hive:1 --build-arg HIVE_VERSION=$HIVE1_VERSION --build-arg HUDI_HIVE1_VERSION=$HUDI_HIVE1_VERSION -f docker/hive1/Dockerfile docker/hive1 -docker build -t metorikku/hive --build-arg HIVE_VERSION=$HIVE_VERSION --build-arg HUDI_VERSION=$HUDI_VERSION -f docker/hive/Dockerfile docker/hive -docker build -t metorikku/metorikku -f docker/metorikku/Dockerfile . diff --git a/scripts/docker_load_dev.sh b/scripts/docker_load_dev.sh new file mode 100755 index 000000000..39eb7c495 --- /dev/null +++ b/scripts/docker_load_dev.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +tag=$(([ ! "${TRAVIS_PULL_REQUEST}" = "false" ] && echo "${TRAVIS_PULL_REQUEST}") || echo "${TRAVIS_BRANCH}") + +docker pull metorikku/dev:metorikku_$tag +docker pull metorikku/dev:spark_$tag +docker pull metorikku/dev:hive_$tag +docker pull metorikku/dev:hive1_$tag + +docker tag metorikku/dev:metorikku_$tag metorikku/metorikku +docker tag metorikku/dev:spark_$tag metorikku/spark +docker tag metorikku/dev:hive_$tag metorikku/hive +docker tag metorikku/dev:hive1_$tag metorikku/hive:1 diff --git a/scripts/docker_publish_dev.sh b/scripts/docker_publish_dev.sh index f08edf5e6..8ac156752 100755 --- a/scripts/docker_publish_dev.sh +++ b/scripts/docker_publish_dev.sh @@ -10,9 +10,11 @@ else docker tag metorikku/metorikku metorikku/dev:metorikku_$tag docker tag metorikku/spark metorikku/dev:spark_$tag docker tag metorikku/hive metorikku/dev:hive_$tag + docker tag metorikku/hive:1 metorikku/dev:hive1_$tag docker push metorikku/dev:metorikku_$tag docker push metorikku/dev:spark_$tag docker push metorikku/dev:hive_$tag + docker push metorikku/dev:hive1_$tag fi diff --git a/scripts/load_docker_from_cache.sh b/scripts/load_docker_from_cache.sh new file mode 100755 index 000000000..c87f37d28 --- /dev/null +++ b/scripts/load_docker_from_cache.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +echo "Loading docker cache" +docker load -i $DOCKER_CACHE/images.tar + +while read REPOSITORY TAG IMAGE_ID +do + echo "== Tagging $REPOSITORY $TAG $IMAGE_ID ==" + docker tag "$IMAGE_ID" "$REPOSITORY:$TAG" +done < $DOCKER_CACHE/images.list diff --git a/scripts/load_from_cache.sh b/scripts/load_from_cache.sh new file mode 100755 index 000000000..6ac5afc03 --- /dev/null +++ b/scripts/load_from_cache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +echo "Loading metorikku JAR" +cp -r $TARGET_CACHE/target . diff --git a/scripts/save_docker_to_cache.sh b/scripts/save_docker_to_cache.sh new file mode 100755 index 000000000..b4bc4efac --- /dev/null +++ b/scripts/save_docker_to_cache.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +echo "Saving docker images to cache" +docker save -o $DOCKER_CACHE/images.tar $(docker images -a -q) +docker images | grep -v '' | sed '1d' | awk '{print $1 " " $2 " " $3}' > $DOCKER_CACHE/images.list diff --git a/scripts/test.sh b/scripts/test.sh index f420d4fea..041c9c14c 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +sbt test + # Metorikku main test example java -Dspark.master=local[*] -cp target/scala-2.11/metorikku-standalone.jar com.yotpo.metorikku.MetorikkuTester --test-settings examples/movies_test.yaml # Metorikku kafka test example diff --git a/scripts/travis_build.sh b/scripts/travis_build.sh index 4e9d3638d..fd23551cb 100755 --- a/scripts/travis_build.sh +++ b/scripts/travis_build.sh @@ -13,54 +13,6 @@ travis_fold start "docker" travis_time_finish travis_fold end "docker" -travis_fold start "tests" - travis_time_start - ./scripts/test.sh - travis_time_finish -travis_fold end "tests" - -travis_fold start "influxdb" - travis_time_start - (cd e2e/influxdb && ./test.sh) - travis_time_finish -travis_fold end "influxdb" - -travis_fold start "kafka" - travis_time_start - (cd e2e/kafka && ./test.sh) - travis_time_finish -travis_fold end "kafka" - -travis_fold start "elasticsearch" - travis_time_start - (cd e2e/elasticsearch && ./test.sh) - travis_time_finish -travis_fold end "elasticsearch" - -travis_fold start "hive1" - travis_time_start - (cd e2e/hive1 && ./test.sh) - travis_time_finish -travis_fold end "hive1" - -travis_fold start "hive" - travis_time_start - (cd e2e/hive && ./test.sh) - travis_time_finish -travis_fold end "hive" - -travis_fold start "hudi" - travis_time_start - (cd e2e/hudi && ./test.sh) - travis_time_finish -travis_fold end "hudi" - -travis_fold start "cdc" - travis_time_start - (cd e2e/cdc && ./test.sh) - travis_time_finish -travis_fold end "cdc" - travis_fold start "docker_publish_dev" travis_time_start ./scripts/docker_publish_dev.sh