fix(travis): parallelise build (#296)

* fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix(travis): refactor * fix
YotpoLtd · May 24, 2020 · 6dfeb12 · 6dfeb12
1 parent 4145646
commit 6dfeb12
Show file tree

Hide file tree

Showing 12 changed files with 115 additions and 84 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -9,6 +9,7 @@ cache:
   directories:
     - $HOME/.ivy2/cache
     - $HOME/.sbt
+    - $TARGET_CACHE
 branches:
   only:
     - master
@@ -22,32 +23,69 @@ env:
     - HUDI_HIVE1_VERSION=0.4.7
     - HIVE_VERSION=2.3.3
     - HUDI_VERSION=0.5.2-incubating
+    - TARGET_CACHE=$HOME/target-cache/${TRAVIS_COMMIT}
 before_script:
   - export -f travis_fold
   - export -f travis_time_start
   - export -f travis_time_finish
   - export -f travis_nanoseconds
-script: scripts/travis_build.sh
-before_cache: scripts/before_cache.sh
-deploy:
-  - provider: script
-    skip_cleanup: true
-    script: scripts/publish.sh
-    on:
-      all_branches: true
-      condition: ($TRAVIS_BRANCH = master) || ($TRAVIS_TAG =~ ^v.*)
-  - provider: script
-    skip_cleanup: true
-    script: scripts/docker_publish.sh
-    on:
-      all_branches: true
-      condition: $TRAVIS_TAG =~ ^v.*
-  - provider: releases
-    skip_cleanup: true
-    token:
-      secure: i+W1C7xBogQ5XUxctAV92+YaraEGrHscjnuIkXHesfGEvF5rz9AdwGvpLzRIdLOHH0S4tR5uPFVUF+w86IWQOHLyZhNDybbkIpoQqRRuB0xiOwsqlcjz5IwNkhraTRF+8sPl9FBI5OuGfOKh8jy91xLpYvcMt+EKsai+5rugDxyb8/L3G5nQcH+/5c40dflDmDChkJF/qoNB7iLofMoW9uG72QHKeO87JoVJKg6o73anpuqQ9qZA7W4GYZBmU5QrHNMfdQqt/8XgrPvxq4DW8EnH1Tzg4nXuBREvY92rBxBYpAuff0uxlRJmd8n19Nk4hNRNNpwBIBlHwmbzCAY4AnRJmZT1G/A3E5wznfUt1JekIdwCfaVBbIaAbeFwrTV2DjJYSWdmeB79HF82CYRHuWY4tkxUzUMuBUK2KKFovfMvNG2Lx33cQRwC1YrCb+9W587rWx3dNfIY4vj9aWbCcfZnfrM0N00rlQKZH16dkdYIEZlg396h3v9M41MmvNgez7O/ftUILF3cIc6AG2X8xzcpbXB0lGCFwNdMXKtEpUf04Nw5Lu84Ov662OWfPAM2G/edSnXF1d56XxLnax8dQNcoqc25wcn+1VmpguzLr530rI9dNREZC2sQ7C0/cLnk6SAP12PKXk9nmigyWIbbGHe2FBC4WBSssb8xMg+bgLs=
-    file_glob: true
-    file: "$TRAVIS_BUILD_DIR/target/scala-2.11/*.jar"
-    on:
-      all_branches: true
-      condition: $TRAVIS_TAG =~ ^v.*
+jobs:
+  include:
+    - stage: "Build"
+      script: scripts/travis_build.sh
+      before_cache: scripts/before_cache.sh
+    - stage: "Tests"
+      name: "Unit & integration tests"
+      script: scripts/test.sh
+      before_script: scripts/load_from_cache.sh
+      before_cache: scripts/clear_from_cache.sh
+    - name: "influxdb"
+      script: (cd e2e/influxdb && ./test.sh)
+      before_script: scripts/docker_load_dev.sh
+    - name: "kafka"
+      script: (cd e2e/kafka && ./test.sh)
+      before_script: scripts/docker_load_dev.sh
+    - name: "elasticsearch"
+      script: (cd e2e/elasticsearch && ./test.sh)
+      before_script: scripts/docker_load_dev.sh
+    - name: "hive1"
+      script: (cd e2e/hive1 && ./test.sh)
+      before_script: scripts/docker_load_dev.sh
+    - name: "hive"
+      script: (cd e2e/hive && ./test.sh)
+      before_script: scripts/docker_load_dev.sh
+    - name: "hudi"
+      script: (cd e2e/hudi && ./test.sh)
+      before_script: scripts/docker_load_dev.sh
+    - name: "cdc"
+      script: (cd e2e/cdc && ./test.sh)
+      before_script: scripts/docker_load_dev.sh
+    - stage: "Deploy"
+      if: type != pull_request and (branch = master or tag =~ ^v)
+      before_script:
+        - scripts/load_from_cache.sh
+        - scripts/docker_load_dev.sh
+      script: skip
+      deploy:
+        - provider: script
+          skip_cleanup: true
+          script: scripts/publish.sh
+          on:
+            all_branches: true
+            condition: ($TRAVIS_BRANCH = master) || ($TRAVIS_TAG =~ ^v.*)
+        - provider: script
+          skip_cleanup: true
+          script: scripts/docker_publish.sh
+          on:
+            all_branches: true
+            condition: $TRAVIS_TAG =~ ^v.*
+        - provider: releases
+          skip_cleanup: true
+          token:
+            secure: i+W1C7xBogQ5XUxctAV92+YaraEGrHscjnuIkXHesfGEvF5rz9AdwGvpLzRIdLOHH0S4tR5uPFVUF+w86IWQOHLyZhNDybbkIpoQqRRuB0xiOwsqlcjz5IwNkhraTRF+8sPl9FBI5OuGfOKh8jy91xLpYvcMt+EKsai+5rugDxyb8/L3G5nQcH+/5c40dflDmDChkJF/qoNB7iLofMoW9uG72QHKeO87JoVJKg6o73anpuqQ9qZA7W4GYZBmU5QrHNMfdQqt/8XgrPvxq4DW8EnH1Tzg4nXuBREvY92rBxBYpAuff0uxlRJmd8n19Nk4hNRNNpwBIBlHwmbzCAY4AnRJmZT1G/A3E5wznfUt1JekIdwCfaVBbIaAbeFwrTV2DjJYSWdmeB79HF82CYRHuWY4tkxUzUMuBUK2KKFovfMvNG2Lx33cQRwC1YrCb+9W587rWx3dNfIY4vj9aWbCcfZnfrM0N00rlQKZH16dkdYIEZlg396h3v9M41MmvNgez7O/ftUILF3cIc6AG2X8xzcpbXB0lGCFwNdMXKtEpUf04Nw5Lu84Ov662OWfPAM2G/edSnXF1d56XxLnax8dQNcoqc25wcn+1VmpguzLr530rI9dNREZC2sQ7C0/cLnk6SAP12PKXk9nmigyWIbbGHe2FBC4WBSssb8xMg+bgLs=
+          file_glob: true
+          file: "$TRAVIS_BUILD_DIR/target/scala-2.11/*.jar"
+          on:
+            all_branches: true
+            condition: $TRAVIS_TAG =~ ^v.*
+
diff --git a/scripts/before_cache.sh b/scripts/before_cache.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 set -e
 
-find $HOME/.sbt -name "*.lock" | xargs rm
-find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm
-rm -f $HOME/.ivy2/.sbt.ivy.lock
-rm -f $HOME/.ivy2/cache/org.apache.hbase/hbase-server/ivy-1.1.1.xml
+./scripts/clear_from_cache.sh
+
+echo "Saving metorikku JAR to cache"
+rm -rf target/streams/\$global/assembly target/streams/\$global/assemblyOption target/streams/test/assemblyOption target/streams/test/assembly
+cp -r target $TARGET_CACHE
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 set -e
 
-sbt -DsparkVersion=$SPARK_VERSION clean scalastyle assembly "set test in (Test, assembly) := {}" test:assembly
+sbt -DsparkVersion=$SPARK_VERSION clean scalastyle "set test in (assembly) := {}" assembly "set test in (Test, assembly) := {}" test:assembly
diff --git a/scripts/clear_from_cache.sh b/scripts/clear_from_cache.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+find $HOME/.sbt -name "*.lock" | xargs rm
+find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm
+rm -f $HOME/.ivy2/.sbt.ivy.lock
+rm -f $HOME/.ivy2/cache/org.apache.hbase/hbase-server/ivy-1.1.1.xml
diff --git a/scripts/docker-without-cache.sh b/scripts/docker-without-cache.sh
diff --git a/scripts/docker_load_dev.sh b/scripts/docker_load_dev.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+tag=$(([ ! "${TRAVIS_PULL_REQUEST}" = "false" ] && echo "${TRAVIS_PULL_REQUEST}") || echo "${TRAVIS_BRANCH}")
+
+docker pull metorikku/dev:metorikku_$tag
+docker pull metorikku/dev:spark_$tag
+docker pull metorikku/dev:hive_$tag
+docker pull metorikku/dev:hive1_$tag
+
+docker tag metorikku/dev:metorikku_$tag metorikku/metorikku
+docker tag metorikku/dev:spark_$tag metorikku/spark
+docker tag metorikku/dev:hive_$tag metorikku/hive
+docker tag metorikku/dev:hive1_$tag metorikku/hive:1
diff --git a/scripts/docker_publish_dev.sh b/scripts/docker_publish_dev.sh
@@ -10,9 +10,11 @@ else
   docker tag metorikku/metorikku metorikku/dev:metorikku_$tag
   docker tag metorikku/spark metorikku/dev:spark_$tag
   docker tag metorikku/hive metorikku/dev:hive_$tag
+  docker tag metorikku/hive:1 metorikku/dev:hive1_$tag
 
   docker push metorikku/dev:metorikku_$tag
   docker push metorikku/dev:spark_$tag
   docker push metorikku/dev:hive_$tag
+  docker push metorikku/dev:hive1_$tag
 fi
 
diff --git a/scripts/load_docker_from_cache.sh b/scripts/load_docker_from_cache.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+echo "Loading docker cache"
+docker load -i $DOCKER_CACHE/images.tar
+
+while read REPOSITORY TAG IMAGE_ID
+do
+        echo "== Tagging $REPOSITORY $TAG $IMAGE_ID =="
+        docker tag "$IMAGE_ID" "$REPOSITORY:$TAG"
+done < $DOCKER_CACHE/images.list
diff --git a/scripts/load_from_cache.sh b/scripts/load_from_cache.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+echo "Loading metorikku JAR"
+cp -r $TARGET_CACHE/target .
diff --git a/scripts/save_docker_to_cache.sh b/scripts/save_docker_to_cache.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+echo "Saving docker images to cache"
+docker save -o $DOCKER_CACHE/images.tar $(docker images -a -q)
+docker images | grep -v '<none>' | sed '1d' | awk '{print $1 " " $2 " " $3}' > $DOCKER_CACHE/images.list
diff --git a/scripts/test.sh b/scripts/test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+sbt test
+
 # Metorikku main test example
 java -Dspark.master=local[*] -cp target/scala-2.11/metorikku-standalone.jar com.yotpo.metorikku.MetorikkuTester --test-settings examples/movies_test.yaml
 # Metorikku kafka test example

diff --git a/scripts/travis_build.sh b/scripts/travis_build.sh
@@ -13,54 +13,6 @@ travis_fold start "docker"
   travis_time_finish
 travis_fold end "docker"
 
-travis_fold start "tests"
-  travis_time_start
-    ./scripts/test.sh
-  travis_time_finish
-travis_fold end "tests"
-
-travis_fold start "influxdb"
-  travis_time_start
-    (cd e2e/influxdb && ./test.sh)
-  travis_time_finish
-travis_fold end "influxdb"
-
-travis_fold start "kafka"
-  travis_time_start
-    (cd e2e/kafka && ./test.sh)
-  travis_time_finish
-travis_fold end "kafka"
-
-travis_fold start "elasticsearch"
-  travis_time_start
-    (cd e2e/elasticsearch && ./test.sh)
-  travis_time_finish
-travis_fold end "elasticsearch"
-
-travis_fold start "hive1"
-  travis_time_start
-    (cd e2e/hive1 && ./test.sh)
-  travis_time_finish
-travis_fold end "hive1"
-
-travis_fold start "hive"
-  travis_time_start
-    (cd e2e/hive && ./test.sh)
-  travis_time_finish
-travis_fold end "hive"
-
-travis_fold start "hudi"
-  travis_time_start
-    (cd e2e/hudi && ./test.sh)
-  travis_time_finish
-travis_fold end "hudi"
-
-travis_fold start "cdc"
-  travis_time_start
-    (cd e2e/cdc && ./test.sh)
-  travis_time_finish
-travis_fold end "cdc"
-
 travis_fold start "docker_publish_dev"
   travis_time_start
     ./scripts/docker_publish_dev.sh