SNOW-1825584: Cache deps to speed up the builds (#781)

snowflakedb · Nov 26, 2024 · 6e20666 · 6e20666
1 parent 64e0f71
commit 6e20666
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 12 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -21,11 +21,25 @@ jobs:
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
         steps:
             - uses: actions/checkout@v1
+            - name: Restore cached deps
+              id: cache-restore-deps
+              uses: actions/cache/restore@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ github.event.pull_request.base.sha }}-Linux-dep-cache
+              if: github.event_name == 'pull_request'
             - name: Build
               shell: bash
               env:
                   BUILD_TYPE: ${{ matrix.build_type }}
               run: ci/build_linux.sh
+            - name: Cache deps
+              id: cache-save-deps
+              uses: actions/cache/save@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ github.sha }}-Linux-dep-cache
+              if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS'
             - uses: actions/setup-python@v1
               with:
                 python-version: '3.7'
@@ -37,6 +51,7 @@ jobs:
                   CLOUD_PROVIDER: ${{ matrix.cloud_provider }}
                   PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
               run: ci/test_linux.sh
+
     build-test-win:
         name: Build-Test-Win
         runs-on: windows-2019
@@ -49,13 +64,27 @@ jobs:
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
         steps:
             - uses: actions/checkout@v1
+            - name: Restore cached deps
+              id: cache-restore-deps
+              uses: actions/cache/restore@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.event.pull_request.base.sha }}-Win-dep-cache
+              if: github.event_name == 'pull_request'
             - name: Build
               shell: cmd
               env:
                   PLATFORM: ${{ matrix.platform }}
                   BUILD_TYPE: ${{ matrix.build_type }}
                   VS_VERSION: ${{ matrix.vs_version }}
               run: ci\build_win.bat
+            - name: Cache deps
+              id: cache-save-deps
+              uses: actions/cache/save@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.sha }}-Win-dep-cache
+              if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS'
             - uses: actions/setup-python@v1
               with:
                 python-version: '3.7'
@@ -81,13 +110,27 @@ jobs:
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
         steps:
             - uses: actions/checkout@v1
+            - name: Restore cached deps
+              id: cache-restore-deps
+              uses: actions/cache/restore@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.event.pull_request.base.sha }}-Win-dep-cache
+              if: github.event_name == 'pull_request'
             - name: Build
               shell: cmd
               env:
                   PLATFORM: ${{ matrix.platform }}
                   BUILD_TYPE: ${{ matrix.build_type }}
                   VS_VERSION: ${{ matrix.vs_version }}
               run: ci\build_win.bat
+            - name: Cache deps
+              id: cache-save-deps
+              uses: actions/cache/save@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.sha }}-Win-dep-cache
+              if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS'
             - uses: actions/setup-python@v1
               with:
                 python-version: '3.7'
@@ -114,11 +157,25 @@ jobs:
             - name: Install Homebrew Bash
               shell: bash
               run: brew install bash
+            - name: Restore cached deps
+              id: cache-restore-deps
+              uses: actions/cache/restore@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ github.event.pull_request.base.sha }}-Mac-dep-cache
+              if: github.event_name == 'pull_request'
             - name: Build
               shell: bash
               env:
                 BUILD_TYPE: ${{ matrix.build_type }}
               run: ./ci/build_mac.sh
+            - name: Cache deps
+              id: cache-save-deps
+              uses: actions/cache/save@v4
+              with:
+                path: dep-cache
+                key: ${{ matrix.build_type }}-${{ github.sha }}-Mac-dep-cache
+              if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS'
             - name: Test
               shell: bash
               env:

diff --git a/ci/build/build.sh b/ci/build/build.sh
@@ -23,7 +23,17 @@ function download_build_component()
     local component_version=$($component_script -v)
     local zip_file_name=$(get_zip_file_name $component_name $component_version $build_type)
     if [[ -n "$GITHUB_ACTIONS" ]]; then
-        "$component_script" -t "$build_type"
+        if cp "$CACHE_DIR/$zip_file_name" "$ARTIFACTS_DIR";
+        then
+            echo "=== using cached: $component_name ==="
+            pushd $DEPENDENCY_DIR >& /dev/null
+                tar xvfz $ARTIFACTS_DIR/$zip_file_name
+            popd >& /dev/null
+        else
+            echo "=== building dep: $component_name ==="
+            "$component_script" -t "$build_type"
+            cache_dependency $component_name $component_version $build_type
+        fi
     else
         echo "=== download or build $component_name ==="
         ret="$(check_directory $component_name $build_type)"
@@ -63,6 +73,7 @@ function build_component()
     echo "=== build: $component_name ==="
     "$component_script" -t "$build_type" "$other_args"
     local component_version=$("$component_script" -v)
+
     if [[ -z "$GITHUB_ACTIONS" ]] && [[ -n "$GIT_BRANCH" ]]; then
         upload_to_sfc_jenkins $component_name $component_version $build_type
         if [[ "$GIT_BRANCH" == "origin/master" || "$GIT_BRANCH" == "master" ]]; then

diff --git a/ci/build_win.bat b/ci/build_win.bat
@@ -82,7 +82,20 @@ goto :EOF
     call %build_script% :get_version
     call %utils_script% :get_zip_file_name %component_name% %version%
     if defined GITHUB_ACTIONS (
-        call %build_script% :build %platform% %build_type% %vs_version% %dynamic_runtime%
+        cmd /c copy %curdir%\dep-cache\%zip_file_name% %curdir%\artifacts\
+        if !ERRORLEVEL! NEQ 0 (
+            call %build_script% :build %platform% %build_type% %vs_version% %dynamic_runtime%
+            echo === copying %curdir%\artifacts\%zip_file_name% ===
+            if not exist "%curdir%\dep-cache" md "%curdir%\dep-cache"
+            cmd /c copy "artifacts\%zip_file_name%" "%curdir%\dep-cache\"
+        ) else (
+            if not exist deps-build\%arcdir%\%vsdir%\%build_type% md deps-build\%arcdir%\%vsdir%\%build_type%
+            pushd deps-build\%arcdir%\%vsdir%\%build_type%
+                if !ERRORLEVEL! NEQ 0 goto :error
+                rd /s /q %component_name%
+                7z x "%curdir%\artifacts\%zip_file_name%"
+            popd
+        )
         if !ERRORLEVEL! NEQ 0 goto :error
     ) else (
         echo === download or build: %component_name% ===

diff --git a/scripts/_init.sh b/scripts/_init.sh
@@ -10,6 +10,7 @@ export TERM=vt100
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 DEPS_DIR=$(cd $DIR/../deps && pwd)
 ARTIFACTS_DIR=$DIR/../artifacts
+CACHE_DIR="$DIR/../dep-cache"
 mkdir -p $ARTIFACTS_DIR
 
 PLATFORM=$(echo $(uname) | tr '[:upper:]' '[:lower:]')

diff --git a/scripts/utils.bat b/scripts/utils.bat
@@ -75,10 +75,6 @@ goto :EOF
 
 :zip_file
     setlocal
-    if not defined JENKINS_URL (
-        echo === No zip file is created if not Jenkins
-        goto :EOF
-    )
     set component_name=%~1
     set component_version=%~2
     if not exist artifacts md artifacts
@@ -94,10 +90,6 @@ goto :EOF
 
 :zip_files
     setlocal
-    if not defined JENKINS_URL (
-        echo === No zip file is created if not Jenkins
-        goto :EOF
-    )
     set component_name=%~1
     set component_version=%~2
     set files=%~3

diff --git a/scripts/utils.sh b/scripts/utils.sh
@@ -65,7 +65,7 @@ function zip_file()
 
     local zip_file_name=$(get_zip_file_name "$component_name" "$component_version" "$build_type")
 
-    if [[ -z "$GITHUB_ACTIONS" ]] && [[ -n "$GIT_BRANCH" ]]; then
+    if [[ -n "$GIT_BRANCH" ]]; then
         local f=$UTILS_DIR/../artifacts/$zip_file_name
         rm -f $f
         pushd $DEPENDENCY_DIR/
@@ -85,7 +85,7 @@ function zip_files()
 
     local zip_file_name=$(get_zip_file_name "$component_name" "$component_version" "$build_type")
 
-    if [[ -z "$GITHUB_ACTIONS" ]] && [[ -n "$GIT_BRANCH" ]]; then
+    if [[ -n "$GIT_BRANCH" ]]; then
         local f=$UTILS_DIR/../artifacts/$zip_file_name
         rm -f $f
         pushd $DEPENDENCY_DIR/
@@ -117,6 +117,18 @@ function upload_to_sfc_dev1_data()
     aws s3 cp --only-show-errors $UTILS_DIR/../artifacts/$zip_file_name $DEP_URL_PREFIX/$component_name/
 }
 
+function cache_dependency()
+{
+
+    local component_name=$1
+    local component_version=$2
+    local build_type=$3
+
+    local zip_file_name=$(get_zip_file_name $component_name $component_version $build_type)
+    mkdir -p $CACHE_DIR
+    cp $UTILS_DIR/../artifacts/$zip_file_name "$CACHE_DIR/"
+}
+
 function upload_to_sfc_jenkins()
 {
     local component_name=$1
@@ -180,3 +192,4 @@ function set_parameters()
         exit 1
     fi
 }
+