From 6e206663f61e8a0c8c0827a2af3f1559a4c5ca0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Szczerbi=C5=84ski?= Date: Tue, 26 Nov 2024 11:06:37 +0100 Subject: [PATCH] SNOW-1825584: Cache deps to speed up the builds (#781) --- .github/workflows/build-test.yml | 57 ++++++++++++++++++++++++++++++++ ci/build/build.sh | 13 +++++++- ci/build_win.bat | 15 ++++++++- scripts/_init.sh | 1 + scripts/utils.bat | 8 ----- scripts/utils.sh | 17 ++++++++-- 6 files changed, 99 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 94f4f15e5c..f71e7f1093 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -21,11 +21,25 @@ jobs: cloud_provider: [ 'AWS', 'AZURE', 'GCP' ] steps: - uses: actions/checkout@v1 + - name: Restore cached deps + id: cache-restore-deps + uses: actions/cache/restore@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ github.event.pull_request.base.sha }}-Linux-dep-cache + if: github.event_name == 'pull_request' - name: Build shell: bash env: BUILD_TYPE: ${{ matrix.build_type }} run: ci/build_linux.sh + - name: Cache deps + id: cache-save-deps + uses: actions/cache/save@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ github.sha }}-Linux-dep-cache + if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS' - uses: actions/setup-python@v1 with: python-version: '3.7' @@ -37,6 +51,7 @@ jobs: CLOUD_PROVIDER: ${{ matrix.cloud_provider }} PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} run: ci/test_linux.sh + build-test-win: name: Build-Test-Win runs-on: windows-2019 @@ -49,6 +64,13 @@ jobs: cloud_provider: [ 'AWS', 'AZURE', 'GCP' ] steps: - uses: actions/checkout@v1 + - name: Restore cached deps + id: cache-restore-deps + uses: actions/cache/restore@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.event.pull_request.base.sha }}-Win-dep-cache + if: github.event_name == 'pull_request' - name: Build shell: cmd env: @@ -56,6 +78,13 @@ jobs: BUILD_TYPE: ${{ matrix.build_type }} VS_VERSION: ${{ matrix.vs_version }} run: ci\build_win.bat + - name: Cache deps + id: cache-save-deps + uses: actions/cache/save@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.sha }}-Win-dep-cache + if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS' - uses: actions/setup-python@v1 with: python-version: '3.7' @@ -81,6 +110,13 @@ jobs: cloud_provider: [ 'AWS', 'AZURE', 'GCP' ] steps: - uses: actions/checkout@v1 + - name: Restore cached deps + id: cache-restore-deps + uses: actions/cache/restore@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.event.pull_request.base.sha }}-Win-dep-cache + if: github.event_name == 'pull_request' - name: Build shell: cmd env: @@ -88,6 +124,13 @@ jobs: BUILD_TYPE: ${{ matrix.build_type }} VS_VERSION: ${{ matrix.vs_version }} run: ci\build_win.bat + - name: Cache deps + id: cache-save-deps + uses: actions/cache/save@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ matrix.platform }}-${{ matrix.vs_version }}-${{ github.sha }}-Win-dep-cache + if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS' - uses: actions/setup-python@v1 with: python-version: '3.7' @@ -114,11 +157,25 @@ jobs: - name: Install Homebrew Bash shell: bash run: brew install bash + - name: Restore cached deps + id: cache-restore-deps + uses: actions/cache/restore@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ github.event.pull_request.base.sha }}-Mac-dep-cache + if: github.event_name == 'pull_request' - name: Build shell: bash env: BUILD_TYPE: ${{ matrix.build_type }} run: ./ci/build_mac.sh + - name: Cache deps + id: cache-save-deps + uses: actions/cache/save@v4 + with: + path: dep-cache + key: ${{ matrix.build_type }}-${{ github.sha }}-Mac-dep-cache + if: github.ref_name == github.event.repository.default_branch && matrix.cloud_provider == 'AWS' - name: Test shell: bash env: diff --git a/ci/build/build.sh b/ci/build/build.sh index 256be4a3e9..e42206d14b 100755 --- a/ci/build/build.sh +++ b/ci/build/build.sh @@ -23,7 +23,17 @@ function download_build_component() local component_version=$($component_script -v) local zip_file_name=$(get_zip_file_name $component_name $component_version $build_type) if [[ -n "$GITHUB_ACTIONS" ]]; then - "$component_script" -t "$build_type" + if cp "$CACHE_DIR/$zip_file_name" "$ARTIFACTS_DIR"; + then + echo "=== using cached: $component_name ===" + pushd $DEPENDENCY_DIR >& /dev/null + tar xvfz $ARTIFACTS_DIR/$zip_file_name + popd >& /dev/null + else + echo "=== building dep: $component_name ===" + "$component_script" -t "$build_type" + cache_dependency $component_name $component_version $build_type + fi else echo "=== download or build $component_name ===" ret="$(check_directory $component_name $build_type)" @@ -63,6 +73,7 @@ function build_component() echo "=== build: $component_name ===" "$component_script" -t "$build_type" "$other_args" local component_version=$("$component_script" -v) + if [[ -z "$GITHUB_ACTIONS" ]] && [[ -n "$GIT_BRANCH" ]]; then upload_to_sfc_jenkins $component_name $component_version $build_type if [[ "$GIT_BRANCH" == "origin/master" || "$GIT_BRANCH" == "master" ]]; then diff --git a/ci/build_win.bat b/ci/build_win.bat index 58fb19422e..67fb0e9ff5 100644 --- a/ci/build_win.bat +++ b/ci/build_win.bat @@ -82,7 +82,20 @@ goto :EOF call %build_script% :get_version call %utils_script% :get_zip_file_name %component_name% %version% if defined GITHUB_ACTIONS ( - call %build_script% :build %platform% %build_type% %vs_version% %dynamic_runtime% + cmd /c copy %curdir%\dep-cache\%zip_file_name% %curdir%\artifacts\ + if !ERRORLEVEL! NEQ 0 ( + call %build_script% :build %platform% %build_type% %vs_version% %dynamic_runtime% + echo === copying %curdir%\artifacts\%zip_file_name% === + if not exist "%curdir%\dep-cache" md "%curdir%\dep-cache" + cmd /c copy "artifacts\%zip_file_name%" "%curdir%\dep-cache\" + ) else ( + if not exist deps-build\%arcdir%\%vsdir%\%build_type% md deps-build\%arcdir%\%vsdir%\%build_type% + pushd deps-build\%arcdir%\%vsdir%\%build_type% + if !ERRORLEVEL! NEQ 0 goto :error + rd /s /q %component_name% + 7z x "%curdir%\artifacts\%zip_file_name%" + popd + ) if !ERRORLEVEL! NEQ 0 goto :error ) else ( echo === download or build: %component_name% === diff --git a/scripts/_init.sh b/scripts/_init.sh index 4fee10d77a..534a8a488f 100755 --- a/scripts/_init.sh +++ b/scripts/_init.sh @@ -10,6 +10,7 @@ export TERM=vt100 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" DEPS_DIR=$(cd $DIR/../deps && pwd) ARTIFACTS_DIR=$DIR/../artifacts +CACHE_DIR="$DIR/../dep-cache" mkdir -p $ARTIFACTS_DIR PLATFORM=$(echo $(uname) | tr '[:upper:]' '[:lower:]') diff --git a/scripts/utils.bat b/scripts/utils.bat index d9919cbf36..7ada8f3944 100644 --- a/scripts/utils.bat +++ b/scripts/utils.bat @@ -75,10 +75,6 @@ goto :EOF :zip_file setlocal - if not defined JENKINS_URL ( - echo === No zip file is created if not Jenkins - goto :EOF - ) set component_name=%~1 set component_version=%~2 if not exist artifacts md artifacts @@ -94,10 +90,6 @@ goto :EOF :zip_files setlocal - if not defined JENKINS_URL ( - echo === No zip file is created if not Jenkins - goto :EOF - ) set component_name=%~1 set component_version=%~2 set files=%~3 diff --git a/scripts/utils.sh b/scripts/utils.sh index 449cfbfbda..5617ce6e31 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -65,7 +65,7 @@ function zip_file() local zip_file_name=$(get_zip_file_name "$component_name" "$component_version" "$build_type") - if [[ -z "$GITHUB_ACTIONS" ]] && [[ -n "$GIT_BRANCH" ]]; then + if [[ -n "$GIT_BRANCH" ]]; then local f=$UTILS_DIR/../artifacts/$zip_file_name rm -f $f pushd $DEPENDENCY_DIR/ @@ -85,7 +85,7 @@ function zip_files() local zip_file_name=$(get_zip_file_name "$component_name" "$component_version" "$build_type") - if [[ -z "$GITHUB_ACTIONS" ]] && [[ -n "$GIT_BRANCH" ]]; then + if [[ -n "$GIT_BRANCH" ]]; then local f=$UTILS_DIR/../artifacts/$zip_file_name rm -f $f pushd $DEPENDENCY_DIR/ @@ -117,6 +117,18 @@ function upload_to_sfc_dev1_data() aws s3 cp --only-show-errors $UTILS_DIR/../artifacts/$zip_file_name $DEP_URL_PREFIX/$component_name/ } +function cache_dependency() +{ + + local component_name=$1 + local component_version=$2 + local build_type=$3 + + local zip_file_name=$(get_zip_file_name $component_name $component_version $build_type) + mkdir -p $CACHE_DIR + cp $UTILS_DIR/../artifacts/$zip_file_name "$CACHE_DIR/" +} + function upload_to_sfc_jenkins() { local component_name=$1 @@ -180,3 +192,4 @@ function set_parameters() exit 1 fi } +