From ef1996afbcf5b22dbcccc09435030bbda90528b8 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 7 Oct 2024 19:27:06 +0200 Subject: [PATCH] cirrus: use zstd over bzip2 for repo archive The repo tar process took over 1:20 min, with zstd it takes less than 10s so we safe over a minute by doing this. Signed-off-by: Paul Holzinger --- .cirrus.yml | 57 ++++++++++++---------- contrib/cirrus/win-installer-main.ps1 | 2 +- contrib/cirrus/win-podman-machine-test.ps1 | 2 +- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 78bd85d04b99..d1f158ad350b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -178,9 +178,9 @@ build_task: # Cirrus-CI is very slow uploading one file at time, and the repo contains # thousands of files. Speed this up by archiving into tarball first. repo_prep_script: &repo_prep >- - tar cjf /tmp/repo.tbz -C $GOSRC . && mv /tmp/repo.tbz $GOSRC/ + tar --zstd -cf /tmp/repo.tar.zst -C $GOSRC . && mv /tmp/repo.tar.zst $GOSRC/ repo_artifacts: &repo_artifacts - path: ./repo.tbz + path: ./repo.tar.zst type: application/octet-stream always: &runner_stats runner_stats_artifacts: @@ -247,11 +247,11 @@ alt_build_task: ALT_NAME: 'Alt Arch. MIPS64 Cross' - env: ALT_NAME: 'Alt Arch. Other Cross' - # This task cannot make use of the shared repo.tbz artifact. + # This task cannot make use of the shared repo.tar.zst artifact. clone_script: *full_clone setup_script: *setup main_script: *main - # Produce a new repo.tbz artifact for consumption by 'artifacts' task. + # Produce a new repo.tar.zst artifact for consumption by 'artifacts' task. repo_prep_script: *repo_prep repo_artifacts: *repo_artifacts always: *runner_stats @@ -298,7 +298,7 @@ osx_alt_build_task: # The Mac tests rely this Podman binary to run, and the CI Mac is ARM-based build_arm64_script: - make podman-remote-release-darwin_arm64.zip - # Produce a new repo.tbz artifact for consumption by dependent tasks. + # Produce a new repo.tar.zst artifact for consumption by dependent tasks. repo_prep_script: *repo_prep repo_artifacts: *repo_artifacts # This host is/was shared with potentially many other CI tasks. @@ -326,11 +326,11 @@ freebsd_alt_build_task: freebsd_instance: image_family: freebsd-13-3 setup_script: - - pkg install -y gpgme bash go-md2man gmake gsed gnugrep go pkgconf + - pkg install -y gpgme bash go-md2man gmake gsed gnugrep go pkgconf zstd build_amd64_script: - gmake podman-release - # This task cannot make use of the shared repo.tbz artifact and must - # produce a new repo.tbz artifact for consumption by 'artifacts' task. + # This task cannot make use of the shared repo.tar.zst artifact and must + # produce a new repo.tar.zst artifact for consumption by 'artifacts' task. repo_prep_script: *repo_prep repo_artifacts: *repo_artifacts @@ -386,9 +386,9 @@ bindings_task: # N/B: This script depends on ${DISTRO_NV} being defined for the task. clone_script: &get_gosrc | cd /tmp - echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz" - time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz - time tar xjf /tmp/repo.tbz -C $GOSRC + echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst" + time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst + time tar --zstd -xf /tmp/repo.tar.zst -C $GOSRC setup_script: *setup main_script: *main always: &logs_artifacts @@ -465,11 +465,11 @@ win_installer_task: $ProgressPreference = 'SilentlyContinue' New-Item -ItemType Directory -Force -Path "$ENV:CIRRUS_WORKING_DIR" Set-Location "$ENV:CIRRUS_WORKING_DIR" - $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tbz" + $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tar.zst" Write-Host "Downloading $uri" For($i = 0;;) { Try { - Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tbz2" ` + Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tar.zst" ` -Uri "$uri" Break } Catch { @@ -480,9 +480,14 @@ win_installer_task: Start-Sleep -Seconds 10 } } - arc unarchive repo.tbz2 .\ + zstd -d repo.tar.zst if ($LASTEXITCODE -ne 0) { - throw "Unarchive repo.tbz2 failed" + throw "Extract repo.tar.zst failed" + Exit 1 + } + arc unarchive repo.tar .\ + if ($LASTEXITCODE -ne 0) { + throw "Unarchive repo.tar failed" Exit 1 } Get-ChildItem -Path .\repo @@ -731,9 +736,9 @@ podman_machine_aarch64_task: VM_IMAGE_NAME: "${FEDORA_AARCH64_AMI}" clone_script: &get_gosrc_aarch64 | cd /tmp - echo "$ARTCURL/build_aarch64/repo/repo.tbz" - time $ARTCURL/build_aarch64/repo/repo.tbz - time tar xjf /tmp/repo.tbz -C $GOSRC + echo "$ARTCURL/build_aarch64/repo/repo.tar.zst" + time $ARTCURL/build_aarch64/repo/repo.tar.zst + time tar --zstd -xf /tmp/repo.tar.zst -C $GOSRC setup_script: *setup main_script: *main always: *int_logs_artifacts @@ -795,8 +800,8 @@ podman_machine_mac_task: clone_script: # artifacts from osx_alt_build_task - mkdir -p $CIRRUS_WORKING_DIR - cd $CIRRUS_WORKING_DIR - - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst # This host is/was shared with potentially many other CI tasks. # The previous task may have been canceled or aborted. prep_script: *mac_cleanup @@ -1105,20 +1110,20 @@ artifacts_task: fedora_binaries_script: - mkdir -p /tmp/fed - cd /tmp/fed - - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst - cp ./bin/* $CIRRUS_WORKING_DIR/ win_binaries_script: - mkdir -p /tmp/win - cd /tmp/win - - $ARTCURL/Windows%20Cross/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Windows%20Cross/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst - mv ./podman-remote*.zip $CIRRUS_WORKING_DIR/ osx_binaries_script: - mkdir -p /tmp/osx - cd /tmp/osx - - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/ - mv ./contrib/pkginstaller/out/podman-installer-macos-*.pkg $CIRRUS_WORKING_DIR/ always: diff --git a/contrib/cirrus/win-installer-main.ps1 b/contrib/cirrus/win-installer-main.ps1 index 619d72242992..033cf0df5e59 100644 --- a/contrib/cirrus/win-installer-main.ps1 +++ b/contrib/cirrus/win-installer-main.ps1 @@ -15,7 +15,7 @@ if ($Env:CI -eq "true") { Push-Location $WIN_INST_FOLDER # Build Installer -# Note: consumes podman-remote-release-windows_amd64.zip from repo.tbz2 +# Note: consumes podman-remote-release-windows_amd64.zip from repo.tar.zst Run-Command ".\build.ps1 $Env:WIN_INST_VER dev `"$RELEASE_DIR`"" Pop-Location diff --git a/contrib/cirrus/win-podman-machine-test.ps1 b/contrib/cirrus/win-podman-machine-test.ps1 index 4d8ff5966769..965163b6a561 100644 --- a/contrib/cirrus/win-podman-machine-test.ps1 +++ b/contrib/cirrus/win-podman-machine-test.ps1 @@ -24,7 +24,7 @@ if ($Env:TEST_FLAVOR -eq "machine-wsl") { Write-Host " CONTAINERS_MACHINE_PROVIDER = $Env:CONTAINERS_MACHINE_PROVIDER" Write-Host "`n" -# The repo.tbz artifact was extracted here +# The repo.tar.zst artifact was extracted here Set-Location "$ENV:CIRRUS_WORKING_DIR\repo" # Tests hard-code this location for podman-remote binary, make sure it actually runs. Run-Command ".\bin\windows\podman.exe --version"