From 3b5832c1bb0dde0fc7853ef95596a475c238f97f Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:29:47 -0500 Subject: [PATCH 1/7] Add python 3.12 --- .github/workflows/development.yaml | 42 +++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index 64dfaec..0cecd99 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -11,12 +11,19 @@ jobs: strategy: matrix: include: + - distro: alpine + conda_ver: '23.5.2' + conda_suffix: '-0' + py_ver: '3.12' + platform: linux/amd64 + py_label: py312_ + conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: alpine conda_ver: '23.5.2' conda_suffix: '-0' py_ver: '3.11' platform: linux/amd64 - py_label: py310_ + py_label: py311_ conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: alpine conda_ver: '23.5.2' @@ -39,12 +46,19 @@ jobs: platform: linux/amd64 py_label: py38_ conda_sha256: e2a4438671e0e42c5bba14cb51de6ce9763938184d6ca2967340bbe972bbe7e6 + - distro: debian + conda_ver: '23.5.2' + conda_suffix: '-0' + py_ver: '3.12' + platform: linux/amd64 + py_label: py312_ + conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: debian conda_ver: '23.5.2' conda_suffix: '-0' py_ver: '3.11' platform: linux/amd64 - py_label: py310_ + py_label: py311_ conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: debian conda_ver: '23.5.2' @@ -119,6 +133,12 @@ jobs: strategy: matrix: include: + - distro: alpine + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' + py_ver: '3.12' + platform: linux/amd64 + package_manager: apk - distro: alpine last_build_conda_ver: '22.11.1' conda_ver: '23.5.2' @@ -143,6 +163,12 @@ jobs: py_ver: '3.8' platform: linux/amd64 package_manager: apk + - distro: debian + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' + py_ver: '3.12' + platform: linux/amd64 + package_manager: apt - distro: debian last_build_conda_ver: '22.11.1' conda_ver: '23.5.2' @@ -207,6 +233,10 @@ jobs: strategy: matrix: include: + - distro: alpine + conda_ver: '23.5.2' + py_ver: '3.12' + platform: linux/amd64 - distro: alpine conda_ver: '23.5.2' py_ver: '3.11' @@ -223,6 +253,10 @@ jobs: conda_ver: '23.5.2' py_ver: '3.8' platform: linux/amd64 + - distro: debian + conda_ver: '23.5.2' + py_ver: '3.12' + platform: linux/amd64 - distro: debian conda_ver: '23.5.2' py_ver: '3.11' @@ -269,7 +303,7 @@ jobs: docker push "${IMAGE}:${TAG}" docker tag "${IMAGE}:${TAG}" "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}" docker push "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}" - [ "$PY_VER" == "3.11" ] && [ "$DISTRO" == "debian" ] \ + [ "$PY_VER" == "3.12" ] && [ "$DISTRO" == "debian" ] \ && docker tag "${IMAGE}:${TAG}" "${IMAGE}:latest" \ && docker push "${IMAGE}:latest" \ || echo "skipping 'latest' tag..." @@ -314,4 +348,4 @@ jobs: with: branch: gh-pages directory: gh-pages - github_token: ${{secrets.GITHUB_TOKEN}} \ No newline at end of file + github_token: ${{secrets.GITHUB_TOKEN}} From 5f2733b7aa1cad68ba6df24561a109ea935b7058 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:38:59 -0500 Subject: [PATCH 2/7] Fix conda_sha256 and py_label --- .github/workflows/development.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index 0cecd99..7d28381 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -16,14 +16,14 @@ jobs: conda_suffix: '-0' py_ver: '3.12' platform: linux/amd64 - py_label: py312_ + py_label: py310_ conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: alpine conda_ver: '23.5.2' conda_suffix: '-0' py_ver: '3.11' platform: linux/amd64 - py_label: py311_ + py_label: py310_ conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: alpine conda_ver: '23.5.2' @@ -51,14 +51,14 @@ jobs: conda_suffix: '-0' py_ver: '3.12' platform: linux/amd64 - py_label: py312_ + py_label: py310_ conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: debian conda_ver: '23.5.2' conda_suffix: '-0' py_ver: '3.11' platform: linux/amd64 - py_label: py311_ + py_label: py310_ conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: debian conda_ver: '23.5.2' From 1a5418b375cbdfa3c4408ac980d3cc17b7baf320 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:27:59 -0500 Subject: [PATCH 3/7] Print SHA on failure --- dist/alpine/Dockerfile | 2 +- dist/debian/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/alpine/Dockerfile b/dist/alpine/Dockerfile index 2c1260b..44db8e2 100644 --- a/dist/alpine/Dockerfile +++ b/dist/alpine/Dockerfile @@ -94,7 +94,7 @@ RUN \ # tr -d '\n' | tr -d '\t') && \ echo "${CONDA_SHA256} Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" \ > miniconda.sha256 && \ - if [ $(sha256sum -c miniconda.sha256 | awk '{print $2}') != "OK" ] ; then exit 1; fi && \ + if [ $(sha256sum -c miniconda.sha256 | awk '{print $2}') != "OK" ] ; then exit 1; else echo 'Failed SHA256 comparison:' && sha256sum "Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" ; fi && \ mv Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh miniconda.sh && \ mkdir -p /opt && \ sh ./miniconda.sh -b -p /opt/conda && \ diff --git a/dist/debian/Dockerfile b/dist/debian/Dockerfile index 0f3f5d1..324992e 100644 --- a/dist/debian/Dockerfile +++ b/dist/debian/Dockerfile @@ -48,7 +48,7 @@ RUN \ tr -d "\n" | tr -d "\t") && \ echo "${CONDA_SHA256} Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" \ > miniconda.sha256 && \ - if [ $(sha256sum -c miniconda.sha256 | awk '{print $2}') != "OK" ] ; then exit 1; fi && \ + if [ $(sha256sum -c miniconda.sha256 | awk '{print $2}') != "OK" ] ; then exit 1; else echo 'Failed SHA256 comparison:' && sha256sum "Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" ; fi && \ mv Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ From 693da3bc5212e2d6a330ee76ffe040a56d2651b1 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:31:32 -0500 Subject: [PATCH 4/7] Disable SHA256 check --- dist/alpine/Dockerfile | 3 --- dist/debian/Dockerfile | 3 --- 2 files changed, 6 deletions(-) diff --git a/dist/alpine/Dockerfile b/dist/alpine/Dockerfile index 44db8e2..1dad730 100644 --- a/dist/alpine/Dockerfile +++ b/dist/alpine/Dockerfile @@ -92,9 +92,6 @@ RUN \ # wget --quiet $(echo "https://repo.continuum.io/miniconda\ # /Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" | \ # tr -d '\n' | tr -d '\t') && \ - echo "${CONDA_SHA256} Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" \ - > miniconda.sha256 && \ - if [ $(sha256sum -c miniconda.sha256 | awk '{print $2}') != "OK" ] ; then exit 1; else echo 'Failed SHA256 comparison:' && sha256sum "Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" ; fi && \ mv Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh miniconda.sh && \ mkdir -p /opt && \ sh ./miniconda.sh -b -p /opt/conda && \ diff --git a/dist/debian/Dockerfile b/dist/debian/Dockerfile index 324992e..fb4651b 100644 --- a/dist/debian/Dockerfile +++ b/dist/debian/Dockerfile @@ -46,9 +46,6 @@ RUN \ wget --quiet $(echo "https://repo.anaconda.com/miniconda\ /Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" | \ tr -d "\n" | tr -d "\t") && \ - echo "${CONDA_SHA256} Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" \ - > miniconda.sha256 && \ - if [ $(sha256sum -c miniconda.sha256 | awk '{print $2}') != "OK" ] ; then exit 1; else echo 'Failed SHA256 comparison:' && sha256sum "Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" ; fi && \ mv Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ From 5ffacb0ec77957615469be91317b09330e1581ed Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:33:33 -0500 Subject: [PATCH 5/7] Remove rm sha command --- dist/alpine/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/alpine/Dockerfile b/dist/alpine/Dockerfile index 1dad730..b651443 100644 --- a/dist/alpine/Dockerfile +++ b/dist/alpine/Dockerfile @@ -95,7 +95,7 @@ RUN \ mv Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh miniconda.sh && \ mkdir -p /opt && \ sh ./miniconda.sh -b -p /opt/conda && \ - rm miniconda.sh miniconda.sha256 && \ + rm miniconda.sh && \ ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ find /opt/conda/ -follow -type f -name '*.a' -delete && \ find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ From d88f35a70c8b42d64b9566309ad0052a3ac8c828 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:12:47 -0500 Subject: [PATCH 6/7] No such var LAST_BUILD_CONDA_VER --- tests/main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main.sh b/tests/main.sh index 0c315cc..d733a6b 100755 --- a/tests/main.sh +++ b/tests/main.sh @@ -158,7 +158,7 @@ SHELL_CMD_TEMPLATE="docker run --rm -i \$SHELL_CMD_FLAGS $REF \ $(docker inspect "$REF" --format '{{join .Config.Cmd " "}}') -c" # Get the compressed size of the last build from docker hub LAST_BUILD_SIZE=$(curl -s https://hub.docker.com/v2/repositories/$IMAGE/tags \ - | jq -r '.results[] | select(.name=="'"$LAST_BUILD_CONDA_VER"'-py'"$PY_VER"'-'"$DISTRO"'") | .images[0].size') + | jq -r '.results[] | select(.name=="'"$CONDA_VER"'-py'"$PY_VER"'-'"$DISTRO"'") | .images[0].size') SIZE_INCRESE_FACTOR=1.5 SIZE_LIMIT=$(echo "scale=4; $LAST_BUILD_SIZE * $SIZE_INCRESE_FACTOR" | bc) # Verify size minimal From c68f4d03816aac06190cdc816177e814220532b9 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:04:56 -0500 Subject: [PATCH 7/7] Debugging statements for fetching LAST_BUILD_SIZE --- tests/main.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/main.sh b/tests/main.sh index d733a6b..581e40f 100755 --- a/tests/main.sh +++ b/tests/main.sh @@ -157,8 +157,13 @@ IMAGE=$(echo $REF | awk -F':' '{print $1}') SHELL_CMD_TEMPLATE="docker run --rm -i \$SHELL_CMD_FLAGS $REF \ $(docker inspect "$REF" --format '{{join .Config.Cmd " "}}') -c" # Get the compressed size of the last build from docker hub +[[ -z $LAST_BUILD_CONDA_VER ]] && echo "ERROR: LAST_BUILD_CONDA_VER not set" && exit 1 +[[ -z $PY_VER ]] && echo "ERROR: PY_VER not set" && exit 1 +[[ -z $DISTRO ]] && echo "ERROR: DISTRO not set" && exit 1 +echo "Fetching LAST_BUILD_SIZE for image $IMAGE with LAST_BUILD_CONDA_VER=$LAST_BUILD_CONDA_VER, PY_VER=$PY_VER, DISTRO=$DISTRO" LAST_BUILD_SIZE=$(curl -s https://hub.docker.com/v2/repositories/$IMAGE/tags \ - | jq -r '.results[] | select(.name=="'"$CONDA_VER"'-py'"$PY_VER"'-'"$DISTRO"'") | .images[0].size') + | jq -r '.results[] | select(.name=="'"$LAST_BUILD_CONDA_VER"'-py'"$PY_VER"'-'"$DISTRO"'") | .images[0].size') +[[ -z $LAST_BUILD_SIZE ]] && echo "ERROR: could not fetch LAST_BUILD_SIZE" && exit 1 SIZE_INCRESE_FACTOR=1.5 SIZE_LIMIT=$(echo "scale=4; $LAST_BUILD_SIZE * $SIZE_INCRESE_FACTOR" | bc) # Verify size minimal