diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index d773a7b..64dfaec 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -1,7 +1,10 @@ name: Development on: push: - pull_request: + tags: + - '*.*.*' + # pull_request: + workflow_dispatch: jobs: build: runs-on: ubuntu-latest @@ -9,85 +12,61 @@ jobs: matrix: include: - distro: alpine - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.11' platform: linux/amd64 py_label: py310_ - conda_sha256: 00938c3534750a0e4069499baf8f4e6dc1c2e471c86a59caa0dd03f4a9269db6 + conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: alpine - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.10' platform: linux/amd64 py_label: py310_ - conda_sha256: 00938c3534750a0e4069499baf8f4e6dc1c2e471c86a59caa0dd03f4a9269db6 + conda_sha256: ea5e6e8a3d5a0247b9df85382d27220fac8e59b5778fd313c5913879cd9baafc - distro: alpine - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.9' platform: linux/amd64 py_label: py39_ - conda_sha256: e685005710679914a909bfb9c52183b3ccc56ad7bb84acc861d596fcbe5d28bb + conda_sha256: 9829d95f639bd0053b2ed06d1204e60644617bf37dd5cc57523732e0e8d64516 - distro: alpine - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.8' platform: linux/amd64 py_label: py38_ - conda_sha256: 473e5ecc8e078e9ef89355fbca21f8eefa5f9081544befca99867c7beac3150d - - distro: alpine - conda_ver: '22.11.1' - conda_suffix: '-1' - py_ver: '3.7' - platform: linux/amd64 - py_label: py37_ - conda_sha256: 22b14d52265b4e609c6ce78e2f2884b277d976b83b5f9c8a83423e3eba2ccfbe + conda_sha256: e2a4438671e0e42c5bba14cb51de6ce9763938184d6ca2967340bbe972bbe7e6 - distro: debian - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.11' platform: linux/amd64 py_label: py310_ + conda_sha256: 634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 - distro: debian - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.10' platform: linux/amd64 py_label: py310_ + conda_sha256: ea5e6e8a3d5a0247b9df85382d27220fac8e59b5778fd313c5913879cd9baafc - distro: debian - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.9' platform: linux/amd64 py_label: py39_ + conda_sha256: 9829d95f639bd0053b2ed06d1204e60644617bf37dd5cc57523732e0e8d64516 - distro: debian - conda_ver: '22.11.1' - conda_suffix: '-1' + conda_ver: '23.5.2' + conda_suffix: '-0' py_ver: '3.8' platform: linux/amd64 py_label: py38_ - - distro: debian - conda_ver: '22.11.1' - conda_suffix: '-1' - py_ver: '3.7' - platform: linux/amd64 - py_label: py37_ - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.9 - # platform: linux/arm64 - # py_label: py39_ - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.8 - # platform: linux/arm64 - # py_label: py38_ - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.7 - # platform: linux/arm64 - # py_label: py37_ + conda_sha256: e2a4438671e0e42c5bba14cb51de6ce9763938184d6ca2967340bbe972bbe7e6 env: DISTRO: ${{matrix.distro}} CONDA_VER: ${{matrix.conda_ver}} @@ -141,72 +120,56 @@ jobs: matrix: include: - distro: alpine - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.11' platform: linux/amd64 package_manager: apk - distro: alpine - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.10' platform: linux/amd64 package_manager: apk - distro: alpine - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.9' platform: linux/amd64 package_manager: apk - distro: alpine - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.8' platform: linux/amd64 package_manager: apk - - distro: alpine - conda_ver: '22.11.1' - py_ver: '3.7' - platform: linux/amd64 - package_manager: apk - distro: debian - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.11' platform: linux/amd64 package_manager: apt - distro: debian - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.10' platform: linux/amd64 package_manager: apt - distro: debian - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.9' platform: linux/amd64 package_manager: apt - distro: debian - conda_ver: '22.11.1' + last_build_conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.8' platform: linux/amd64 package_manager: apt - - distro: debian - conda_ver: '22.11.1' - py_ver: '3.7' - platform: linux/amd64 - package_manager: apt - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.9 - # platform: linux/arm64 - # package_manager: apt - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.8 - # platform: linux/arm64 - # package_manager: apt - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.7 - # platform: linux/arm64 - # package_manager: apt env: DISTRO: ${{matrix.distro}} + LAST_BUILD_CONDA_VER: ${{matrix.last_build_conda_ver}} CONDA_VER: ${{matrix.conda_ver}} PY_VER: ${{matrix.py_ver}} PLATFORM: ${{matrix.platform}} @@ -245,57 +208,37 @@ jobs: matrix: include: - distro: alpine - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.11' platform: linux/amd64 - distro: alpine - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.10' platform: linux/amd64 - distro: alpine - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.9' platform: linux/amd64 - distro: alpine - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.8' platform: linux/amd64 - - distro: alpine - conda_ver: '22.11.1' - py_ver: '3.7' - platform: linux/amd64 - distro: debian - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.11' platform: linux/amd64 - distro: debian - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.10' platform: linux/amd64 - distro: debian - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.9' platform: linux/amd64 - distro: debian - conda_ver: '22.11.1' + conda_ver: '23.5.2' py_ver: '3.8' platform: linux/amd64 - - distro: debian - conda_ver: '22.11.1' - py_ver: '3.7' - platform: linux/amd64 - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.9 - # platform: linux/arm64 - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.8 - # platform: linux/arm64 - # - distro: debian - # conda_ver: 4.10.3 - # py_ver: 3.7 - # platform: linux/arm64 env: DISTRO: ${{matrix.distro}} CONDA_VER: ${{matrix.conda_ver}} diff --git a/config/.env b/config/.env index db3ab7f..4347e0e 100644 --- a/config/.env +++ b/config/.env @@ -1,35 +1,32 @@ ## build -# CONDA_VER=22.11.1 -# CONDA_SUFFIX=-1 +# LAST_BUILD_CONDA_VER=22.11.1 +# CONDA_VER=23.5.2 +# CONDA_SUFFIX=-0 +# CONDA_SHA256=634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817 # PY_VER='3.11' -# PY_LABEL=py310_ -# CONDA_SHA256=00938c3534750a0e4069499baf8f4e6dc1c2e471c86a59caa0dd03f4a9269db6 +# PY_LABEL=py311_ -CONDA_VER=22.11.1 -CONDA_SUFFIX=-1 +LAST_BUILD_CONDA_VER=22.11.1 +CONDA_VER=23.5.2 +CONDA_SUFFIX=-0 +CONDA_SHA256=ea5e6e8a3d5a0247b9df85382d27220fac8e59b5778fd313c5913879cd9baafc PY_VER='3.10' PY_LABEL=py310_ -CONDA_SHA256=00938c3534750a0e4069499baf8f4e6dc1c2e471c86a59caa0dd03f4a9269db6 -# CONDA_VER=22.11.1 -# CONDA_SUFFIX=-1 +# LAST_BUILD_CONDA_VER=22.11.1 +# CONDA_VER=23.5.2 +# CONDA_SUFFIX=-0 +# CONDA_SHA256=9829d95f639bd0053b2ed06d1204e60644617bf37dd5cc57523732e0e8d64516 # PY_VER='3.9' # PY_LABEL=py39_ -# CONDA_SHA256=e685005710679914a909bfb9c52183b3ccc56ad7bb84acc861d596fcbe5d28bb -# CONDA_VER=22.11.1 -# CONDA_SUFFIX=-1 +# LAST_BUILD_CONDA_VER=22.11.1 +# CONDA_VER=23.5.2 +# CONDA_SUFFIX=-0 +# CONDA_SHA256=e2a4438671e0e42c5bba14cb51de6ce9763938184d6ca2967340bbe972bbe7e6 # PY_VER='3.8' # PY_LABEL=py38_ -# CONDA_SHA256=473e5ecc8e078e9ef89355fbca21f8eefa5f9081544befca99867c7beac3150d - -# CONDA_VER=22.11.1 -# CONDA_SUFFIX=-1 -# PY_VER='3.7' -# PY_LABEL=py37_ -# CONDA_SHA256=22b14d52265b4e609c6ce78e2f2884b277d976b83b5f9c8a83423e3eba2ccfbe - ## test diff --git a/dist/alpine/Dockerfile b/dist/alpine/Dockerfile index 64e1452..2c1260b 100644 --- a/dist/alpine/Dockerfile +++ b/dist/alpine/Dockerfile @@ -120,6 +120,7 @@ RUN \ find /opt/conda/conda-meta -user anaconda -exec chmod u+rwx,g+rwx,o-rwx "{}" \; && \ conda clean -ya +ENV HOME /home/anaconda ENV APK_REQUIREMENTS /tmp/apk_requirements.txt ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt ENV CONDA_REQUIREMENTS /tmp/conda_requirements.txt diff --git a/dist/debian/Dockerfile b/dist/debian/Dockerfile index daf97b5..0f3f5d1 100644 --- a/dist/debian/Dockerfile +++ b/dist/debian/Dockerfile @@ -6,17 +6,19 @@ FROM golang:alpine as go_tmp COPY ./utilities/startup.go /startup.go RUN cd / && go build startup.go -FROM debian:11-slim as miniconda-build +FROM debian:12-slim as miniconda-build LABEL maintainerName="Raphael Guzman" \ maintainerEmail="raphael@datajoint.com" \ maintainerCompany="DataJoint" ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive ARG CONDA_VER ARG CONDA_SUFFIX ARG PY_LABEL ARG PY_VER +ARG CONDA_SHA256 # Create anaconda group user, install dependencies, install Conda RUN \ groupadd -r anaconda && \ @@ -42,8 +44,12 @@ RUN \ # libxrender1 git mercurial subversion && \ apt-get clean && \ wget --quiet $(echo "https://repo.anaconda.com/miniconda\ - /Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-$(uname -m).sh" | \ - tr -d "\n" | tr -d "\t") -O ~/miniconda.sh && \ + /Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" | \ + tr -d "\n" | tr -d "\t") && \ + echo "${CONDA_SHA256} Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh" \ + > miniconda.sha256 && \ + if [ $(sha256sum -c miniconda.sha256 | awk '{print $2}') != "OK" ] ; then exit 1; fi && \ + mv Miniconda3-${PY_LABEL}${CONDA_VER}${CONDA_SUFFIX}-Linux-x86_64.sh ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ /opt/conda/bin/conda clean -tipy && \ @@ -67,6 +73,7 @@ RUN \ find /opt/conda/conda-meta -user anaconda -exec chmod u+rwx,g+rwx,o-rwx "{}" \; && \ conda clean -ya +ENV HOME /home/anaconda ENV APT_REQUIREMENTS /tmp/apt_requirements.txt ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt ENV CONDA_REQUIREMENTS /tmp/conda_requirements.txt diff --git a/dist/debian/docker-compose.yaml b/dist/debian/docker-compose.yaml index 42edfd6..565a5bd 100644 --- a/dist/debian/docker-compose.yaml +++ b/dist/debian/docker-compose.yaml @@ -7,6 +7,7 @@ services: args: - CONDA_VER - CONDA_SUFFIX + - CONDA_SHA256 - PY_LABEL - PY_VER image: datajoint/miniconda3:${CONDA_VER}-py${PY_VER}-debian diff --git a/tests/main.sh b/tests/main.sh index de3b391..0c315cc 100755 --- a/tests/main.sh +++ b/tests/main.sh @@ -156,38 +156,23 @@ TAG=$(echo $REF | awk -F':' '{print $2}') IMAGE=$(echo $REF | awk -F':' '{print $1}') SHELL_CMD_TEMPLATE="docker run --rm -i \$SHELL_CMD_FLAGS $REF \ $(docker inspect "$REF" --format '{{join .Config.Cmd " "}}') -c" -# Determine reference size -if [ $DISTRO == alpine ] && [ $PY_VER == '3.11' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=469 -elif [ $DISTRO == alpine ] && [ $PY_VER == '3.10' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=270 -elif [ $DISTRO == alpine ] && [ $PY_VER == '3.9' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=259 -elif [ $DISTRO == alpine ] && [ $PY_VER == '3.8' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=254 -elif [ $DISTRO == alpine ] && [ $PY_VER == '3.7' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=223 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.11' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=560 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.10' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=362 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.9' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=351 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.8' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=346 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.7' ] && [ $PLATFORM == 'linux/amd64' ]; then - SIZE_LIMIT=316 -# elif [ $DISTRO == debian ] && [ $PY_VER == '3.9' ] && [ $PLATFORM == 'linux/arm64' ]; then -# SIZE_LIMIT=505 -# elif [ $DISTRO == debian ] && [ $PY_VER == '3.8' ] && [ $PLATFORM == 'linux/arm64' ]; then -# SIZE_LIMIT=450 -# elif [ $DISTRO == debian ] && [ $PY_VER == '3.7' ] && [ $PLATFORM == 'linux/arm64' ]; then -# SIZE_LIMIT=460 -fi -SIZE_LIMIT=$(echo "scale=4; $SIZE_LIMIT * 1.05" | bc) +# Get the compressed size of the last build from docker hub +LAST_BUILD_SIZE=$(curl -s https://hub.docker.com/v2/repositories/$IMAGE/tags \ + | jq -r '.results[] | select(.name=="'"$LAST_BUILD_CONDA_VER"'-py'"$PY_VER"'-'"$DISTRO"'") | .images[0].size') +SIZE_INCRESE_FACTOR=1.5 +SIZE_LIMIT=$(echo "scale=4; $LAST_BUILD_SIZE * $SIZE_INCRESE_FACTOR" | bc) # Verify size minimal -SIZE=$(docker images --filter "reference=$REF" --format "{{.Size}}" | awk -F'MB' '{print $1}') +echo Compressing image for size verification... +docker save $REF | gzip > /tmp/$TAG.tar.gz +SIZE=$(ls -al /tmp | grep $TAG.tar.gz | awk '{ print $5 }') +echo -e \ + Size comparison:\\n\ + Current size: $(numfmt --to iec --format "%8.4f" $SIZE)\\n\ + Last build size: $(numfmt --to iec --format "%8.4f" $LAST_BUILD_SIZE)\\n\ + Size factor: $SIZE_INCRESE_FACTOR\\n\ + Size limit: $(numfmt --to iec --format "%8.4f" $SIZE_LIMIT) assert "minimal footprint" "(( $(echo "$SIZE <= $SIZE_LIMIT" | bc -l) ))" $LINENO +rm /tmp/$TAG.tar.gz # Run tests SHELL_CMD=$(eval "echo \"$SHELL_CMD_TEMPLATE\"") validate