From e3e3e8512c79812ebb6f70e270b9c86e7e0e2b3a Mon Sep 17 00:00:00 2001 From: duncdrum Date: Sat, 2 Nov 2024 10:52:39 +0100 Subject: [PATCH 1/6] [refactor] remove dead code dependencies are part of distroless base image enable experimental image push for testing --- .github/workflows/ci-deploy.yml | 16 ++++++++-------- .../src/main/resources-filtered/Dockerfile | 18 ------------------ 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci-deploy.yml b/.github/workflows/ci-deploy.yml index 94b1ee46c48..6dad561fbf4 100644 --- a/.github/workflows/ci-deploy.yml +++ b/.github/workflows/ci-deploy.yml @@ -60,12 +60,12 @@ jobs: run: mvn -q -Ddocker.tag=release -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push working-directory: ./exist-docker # NOTE (DP): This is for debugging, publishes an experimental image from inside PRs against develop - # - name: Publish experimental images - # if: github.base_ref == 'develop' - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - # DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - # run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push - # working-directory: ./exist-docker + - name: Publish experimental images + if: github.base_ref == 'develop' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push + working-directory: ./exist-docker diff --git a/exist-docker/src/main/resources-filtered/Dockerfile b/exist-docker/src/main/resources-filtered/Dockerfile index 16887291004..6ef7d5dab95 100644 --- a/exist-docker/src/main/resources-filtered/Dockerfile +++ b/exist-docker/src/main/resources-filtered/Dockerfile @@ -20,26 +20,8 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # -# Use Debian Bullseye (which is the base of gcr.io/distroless/java:17) for additional library dependencies that we need -# FROM debian:bullseye-slim as debian-slim -# RUN apt-get update && apt-get -y dist-upgrade -# RUN apt-get install -y openjdk-17-jre-headless -# RUN apt-get install -y expat fontconfig # Install tools required by FOP - FROM gcr.io/distroless/java17:latest -# Copy over dependencies for Apache FOP, missing from GCR's JRE -# COPY --from=debian-slim /usr/lib/x86_64-linux-gnu/libfreetype.so.6 /usr/lib/x86_64-linux-gnu/libfreetype.so.6 -# COPY --from=debian-slim /usr/lib/x86_64-linux-gnu/liblcms2.so.2 /usr/lib/x86_64-linux-gnu/liblcms2.so.2 -# COPY --from=debian-slim /usr/lib/x86_64-linux-gnu/libpng16.so.16 /usr/lib/x86_64-linux-gnu/libpng16.so.16 -# COPY --from=debian-slim /usr/lib/x86_64-linux-gnu/libfontconfig.so.1 /usr/lib/x86_64-linux-gnu/libfontconfig.so.1 - -# Copy dependencies for Apache Batik (used by Apache FOP to handle SVG rendering) -# COPY --from=debian-slim /etc/fonts /etc/fonts -# COPY --from=debian-slim /lib/x86_64-linux-gnu/libexpat.so.1 /lib/x86_64-linux-gnu/libexpat.so.1 -# COPY --from=debian-slim /usr/share/fontconfig /usr/share/fontconfig -# COPY --from=debian-slim /usr/share/fonts/truetype/dejavu /usr/share/fonts/truetype/dejavu - # Copy eXist-db COPY LICENSE /exist/LICENSE COPY autodeploy /exist/autodeploy From 75981d8de6cb3d869d38ed7411381e8ee85a1e38 Mon Sep 17 00:00:00 2001 From: duncdrum Date: Sat, 2 Nov 2024 12:19:17 +0100 Subject: [PATCH 2/6] [feature] add multi-arch images see #5450 close #4151 close #5532 close #4153 --- .github/workflows/ci-deploy.yml | 5 ++++ exist-docker/pom.xml | 49 ++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-deploy.yml b/.github/workflows/ci-deploy.yml index 6dad561fbf4..d75ac5195f3 100644 --- a/.github/workflows/ci-deploy.yml +++ b/.github/workflows/ci-deploy.yml @@ -5,6 +5,7 @@ jobs: name: Build and Test Images runs-on: ubuntu-latest # NOTE (DP): Publish on develop and master, test on PRs against these + # TODO(DP) Reinstate CRONed release builds to update stock apps regularly if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master' || github.base_ref == 'develop' || github.base_ref == 'master' steps: - uses: actions/checkout@v4 @@ -15,6 +16,10 @@ jobs: with: distribution: liberica java-version: '17' + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 - name: Make buildkit default uses: docker/setup-buildx-action@v3 id: buildx diff --git a/exist-docker/pom.xml b/exist-docker/pom.xml index 13c9c1a4dd6..d49f8e54d03 100644 --- a/exist-docker/pom.xml +++ b/exist-docker/pom.xml @@ -22,7 +22,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA --> - + 4.0.0 @@ -42,14 +44,15 @@ scm:git:https://github.com/exist-db/exist.git scm:git:https://github.com/exist-db/exist.git scm:git:https://github.com/exist-db/exist.git - HEAD - + HEAD + ${project.build.directory}/exist-docker-${project.version}-docker-dir exist.uber.jar latest debug + linux/amd64, linux/arm64 @@ -106,7 +109,9 @@ true - ${project.groupId}:exist-distribution:pom:${project.version} + + ${project.groupId}:exist-distribution:pom:${project.version} @@ -124,7 +129,8 @@ single - ${project.basedir}/../exist-distribution/target/exist-distribution-${project.version}-dir + + ${project.basedir}/../exist-distribution/target/exist-distribution-${project.version}-dir false src/assembly/dist-assembly-docker.xml @@ -183,18 +189,24 @@ - - + + false - + META-INF/mailcap - + META-INF/mailcap.default - - + + true @@ -210,12 +222,18 @@ docker-maven-plugin 0.45.1 - true + true + registry.hub.docker.com existdb/existdb:%v exist + + + ${docker.platforms} + + ${docker.tag} @@ -227,6 +245,11 @@ existdb/existdb:%v-DEBUG exist-debug + + + ${docker.platforms} + + ${docker.debug.tag} @@ -256,4 +279,4 @@ - + \ No newline at end of file From 9bc97a10443ad014fad96373c271c87559e6c2ac Mon Sep 17 00:00:00 2001 From: duncdrum Date: Tue, 12 Nov 2024 12:57:57 +0100 Subject: [PATCH 3/6] [chore] bump base image --- exist-docker/src/main/resources-filtered/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exist-docker/src/main/resources-filtered/Dockerfile b/exist-docker/src/main/resources-filtered/Dockerfile index 6ef7d5dab95..66307a5c5da 100644 --- a/exist-docker/src/main/resources-filtered/Dockerfile +++ b/exist-docker/src/main/resources-filtered/Dockerfile @@ -20,7 +20,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # -FROM gcr.io/distroless/java17:latest +FROM gcr.io/distroless/java17-debian12:latest # Copy eXist-db COPY LICENSE /exist/LICENSE From bd809c94828c84bdde3ea50bf02b63d76ecbb334 Mon Sep 17 00:00:00 2001 From: duncdrum Date: Tue, 12 Nov 2024 13:02:36 +0100 Subject: [PATCH 4/6] [bugfix] workaround docker maven plugin issue fixes #5450 see [docker maven plugin bug](https://github.com/fabric8io/docker-maven-plugin/issues/1835) --- .github/workflows/ci-deploy.yml | 6 +++++- exist-docker/pom.xml | 2 ++ exist-docker/src/main/resources-filtered/Dockerfile-DEBUG | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-deploy.yml b/.github/workflows/ci-deploy.yml index d75ac5195f3..26ca3c4f0d6 100644 --- a/.github/workflows/ci-deploy.yml +++ b/.github/workflows/ci-deploy.yml @@ -32,7 +32,11 @@ jobs: key: deploy-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: deploy-${{ runner.os }}-maven - name: Install bats - run: sudo apt-get install bats + run: sudo apt-get install bats + # Hack around #5450 + - name: pull base image + run: | + docker pull --platform linux/amd64 --platform linux/arm64 gcr.io/distroless/java17-debian12:latest - name: Build images run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true -P !mac-dmg-on-unix,!installer,!concurrency-stress-tests,!micro-benchmarks,skip-build-dist-archives clean package - name: Check local images diff --git a/exist-docker/pom.xml b/exist-docker/pom.xml index d49f8e54d03..eb89bb1d1a5 100644 --- a/exist-docker/pom.xml +++ b/exist-docker/pom.xml @@ -227,6 +227,7 @@ existdb/existdb:%v + registry.hub.docker.com exist @@ -243,6 +244,7 @@ existdb/existdb:%v-DEBUG + registry.hub.docker.com exist-debug diff --git a/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG b/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG index 17bf0239765..d68aa8b85e3 100644 --- a/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG +++ b/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG @@ -21,7 +21,7 @@ # # Use JDK 17 in Debian Bullseye (as our production image gcr.io/distroless/java:17 is based on Debian Bullseye with just a JRE) -FROM debian:bullseye-slim +FROM debian:bookworm-slim RUN apt-get update && apt-get -y dist-upgrade RUN apt-get install -y openjdk-17-jdk-headless RUN apt-get install -y expat fontconfig # Install tools required by FOP From 5cf787ad827fed3be79ee97ba95188a76c6b5ae5 Mon Sep 17 00:00:00 2001 From: duncdrum Date: Tue, 12 Nov 2024 14:12:01 +0100 Subject: [PATCH 5/6] [refactor] LegacyKeyValueFormat --- exist-docker/src/main/resources-filtered/Dockerfile | 6 +++--- exist-docker/src/main/resources-filtered/Dockerfile-DEBUG | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exist-docker/src/main/resources-filtered/Dockerfile b/exist-docker/src/main/resources-filtered/Dockerfile index 66307a5c5da..72cbcda71eb 100644 --- a/exist-docker/src/main/resources-filtered/Dockerfile +++ b/exist-docker/src/main/resources-filtered/Dockerfile @@ -48,10 +48,10 @@ ARG CACHE_MEM ARG MAX_BROKER ARG JVM_MAX_RAM_PERCENTAGE -ENV EXIST_HOME "/exist" +ENV EXIST_HOME=/exist ENV CLASSPATH=/exist/lib/${exist.uber.jar.filename} -ENV JAVA_TOOL_OPTIONS \ +ENV JAVA_TOOL_OPTIONS="\ -Dfile.encoding=UTF8 \ -Dsun.jnu.encoding=UTF-8 \ -Djava.awt.headless=true \ @@ -67,7 +67,7 @@ ENV JAVA_TOOL_OPTIONS \ -XX:+UseStringDeduplication \ -XX:+UseContainerSupport \ -XX:MaxRAMPercentage=${JVM_MAX_RAM_PERCENTAGE:-75.0} \ - -XX:+ExitOnOutOfMemoryError + -XX:+ExitOnOutOfMemoryError" HEALTHCHECK CMD [ "java", \ "org.exist.start.Main", "client", \ diff --git a/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG b/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG index d68aa8b85e3..44324782875 100644 --- a/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG +++ b/exist-docker/src/main/resources-filtered/Dockerfile-DEBUG @@ -52,10 +52,10 @@ ARG MAX_BROKER ARG JVM_MAX_RAM_PERCENTAGE ARG JVM_JDWP_SUSPEND -ENV EXIST_HOME "/exist" +ENV EXIST_HOME=/exist ENV CLASSPATH=/exist/lib/${exist.uber.jar.filename} -ENV JAVA_TOOL_OPTIONS \ +ENV JAVA_TOOL_OPTIONS="\ -Dfile.encoding=UTF8 \ -Dsun.jnu.encoding=UTF-8 \ -Djava.awt.headless=true \ @@ -72,7 +72,7 @@ ENV JAVA_TOOL_OPTIONS \ -XX:+UseContainerSupport \ -XX:MaxRAMPercentage=${JVM_MAX_RAM_PERCENTAGE:-75.0} \ -XX:+ExitOnOutOfMemoryError \ - -agentlib:jdwp=transport=dt_socket,server=y,suspend=${JVM_JDWP_SUSPEND:-n},address=5005 + -agentlib:jdwp=transport=dt_socket,server=y,suspend=${JVM_JDWP_SUSPEND:-n},address=5005" HEALTHCHECK CMD [ "java", \ "org.exist.start.Main", "client", \ From 9b1a3cfeb800a4d900fdf10068f723adf310fb68 Mon Sep 17 00:00:00 2001 From: duncdrum Date: Tue, 12 Nov 2024 15:54:31 +0100 Subject: [PATCH 6/6] [chore] no more experimental images --- .github/workflows/ci-deploy.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-deploy.yml b/.github/workflows/ci-deploy.yml index 26ca3c4f0d6..a81af4f361c 100644 --- a/.github/workflows/ci-deploy.yml +++ b/.github/workflows/ci-deploy.yml @@ -69,12 +69,12 @@ jobs: run: mvn -q -Ddocker.tag=release -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push working-directory: ./exist-docker # NOTE (DP): This is for debugging, publishes an experimental image from inside PRs against develop - - name: Publish experimental images - if: github.base_ref == 'develop' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push - working-directory: ./exist-docker + # - name: Publish experimental images + # if: github.base_ref == 'develop' + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + # DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + # run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push + # working-directory: ./exist-docker