From 9f0d8cc148b7f7c94774081fd3fbea15dc993110 Mon Sep 17 00:00:00 2001 From: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:01:28 +0300 Subject: [PATCH 1/6] Working Docker Signed-off-by: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> --- Dockerfile | 61 +++++++++++++++------- apps/climatemappedafrica/payload.config.ts | 2 +- docker-compose.yml | 10 ++++ 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index f8b072f99..5d60b0f10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,6 +66,7 @@ RUN pnpm fetch COPY *.yaml *.json ./ COPY packages/commons-ui-core/package.json ./packages/commons-ui-core/package.json COPY packages/commons-ui-next/package.json ./packages/commons-ui-next/package.json +COPY packages/commons-ui-payload/package.json ./packages/commons-ui-payload/package.json # Next.js lints when doing production build COPY packages/eslint-config-commons-ui/package.json ./packages/eslint-config-commons-ui/package.json # TODO(kilemensi): Figure out why this is needed (charterafrica, codeforafrica) @@ -235,7 +236,7 @@ CMD ["node", "dist/server.js"] # ============================================================================ # -# climatemappedafrica-desp: image with all climatemappedafrica dependencies +# climatemappedafrica-deps: image with all climatemappedafrica dependencies # ------------------------------------------------------------------------- FROM base-deps AS climatemappedafrica-deps @@ -245,7 +246,7 @@ COPY packages/hurumap-next/package.json ./packages/hurumap-next/package.json COPY apps/climatemappedafrica/package.json ./apps/climatemappedafrica/package.json # Use virtual store: https://pnpm.io/cli/fetch#usage-scenario -RUN pnpm --filter "./apps/climatemappedafrica" install --offline --frozen-lockfile +RUN pnpm --filter "./apps/climatemappedafrica/" install --offline --frozen-lockfile # # climatemappedafrica-builder: image that uses deps to build shippable output @@ -264,6 +265,9 @@ ARG NEXT_TELEMETRY_DISABLED \ NEXT_PUBLIC_IMAGE_DOMAINS="cms.dev.codeforafrica.org,hurumap-v2.s3.amazonaws.com" \ NEXT_PUBLIC_IMAGE_SCALE_FACTOR=2 \ NEXT_PUBLIC_GOOGLE_ANALYTICS \ + # Payload (runtime) + MONGO_URL \ + PAYLOAD_SECRET \ # Sentry (build time) SENTRY_AUTH_TOKEN \ SENTRY_ENVIRONMENT \ @@ -277,9 +281,17 @@ COPY --from=climatemappedafrica-deps /workspace/node_modules ./node_modules COPY --from=climatemappedafrica-deps /workspace/apps/climatemappedafrica/node_modules ./apps/climatemappedafrica/node_modules -COPY apps/climatemappedafrica ./apps/climatemappedafrica +COPY apps/climatemappedafrica ./apps/climatemappedafrica/ -RUN pnpm --filter "./apps/climatemappedafrica" build +# When building Next.js app, Next.js needs to connect to local Payload +ENV PAYLOAD_PUBLIC_APP_URL=http://localhost:3000 +ENV NEXT_PUBLIC_SEO_DISABLED=${NEXT_PUBLIC_SEO_DISABLED} +RUN --mount=type=secret,id=sentry_auth_token,env=SENTRY_AUTH_TOKEN \ + pnpm --filter "./apps/climatemappedafrica/" build-next + +# When building Payload app, Payload needs to have final app URL +ENV PAYLOAD_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL} +RUN pnpm --filter "./apps/climatemappedafrica/" build-payload # # climatemappedafrica-runner: final deployable image @@ -287,15 +299,16 @@ RUN pnpm --filter "./apps/climatemappedafrica" build FROM base-runner AS climatemappedafrica-runner -ARG NEXT_PUBLIC_IMAGE_DOMAINS \ - NEXT_PUBLIC_IMAGE_SCALE_FACTOR \ - NEXT_PUBLIC_OPENAFRICA_DOMAINS \ - NEXT_PUBLIC_SOURCEAFRICA_DOMAINS +ARG PAYLOAD_CONFIG_PATH="dist/payload.config.js" \ + PAYLOAD_PUBLIC_APP_URL -ENV NEXT_PUBLIC_IMAGE_DOMAINS=${NEXT_PUBLIC_IMAGE_DOMAINS} \ - NEXT_PUBLIC_IMAGE_SCALE_FACTOR=${NEXT_PUBLIC_IMAGE_SCALE_FACTOR} \ - NEXT_PUBLIC_OPENAFRICA_DOMAINS=${NEXT_PUBLIC_OPENAFRICA_DOMAINS} \ - NEXT_PUBLIC_SOURCEAFRICA_DOMAINS=${NEXT_PUBLIC_SOURCEAFRICA_DOMAINS} +# DO NOT initialise ENV vars using ARG for secrets!!!! +# These include: +# i. Mongo DB URL +# ii. Payload Secret +# ENV are persisted in the image & may lead to security issues +ENV PAYLOAD_PUBLIC_APP_URL=${PAYLOAD_PUBLIC_APP_URL} \ + PAYLOAD_CONFIG_PATH=${PAYLOAD_CONFIG_PATH} RUN set -ex \ # Create nextjs cache dir w/ correct permissions @@ -306,21 +319,29 @@ RUN set -ex \ # symlink some dependencies COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/node_modules ./node_modules +# Since we can't use output: "standalone", copy all app's dependencies +COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/apps/climatemappedafrica/node_modules ./apps/climatemappedafrica/node_modules +COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/apps/climatemappedafrica/next.config.js ./apps/climatemappedafrica/next.config.js +COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/apps/climatemappedafrica/.env ./apps/climatemappedafrica/.env # Next.js # Public assets COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/apps/climatemappedafrica/public ./apps/climatemappedafrica/public -# Automatically leverage output traces to reduce image size -# https://nextjs.org/docs/advanced-features/output-file-tracing -COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/apps/climatemappedafrica/.next/standalone ./apps/climatemappedafrica -COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/apps/climatemappedafrica/.next/static ./apps/climatemappedafrica/.next/static +# Since we can't use output: "standalone", copy the whole app's .next folder +# TODO(kilemensi): Figure out which files in .next folder are not needed +COPY --from=climatemappedafrica-builder --chown=nextjs:nodejs /workspace/apps/climatemappedafrica/.next ./apps/climatemappedafrica/.next -USER nextjs +# Payload +COPY --from=climatemappedafrica-builder /workspace/apps/climatemappedafrica/dist ./apps/climatemappedafrica/dist +COPY --from=climatemappedafrica-builder /workspace/apps/climatemappedafrica/build ./apps/climatemappedafrica/build -# server.js is created by next build from the standalone output -# https://nextjs.org/docs/pages/api-reference/next-config-js/output -CMD ["node", "apps/climatemappedafrica/server.js"] +# Since we can't use output: "standalone", switch to specific app's folder +WORKDIR /workspace/apps/climatemappedafrica + +USER nextjs +# Custom server to run Payload and Next.js in the same app +CMD ["node", "dist/server.js"] # ============================================================================ # CivicSignal Blog diff --git a/apps/climatemappedafrica/payload.config.ts b/apps/climatemappedafrica/payload.config.ts index 9f7505efe..6d65eb688 100644 --- a/apps/climatemappedafrica/payload.config.ts +++ b/apps/climatemappedafrica/payload.config.ts @@ -16,7 +16,7 @@ import Members from "./src/payload/collections/Members"; import Pages from "./src/payload/collections/Pages"; import Users from "./src/payload/collections/Users"; -import Site from "./src/payload/globals/Site"; +import Site from "./src/payload/globals/site"; const projectDir = process.cwd(); loadEnvConfig(projectDir); diff --git a/docker-compose.yml b/docker-compose.yml index 4f0f5ec5d..6c4d40382 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,12 +83,22 @@ services: target: climatemappedafrica-runner args: - HURUMAP_API_URL + - MONGO_URL=mongodb://${MONGO_INITDB_ROOT_USERNAME:-root}:${MONGO_INITDB_ROOT_PASSWORD:-rootpassword}@host.docker.internal:${MONGODB_PORT:-27017}/climatemappedafrica?authSource=admin&directConnection=true + - PAYLOAD_SECRET + - SENTRY_ENVIRONMENT + - SENTRY_ORG + - SENTRY_PROJECT environment: HURUMAP_API_URL: ${HURUMAP_API_URL} S3_UPLOAD_KEY: ${S3_UPLOAD_KEY} S3_UPLOAD_SECRET: ${S3_UPLOAD_SECRET} S3_UPLOAD_BUCKET: ${S3_UPLOAD_BUCKET} S3_UPLOAD_REGION: ${S3_UPLOAD_REGION} + MONGO_URL: mongodb://${MONGO_INITDB_ROOT_USERNAME:-root}:${MONGO_INITDB_ROOT_PASSWORD:-rootpassword}@host.docker.internal:${MONGODB_PORT:-27017}/climatemappedafrica?authSource=admin&directConnection=true + PAYLOAD_SECRET: ${PAYLOAD_SECRET} + SENTRY_ORG: ${SENTRY_ORG} + SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT} + SENTRY_PROJECT: ${SENTRY_PROJECT} ports: - 3000:3000 From 88ffd121a126af709ad9ca10c29b204e217ac880 Mon Sep 17 00:00:00 2001 From: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:51:06 +0300 Subject: [PATCH 2/6] Test docker build Signed-off-by: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> --- .github/workflows/climatemappedafrica-deploy-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/climatemappedafrica-deploy-dev.yml b/.github/workflows/climatemappedafrica-deploy-dev.yml index 804cbaede..d67bfad98 100644 --- a/.github/workflows/climatemappedafrica-deploy-dev.yml +++ b/.github/workflows/climatemappedafrica-deploy-dev.yml @@ -2,7 +2,7 @@ name: Climate Mapped Africa | Deploy | DEV on: push: - branches: [main] + branches: [ft/climatemapped-fix-docker-build] paths: - "apps/climatemappedafrica/**" - "Dockerfile" From 343107069334e7a90383aaab479d9ad282be59c8 Mon Sep 17 00:00:00 2001 From: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:07:45 +0300 Subject: [PATCH 3/6] Update secrets Signed-off-by: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> --- .github/workflows/climatemappedafrica-deploy-dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/climatemappedafrica-deploy-dev.yml b/.github/workflows/climatemappedafrica-deploy-dev.yml index d67bfad98..f01da0907 100644 --- a/.github/workflows/climatemappedafrica-deploy-dev.yml +++ b/.github/workflows/climatemappedafrica-deploy-dev.yml @@ -55,6 +55,8 @@ jobs: with: build-args: | HURUMAP_API_URL=${{ secrets.HURUMAP_HURUMAP_API_URL }} + MONGO_URL=${{ secrets.CLIMATEMAPPEDAFRICA_MONGO_URL }} + PAYLOAD_SECRET=${{ secrets.CLIMATEMAPPEDAFRICA_PAYLOAD_SECRET }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new context: . From 95604c9d47f69afc899a50240a5db2a36fbf9d0a Mon Sep 17 00:00:00 2001 From: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:14:37 +0300 Subject: [PATCH 4/6] Update HURUMAP API URL Signed-off-by: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> --- .github/workflows/climatemappedafrica-deploy-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/climatemappedafrica-deploy-dev.yml b/.github/workflows/climatemappedafrica-deploy-dev.yml index f01da0907..ce803d64c 100644 --- a/.github/workflows/climatemappedafrica-deploy-dev.yml +++ b/.github/workflows/climatemappedafrica-deploy-dev.yml @@ -54,7 +54,7 @@ jobs: uses: docker/build-push-action@v3 with: build-args: | - HURUMAP_API_URL=${{ secrets.HURUMAP_HURUMAP_API_URL }} + HURUMAP_API_URL=${{ secrets.CLIMATEMAPPEDAFRICA_HURUMAP_API_URL }} MONGO_URL=${{ secrets.CLIMATEMAPPEDAFRICA_MONGO_URL }} PAYLOAD_SECRET=${{ secrets.CLIMATEMAPPEDAFRICA_PAYLOAD_SECRET }} cache-from: type=local,src=/tmp/.buildx-cache From 66f116295546f6b39308496dd5a075cb2fe42e1d Mon Sep 17 00:00:00 2001 From: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:16:02 +0300 Subject: [PATCH 5/6] Review Fixex Signed-off-by: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d60b0f10..4fe65a578 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,6 @@ RUN pnpm fetch COPY *.yaml *.json ./ COPY packages/commons-ui-core/package.json ./packages/commons-ui-core/package.json COPY packages/commons-ui-next/package.json ./packages/commons-ui-next/package.json -COPY packages/commons-ui-payload/package.json ./packages/commons-ui-payload/package.json # Next.js lints when doing production build COPY packages/eslint-config-commons-ui/package.json ./packages/eslint-config-commons-ui/package.json # TODO(kilemensi): Figure out why this is needed (charterafrica, codeforafrica) @@ -244,9 +243,10 @@ FROM base-deps AS climatemappedafrica-deps COPY packages/hurumap-core/package.json ./packages/hurumap-core/package.json COPY packages/hurumap-next/package.json ./packages/hurumap-next/package.json COPY apps/climatemappedafrica/package.json ./apps/climatemappedafrica/package.json +COPY packages/commons-ui-payload/package.json ./packages/commons-ui-payload/package.json # Use virtual store: https://pnpm.io/cli/fetch#usage-scenario -RUN pnpm --filter "./apps/climatemappedafrica/" install --offline --frozen-lockfile +RUN pnpm --filter "./apps/climatemappedafrica" install --offline --frozen-lockfile # # climatemappedafrica-builder: image that uses deps to build shippable output @@ -281,17 +281,17 @@ COPY --from=climatemappedafrica-deps /workspace/node_modules ./node_modules COPY --from=climatemappedafrica-deps /workspace/apps/climatemappedafrica/node_modules ./apps/climatemappedafrica/node_modules -COPY apps/climatemappedafrica ./apps/climatemappedafrica/ +COPY apps/climatemappedafrica ./apps/climatemappedafrica # When building Next.js app, Next.js needs to connect to local Payload ENV PAYLOAD_PUBLIC_APP_URL=http://localhost:3000 ENV NEXT_PUBLIC_SEO_DISABLED=${NEXT_PUBLIC_SEO_DISABLED} RUN --mount=type=secret,id=sentry_auth_token,env=SENTRY_AUTH_TOKEN \ - pnpm --filter "./apps/climatemappedafrica/" build-next + pnpm --filter "./apps/climatemappedafrica" build-next # When building Payload app, Payload needs to have final app URL ENV PAYLOAD_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL} -RUN pnpm --filter "./apps/climatemappedafrica/" build-payload +RUN pnpm --filter "./apps/climatemappedafrica" build-payload # # climatemappedafrica-runner: final deployable image From a7f7ba1668e841a820182ef32160f35bd3b99df0 Mon Sep 17 00:00:00 2001 From: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:32:14 +0300 Subject: [PATCH 6/6] Traget Main Signed-off-by: Kipruto <43873157+kelvinkipruto@users.noreply.github.com> --- .github/workflows/climatemappedafrica-deploy-dev.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/climatemappedafrica-deploy-dev.yml b/.github/workflows/climatemappedafrica-deploy-dev.yml index ce803d64c..4c9d24776 100644 --- a/.github/workflows/climatemappedafrica-deploy-dev.yml +++ b/.github/workflows/climatemappedafrica-deploy-dev.yml @@ -2,7 +2,7 @@ name: Climate Mapped Africa | Deploy | DEV on: push: - branches: [ft/climatemapped-fix-docker-build] + branches: [main] paths: - "apps/climatemappedafrica/**" - "Dockerfile" @@ -33,6 +33,10 @@ jobs: with: fetch-depth: 0 + # Add support for more platforms with QEMU (optional) + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -60,6 +64,7 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new context: . + platforms: linux/amd64,linux/arm64 target: climatemappedafrica-runner push: true tags: "${{ env.IMAGE_NAME }}:${{ github.sha }}"