feat: stack upgrade

.bin/scripts/build-images.sh

@@ -24,7 +24,7 @@ if [[ $# == "0" ]]; then
 fi;
 
 set +e
-docker buildx create --name mna --driver docker-container --bootstrap --use 2> /dev/null
+docker buildx create --name mna-tdb --driver docker-container --config "$SCRIPT_DIR/buildkitd.toml" 2> /dev/null
 set -e
 
 if [[ ! -z "${CI:-}" ]]; then
@@ -36,4 +36,6 @@ fi
 export CHANNEL=$(get_channel $VERSION)
 
 # "$@" is the list of environements
-docker buildx bake --builder mna --${mode} "$@"
+docker buildx bake --builder mna-tdb --${mode} "$@"
+docker builder prune --builder mna-tdb --keep-storage 20GB --force
+docker buildx stop --builder mna-tdb

.bin/scripts/buildkitd.toml

@@ -0,0 +1,7 @@
+[worker.oci]
+  max-parallelism = 2
+
+[[worker.oci.gcpolicy]]
+  all = true
+  keepBytes = "20GB"
+  keepDuration = "72h"

.bin/scripts/seed-update.sh

@@ -9,6 +9,8 @@ else
     shift
 fi
 
+echo "base de donnée cible: $TARGET_DB"
+
 read -p "La base de donnée contient-elle des données sensible ? [Y/n]: " response
 case $response in
   [nN][oO]|[nN])

.bin/scripts/setup-local-env.sh

@@ -10,6 +10,12 @@ ANSIBLE_CONFIG="${ROOT_DIR}/.infra/ansible/ansible.cfg" ansible all \
   --extra-vars "@${ROOT_DIR}/.infra/vault/vault.yml" \
   --vault-password-file="${SCRIPT_DIR}/get-vault-password-client.sh"
 
+echo "PUBLIC_VERSION=0.0.0-local" >> "${ROOT_DIR}/server/.env"
+
 echo "NEXT_PUBLIC_ENV=local" >> "${ROOT_DIR}/ui/.env"
 echo "NEXT_PUBLIC_VERSION=0.0.0-local" >> "${ROOT_DIR}/ui/.env"
 echo "NEXT_PUBLIC_API_PORT=5001" >> "${ROOT_DIR}/ui/.env"
+
+yarn build:dev
+yarn cli migrations:up
+yarn cli indexes:create

.dockerignore

@@ -1,14 +1,18 @@
 *
 
-!/server
-/server/.env
-/server/.env.local
+!/server/src
+!/server/static
+!/server/package.json
+!/server/tsconfig.json
+!/server/tsoa.json
+!/server/tsup.config.ts
 
 !/shared
 
 !/ui
 /ui/.env
 /ui/.next
+/ui/.eslintrc.js
 
 !/package.json
 !/.yarn/cache

.github/workflows/deploy_preview.yml

@@ -0,0 +1,138 @@
+name: Deploy Preview
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  debug:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hmarr/debug-action@v2
+
+  deploy_preview:
+    if: github.event.comment.body == ':rocket:' && github.event.issue.pull_request
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.event.issue.id }}
+      cancel-in-progress: true
+    name: Deploy Preview ${{ github.event.issue.number }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: React to comment
+        uses: dkershner6/reaction-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commentId: ${{ github.event.comment.id }}
+          reaction: "+1"
+
+      - id: "get-branch"
+        run: echo "branch=$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT
+        env:
+          REPO: ${{ github.repository }}
+          PR_NO: ${{ github.event.issue.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get-branch.outputs.branch }}
+
+      - name: Create LFS file list
+        run: git lfs ls-files --long | cut -d ' ' -f1 | sort > .lfs-assets-id
+
+      - name: LFS Cache
+        uses: actions/cache@v3
+        with:
+          path: .git/lfs/objects
+          key: ${{ runner.os }}-lfs-${{ hashFiles('.lfs-assets-id') }}
+          restore-keys: |
+            ${{ runner.os }}-lfs-
+
+      - name: Git LFS Pull
+        run: git lfs pull
+
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          name: github_actions
+          key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}
+          known_hosts: ${{ vars.SSH_KNOWN_HOSTS }}
+          config: |
+            Host *
+              IdentityFile ~/.ssh/github_actions
+
+      - name: Create vault pwd file
+        run: echo ${{ secrets.VAULT_PWD }} > .infra/.vault_pwd.txt
+
+      - name: Install jmespath
+        run: |
+          sudo pipx inject ansible-core jmespath
+
+      - name: Run playbook
+        run: .bin/mna-lba deploy preview "${{ github.event.issue.number }}"
+        env:
+          ANSIBLE_VAULT_PASSWORD_FILE: .infra/.vault_pwd.txt
+          ANSIBLE_REMOTE_USER: deploy
+          ANSIBLE_BECOME_PASS: ${{ secrets.DEPLOY_PASS }}
+
+      - name: Encrypt Error log on failure
+        run: .bin/mna-lba deploy:log:encrypt
+        if: failure()
+        env:
+          ANSIBLE_VAULT_PASSWORD_FILE: .infra/.vault_pwd.txt
+
+      - name: Upload failure artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: error-logs
+          path: /tmp/deploy_error.log.gpg
+
+      - name: Preview Summary when failed
+        if: failure()
+        run: echo 'You can get error logs using `.bin/mna-lba deploy:log:decrypt ${{ github.run_id }}`' >> $GITHUB_STEP_SUMMARY
+
+      - name: Preview Summary
+        run: echo 'https://${{ github.event.issue.number }}.labonnealternance-preview.apprentissage.beta.gouv.fr/ 🚀' >> $GITHUB_STEP_SUMMARY
+
+      - name: Comment PR Preview
+        if: github.event.issue.state != 'closed'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            ### :rocket: Prévisualisation
+            https://${{ github.event.issue.number }}.labonnealternance-preview.apprentissage.beta.gouv.fr/
+
+            To re-deploy just add a comment with :rocket:
+          comment_tag: deployment
+          mode: recreate
+          pr_number: ${{ github.event.issue.number }}
+
+      - name: Comment PR Preview when failed
+        if: failure() && github.event.issue.state != 'closed'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            ### :ambulance: Prévisualisation failed
+
+            https://${{ github.event.issue.number }}.labonnealternance-preview.apprentissage.beta.gouv.fr/
+
+            You can get error logs using `.bin/mna-lba deploy:log:decrypt ${{ github.run_id }}`
+
+            To re-deploy just add a comment with :rocket:
+          comment_tag: deployment
+          mode: recreate
+          pr_number: ${{ github.event.issue.number }}
+
+      - name: Comment PR Preview when cancelled
+        if: cancelled() && github.event.issue.state != 'closed'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            ### :ambulance: Prévisualisation cancelled
+
+            https://${{ github.event.issue.number }}.labonnealternance-preview.apprentissage.beta.gouv.fr/
+
+            To re-deploy just add a comment with :rocket:
+          comment_tag: deployment
+          mode: recreate
+          pr_number: ${{ github.event.issue.number }}

.github/workflows/gitguardian.yml

@@ -0,0 +1,21 @@
+name: GitGuardian scan
+
+on: [push, pull_request]
+
+jobs:
+  scanning:
+    name: GitGuardian scan
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # fetch all history so multiple commits can be scanned
+      - name: GitGuardian scan
+        uses: GitGuardian/[email protected]
+        env:
+          GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }}
+          GITHUB_PUSH_BASE_SHA: ${{ github.event.base }}
+          GITHUB_PULL_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}

.github/workflows/merge_queue.yml

@@ -0,0 +1,10 @@
+name: Merge Queue
+on:
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  tests:
+    uses: "./.github/workflows/ci.yml"
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/preview.yml

@@ -10,69 +10,15 @@ jobs:
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-  deploy:
-    if: github.event.pull_request.draft == false
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
-    name: Deploy Preview ${{ github.event.pull_request.number }}
+  deploy_comment:
+    name: Add deploy comment
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout project
-        uses: actions/checkout@v4
-        with:
-          lfs: true
-
-      - name: Install SSH key
-        uses: shimataro/ssh-key-action@v2
-        with:
-          name: github_actions
-          key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}
-          known_hosts: ${{ vars.SSH_KNOWN_HOSTS }}
-          config: |
-            Host *
-              IdentityFile ~/.ssh/github_actions
-
-      - name: Create vault pwd file
-        run: echo ${{ secrets.VAULT_PWD }} > .infra/.vault_pwd.txt
-
-      - name: Install jmespath
-        run: |
-          sudo pipx inject ansible-core jmespath
-
-      - name: Run playbook
-        run: .bin/mna-tdb deploy preview "${{ github.event.pull_request.number }}"
-        env:
-          ANSIBLE_VAULT_PASSWORD_FILE: .infra/.vault_pwd.txt
-          ANSIBLE_REMOTE_USER: deploy
-          ANSIBLE_BECOME_PASS: ${{ secrets.DEPLOY_PASS }}
-
-      - name: Encrypt Error log on failure
-        run: .bin/mna-tdb deploy:log:encrypt
-        if: failure()
-        env:
-          ANSIBLE_VAULT_PASSWORD_FILE: .infra/.vault_pwd.txt
-
-      - name: Upload failure artifacts on failure
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: error-logs
-          path: /tmp/deploy_error.log.gpg
-
-      - name: Preview Summary when failed
-        if: failure()
-        run: echo 'You can get error logs using `.bin/mna-tdb deploy:log:decrypt ${{ github.run_id }}`' >> $GITHUB_STEP_SUMMARY
-
-      - name: Preview Summary
-        run: echo 'https://${{ github.event.pull_request.number }}.tdb-preview.apprentissage.beta.gouv.fr/ 🚀' >> $GITHUB_STEP_SUMMARY
-
       - name: Comment PR Preview
         if: github.event.pull_request.state != 'closed'
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
-            ### :rocket: Prévisualisation
-            https://${{ github.event.pull_request.number }}.tdb-preview.apprentissage.beta.gouv.fr/
-          comment_tag: execution
-          mode: recreate
+            To deploy this PR just add a comment with a simple :rocket:
+          comment_tag: deployment_instructions
+          mode: upsert

.github/workflows/release.yml

@@ -10,6 +10,8 @@ jobs:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   release:
+    concurrency:
+      group: "release-${{ github.workflow }}-${{ github.ref }}"
     permissions: write-all
     outputs:
       VERSION: ${{ steps.get-version.outputs.VERSION }}
@@ -66,7 +68,7 @@ jobs:
 
   deploy:
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
+      group: "deploy-${{ github.workflow }}-${{ github.ref }}"
     needs: ["release"]
     name: Deploy ${{ needs.release.outputs.VERSION }} on recette
     uses: "./.github/workflows/_deploy.yml"

.infra/ansible/tasks/preview_pr.yml

@@ -68,7 +68,7 @@
 
     - name: "[{{ pr_number }}] Build local images 0.0.0-{{ pr_number }}"
       shell:
-        cmd: ".bin/mna-tdb preview:build 0.0.0-{{ pr_number }} load preview"
+        cmd: "flock --verbose --close /tmp/deployment_build.lock .bin/mna-tdb preview:build 0.0.0-{{ pr_number }} load preview"
         chdir: "/opt/app/projects/{{ pr_number }}/repository"
       async: 900 # max 15 minutes
       poll: 15 # check every 15s
@@ -81,18 +81,20 @@
     - name: "[{{ pr_number }}] Trigger ACME companion"
       shell:
         chdir: /opt/app
-        cmd: docker exec nginx-proxy-acme /app/force_renew
+        cmd: docker exec nginx-proxy-acme /app/signal_le_service
+
+    - name: "[{{ pr_number }}] Seed database"
+      shell:
+        chdir: "/opt/app"
+        cmd: "flock --verbose --close /tmp/deployment_seed.lock /opt/app/scripts/seed.sh preview_{{ pr_number | default('00') }}"
+      async: 900 # max 15 minutes
+      poll: 15 # check every 15s
 
     - name: "[{{ pr_number }}] Execute MongoDB migrations"
       shell:
         chdir: "/opt/app/projects/{{ pr_number }}"
         cmd: "docker compose run --rm server yarn cli migrations:up"
 
-    - name: "[{{ pr_number }}] Seed database"
-      shell:
-        chdir: "/opt/app"
-        cmd: "/opt/app/scripts/seed.sh preview_{{ pr_number | default('00') }}"
-
     - name: "[{{ pr_number }}] Preview URL"
       debug:
         msg: "{{ vault[env_type].MNA_TDB_PUBLIC_URL }}"

.infra/docker-compose.preview-system.yml

@@ -4,7 +4,7 @@ x-default: &default
   deploy:
     resources:
       limits:
-        memory: 256m
+        memory: 1g
   restart: always
   networks:
     - mna_network

.infra/docker-compose.preview.yml

@@ -4,7 +4,7 @@ x-default: &default
   deploy:
     resources:
       limits:
-        memory: 256m
+        memory: 2g
   restart: always
   networks:
     - mna_network