From e0066cd11405eb827f8b730e8f3c7fed1c5c58d2 Mon Sep 17 00:00:00 2001 From: Pyry Date: Tue, 18 Jun 2024 10:44:04 +0300 Subject: [PATCH 1/2] feat: shard client and login tests --- .../lint-and-client-and-login-test.yml | 135 ++++++++++++++++++ .github/workflows/lint-and-test.yml | 9 +- 2 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/lint-and-client-and-login-test.yml diff --git a/.github/workflows/lint-and-client-and-login-test.yml b/.github/workflows/lint-and-client-and-login-test.yml new file mode 100644 index 0000000000..c4f8e20bc2 --- /dev/null +++ b/.github/workflows/lint-and-client-and-login-test.yml @@ -0,0 +1,135 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# OpenCRVS is also distributed under the terms of the Civil Registration +# & Healthcare Disclaimer located at http://opencrvs.org/license. +# +# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. + +name: Lint, run client and login tests and security scans + +on: [pull_request] + +jobs: + setup: + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get list of packages + id: set-matrix + run: | + PACKAGES='["packages/client", "packages/login"]' + echo "Found packages: $PACKAGES" + echo "matrix=${PACKAGES}" >> $GITHUB_OUTPUT + + test: + needs: setup + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + package: ${{fromJson(needs.setup.outputs.matrix)}} + shard: [1/4, 2/4, 3/4, 4/4] + + steps: + - name: Checking out git repo + uses: actions/checkout@v4 + + - name: Check package.json and scripts + id: check-scripts + run: | + if [ ! -f "${{ matrix.package }}/package.json" ]; then + echo "No package.json found for ${{ matrix.package }}. Stopping pipeline." + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "skip=false" >> $GITHUB_OUTPUT + + if ! grep -q "\"test\":" "${{ matrix.package }}/package.json"; then + echo "Test not found in ${{ matrix.package }}" + echo "skip-test=true" >> $GITHUB_OUTPUT + else + echo "skip=false" >> $GITHUB_OUTPUT + fi + + if ! grep -q "\"lint\":" "${{ matrix.package }}/package.json"; then + echo "Lint scripts not found in ${{ matrix.package }}. Stopping pipeline." + echo "skip-lint=true" >> $GITHUB_OUTPUT + else + echo "skip-lint=false" >> $GITHUB_OUTPUT + fi + fi + + - name: Use Node.js from .nvmrc + if: steps.check-scripts.outputs.skip != 'true' + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + + - name: Remove other package directories + if: steps.check-scripts.outputs.skip != 'true' + run: | + for dir in packages/*; do + if [ "$dir" != "${{ matrix.package }}" ] && [ "$dir" != "packages/commons" ] && [ "$dir" != "packages/components" ]; then + if [ "${{ matrix.package }}" == "packages/client" ] && [ "$dir" == "packages/gateway" ] ; then + echo "Skipping $dir" + else + echo "Removing $dir" + rm -rf "$dir" + fi + fi + done + + - name: Cache Node.js dependencies + uses: actions/cache@v4 + with: + path: | + **/node_modules + ~/.cache/yarn/v6 + key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}',matrix.package,'package.json')) }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Verify every file has a license header + if: steps.check-scripts.outputs.skip != 'true' + run: npx license-check-and-add check -f license-config.json + + - name: Runs dependency installation + if: steps.check-scripts.outputs.skip != 'true' + run: CI="" yarn install --frozen-lockfile + + # TODO: Move out of the matrix to be built once and shared + - name: Build common package + if: steps.check-scripts.outputs.skip != 'true' + run: cd packages/commons && yarn build + + - name: Build components client and login + if: steps.check-scripts.outputs.skip != 'true' + run: cd packages/components && yarn build + + # TODO: should run parallel to unit tests as can take as much as unit tests + - name: Run linting + if: steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-lint != 'true' + run: cd ${{ matrix.package }} && yarn lint + + - name: Run Unit Test + if: steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-test != 'true' + run: cd ${{ matrix.package }} && yarn test --shard ${{ matrix.shard }} + + security-scans: + needs: setup + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run Trivy vulnerability scanner in fs mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + trivy-config: trivy.yaml diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 6bb71f79e9..ee5af03955 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -24,7 +24,7 @@ jobs: - name: Get list of packages id: set-matrix run: | - PACKAGES=$(ls -d packages/* | jq -R -s -c 'split("\n")[:-1]') + PACKAGES=$(ls -d packages/* | grep -v 'packages/client\|packages/login' | jq -R -s -c 'split("\n")[:-1]') echo "Found packages: $PACKAGES" echo "matrix=${PACKAGES}" >> $GITHUB_OUTPUT @@ -107,13 +107,6 @@ jobs: if: steps.check-scripts.outputs.skip != 'true' run: cd packages/commons && yarn build - - name: Build components client and login - if: steps.check-scripts.outputs.skip != 'true' - run: | - if [[ "${{ matrix.package }}" == "packages/client" || "${{ matrix.package }}" == "packages/login" ]]; then - cd packages/components && yarn build - fi - # TODO: should run parallel to unit tests as can take as much as unit tests - name: Run linting if: steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-lint != 'true' From c09a8c19e9a5152cd37faeec65bb6dce417c7958 Mon Sep 17 00:00:00 2001 From: Pyry Date: Tue, 18 Jun 2024 10:53:15 +0300 Subject: [PATCH 2/2] refactor: only run client tests on it's own --- ...ogin-test.yml => lint-and-test-client.yml} | 114 ++++++++++-------- .github/workflows/lint-and-test.yml | 2 +- 2 files changed, 64 insertions(+), 52 deletions(-) rename .github/workflows/{lint-and-client-and-login-test.yml => lint-and-test-client.yml} (57%) diff --git a/.github/workflows/lint-and-client-and-login-test.yml b/.github/workflows/lint-and-test-client.yml similarity index 57% rename from .github/workflows/lint-and-client-and-login-test.yml rename to .github/workflows/lint-and-test-client.yml index c4f8e20bc2..d0c2d07e02 100644 --- a/.github/workflows/lint-and-client-and-login-test.yml +++ b/.github/workflows/lint-and-test-client.yml @@ -7,58 +7,42 @@ # # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. -name: Lint, run client and login tests and security scans +name: Lint, run client tests and security scans on: [pull_request] jobs: setup: runs-on: ubuntu-22.04 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - name: Checkout code uses: actions/checkout@v4 - - name: Get list of packages - id: set-matrix - run: | - PACKAGES='["packages/client", "packages/login"]' - echo "Found packages: $PACKAGES" - echo "matrix=${PACKAGES}" >> $GITHUB_OUTPUT - - test: - needs: setup + build: runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - package: ${{fromJson(needs.setup.outputs.matrix)}} - shard: [1/4, 2/4, 3/4, 4/4] - + needs: setup steps: - - name: Checking out git repo + - name: Checkout code uses: actions/checkout@v4 - name: Check package.json and scripts id: check-scripts run: | - if [ ! -f "${{ matrix.package }}/package.json" ]; then - echo "No package.json found for ${{ matrix.package }}. Stopping pipeline." + if [ ! -f "packages/client/package.json" ]; then + echo "No package.json found for packages/client. Stopping pipeline." echo "skip=true" >> $GITHUB_OUTPUT else echo "skip=false" >> $GITHUB_OUTPUT - if ! grep -q "\"test\":" "${{ matrix.package }}/package.json"; then - echo "Test not found in ${{ matrix.package }}" + if ! grep -q "\"test\":" "packages/client/package.json"; then + echo "Test not found in packages/client" echo "skip-test=true" >> $GITHUB_OUTPUT else echo "skip=false" >> $GITHUB_OUTPUT fi - if ! grep -q "\"lint\":" "${{ matrix.package }}/package.json"; then - echo "Lint scripts not found in ${{ matrix.package }}. Stopping pipeline." + if ! grep -q "\"lint\":" "packages/client/package.json"; then + echo "Lint scripts not found in packages/client. Stopping pipeline." echo "skip-lint=true" >> $GITHUB_OUTPUT else echo "skip-lint=false" >> $GITHUB_OUTPUT @@ -71,39 +55,22 @@ jobs: with: node-version-file: .nvmrc - - name: Remove other package directories + - name: Cache node modules if: steps.check-scripts.outputs.skip != 'true' - run: | - for dir in packages/*; do - if [ "$dir" != "${{ matrix.package }}" ] && [ "$dir" != "packages/commons" ] && [ "$dir" != "packages/components" ]; then - if [ "${{ matrix.package }}" == "packages/client" ] && [ "$dir" == "packages/gateway" ] ; then - echo "Skipping $dir" - else - echo "Removing $dir" - rm -rf "$dir" - fi - fi - done - - - name: Cache Node.js dependencies uses: actions/cache@v4 with: path: | - **/node_modules - ~/.cache/yarn/v6 - key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}',matrix.package,'package.json')) }} + packages/client/node_modules + packages/commons/node_modules + packages/components/node_modules + key: ${{ runner.os }}-node-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-node- - - name: Verify every file has a license header - if: steps.check-scripts.outputs.skip != 'true' - run: npx license-check-and-add check -f license-config.json - - name: Runs dependency installation if: steps.check-scripts.outputs.skip != 'true' run: CI="" yarn install --frozen-lockfile - # TODO: Move out of the matrix to be built once and shared - name: Build common package if: steps.check-scripts.outputs.skip != 'true' run: cd packages/commons && yarn build @@ -112,14 +79,59 @@ jobs: if: steps.check-scripts.outputs.skip != 'true' run: cd packages/components && yarn build - # TODO: should run parallel to unit tests as can take as much as unit tests + - name: Upload build artifacts + if: steps.check-scripts.outputs.skip != 'true' + uses: actions/upload-artifact@v3 + with: + name: client-build + path: | + packages/commons/build + packages/components/build + + test: + needs: [setup, build] + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + shard: [1/4, 2/4, 3/4, 4/4] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v3 + with: + name: client-build + path: packages/ + + - name: Use Node.js from .nvmrc + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + + - name: Cache node modules + uses: actions/cache@v4 + with: + path: | + packages/client/node_modules + packages/commons/node_modules + packages/components/node_modules + key: ${{ runner.os }}-node-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Runs dependency installation + run: CI="" yarn install --frozen-lockfile + - name: Run linting if: steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-lint != 'true' - run: cd ${{ matrix.package }} && yarn lint + run: cd packages/client && yarn lint - name: Run Unit Test if: steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-test != 'true' - run: cd ${{ matrix.package }} && yarn test --shard ${{ matrix.shard }} + run: cd packages/client && yarn test --shard ${{ matrix.shard }} security-scans: needs: setup diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index ee5af03955..5fa01ba4e8 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -24,7 +24,7 @@ jobs: - name: Get list of packages id: set-matrix run: | - PACKAGES=$(ls -d packages/* | grep -v 'packages/client\|packages/login' | jq -R -s -c 'split("\n")[:-1]') + PACKAGES=$(ls -d packages/* | grep -v 'packages/client' | jq -R -s -c 'split("\n")[:-1]') echo "Found packages: $PACKAGES" echo "matrix=${PACKAGES}" >> $GITHUB_OUTPUT