From 8b8104799adc6a2282c07338684a614937fc6eed Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Tue, 10 Sep 2024 16:20:35 -0700 Subject: [PATCH 01/17] adding unit test workflow --- .github/workflows/test_unit.yml | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/test_unit.yml diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml new file mode 100644 index 0000000..428362f --- /dev/null +++ b/.github/workflows/test_unit.yml @@ -0,0 +1,66 @@ +name: Run Unit Tests + +# When the workflow will be triggered +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest # You can use ubuntu, macos, or windows depending on your project needs + + services: + postgres: + image: postgres:13 # Use the latest version of PostgreSQL you need + ports: + - 5432:5432 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: pw + POSTGRES_DB: testdb + options: >- + --health-cmd "pg_isready -U testuser" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + # Step 1: Check out the repository + - name: Checkout code + uses: actions/checkout@v3 + + # Step 2: Set up Python + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' # Specify your Python version + + # Step 3: Install dependencies + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + # Step 4: Wait for PostgreSQL to be ready + - name: Wait for PostgreSQL to be ready + run: | + until pg_isready -h localhost -p 5432 -U postgres; do + echo "Waiting for PostgreSQL to be ready..." + sleep 1 + done + + # Step 5: Run the tests (you may need to adjust this depending on your test setup) + - name: Run unit tests + env: + MPI_DB_TYPE: postgres + MPI_DBNAME: testdb + MPI_HOST: localhost + MPI_PORT: 5432 + MPI_USER: postgres + MPI_PASSWORD: pw + run: | + pytest tests/unit From 2a518c4c6c9d3514450883fbe0a3486753dce8b7 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Tue, 10 Sep 2024 16:24:26 -0700 Subject: [PATCH 02/17] fixing pip install step --- .github/workflows/test_unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index 428362f..ca71407 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -43,7 +43,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install '.[dev]' # Step 4: Wait for PostgreSQL to be ready - name: Wait for PostgreSQL to be ready From 3226327db85d177e78672f255a59080e5aba9aa1 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Tue, 10 Sep 2024 16:48:55 -0700 Subject: [PATCH 03/17] adding lint check --- .github/workflows/lint.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..2ad1ce2 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,36 @@ +name: Lint check + +# When the workflow will be triggered +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest # You can use ubuntu, macos, or windows depending on your project needs + + steps: + # Step 1: Check out the repository + - name: Checkout code + uses: actions/checkout@v3 + + # Step 2: Set up Python + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' # Specify your Python version + + # Step 3: Install dependencies + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install '.[dev]' + + # Step 4: Run lint checks + - name: Run lint checks + run: | + ruff check src/ From 8101db8650b6a9abb3b95ea2df757901d269b2e8 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Tue, 10 Sep 2024 16:49:53 -0700 Subject: [PATCH 04/17] adding coverage output to unit test workflow --- .github/workflows/test_unit.yml | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index ca71407..5f8d7b9 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -53,7 +53,7 @@ jobs: sleep 1 done - # Step 5: Run the tests (you may need to adjust this depending on your test setup) + # Step 5: Run the tests - name: Run unit tests env: MPI_DB_TYPE: postgres @@ -63,4 +63,4 @@ jobs: MPI_USER: postgres MPI_PASSWORD: pw run: | - pytest tests/unit + pytest --cov=recordlinker --cov-report=xml tests/unit diff --git a/pyproject.toml b/pyproject.toml index bb8ecb9..02a58c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ [project.optional-dependencies] dev = [ "pytest>=8.3", + "pytest-cov", "ruff", "pip-audit", "bandit", From cb573ef684aaf94339d4a45ae3ea577872bb276b Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 07:46:14 -0700 Subject: [PATCH 05/17] fixing linting issues --- src/recordlinker/config.py | 2 +- src/recordlinker/linkage/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/recordlinker/config.py b/src/recordlinker/config.py index ce39b6e..0afaf50 100644 --- a/src/recordlinker/config.py +++ b/src/recordlinker/config.py @@ -1,8 +1,8 @@ from functools import lru_cache from typing import Optional -from pydantic_settings import BaseSettings from pydantic import Field +from pydantic_settings import BaseSettings class Settings(BaseSettings): diff --git a/src/recordlinker/linkage/config.py b/src/recordlinker/linkage/config.py index 2908a5c..d445574 100644 --- a/src/recordlinker/linkage/config.py +++ b/src/recordlinker/linkage/config.py @@ -2,6 +2,7 @@ from pydantic_settings import BaseSettings + class DBSettings(BaseSettings): mpi_db_type: str mpi_dbname: str From 6d26e26a4ad3e2fb272fef3807e6fcd7bda4fbeb Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 08:47:58 -0700 Subject: [PATCH 06/17] adding pip caching --- .github/workflows/lint.yml | 7 ++++--- .github/workflows/test_unit.yml | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2ad1ce2..830d9f5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Lint check +name: lint check # When the workflow will be triggered on: @@ -20,9 +20,10 @@ jobs: # Step 2: Set up Python - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' # Specify your Python version + python-version: '3.11' + cache: 'pip' # Step 3: Install dependencies - name: Install dependencies diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index 5f8d7b9..5157ebf 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -1,4 +1,4 @@ -name: Run Unit Tests +name: unit tests # When the workflow will be triggered on: @@ -35,9 +35,10 @@ jobs: # Step 2: Set up Python - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' # Specify your Python version + python-version: '3.11' + cache: 'pip' # Step 3: Install dependencies - name: Install dependencies From 980ce5ecc841a4d15330df218a04b8b8447411d5 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 08:56:21 -0700 Subject: [PATCH 07/17] adding code vulnerabilities check --- .../workflows/check_code_vulnerabilities.yml | 38 +++++++++++++++++++ .../workflows/{lint.yml => check_lint.yml} | 10 ++--- .../{test_unit.yml => check_unit_tests.yml} | 10 ++--- 3 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/check_code_vulnerabilities.yml rename .github/workflows/{lint.yml => check_lint.yml} (79%) rename .github/workflows/{test_unit.yml => check_unit_tests.yml} (89%) diff --git a/.github/workflows/check_code_vulnerabilities.yml b/.github/workflows/check_code_vulnerabilities.yml new file mode 100644 index 0000000..f429684 --- /dev/null +++ b/.github/workflows/check_code_vulnerabilities.yml @@ -0,0 +1,38 @@ +name: code vulnerabilities check + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + codeql: + runs-on: ubuntu-latest + + permissions: + actions: read + contents: read + security-events: write + + strategy: + matrix: + language: ['python'] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + run: codeql database analyze --format=sarif --output=results.sarif + + - name: Upload CodeQL Results + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: results.sarif + diff --git a/.github/workflows/lint.yml b/.github/workflows/check_lint.yml similarity index 79% rename from .github/workflows/lint.yml rename to .github/workflows/check_lint.yml index 830d9f5..a1c4508 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/check_lint.yml @@ -3,15 +3,13 @@ name: lint check # When the workflow will be triggered on: push: - branches: - - main + branches: [main] pull_request: - branches: - - main + branches: [main] jobs: - test: - runs-on: ubuntu-latest # You can use ubuntu, macos, or windows depending on your project needs + lint: + runs-on: ubuntu-latest steps: # Step 1: Check out the repository diff --git a/.github/workflows/test_unit.yml b/.github/workflows/check_unit_tests.yml similarity index 89% rename from .github/workflows/test_unit.yml rename to .github/workflows/check_unit_tests.yml index 5157ebf..4e5b8e5 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/check_unit_tests.yml @@ -1,17 +1,15 @@ -name: unit tests +name: unit tests check # When the workflow will be triggered on: push: - branches: - - main + branches: [main] pull_request: - branches: - - main + branches: [main] jobs: test: - runs-on: ubuntu-latest # You can use ubuntu, macos, or windows depending on your project needs + runs-on: ubuntu-latest services: postgres: From 4a537a62aefca5a9bc02c7418a74cd8e62d7cfac Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 09:35:59 -0700 Subject: [PATCH 08/17] updating code vulnerability check --- .github/workflows/check_code_vulnerabilities.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check_code_vulnerabilities.yml b/.github/workflows/check_code_vulnerabilities.yml index f429684..d3efb5c 100644 --- a/.github/workflows/check_code_vulnerabilities.yml +++ b/.github/workflows/check_code_vulnerabilities.yml @@ -11,6 +11,7 @@ jobs: runs-on: ubuntu-latest permissions: + packages: read actions: read contents: read security-events: write @@ -24,15 +25,11 @@ jobs: uses: actions/checkout@v3 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - run: codeql database analyze --format=sarif --output=results.sarif - - - name: Upload CodeQL Results - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/analyze@v3 with: - sarif_file: results.sarif - + category: "/language:${{matrix.language}}" From dee2b4446ce2186fd435f76ee3493d4a7fcb15fd Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 09:44:30 -0700 Subject: [PATCH 09/17] specifying code vulnerabilty path in config --- .github/workflows/check_code_vulnerabilities.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/check_code_vulnerabilities.yml b/.github/workflows/check_code_vulnerabilities.yml index d3efb5c..d529d9a 100644 --- a/.github/workflows/check_code_vulnerabilities.yml +++ b/.github/workflows/check_code_vulnerabilities.yml @@ -33,3 +33,5 @@ jobs: uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" + config: | + paths: ["src"] From 90eab68bc158fae7babd1a5ee506491eeac5e160 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 14:06:41 -0700 Subject: [PATCH 10/17] adding dependabot workflow to upgrade dependencies weekly --- .github/dependabot.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4198f1a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "[deps] " + open-pull-requests-limit: 3 From f36befe7b27da928c62d7c96ca3837a30c2ef0f4 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 14:07:31 -0700 Subject: [PATCH 11/17] removing vulnerabilty check script --- pyproject.toml | 2 -- scripts/vulnerability_check.sh | 11 ----------- 2 files changed, 13 deletions(-) delete mode 100644 scripts/vulnerability_check.sh diff --git a/pyproject.toml b/pyproject.toml index 02a58c7..e49e379 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,6 @@ dev = [ "pytest>=8.3", "pytest-cov", "ruff", - "pip-audit", - "bandit", "mypy", "pyarrow", "httpx" diff --git a/scripts/vulnerability_check.sh b/scripts/vulnerability_check.sh deleted file mode 100644 index be88417..0000000 --- a/scripts/vulnerability_check.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# -# Run vulnerability checks on the project and its dependencies. -# -# Usage: scripts/vulnerability_check.sh - -set -e - -cd "$(dirname "$0")/.." - -pip-audit . && bandit -r src/ From 11157607a95e58e2a8967e5ba54a5f72799388ca Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Wed, 11 Sep 2024 14:12:39 -0700 Subject: [PATCH 12/17] removing old section from README --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index fa2ff4d..d3ebb7d 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,6 @@ To run a single unit test, use the following command: ./scripts/test_unit.sh tests/unit/test_linkage.py::test_link_record_against_mpi ``` -### Building the Docker Image - -To build the Docker image for the record linkage service from source code instead of downloading it from the DIBBs repository follow these steps. -1. Ensure that both [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker](https://docs.docker.com/get-docker/) are installed. -2. Clone the DIBBs repository with `git clone https://github.com/CDCgov/phdi`. -3. Navigate to `/phdi/containers/record-linkage/`. -4. Run `docker build -t record-linkage .`. - ## Standard Notices ### Public Domain Standard Notice From 59b16668eecaadf386a939a1fc2e0bb80241985a Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Fri, 13 Sep 2024 13:01:48 -0700 Subject: [PATCH 13/17] removing redundant comments --- .../workflows/check_code_vulnerabilities.yml | 2 ++ .github/workflows/check_lint.yml | 4 ---- .github/workflows/check_unit_tests.yml | 22 +++---------------- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/.github/workflows/check_code_vulnerabilities.yml b/.github/workflows/check_code_vulnerabilities.yml index d529d9a..e78327a 100644 --- a/.github/workflows/check_code_vulnerabilities.yml +++ b/.github/workflows/check_code_vulnerabilities.yml @@ -18,6 +18,7 @@ jobs: strategy: matrix: + # Using a matrix in case we need to test Javascript code in the future language: ['python'] steps: @@ -34,4 +35,5 @@ jobs: with: category: "/language:${{matrix.language}}" config: | + # only scan the code in the src directory paths: ["src"] diff --git a/.github/workflows/check_lint.yml b/.github/workflows/check_lint.yml index a1c4508..2fa4523 100644 --- a/.github/workflows/check_lint.yml +++ b/.github/workflows/check_lint.yml @@ -12,24 +12,20 @@ jobs: runs-on: ubuntu-latest steps: - # Step 1: Check out the repository - name: Checkout code uses: actions/checkout@v3 - # Step 2: Set up Python - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.11' cache: 'pip' - # Step 3: Install dependencies - name: Install dependencies run: | python -m pip install --upgrade pip pip install '.[dev]' - # Step 4: Run lint checks - name: Run lint checks run: | ruff check src/ diff --git a/.github/workflows/check_unit_tests.yml b/.github/workflows/check_unit_tests.yml index 4e5b8e5..59f6941 100644 --- a/.github/workflows/check_unit_tests.yml +++ b/.github/workflows/check_unit_tests.yml @@ -1,6 +1,5 @@ name: unit tests check -# When the workflow will be triggered on: push: branches: [main] @@ -13,50 +12,35 @@ jobs: services: postgres: - image: postgres:13 # Use the latest version of PostgreSQL you need + image: postgres:13 ports: - 5432:5432 env: - POSTGRES_USER: postgres POSTGRES_PASSWORD: pw - POSTGRES_DB: testdb - options: >- - --health-cmd "pg_isready -U testuser" - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - # Step 1: Check out the repository - name: Checkout code uses: actions/checkout@v3 - # Step 2: Set up Python - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.11' cache: 'pip' - # Step 3: Install dependencies - name: Install dependencies run: | python -m pip install --upgrade pip pip install '.[dev]' - # Step 4: Wait for PostgreSQL to be ready - name: Wait for PostgreSQL to be ready run: | - until pg_isready -h localhost -p 5432 -U postgres; do - echo "Waiting for PostgreSQL to be ready..." - sleep 1 - done + until pg_isready -U postgres; do sleep 1; done - # Step 5: Run the tests - name: Run unit tests env: MPI_DB_TYPE: postgres - MPI_DBNAME: testdb + MPI_DBNAME: postgres MPI_HOST: localhost MPI_PORT: 5432 MPI_USER: postgres From ca08dde590560bbbdb57bd19e29311147a5cbaae Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Fri, 13 Sep 2024 21:18:21 -0700 Subject: [PATCH 14/17] fixing postgres ready test --- .github/workflows/check_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_unit_tests.yml b/.github/workflows/check_unit_tests.yml index 59f6941..5013b99 100644 --- a/.github/workflows/check_unit_tests.yml +++ b/.github/workflows/check_unit_tests.yml @@ -35,7 +35,7 @@ jobs: - name: Wait for PostgreSQL to be ready run: | - until pg_isready -U postgres; do sleep 1; done + until pg_isready -h localhost -U postgres; do sleep 1; done - name: Run unit tests env: From 4669578f3a96caf2a96dcc18c22b0856a6808c69 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Fri, 13 Sep 2024 21:51:47 -0700 Subject: [PATCH 15/17] adding back testdb name since its hardcoded in the unit tests --- .github/workflows/check_unit_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check_unit_tests.yml b/.github/workflows/check_unit_tests.yml index 5013b99..78c500a 100644 --- a/.github/workflows/check_unit_tests.yml +++ b/.github/workflows/check_unit_tests.yml @@ -17,6 +17,7 @@ jobs: - 5432:5432 env: POSTGRES_PASSWORD: pw + POSTGRES_DB: testdb steps: - name: Checkout code From 2ebd2f91cac5b4710303d363d9f249a7777b114b Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Fri, 13 Sep 2024 21:55:42 -0700 Subject: [PATCH 16/17] fixing linting issue --- src/recordlinker/linkage/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/recordlinker/linkage/models.py b/src/recordlinker/linkage/models.py index fa11e79..1d4c30a 100644 --- a/src/recordlinker/linkage/models.py +++ b/src/recordlinker/linkage/models.py @@ -1,6 +1,9 @@ import uuid -from sqlalchemy import orm, ForeignKey, String, JSON +from sqlalchemy import ForeignKey +from sqlalchemy import JSON +from sqlalchemy import orm +from sqlalchemy import String class Base(orm.DeclarativeBase): From b5332a731ca0e6c78ddd3a16e00f4e412fddba3f Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Fri, 13 Sep 2024 22:00:03 -0700 Subject: [PATCH 17/17] adding the dbname environment variable back --- .github/workflows/check_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_unit_tests.yml b/.github/workflows/check_unit_tests.yml index 78c500a..e8fd8a7 100644 --- a/.github/workflows/check_unit_tests.yml +++ b/.github/workflows/check_unit_tests.yml @@ -41,7 +41,7 @@ jobs: - name: Run unit tests env: MPI_DB_TYPE: postgres - MPI_DBNAME: postgres + MPI_DBNAME: testdb MPI_HOST: localhost MPI_PORT: 5432 MPI_USER: postgres