[GEN-1653] Add integration tests to GH actions workflow

.github/workflows/ci.yml

@@ -6,8 +6,16 @@ name: build
 on:
   push:
     branches: [main, develop, 'GEN*', 'gen*']
+    paths-ignore:
+      - '**.md'                            # All Markdown files
+      - '**/docs/**'                       # Documentation directory
+      - '.github/workflows/codeql.yml'     # Code scanner
+      - '.github/workflows/build_docs.yml' # mkdocs GH workflow
 
   pull_request:
+      types:
+        - opened
+        - reopened
 
   release:
     types:
@@ -16,9 +24,36 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  TEST_PROJECT_SYNID: syn7208886
 
 jobs:
 
+  determine-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      non-test-changes: ${{ steps.get-changes.outputs.non-test-changes }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Determine Changes
+        id: get-changes
+        run: |
+          if [ "${{ github.event_name }}" == "push" ]; then
+            echo "Handling a push event..."
+            ALL_CHANGED_FILES=$(git diff --name-only HEAD^ HEAD)
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Handling a pull request event..."
+            git fetch origin ${{ github.base_ref }}
+            ALL_CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }} HEAD)
+          fi
+          echo "All Changed Files: $ALL_CHANGED_FILES"
+          NON_TEST_CHANGES=$(echo "$ALL_CHANGED_FILES" | grep -v '^tests/' || true)
+          echo "Non-Test Changes: $NON_TEST_CHANGES"
+          echo "::set-output name=non-test-changes::$NON_TEST_CHANGES"
+
   test:
     runs-on: ubuntu-latest
     strategy:
@@ -54,34 +89,14 @@ jobs:
       # Use always() to always run this step to publish test results when there are test failures
       if: ${{ always() }}
 
+
   lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
             version: "~=23.12"
-  deploy:
-    needs: [test, lint]
-    runs-on: ubuntu-latest
-    if: github.event_name == 'release'
-    permissions:
-      id-token: write
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel build
-    - name: Build distributions
-      run: python -m build
-    - name: Publish to pypi
-      uses: pypa/gh-action-pypi-publish@release/v1
-
 
   build-container:
     needs: [test, lint]
@@ -109,7 +124,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
+      
       - name: Format tags as registry refs
         id: registry_refs
         env:
@@ -129,3 +144,87 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: ${{ steps.registry_refs.outputs.tags }},mode=max
           cache-to: ${{ steps.registry_refs.outputs.tags }},mode=max
+
+  integration-tests:
+    needs: [determine-changes, lint, test, build-container]
+    runs-on: ubuntu-latest
+    if: ${{ needs.determine-changes.outputs.non-test-changes }}
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Extract Branch Name
+      run: |
+        if [ "$GITHUB_HEAD_REF" != "" ]; then
+          echo "BRANCH_NAME=$GITHUB_HEAD_REF" >> $GITHUB_ENV
+        else
+          echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+        fi
+
+    - name: Pull Public Docker Image from GHCR
+      run: |
+        docker pull ghcr.io/sage-bionetworks/genie:${{ env.BRANCH_NAME }}
+
+    - name: Start Docker Container
+      run: |
+        docker run -d --name genie-container \
+          -e SYNAPSE_AUTH_TOKEN="${{ secrets.SYNAPSE_AUTH_TOKEN }}" \
+          ghcr.io/sage-bionetworks/genie:${{ env.BRANCH_NAME }} \
+          sh -c "while true; do sleep 1; done"
+
+    - name: Run validation in test pipeline
+      run: |
+        docker exec genie-container \
+        python3 /root/Genie/bin/input_to_database.py \
+            mutation \
+            --project_id ${{ env.TEST_PROJECT_SYNID }} \
+            --onlyValidate \
+            --genie_annotation_pkg /root/annotation-tools
+
+    - name: Run processing on mutation data in test pipeline
+      run: |
+        docker exec genie-container \
+        python3 /root/Genie/bin/input_to_database.py mutation \
+            --project_id ${{ env.TEST_PROJECT_SYNID }} \
+            --genie_annotation_pkg /root/annotation-tools \
+            --createNewMafDatabase
+
+    - name: Run processing on non-mutation data in test pipeline
+      run: |
+        docker exec genie-container \
+        python3 /root/Genie/bin/input_to_database.py main \
+            --project_id ${{ env.TEST_PROJECT_SYNID }}
+
+    - name: Run consortium release in test pipeline
+      run: |
+        docker exec genie-container \
+        python3 /root/Genie/bin/database_to_staging.py Jan-2017 ../cbioportal TEST --test
+
+    - name: Run public release in test pipeline
+      run: |
+        docker exec genie-container \
+        python3 /root/Genie/bin/consortium_to_public.py Jan-2017 ../cbioportal TEST --test
+
+    - name: Stop and Remove Docker Container
+      run: docker stop genie-container && docker rm genie-container
+
+  deploy:
+    needs: [test, lint, build-container, integration-tests]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+    permissions:
+      id-token: write
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel build
+    - name: Build distributions
+      run: python -m build
+    - name: Publish to pypi
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,4 +69,4 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

tests/test_clinical.py

@@ -1,3 +1,5 @@
+"""Test GENIE Clinical class"""
+
 import datetime
 import json
 from collections import Counter