Merge pull request #107 from Knowledge-Graph-Hub/uniprot-workflow

Use GH workflow to populate Uniprot data into an S3 bucket

.github/workflows/qc.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Run CodeSpell
         run: poetry run tox -e codespell
 
-      - name: Check static typing with MyPy
+      - name: Linting
         run: poetry run tox -e lint
 
       - name: Test with pytest and generate coverage file

.github/workflows/uniprot_s3_dump.yml

@@ -4,3 +4,44 @@ on:
 #   schedule:
 #     - cron: '0 0 1 * *' # Runs on the 1st of every month at midnight
   workflow_dispatch: # Allows manual triggering
+
+jobs:
+  fetch-and-upload:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.10" ]
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Install additional dependencies
+        run: poetry install --with s3
+
+      # - name: Fetch data from Uniprot API
+      #   run: |
+      #     kg download
+      #     kg transform -s OntologyTransform
+      #     kg get-via-api -a uniprot
+
+      - name: Upload to S3
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Sync files to S3 bucket
+        run: |
+          aws s3 sync data/raw/uniprot/s3 s3://${{ secrets.AWS_BUCKET_NAME }} --delete
+

.gitignore

@@ -146,3 +146,5 @@ data/merged/*
 notebooks/*.db
 kg_microbe/transform_utils/mediadive/tmp/ingredient_overlap.*
 kg_microbe/transform_utils/bacdive/tmp/bacdive_physiology_metabolism.tsv
+data/raw/uniprot/s3/*.tsv
+uniprot_cache.sqlite

Jenkinsfile

@@ -98,7 +98,7 @@ pipeline {
         stage('Transform') {
             steps {
                 dir('./gitrepo') {
-                    sh '. venv/bin/activate && env && kg transform'
+                    sh '. venv/bin/activate && env && kg transform --no-show-status'
                 }
             }
         }