From a0a5eada6fc1c7b1382930eef2bba3fd2fba3d64 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 11:24:07 -0700 Subject: [PATCH 1/7] Made run-tests workflow dependent on update-testing-data workflow --- .github/workflows/run-tests.yml | 40 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b68e169f..276921f5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,6 +4,10 @@ on: - cron: "0 0 * * *" # daily pull_request: workflow_dispatch: + workflow_run: + workflows: ["Update testing data"] + types: + - completed jobs: run: @@ -41,24 +45,24 @@ jobs: - - name: Get ophys_testing_data current head hash - id: ophys - run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - uses: actions/cache@v2 - id: cache-ophys-datasets - with: - path: ./ophys_testing_data - key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Install and configure AWS CLI - run: | - pip install awscli==1.25.27 - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Download data from S3 - run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data + # - name: Get ophys_testing_data current head hash + # id: ophys + # run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + # - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + # uses: actions/cache@v2 + # id: cache-ophys-datasets + # with: + # path: ./ophys_testing_data + # key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Install and configure AWS CLI + # run: | + # pip install awscli==1.25.27 + # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Download data from S3 + # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data From ce624f8dfc07bd0e89dbeb2a59e4ffc0125e082e Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 11:36:20 -0700 Subject: [PATCH 2/7] update awscli version --- .github/workflows/update-testing-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index a2551302..6d327aea 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -38,7 +38,7 @@ jobs: - if: steps.cache-ophys-datasets.outputs.cache-hit == false name: Install and configure AWS CLI run: | - pip install awscli==1.25.27 + pip install awscli==1.29.56 aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: steps.cache-ophys-datasets.outputs.cache-hit == false From cfab180c92a7a40a05e84d21bf469fc8dd54aed6 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 11:47:47 -0700 Subject: [PATCH 3/7] pin wheel version to avoid future dependency conflicts --- .github/workflows/update-testing-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index 6d327aea..91342a0e 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -24,7 +24,7 @@ jobs: pip install -U pip git config --global user.email "CI@example.com" git config --global user.name "CI Almighty" - pip install wheel # needed for scanimage + pip install wheel==0.41.2 # needed for scanimage - name: Get ophys_testing_data current head hash id: ophys From 215802573c9acefb7593036d265030d58280365c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 12:03:07 -0700 Subject: [PATCH 4/7] updated awscli version in run-tests --- .github/workflows/run-tests.yml | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 276921f5..ce2dc45f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -33,7 +33,7 @@ jobs: pip install pytest-xdist git config --global user.email "CI@example.com" git config --global user.name "CI Almighty" - pip install wheel # needed for scanimage + pip install wheel==0.41.2 # needed for scanimage - name: Install roiextractors with minimal requirements run: pip install .[test] @@ -45,24 +45,24 @@ jobs: - # - name: Get ophys_testing_data current head hash - # id: ophys - # run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - # - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - # uses: actions/cache@v2 - # id: cache-ophys-datasets - # with: - # path: ./ophys_testing_data - # key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Install and configure AWS CLI - # run: | - # pip install awscli==1.25.27 - # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Download data from S3 - # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data + - name: Get ophys_testing_data current head hash + id: ophys + run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + uses: actions/cache@v2 + id: cache-ophys-datasets + with: + path: ./ophys_testing_data + key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + - if: steps.cache-ophys-datasets.outputs.cache-hit == false + name: Install and configure AWS CLI + run: | + pip install awscli==1.29.56 + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - if: steps.cache-ophys-datasets.outputs.cache-hit == false + name: Download data from S3 + run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data From 56cd15e501e2e2427a11ada3c3c8d01b1ae4ee8f Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 13:05:23 -0700 Subject: [PATCH 5/7] removed dependent run of run-tests --- .github/workflows/run-tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ce2dc45f..5941f1c6 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,10 +4,6 @@ on: - cron: "0 0 * * *" # daily pull_request: workflow_dispatch: - workflow_run: - workflows: ["Update testing data"] - types: - - completed jobs: run: From 587865d2f9a67b8b26928de8364164115b75c54f Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 13:33:01 -0700 Subject: [PATCH 6/7] commented out awscli commands only --- .github/workflows/run-tests.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 5941f1c6..ce0e6463 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -50,15 +50,15 @@ jobs: with: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Install and configure AWS CLI - run: | - pip install awscli==1.29.56 - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Download data from S3 - run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Install and configure AWS CLI + # run: | + # pip install awscli==1.29.56 + # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Download data from S3 + # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data From 94ddfa059a623a9120b8556c2d259c5cec599bb2 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 13:55:36 -0700 Subject: [PATCH 7/7] refactored to trigger run-tests after update-testing-data --- .github/workflows/run-tests.yml | 19 ++++--------------- .github/workflows/update-testing-data.yml | 4 +++- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ce0e6463..0b26dcc0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,9 +1,10 @@ name: Full Tests on: - schedule: - - cron: "0 0 * * *" # daily pull_request: workflow_dispatch: + workflow_run: + workflows: [update-testing-data] + types: [completed] jobs: run: @@ -33,14 +34,13 @@ jobs: - name: Install roiextractors with minimal requirements run: pip install .[test] + - name: Run minimal tests run: pytest tests/test_internals -n auto --dist loadscope - name: Test full installation (-e needed for codecov report) run: pip install -e .[full] - - - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" @@ -50,17 +50,6 @@ jobs: with: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Install and configure AWS CLI - # run: | - # pip install awscli==1.29.56 - # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Download data from S3 - # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data - - - name: Run full pytest with coverage run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index 91342a0e..d012a203 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -1,5 +1,7 @@ -name: Update testing data +name: update-testing-data on: + schedule: + - cron: "0 0 * * *" # daily workflow_dispatch: jobs: