From 208bd38a1dc559faab4814838ce9469dad8f22d7 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 17:15:09 -0600 Subject: [PATCH 01/15] add macos and windows to cron jobs --- .github/workflows/caches_cron_job.yml | 62 ++++++++++----------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 20e2a55178..6a5ba668d5 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -2,64 +2,35 @@ name: Create caches for gin ecephys data and virtual env on: workflow_dispatch: - push: # When someting is pushed into main this checks if caches need to re-created + push: # When something is pushed into main this checks if caches need to be re-created branches: - main schedule: - cron: "0 12 * * *" # Daily at noon UTC jobs: - - - - create-virtual-env-cache-if-missing: - name: Caching virtual env - runs-on: "ubuntu-latest" - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Get current year-month - id: date - run: | - echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT - - name: Get current dependencies hash - id: dependencies - run: | - echo "hash=${{hashFiles('**/pyproject.toml')}}" >> $GITHUB_OUTPUT - - uses: actions/cache@v4 - id: cache-venv - with: - path: ${{ github.workspace }}/test_env - key: ${{ runner.os }}-venv-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }} - lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - - name: Cache found? - run: echo "Cache-hit == ${{steps.cache-venv.outputs.cache-hit == 'true'}}" - - name: Create the virtual environment to be cached - if: steps.cache-venv.outputs.cache-hit != 'true' - uses: ./.github/actions/build-test-environment - - - - create-gin-data-cache-if-missing: name: Caching data env - runs-on: "ubuntu-latest" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Create the directory to store the data run: | - mkdir --parents --verbose $HOME/spikeinterface_datasets/ephy_testing_data/ - chmod -R 777 $HOME/spikeinterface_datasets + mkdir -p $HOME/spikeinterface_datasets/ephy_testing_data/ ls -l $HOME/spikeinterface_datasets + shell: bash - name: Get current hash (SHA) of the ephy_testing_data repo id: repo_hash run: | echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT + shell: bash - uses: actions/cache@v4 id: cache-datasets with: @@ -68,6 +39,7 @@ jobs: lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? run: echo "Cache-hit == ${{steps.cache-datasets.outputs.cache-hit == 'true'}}" + shell: bash - name: Installing datalad and git-annex if: steps.cache-datasets.outputs.cache-hit != 'true' run: | @@ -75,17 +47,26 @@ jobs: git config --global user.name "CI Almighty" python -m pip install -U pip # Official recommended way pip install datalad-installer - datalad-installer --sudo ok git-annex --method datalad/packages + if [ ${{ runner.os }} == 'Linux' ]; then + datalad-installer --sudo ok git-annex --method datalad/packages + elif [ ${{ runner.os }} == 'macOS' ]; then + datalad-installer --sudo ok git-annex --method brew + elif [ ${{ runner.os }} == 'Windows' ]; then + datalad-installer --sudo ok git-annex --method datalad/git-annex:release + fi pip install datalad git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency + shell: bash - name: Download dataset if: steps.cache-datasets.outputs.cache-hit != 'true' run: | datalad install --recursive --get-data https://gin.g-node.org/NeuralEnsemble/ephy_testing_data + shell: bash - name: Move the downloaded data to the right directory if: steps.cache-datasets.outputs.cache-hit != 'true' run: | mv --force ./ephy_testing_data $HOME/spikeinterface_datasets/ + shell: bash - name: Show size of the cache to assert data is downloaded run: | cd $HOME @@ -96,3 +77,4 @@ jobs: ls -lh # Should show ephy_testing_data cd ephy_testing_data ls -lh + shell: bash From 17a5deffa5497141270847e95221a59a7ec34700 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 17:29:34 -0600 Subject: [PATCH 02/15] add more specific paths and testing functionality --- .github/workflows/caches_cron_job.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 6a5ba668d5..a029ab7942 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -7,7 +7,10 @@ on: - main schedule: - cron: "0 12 * * *" # Daily at noon UTC - + pull_request: # Temporary, will remove this line before merging + types: [synchronize, opened, reopened] + branches: + - main jobs: create-gin-data-cache-if-missing: name: Caching data env @@ -34,7 +37,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: ~/spikeinterface_datasets + path: $HOME/spikeinterface_datasets key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? From c4d92c4135b5472c7d1dd2165218bdaf465fcca1 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 19:44:15 -0600 Subject: [PATCH 03/15] fix mac and add recursive specification --- .github/workflows/caches_cron_job.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index a029ab7942..2d59f44cdd 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: $HOME/spikeinterface_datasets + path: $HOME/spikeinterface_datasets/* key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? @@ -68,7 +68,8 @@ jobs: - name: Move the downloaded data to the right directory if: steps.cache-datasets.outputs.cache-hit != 'true' run: | - mv --force ./ephy_testing_data $HOME/spikeinterface_datasets/ + rm -rf $HOME/spikeinterface_datasets/ephy_testing_data/ # Remove existing directory if present + mv ./ephy_testing_data $HOME/spikeinterface_datasets/ shell: bash - name: Show size of the cache to assert data is downloaded run: | From 256b8159d16dc7f2e2890dc25853b2afe90e27e7 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:02:31 -0600 Subject: [PATCH 04/15] avoid removing --- .github/workflows/caches_cron_job.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 2d59f44cdd..3165521d92 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: $HOME/spikeinterface_datasets/* + path: $HOME/spikeinterface_datasets/ key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? @@ -68,7 +68,6 @@ jobs: - name: Move the downloaded data to the right directory if: steps.cache-datasets.outputs.cache-hit != 'true' run: | - rm -rf $HOME/spikeinterface_datasets/ephy_testing_data/ # Remove existing directory if present mv ./ephy_testing_data $HOME/spikeinterface_datasets/ shell: bash - name: Show size of the cache to assert data is downloaded From 513c4d30121960e166ad0ec2a5df84f8a5bd515c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:08:10 -0600 Subject: [PATCH 05/15] test avoiding the remove --- .github/workflows/caches_cron_job.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 3165521d92..fa6772ffc0 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -25,7 +25,7 @@ jobs: python-version: '3.11' - name: Create the directory to store the data run: | - mkdir -p $HOME/spikeinterface_datasets/ephy_testing_data/ + mkdir -p $HOME/spikeinterface_datasets/behavior_testing_data/ ls -l $HOME/spikeinterface_datasets shell: bash - name: Get current hash (SHA) of the ephy_testing_data repo @@ -63,12 +63,12 @@ jobs: - name: Download dataset if: steps.cache-datasets.outputs.cache-hit != 'true' run: | - datalad install --recursive --get-data https://gin.g-node.org/NeuralEnsemble/ephy_testing_data + datalad install --recursive --get-data https://gin.g-node.org/CatalystNeuro/behavior_testing_data shell: bash - name: Move the downloaded data to the right directory if: steps.cache-datasets.outputs.cache-hit != 'true' run: | - mv ./ephy_testing_data $HOME/spikeinterface_datasets/ + mv ./behavior_testing_data $HOME/spikeinterface_datasets/ shell: bash - name: Show size of the cache to assert data is downloaded run: | @@ -78,6 +78,6 @@ jobs: cd spikeinterface_datasets pwd ls -lh # Should show ephy_testing_data - cd ephy_testing_data + cd behavior_testing_data ls -lh shell: bash From 0498dbcedcd532b82de5a038a70ee7e2626e38b7 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:12:13 -0600 Subject: [PATCH 06/15] try wildcard again --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index fa6772ffc0..b8765d34fb 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: $HOME/spikeinterface_datasets/ + path: $HOME/spikeinterface_datasets/* key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? From 5957b76e150b3036f8ee7bfb6fc9039328a223b8 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:16:34 -0600 Subject: [PATCH 07/15] add permits --- .github/workflows/caches_cron_job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index b8765d34fb..47ecaedbbe 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -26,6 +26,7 @@ jobs: - name: Create the directory to store the data run: | mkdir -p $HOME/spikeinterface_datasets/behavior_testing_data/ + chmod -R 777 $HOME/spikeinterface_datasets ls -l $HOME/spikeinterface_datasets shell: bash - name: Get current hash (SHA) of the ephy_testing_data repo From aea85aa3742008cc9eccc12d942daaae37668ade Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:21:49 -0600 Subject: [PATCH 08/15] try this one --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 47ecaedbbe..40f44cb5ec 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -25,7 +25,7 @@ jobs: python-version: '3.11' - name: Create the directory to store the data run: | - mkdir -p $HOME/spikeinterface_datasets/behavior_testing_data/ + mkdir --parents --verbose $HOME/spikeinterface_datasets/behavior_testing_data/ chmod -R 777 $HOME/spikeinterface_datasets ls -l $HOME/spikeinterface_datasets shell: bash From a75257aa2cb3c115329b80ec91472833027563fd Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:24:51 -0600 Subject: [PATCH 09/15] try this other one --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 40f44cb5ec..e718b9f801 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: $HOME/spikeinterface_datasets/* + path: ~/spikeinterface_datasets/ key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? From 2825cbf9145390d7cdbd74505587fc55b3d1cf57 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:32:40 -0600 Subject: [PATCH 10/15] try yet another one --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index e718b9f801..c018b709ab 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: ~/spikeinterface_datasets/ + path: ~/spikeinterface_datasets key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? From 5c8003ae9a0ebd11dd35fff96723dd090d45df81 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:37:51 -0600 Subject: [PATCH 11/15] mmm one worked and I did not notice --- .github/workflows/caches_cron_job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index c018b709ab..ee0eef96d9 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -25,7 +25,7 @@ jobs: python-version: '3.11' - name: Create the directory to store the data run: | - mkdir --parents --verbose $HOME/spikeinterface_datasets/behavior_testing_data/ + mkdir -p $HOME/spikeinterface_datasets/behavior_testing_data/ chmod -R 777 $HOME/spikeinterface_datasets ls -l $HOME/spikeinterface_datasets shell: bash @@ -38,7 +38,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: ~/spikeinterface_datasets + path: $HOME/spikeinterface_datasets key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? From 201e43f2ed40d44e5559be1c12ebce5e7225dc5c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:42:45 -0600 Subject: [PATCH 12/15] use ~ notation --- .github/workflows/caches_cron_job.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index ee0eef96d9..2fc0a70877 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -25,9 +25,9 @@ jobs: python-version: '3.11' - name: Create the directory to store the data run: | - mkdir -p $HOME/spikeinterface_datasets/behavior_testing_data/ - chmod -R 777 $HOME/spikeinterface_datasets - ls -l $HOME/spikeinterface_datasets + mkdir -p ~/spikeinterface_datasets/behavior_testing_data/ + chmod -R 777 ~/spikeinterface_datasets + ls -l ~/spikeinterface_datasets shell: bash - name: Get current hash (SHA) of the ephy_testing_data repo id: repo_hash @@ -38,7 +38,7 @@ jobs: - uses: actions/cache@v4 id: cache-datasets with: - path: $HOME/spikeinterface_datasets + path: ~/spikeinterface_datasets key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} lookup-only: 'true' # Avoids downloading the data, saving behavior is not affected. - name: Cache found? @@ -69,11 +69,11 @@ jobs: - name: Move the downloaded data to the right directory if: steps.cache-datasets.outputs.cache-hit != 'true' run: | - mv ./behavior_testing_data $HOME/spikeinterface_datasets/ + mv ./behavior_testing_data ~/spikeinterface_datasets/ shell: bash - name: Show size of the cache to assert data is downloaded run: | - cd $HOME + cd ~ pwd du -hs spikeinterface_datasets # Should show the size of ephy_testing_data cd spikeinterface_datasets From db1d766682399047d0fa49e0868129fa93ed9fb4 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:47:37 -0600 Subject: [PATCH 13/15] try without permits --- .github/workflows/caches_cron_job.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 2fc0a70877..0d0bb8577b 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -26,7 +26,6 @@ jobs: - name: Create the directory to store the data run: | mkdir -p ~/spikeinterface_datasets/behavior_testing_data/ - chmod -R 777 ~/spikeinterface_datasets ls -l ~/spikeinterface_datasets shell: bash - name: Get current hash (SHA) of the ephy_testing_data repo From 63d1050e5c8cd3af838806aaf298d61bf9efdc6e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 24 Jun 2024 22:52:58 -0600 Subject: [PATCH 14/15] restore ephy testing, this is DONE --- .github/workflows/caches_cron_job.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 0d0bb8577b..56b3c03038 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -25,7 +25,7 @@ jobs: python-version: '3.11' - name: Create the directory to store the data run: | - mkdir -p ~/spikeinterface_datasets/behavior_testing_data/ + mkdir -p ~/spikeinterface_datasets/ephy_testing_data/ ls -l ~/spikeinterface_datasets shell: bash - name: Get current hash (SHA) of the ephy_testing_data repo @@ -63,12 +63,12 @@ jobs: - name: Download dataset if: steps.cache-datasets.outputs.cache-hit != 'true' run: | - datalad install --recursive --get-data https://gin.g-node.org/CatalystNeuro/behavior_testing_data + datalad install --recursive --get-data https://gin.g-node.org/NeuralEnsemble/ephy_testing_data shell: bash - name: Move the downloaded data to the right directory if: steps.cache-datasets.outputs.cache-hit != 'true' run: | - mv ./behavior_testing_data ~/spikeinterface_datasets/ + mv ./ephy_testing_data ~/spikeinterface_datasets/ shell: bash - name: Show size of the cache to assert data is downloaded run: | @@ -78,6 +78,6 @@ jobs: cd spikeinterface_datasets pwd ls -lh # Should show ephy_testing_data - cd behavior_testing_data + cd ephy_testing_data ls -lh shell: bash From 37851cddff5fe37af6bfd77c18cbf795f8cb68c7 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 25 Jun 2024 08:31:00 -0600 Subject: [PATCH 15/15] revert activate on push --- .github/workflows/caches_cron_job.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 56b3c03038..2454e97ad7 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -7,10 +7,7 @@ on: - main schedule: - cron: "0 12 * * *" # Daily at noon UTC - pull_request: # Temporary, will remove this line before merging - types: [synchronize, opened, reopened] - branches: - - main + jobs: create-gin-data-cache-if-missing: name: Caching data env