From 60c5e2a9a1736c237ce46f1545abbe0a318070e6 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Thu, 24 Oct 2024 11:25:13 +1100 Subject: [PATCH] Fix Failing Dailies (#1113) --- .github/workflows/all_os_versions.txt | 1 + .github/workflows/all_python_versions.txt | 1 + .github/workflows/dailies.yml | 23 +++++++++++++++++ .github/workflows/deploy-tests.yml | 29 +++++++++++++++------- .github/workflows/dev-testing.yml | 1 - .github/workflows/doctests.yml | 2 -- .github/workflows/live-service-testing.yml | 2 -- .github/workflows/testing.yml | 3 --- CHANGELOG.md | 1 + 9 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/all_os_versions.txt create mode 100644 .github/workflows/all_python_versions.txt diff --git a/.github/workflows/all_os_versions.txt b/.github/workflows/all_os_versions.txt new file mode 100644 index 000000000..93d2aa0a7 --- /dev/null +++ b/.github/workflows/all_os_versions.txt @@ -0,0 +1 @@ +["ubuntu-latest", "macos-latest", "windows-latest", "macos-13"] diff --git a/.github/workflows/all_python_versions.txt b/.github/workflows/all_python_versions.txt new file mode 100644 index 000000000..7d1db34cd --- /dev/null +++ b/.github/workflows/all_python_versions.txt @@ -0,0 +1 @@ +["3.9", "3.10", "3.11", "3.12"] diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 459246091..136590652 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -6,6 +6,18 @@ on: - cron: "0 4 * * *" # Daily at 8PM PST, 11PM EST, 5AM CET to avoid working hours jobs: + load_python_and_os_versions: + runs-on: ubuntu-latest + outputs: + ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} + ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} + steps: + - uses: actions/checkout@v4 + - id: load_python_versions + run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" + - id: load_os_versions + run: echo "os_versions=$(cat ./.github/workflows/all_os_versions.txt)" >> "$GITHUB_OUTPUT" + build-and-upload-docker-image-dev: uses: ./.github/workflows/build_and_upload_docker_image_dev.yml secrets: @@ -13,25 +25,36 @@ jobs: DOCKER_UPLOADER_PASSWORD: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} run-daily-tests: + needs: load_python_and_os_versions uses: ./.github/workflows/testing.yml secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-daily-dev-tests: + needs: load_python_and_os_versions uses: ./.github/workflows/dev-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} run-daily-live-service-testing: + needs: load_python_and_os_versions uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-daily-neuroconv-docker-testing: uses: ./.github/workflows/neuroconv_docker_testing.yml diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 4f67d15de..a18fe8310 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -13,6 +13,17 @@ concurrency: cancel-in-progress: true jobs: + load_python_and_os_versions: + runs-on: ubuntu-latest + outputs: + ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} + ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} + steps: + - uses: actions/checkout@v4 + - id: load_python_versions + run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" + - id: load_os_versions + run: echo "os_versions=$(cat ./.github/workflows/all_os_versions.txt)" >> "$GITHUB_OUTPUT" assess-file-changes: uses: ./.github/workflows/assess-file-changes.yml @@ -31,7 +42,7 @@ jobs: 0 run-tests: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} uses: ./.github/workflows/testing.yml secrets: @@ -40,28 +51,28 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} # If the conversion gallery is the only thing that changed, run doctests only run-doctests-only: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-live-service-tests: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} check-final-status: diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index 31b5329a8..65c011653 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -7,7 +7,6 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' secrets: DANDI_API_KEY: required: true diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index e492eda0c..5d96bf4a3 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -6,12 +6,10 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' os-versions: description: 'List of OS versions to use in matrix, as JSON string' required: true type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' jobs: diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index b9a425a8d..24eda7bc3 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -7,12 +7,10 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' os-versions: description: 'List of OS versions to use in matrix, as JSON string' required: true type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' secrets: DANDI_API_KEY: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 06de82c4c..d8c5bb9fd 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -7,13 +7,10 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' os-versions: description: 'List of OS versions to use in matrix, as JSON string' required: true type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' - secrets: AWS_ACCESS_KEY_ID: diff --git a/CHANGELOG.md b/CHANGELOG.md index 931383066..268a773e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Deprecations ## Bug Fixes +* Fixed dailies [PR #1113](https://github.com/catalystneuro/neuroconv/pull/1113) ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068)