From a6e2e7e82a1e0bcd38dddb95d6230ca8196dc599 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 3 Oct 2023 12:23:01 -0400 Subject: [PATCH 1/8] change webbpsf data caching in github CI download webbpsf data once per week to update a cache ``webbpsf-`` that can be used as a ``cache-restore-keys`` entry for the cache in other ci jobs. This can also be triggered when a PR is specifically labeled to update webbpsf data. Update other ``roman_ci.yml`` to use most recent ``webbpsf-`` to look up hash to then construct a combined cache key (for the combined crds/webbpsf data cache) using the webbpsf data as a restore key to initialize the cache with the pre-fetched webbpsf data. ``roman_ci_cron.yml`` will need similar updates. --- .github/workflows/roman_ci.yml | 34 ++++++++++---------- .github/workflows/webbpsf_data.yml | 51 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/webbpsf_data.yml diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index d85cf38a5..371e148c8 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -26,8 +26,11 @@ jobs: env: OBSERVATORY: roman CRDS_SERVER_URL: https://roman-crds.stsci.edu + # CRDS_PATH here should match DATA_PATH in webbpsf_data.yml CRDS_PATH: /tmp/data + GH_TOKEN: ${{ github.token }} outputs: + # TODO simplify data_path and crds_path as they are the same data_path: ${{ steps.data.outputs.path }} webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} data_hash: ${{ steps.data_hash.outputs.hash }} @@ -36,6 +39,8 @@ jobs: crds_server: ${{ steps.crds_server.outputs.url }} steps: # crds: + - id: data + run: echo "path=/tmp/data" >> $GITHUB_OUTPUT - id: crds_context run: > echo "pmap=$( @@ -49,25 +54,17 @@ jobs: - id: crds_server run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT # webbpsf: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: data - run: | - echo "path=/tmp/data" >> $GITHUB_OUTPUT - echo "webbpsf_url=https://stsci.box.com/shared/static/qxpiaxsjwo15ml6m4pkhtk36c9jgj70k.gz" >> $GITHUB_OUTPUT - - run: | - mkdir -p tmp/data/ - mkdir -p ${{ steps.data.outputs.path }} - - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz - - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/ - id: data_hash - run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT - - run: mv tmp/data/* ${{ steps.data.outputs.path }} - - uses: actions/cache@v3 - with: - path: ${{ steps.data.outputs.path }} - key: data-${{ steps.data_hash.outputs.hash }}-${{ steps.crds_context.outputs.pmap }} + run: | + # use actions/gh-actions-cache to allow filtering by key + gh extension install actions/gh-actions-cache + + RECENT=$(gh actions-cache list -R spacetelescope/romancal --key webbpsf- --sort created-at | cut -f 1 | head -n 1) + echo "RECENT=$RECENT" + HASH=$(echo $RECENT | cut -d '-' -f 2) + echo "HASH=$HASH" + echo "hash=$HASH" >> $GITHUB_OUTPUT + if [ "$HASH" == '' ]; then exit 1; fi - id: webbpsf_path run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT check: @@ -93,6 +90,7 @@ jobs: DD_GIT_BRANCH: ${{ github.ref_name }} cache-path: ${{ needs.data.outputs.data_path }} cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }} + cache-restore-keys: webbpsf-${{ needs.data.outputs.data_hash }} envs: | - linux: py39-oldestdeps-cov pytest-results-summary: true diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml new file mode 100644 index 000000000..06b102388 --- /dev/null +++ b/.github/workflows/webbpsf_data.yml @@ -0,0 +1,51 @@ +name: check/update webbpsf cache + +on: + schedule: + - cron: "42 4 * * 3" + pull_request: + # We also want this workflow triggered if the `update webbpsf data` label is + # added or present when PR is updated + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + webbpsf-data: + if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'update webbpsf data'))) + name: fetch, check, and possibly update webbpsf data cache + runs-on: ubuntu-latest + env: + # DATA_PATH here should match CRDS_PATH in roman_ci.yml and roman_ci_cron.yml + DATA_PATH: /tmp/data + WEBBPSF_DATA_URL: https://stsci.box.com/shared/static/qxpiaxsjwo15ml6m4pkhtk36c9jgj70k.gz + outputs: + path: ${{ steps.cache_path.outputs.path }} + hash: ${{ steps.data_hash.outputs.hash }} + steps: + - id: cache_path + run: | + echo "path=${{ env.DATA_PATH }}" >> $GITHUB_OUTPUT + - id: data_hash + run: | + mkdir -p tmp/data + wget ${{ env.WEBBPSF_DATA_URL }} -O tmp/webbpsf-data.tar.gz + echo "hash=$( shasum tmp/webbpsf-data.tar.gz | cut -d ' ' -f 1 )" >> $GITHUB_OUTPUT + - id: cache_check + uses: actions/cache@v3 + with: + path: ${{ steps.cache_path.outputs.path }} + key: webbpsf-${{ steps.data_hash.outputs.hash }} + - if: ${{ steps.cache_check.outputs.cache-hit != 'true' }} + name: Initialize cache + run: | + mkdir -p ${{ steps.cache_path.outputs.path }} + tar -xzvf tmp/webbpsf-data.tar.gz -C ${{ steps.cache_path.outputs.path }} From 03258f7a6090842c7ade26e6b17d61af9c536e66 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 3 Oct 2023 15:57:08 -0400 Subject: [PATCH 2/8] attempt at reusable workflow --- .github/workflows/roman_ci.yml | 47 +----------------- .github/workflows/webbpsf_data.yml | 80 ++++++++++++++++++++++++------ 2 files changed, 66 insertions(+), 61 deletions(-) diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index 371e148c8..399748cc0 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -21,52 +21,7 @@ concurrency: jobs: data: - name: retrieve current CRDS context, and WebbPSF data - runs-on: ubuntu-latest - env: - OBSERVATORY: roman - CRDS_SERVER_URL: https://roman-crds.stsci.edu - # CRDS_PATH here should match DATA_PATH in webbpsf_data.yml - CRDS_PATH: /tmp/data - GH_TOKEN: ${{ github.token }} - outputs: - # TODO simplify data_path and crds_path as they are the same - data_path: ${{ steps.data.outputs.path }} - webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - crds_path: ${{ steps.crds_path.outputs.path }} - crds_context: ${{ steps.crds_context.outputs.pmap }} - crds_server: ${{ steps.crds_server.outputs.url }} - steps: - # crds: - - id: data - run: echo "path=/tmp/data" >> $GITHUB_OUTPUT - - id: crds_context - run: > - echo "pmap=$( - curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | - python -c "import sys, json; print(json.load(sys.stdin)['result'])" - )" >> $GITHUB_OUTPUT - # Get default CRDS_CONTEXT without installing crds client - # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request - - id: crds_path - run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT - - id: crds_server - run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT - # webbpsf: - - id: data_hash - run: | - # use actions/gh-actions-cache to allow filtering by key - gh extension install actions/gh-actions-cache - - RECENT=$(gh actions-cache list -R spacetelescope/romancal --key webbpsf- --sort created-at | cut -f 1 | head -n 1) - echo "RECENT=$RECENT" - HASH=$(echo $RECENT | cut -d '-' -f 2) - echo "HASH=$HASH" - echo "hash=$HASH" >> $GITHUB_OUTPUT - if [ "$HASH" == '' ]; then exit 1; fi - - id: webbpsf_path - run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT + uses: ./.github/workflows/webbpsf_data.yml check: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml index 06b102388..f6ff5c8e2 100644 --- a/.github/workflows/webbpsf_data.yml +++ b/.github/workflows/webbpsf_data.yml @@ -1,26 +1,25 @@ name: check/update webbpsf cache on: + workflow_call: + outputs: + webbpsf_path: + value: ${{ jobs.data.outputs.webbpsf_path }} + data_hash: + value: ${{ jobs.data.outputs.data_hash }} + crds_path: + value: ${{ jobs.data.outputs.crds_path }} + crds_context: + value: ${{ jobs.data.outputs.crds_context }} + crds_server: + value: ${{ jobs.data.outputs.crds_server }} + workflow_dispatch: schedule: - cron: "42 4 * * 3" - pull_request: - # We also want this workflow triggered if the `update webbpsf data` label is - # added or present when PR is updated - types: - - opened - - reopened - - labeled - - unlabeled - - synchronize - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true jobs: webbpsf-data: - if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'update webbpsf data'))) + if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'update webbpsf data'))) name: fetch, check, and possibly update webbpsf data cache runs-on: ubuntu-latest env: @@ -49,3 +48,54 @@ jobs: run: | mkdir -p ${{ steps.cache_path.outputs.path }} tar -xzvf tmp/webbpsf-data.tar.gz -C ${{ steps.cache_path.outputs.path }} + data: + needs: + [webbpsf-data] + # run data job if webbpsf-data succeeds or is skipped + if: always() && (needs.webbpsf-data.result == 'success' || needs.webbpsf-data.result == 'skipped') + name: retrieve current CRDS context, and WebbPSF data + runs-on: ubuntu-latest + env: + OBSERVATORY: roman + CRDS_SERVER_URL: https://roman-crds.stsci.edu + # CRDS_PATH here should match DATA_PATH in webbpsf_data.yml + CRDS_PATH: /tmp/data + GH_TOKEN: ${{ github.token }} + outputs: + # TODO simplify data_path and crds_path as they are the same + data_path: ${{ steps.data.outputs.path }} + webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} + data_hash: ${{ steps.data_hash.outputs.hash }} + crds_path: ${{ steps.crds_path.outputs.path }} + crds_context: ${{ steps.crds_context.outputs.pmap }} + crds_server: ${{ steps.crds_server.outputs.url }} + steps: + # crds: + - id: data + run: echo "path=/tmp/data" >> $GITHUB_OUTPUT + - id: crds_context + run: > + echo "pmap=$( + curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | + python -c "import sys, json; print(json.load(sys.stdin)['result'])" + )" >> $GITHUB_OUTPUT + # Get default CRDS_CONTEXT without installing crds client + # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request + - id: crds_path + run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT + - id: crds_server + run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT + # webbpsf: + - id: data_hash + run: | + # use actions/gh-actions-cache to allow filtering by key + gh extension install actions/gh-actions-cache + + RECENT=$(gh actions-cache list -R spacetelescope/romancal --key webbpsf- --sort created-at | cut -f 1 | head -n 1) + echo "RECENT=$RECENT" + HASH=$(echo $RECENT | cut -d '-' -f 2) + echo "HASH=$HASH" + echo "hash=$HASH" >> $GITHUB_OUTPUT + if [ "$HASH" == '' ]; then exit 1; fi + - id: webbpsf_path + run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT From c9616e21892db64a7bfdd642101fc35eacdcfaf7 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 3 Oct 2023 16:20:57 -0400 Subject: [PATCH 3/8] simplifying paths and outputs --- .github/workflows/roman_ci.yml | 6 +-- .github/workflows/roman_ci_cron.yaml | 53 ++------------------------- .github/workflows/tests_devdeps.yml | 55 ++-------------------------- .github/workflows/webbpsf_data.yml | 26 +++++-------- 4 files changed, 20 insertions(+), 120 deletions(-) diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index 399748cc0..1e412492c 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -43,9 +43,9 @@ jobs: DD_GIT_REPOSITORY_URL: ${{ github.repositoryUrl }} DD_GIT_COMMIT_SHA: ${{ github.sha }} DD_GIT_BRANCH: ${{ github.ref_name }} - cache-path: ${{ needs.data.outputs.data_path }} - cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }} - cache-restore-keys: webbpsf-${{ needs.data.outputs.data_hash }} + cache-path: ${{ needs.data.outputs.crds_path }} + cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }} + cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }} envs: | - linux: py39-oldestdeps-cov pytest-results-summary: true diff --git a/.github/workflows/roman_ci_cron.yaml b/.github/workflows/roman_ci_cron.yaml index 1e3a1d7be..3359fd650 100644 --- a/.github/workflows/roman_ci_cron.yaml +++ b/.github/workflows/roman_ci_cron.yaml @@ -25,55 +25,7 @@ concurrency: jobs: data: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests'))) - name: retrieve current CRDS context, and WebbPSF data - runs-on: ubuntu-latest - env: - OBSERVATORY: roman - CRDS_SERVER_URL: https://roman-crds.stsci.edu - CRDS_PATH: /tmp/data - outputs: - data_path: ${{ steps.data.outputs.path }} - webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - crds_path: ${{ steps.crds_path.outputs.path }} - crds_context: ${{ steps.crds_context.outputs.pmap }} - crds_server: ${{ steps.crds_server.outputs.url }} - steps: - # crds: - - id: crds_context - run: > - echo "pmap=$( - curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | - python -c "import sys, json; print(json.load(sys.stdin)['result'])" - )" >> $GITHUB_OUTPUT - # Get default CRDS_CONTEXT without installing crds client - # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request - - id: crds_path - run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT - - id: crds_server - run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT - # webbpsf: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: data - run: | - echo "path=/tmp/data" >> $GITHUB_OUTPUT - echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT - - run: | - mkdir -p tmp/data/ - mkdir -p ${{ steps.data.outputs.path }} - - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz - - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/ - - id: data_hash - run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT - - run: mv tmp/data/* ${{ steps.data.outputs.path }} - - uses: actions/cache@v3 - with: - path: ${{ steps.data.outputs.path }} - key: data-${{ steps.data_hash.outputs.hash }}-${{ steps.crds_context.outputs.pmap }} - - id: webbpsf_path - run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT + uses: ./.github/workflows/webbpsf_data.yml test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main needs: [ data ] @@ -85,7 +37,8 @@ jobs: CRDS_CLIENT_RETRY_COUNT: 3 CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 cache-path: ${{ needs.data.outputs.crds_path }} - cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }} + cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }} + cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }} envs: | - macos: py39 pytest-results-summary: true diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml index 43b55d9a8..7ca4ed412 100644 --- a/.github/workflows/tests_devdeps.yml +++ b/.github/workflows/tests_devdeps.yml @@ -26,55 +26,7 @@ concurrency: jobs: data: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) - name: retrieve current CRDS context, and WebbPSF data - runs-on: ubuntu-latest - env: - OBSERVATORY: roman - CRDS_SERVER_URL: https://roman-crds.stsci.edu - CRDS_PATH: /tmp/data - outputs: - data_path: ${{ steps.data.outputs.path }} - webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - crds_path: ${{ steps.crds_path.outputs.path }} - crds_context: ${{ steps.crds_context.outputs.pmap }} - crds_server: ${{ steps.crds_server.outputs.url }} - steps: - # crds: - - id: crds_context - run: > - echo "pmap=$( - curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | - python -c "import sys, json; print(json.load(sys.stdin)['result'])" - )" >> $GITHUB_OUTPUT - # Get default CRDS_CONTEXT without installing crds client - # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request - - id: crds_path - run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT - - id: crds_server - run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT - # webbpsf: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: data - run: | - echo "path=/tmp/data" >> $GITHUB_OUTPUT - echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT - - run: | - mkdir -p tmp/data/ - mkdir -p ${{ steps.data.outputs.path }} - - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz - - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/ - - id: data_hash - run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT - - run: mv tmp/data/* ${{ steps.data.outputs.path }} - - uses: actions/cache@v3 - with: - path: ${{ steps.data.outputs.path }} - key: data-${{ steps.data_hash.outputs.hash }}-${{ steps.crds_context.outputs.pmap }} - - id: webbpsf_path - run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT + uses: ./.github/workflows/webbpsf_data.yml test: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main @@ -86,8 +38,9 @@ jobs: CRDS_SERVER_URL: ${{ needs.data.outputs.crds_server }} CRDS_CLIENT_RETRY_COUNT: 3 CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 - cache-path: ${{ needs.data.outputs.data_path }} - cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }} + cache-path: ${{ needs.data.outputs.crds_path }} + cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }} + cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }} envs: | - linux: py39-devdeps - macos: py39-devdeps diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml index f6ff5c8e2..8b14d00d4 100644 --- a/.github/workflows/webbpsf_data.yml +++ b/.github/workflows/webbpsf_data.yml @@ -3,16 +3,16 @@ name: check/update webbpsf cache on: workflow_call: outputs: - webbpsf_path: - value: ${{ jobs.data.outputs.webbpsf_path }} - data_hash: - value: ${{ jobs.data.outputs.data_hash }} - crds_path: - value: ${{ jobs.data.outputs.crds_path }} crds_context: value: ${{ jobs.data.outputs.crds_context }} + crds_path: + value: ${{ jobs.data.outputs.crds_path }} crds_server: value: ${{ jobs.data.outputs.crds_server }} + webbpsf_hash: + value: ${{ jobs.data.outputs.webbpsf_hash }} + webbpsf_path: + value: ${{ jobs.data.outputs.webbpsf_path }} workflow_dispatch: schedule: - cron: "42 4 * * 3" @@ -23,7 +23,6 @@ jobs: name: fetch, check, and possibly update webbpsf data cache runs-on: ubuntu-latest env: - # DATA_PATH here should match CRDS_PATH in roman_ci.yml and roman_ci_cron.yml DATA_PATH: /tmp/data WEBBPSF_DATA_URL: https://stsci.box.com/shared/static/qxpiaxsjwo15ml6m4pkhtk36c9jgj70k.gz outputs: @@ -58,21 +57,16 @@ jobs: env: OBSERVATORY: roman CRDS_SERVER_URL: https://roman-crds.stsci.edu - # CRDS_PATH here should match DATA_PATH in webbpsf_data.yml CRDS_PATH: /tmp/data GH_TOKEN: ${{ github.token }} outputs: - # TODO simplify data_path and crds_path as they are the same - data_path: ${{ steps.data.outputs.path }} - webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - crds_path: ${{ steps.crds_path.outputs.path }} crds_context: ${{ steps.crds_context.outputs.pmap }} + crds_path: ${{ steps.crds_path.outputs.path }} crds_server: ${{ steps.crds_server.outputs.url }} + webbpsf_hash: ${{ steps.webbpsf_hash.outputs.hash }} + webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} steps: # crds: - - id: data - run: echo "path=/tmp/data" >> $GITHUB_OUTPUT - id: crds_context run: > echo "pmap=$( @@ -86,7 +80,7 @@ jobs: - id: crds_server run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT # webbpsf: - - id: data_hash + - id: webbpsf_hash run: | # use actions/gh-actions-cache to allow filtering by key gh extension install actions/gh-actions-cache From 09bb1c9f65b81271b5ed1884c80ae24de0a0b27a Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 3 Oct 2023 16:37:22 -0400 Subject: [PATCH 4/8] rename webbpsf_data.yml to data.yml --- .github/workflows/{webbpsf_data.yml => data.yml} | 2 +- .github/workflows/roman_ci.yml | 2 +- .github/workflows/roman_ci_cron.yaml | 2 +- .github/workflows/tests_devdeps.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{webbpsf_data.yml => data.yml} (98%) diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/data.yml similarity index 98% rename from .github/workflows/webbpsf_data.yml rename to .github/workflows/data.yml index 8b14d00d4..63a6c0349 100644 --- a/.github/workflows/webbpsf_data.yml +++ b/.github/workflows/data.yml @@ -1,4 +1,4 @@ -name: check/update webbpsf cache +name: check and update webbpsf and crds cache on: workflow_call: diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index 1e412492c..9355430f6 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -21,7 +21,7 @@ concurrency: jobs: data: - uses: ./.github/workflows/webbpsf_data.yml + uses: ./.github/workflows/data.yml check: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: diff --git a/.github/workflows/roman_ci_cron.yaml b/.github/workflows/roman_ci_cron.yaml index 3359fd650..cb74f9217 100644 --- a/.github/workflows/roman_ci_cron.yaml +++ b/.github/workflows/roman_ci_cron.yaml @@ -25,7 +25,7 @@ concurrency: jobs: data: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests'))) - uses: ./.github/workflows/webbpsf_data.yml + uses: ./.github/workflows/data.yml test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main needs: [ data ] diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml index 7ca4ed412..485dd1012 100644 --- a/.github/workflows/tests_devdeps.yml +++ b/.github/workflows/tests_devdeps.yml @@ -26,7 +26,7 @@ concurrency: jobs: data: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) - uses: ./.github/workflows/webbpsf_data.yml + uses: ./.github/workflows/data.yml test: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main From 42cf4c85b28ba7557104c5bf8b39c2e961163a56 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 3 Oct 2023 16:46:51 -0400 Subject: [PATCH 5/8] fix webbpsf path --- .github/workflows/data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml index 63a6c0349..24ca54f76 100644 --- a/.github/workflows/data.yml +++ b/.github/workflows/data.yml @@ -92,4 +92,4 @@ jobs: echo "hash=$HASH" >> $GITHUB_OUTPUT if [ "$HASH" == '' ]; then exit 1; fi - id: webbpsf_path - run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT + run: echo "path=${{ steps.crds_path.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT From e378bc0c463860d6b7b836ec51e8573ed84df1e1 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 3 Oct 2023 17:09:54 -0400 Subject: [PATCH 6/8] debug --- .github/workflows/data.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml index 24ca54f76..56aa11892 100644 --- a/.github/workflows/data.yml +++ b/.github/workflows/data.yml @@ -18,6 +18,14 @@ on: - cron: "42 4 * * 3" jobs: + report: + runs-on: ubuntu-latest + steps: + - name: echo + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: | + echo "$GITHUB_CONTEXT" webbpsf-data: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'update webbpsf data'))) name: fetch, check, and possibly update webbpsf data cache From e14e22813c55d9d1cac28e2c7e1becfaa09ed436 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 3 Oct 2023 17:16:13 -0400 Subject: [PATCH 7/8] remove debug --- .github/workflows/data.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml index 56aa11892..a9f9ba856 100644 --- a/.github/workflows/data.yml +++ b/.github/workflows/data.yml @@ -18,16 +18,8 @@ on: - cron: "42 4 * * 3" jobs: - report: - runs-on: ubuntu-latest - steps: - - name: echo - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: | - echo "$GITHUB_CONTEXT" webbpsf-data: - if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'update webbpsf data'))) + if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'update webbpsf data'))) name: fetch, check, and possibly update webbpsf data cache runs-on: ubuntu-latest env: From 1ed817b84d6313f7086359659bad760c4c4064e2 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 4 Oct 2023 10:27:41 -0400 Subject: [PATCH 8/8] add comment about webbpsf download skip --- .github/workflows/data.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml index a9f9ba856..e2df5122c 100644 --- a/.github/workflows/data.yml +++ b/.github/workflows/data.yml @@ -50,7 +50,9 @@ jobs: data: needs: [webbpsf-data] - # run data job if webbpsf-data succeeds or is skipped + # run data job if webbpsf-data succeeds or is skipped. This allows + # this data job to always fetch the crds context even if the webbpsf data fetching + # was skipped (and an existing cache will be used for the webbpsf data). if: always() && (needs.webbpsf-data.result == 'success' || needs.webbpsf-data.result == 'skipped') name: retrieve current CRDS context, and WebbPSF data runs-on: ubuntu-latest