diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index 399748cc0..1e412492c 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -43,9 +43,9 @@ jobs: DD_GIT_REPOSITORY_URL: ${{ github.repositoryUrl }} DD_GIT_COMMIT_SHA: ${{ github.sha }} DD_GIT_BRANCH: ${{ github.ref_name }} - cache-path: ${{ needs.data.outputs.data_path }} - cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }} - cache-restore-keys: webbpsf-${{ needs.data.outputs.data_hash }} + cache-path: ${{ needs.data.outputs.crds_path }} + cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }} + cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }} envs: | - linux: py39-oldestdeps-cov pytest-results-summary: true diff --git a/.github/workflows/roman_ci_cron.yaml b/.github/workflows/roman_ci_cron.yaml index 1e3a1d7be..3359fd650 100644 --- a/.github/workflows/roman_ci_cron.yaml +++ b/.github/workflows/roman_ci_cron.yaml @@ -25,55 +25,7 @@ concurrency: jobs: data: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests'))) - name: retrieve current CRDS context, and WebbPSF data - runs-on: ubuntu-latest - env: - OBSERVATORY: roman - CRDS_SERVER_URL: https://roman-crds.stsci.edu - CRDS_PATH: /tmp/data - outputs: - data_path: ${{ steps.data.outputs.path }} - webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - crds_path: ${{ steps.crds_path.outputs.path }} - crds_context: ${{ steps.crds_context.outputs.pmap }} - crds_server: ${{ steps.crds_server.outputs.url }} - steps: - # crds: - - id: crds_context - run: > - echo "pmap=$( - curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | - python -c "import sys, json; print(json.load(sys.stdin)['result'])" - )" >> $GITHUB_OUTPUT - # Get default CRDS_CONTEXT without installing crds client - # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request - - id: crds_path - run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT - - id: crds_server - run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT - # webbpsf: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: data - run: | - echo "path=/tmp/data" >> $GITHUB_OUTPUT - echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT - - run: | - mkdir -p tmp/data/ - mkdir -p ${{ steps.data.outputs.path }} - - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz - - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/ - - id: data_hash - run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT - - run: mv tmp/data/* ${{ steps.data.outputs.path }} - - uses: actions/cache@v3 - with: - path: ${{ steps.data.outputs.path }} - key: data-${{ steps.data_hash.outputs.hash }}-${{ steps.crds_context.outputs.pmap }} - - id: webbpsf_path - run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT + uses: ./.github/workflows/webbpsf_data.yml test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main needs: [ data ] @@ -85,7 +37,8 @@ jobs: CRDS_CLIENT_RETRY_COUNT: 3 CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 cache-path: ${{ needs.data.outputs.crds_path }} - cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }} + cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }} + cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }} envs: | - macos: py39 pytest-results-summary: true diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml index 43b55d9a8..7ca4ed412 100644 --- a/.github/workflows/tests_devdeps.yml +++ b/.github/workflows/tests_devdeps.yml @@ -26,55 +26,7 @@ concurrency: jobs: data: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) - name: retrieve current CRDS context, and WebbPSF data - runs-on: ubuntu-latest - env: - OBSERVATORY: roman - CRDS_SERVER_URL: https://roman-crds.stsci.edu - CRDS_PATH: /tmp/data - outputs: - data_path: ${{ steps.data.outputs.path }} - webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - crds_path: ${{ steps.crds_path.outputs.path }} - crds_context: ${{ steps.crds_context.outputs.pmap }} - crds_server: ${{ steps.crds_server.outputs.url }} - steps: - # crds: - - id: crds_context - run: > - echo "pmap=$( - curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | - python -c "import sys, json; print(json.load(sys.stdin)['result'])" - )" >> $GITHUB_OUTPUT - # Get default CRDS_CONTEXT without installing crds client - # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request - - id: crds_path - run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT - - id: crds_server - run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT - # webbpsf: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: data - run: | - echo "path=/tmp/data" >> $GITHUB_OUTPUT - echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT - - run: | - mkdir -p tmp/data/ - mkdir -p ${{ steps.data.outputs.path }} - - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz - - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/ - - id: data_hash - run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT - - run: mv tmp/data/* ${{ steps.data.outputs.path }} - - uses: actions/cache@v3 - with: - path: ${{ steps.data.outputs.path }} - key: data-${{ steps.data_hash.outputs.hash }}-${{ steps.crds_context.outputs.pmap }} - - id: webbpsf_path - run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT + uses: ./.github/workflows/webbpsf_data.yml test: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main @@ -86,8 +38,9 @@ jobs: CRDS_SERVER_URL: ${{ needs.data.outputs.crds_server }} CRDS_CLIENT_RETRY_COUNT: 3 CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 - cache-path: ${{ needs.data.outputs.data_path }} - cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }} + cache-path: ${{ needs.data.outputs.crds_path }} + cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }} + cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }} envs: | - linux: py39-devdeps - macos: py39-devdeps diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml index f6ff5c8e2..8b14d00d4 100644 --- a/.github/workflows/webbpsf_data.yml +++ b/.github/workflows/webbpsf_data.yml @@ -3,16 +3,16 @@ name: check/update webbpsf cache on: workflow_call: outputs: - webbpsf_path: - value: ${{ jobs.data.outputs.webbpsf_path }} - data_hash: - value: ${{ jobs.data.outputs.data_hash }} - crds_path: - value: ${{ jobs.data.outputs.crds_path }} crds_context: value: ${{ jobs.data.outputs.crds_context }} + crds_path: + value: ${{ jobs.data.outputs.crds_path }} crds_server: value: ${{ jobs.data.outputs.crds_server }} + webbpsf_hash: + value: ${{ jobs.data.outputs.webbpsf_hash }} + webbpsf_path: + value: ${{ jobs.data.outputs.webbpsf_path }} workflow_dispatch: schedule: - cron: "42 4 * * 3" @@ -23,7 +23,6 @@ jobs: name: fetch, check, and possibly update webbpsf data cache runs-on: ubuntu-latest env: - # DATA_PATH here should match CRDS_PATH in roman_ci.yml and roman_ci_cron.yml DATA_PATH: /tmp/data WEBBPSF_DATA_URL: https://stsci.box.com/shared/static/qxpiaxsjwo15ml6m4pkhtk36c9jgj70k.gz outputs: @@ -58,21 +57,16 @@ jobs: env: OBSERVATORY: roman CRDS_SERVER_URL: https://roman-crds.stsci.edu - # CRDS_PATH here should match DATA_PATH in webbpsf_data.yml CRDS_PATH: /tmp/data GH_TOKEN: ${{ github.token }} outputs: - # TODO simplify data_path and crds_path as they are the same - data_path: ${{ steps.data.outputs.path }} - webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} - data_hash: ${{ steps.data_hash.outputs.hash }} - crds_path: ${{ steps.crds_path.outputs.path }} crds_context: ${{ steps.crds_context.outputs.pmap }} + crds_path: ${{ steps.crds_path.outputs.path }} crds_server: ${{ steps.crds_server.outputs.url }} + webbpsf_hash: ${{ steps.webbpsf_hash.outputs.hash }} + webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} steps: # crds: - - id: data - run: echo "path=/tmp/data" >> $GITHUB_OUTPUT - id: crds_context run: > echo "pmap=$( @@ -86,7 +80,7 @@ jobs: - id: crds_server run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT # webbpsf: - - id: data_hash + - id: webbpsf_hash run: | # use actions/gh-actions-cache to allow filtering by key gh extension install actions/gh-actions-cache