simplifying paths and outputs

spacetelescope · Oct 3, 2023 · be68c69 · be68c69
1 parent dff3152
commit be68c69
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 120 deletions.
diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml
@@ -43,9 +43,9 @@ jobs:
         DD_GIT_REPOSITORY_URL: ${{ github.repositoryUrl }}
         DD_GIT_COMMIT_SHA: ${{ github.sha }}
         DD_GIT_BRANCH: ${{ github.ref_name }}
-      cache-path: ${{ needs.data.outputs.data_path }}
-      cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }}
-      cache-restore-keys: webbpsf-${{ needs.data.outputs.data_hash }}
+      cache-path: ${{ needs.data.outputs.crds_path }}
+      cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }}
+      cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }}
       envs: |
         - linux: py39-oldestdeps-cov
           pytest-results-summary: true

diff --git a/.github/workflows/roman_ci_cron.yaml b/.github/workflows/roman_ci_cron.yaml
@@ -25,55 +25,7 @@ concurrency:
 jobs:
   data:
     if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests')))
-    name: retrieve current CRDS context, and WebbPSF data
-    runs-on: ubuntu-latest
-    env:
-      OBSERVATORY: roman
-      CRDS_SERVER_URL: https://roman-crds.stsci.edu
-      CRDS_PATH: /tmp/data
-    outputs:
-      data_path: ${{ steps.data.outputs.path }}
-      webbpsf_path: ${{ steps.webbpsf_path.outputs.path }}
-      data_hash: ${{ steps.data_hash.outputs.hash }}
-      crds_path: ${{ steps.crds_path.outputs.path }}
-      crds_context: ${{ steps.crds_context.outputs.pmap }}
-      crds_server: ${{ steps.crds_server.outputs.url }}
-    steps:
-      # crds:
-      - id: crds_context
-        run: >
-          echo "pmap=$(
-          curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ |
-          python -c "import sys, json; print(json.load(sys.stdin)['result'])"
-          )" >> $GITHUB_OUTPUT
-        # Get default CRDS_CONTEXT without installing crds client
-        # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request
-      - id: crds_path
-        run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT
-      - id: crds_server
-        run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT
-      # webbpsf:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - id: data
-        run: |
-          echo "path=/tmp/data" >> $GITHUB_OUTPUT
-          echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT
-      - run: |
-          mkdir -p tmp/data/
-          mkdir -p ${{ steps.data.outputs.path }}
-      - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz
-      - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/
-      - id: data_hash
-        run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT
-      - run: mv tmp/data/* ${{ steps.data.outputs.path }}
-      - uses: actions/cache@v3
-        with:
-          path: ${{ steps.data.outputs.path }}
-          key: data-${{ steps.data_hash.outputs.hash }}-${{ steps.crds_context.outputs.pmap }}
-      - id: webbpsf_path
-        run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT
+    uses: ./.github/workflows/webbpsf_data.yml
   test:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
     needs: [ data ]
@@ -85,7 +37,8 @@ jobs:
         CRDS_CLIENT_RETRY_COUNT: 3
         CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       cache-path: ${{ needs.data.outputs.crds_path }}
-      cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }}
+      cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }}
+      cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }}
       envs: |
         - macos: py39
           pytest-results-summary: true

diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml
@@ -26,55 +26,7 @@ concurrency:
 jobs:
   data:
     if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests')))
-    name: retrieve current CRDS context, and WebbPSF data
-    runs-on: ubuntu-latest
-    env:
-      OBSERVATORY: roman
-      CRDS_SERVER_URL: https://roman-crds.stsci.edu
-      CRDS_PATH: /tmp/data
-    outputs:
-      data_path: ${{ steps.data.outputs.path }}
-      webbpsf_path: ${{ steps.webbpsf_path.outputs.path }}
-      data_hash: ${{ steps.data_hash.outputs.hash }}
-      crds_path: ${{ steps.crds_path.outputs.path }}
-      crds_context: ${{ steps.crds_context.outputs.pmap }}
-      crds_server: ${{ steps.crds_server.outputs.url }}
-    steps:
-      # crds:
-      - id: crds_context
-        run: >
-          echo "pmap=$(
-          curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ |
-          python -c "import sys, json; print(json.load(sys.stdin)['result'])"
-          )" >> $GITHUB_OUTPUT
-        # Get default CRDS_CONTEXT without installing crds client
-        # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request
-      - id: crds_path
-        run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT
-      - id: crds_server
-        run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT
-      # webbpsf:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - id: data
-        run: |
-          echo "path=/tmp/data" >> $GITHUB_OUTPUT
-          echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT
-      - run: |
-          mkdir -p tmp/data/
-          mkdir -p ${{ steps.data.outputs.path }}
-      - run: wget ${{ steps.data.outputs.webbpsf_url }} -O tmp/minimal-webbpsf-data.tar.gz
-      - run: tar -xzvf tmp/minimal-webbpsf-data.tar.gz -C tmp/data/
-      - id: data_hash
-        run: echo "hash=${{ hashFiles( 'tmp/data' ) }}" >> $GITHUB_OUTPUT
-      - run: mv tmp/data/* ${{ steps.data.outputs.path }}
-      - uses: actions/cache@v3
-        with:
-          path: ${{ steps.data.outputs.path }}
-          key: data-${{ steps.data_hash.outputs.hash }}-${{ steps.crds_context.outputs.pmap }}
-      - id: webbpsf_path
-        run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT
+    uses: ./.github/workflows/webbpsf_data.yml
   test:
     if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests')))
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
@@ -86,8 +38,9 @@ jobs:
         CRDS_SERVER_URL: ${{ needs.data.outputs.crds_server }}
         CRDS_CLIENT_RETRY_COUNT: 3
         CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
-      cache-path: ${{ needs.data.outputs.data_path }}
-      cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }}
+      cache-path: ${{ needs.data.outputs.crds_path }}
+      cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }}
+      cache-restore-keys: webbpsf-${{ needs.data.outputs.webbpsf_hash }}
       envs: |
         - linux: py39-devdeps
         - macos: py39-devdeps

diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml
@@ -3,16 +3,16 @@ name: check/update webbpsf cache
 on:
   workflow_call:
     outputs:
-      webbpsf_path:
-        value: ${{ jobs.data.outputs.webbpsf_path }}
-      data_hash:
-        value: ${{ jobs.data.outputs.data_hash }}
-      crds_path:
-        value: ${{ jobs.data.outputs.crds_path }}
       crds_context:
         value: ${{ jobs.data.outputs.crds_context }}
+      crds_path:
+        value: ${{ jobs.data.outputs.crds_path }}
       crds_server:
         value: ${{ jobs.data.outputs.crds_server }}
+      webbpsf_hash:
+        value: ${{ jobs.data.outputs.webbpsf_hash }}
+      webbpsf_path:
+        value: ${{ jobs.data.outputs.webbpsf_path }}
   workflow_dispatch:
   schedule:
     - cron: "42 4 * * 3"
@@ -23,7 +23,6 @@ jobs:
     name: fetch, check, and possibly update webbpsf data cache
     runs-on: ubuntu-latest
     env:
-      # DATA_PATH here should match CRDS_PATH in roman_ci.yml and roman_ci_cron.yml
       DATA_PATH: /tmp/data
       WEBBPSF_DATA_URL: https://stsci.box.com/shared/static/qxpiaxsjwo15ml6m4pkhtk36c9jgj70k.gz
     outputs:
@@ -58,21 +57,16 @@ jobs:
     env:
       OBSERVATORY: roman
       CRDS_SERVER_URL: https://roman-crds.stsci.edu
-      # CRDS_PATH here should match DATA_PATH in webbpsf_data.yml
       CRDS_PATH: /tmp/data
       GH_TOKEN: ${{ github.token }}
     outputs:
-      # TODO simplify data_path and crds_path as they are the same
-      data_path: ${{ steps.data.outputs.path }}
-      webbpsf_path: ${{ steps.webbpsf_path.outputs.path }}
-      data_hash: ${{ steps.data_hash.outputs.hash }}
-      crds_path: ${{ steps.crds_path.outputs.path }}
       crds_context: ${{ steps.crds_context.outputs.pmap }}
+      crds_path: ${{ steps.crds_path.outputs.path }}
       crds_server: ${{ steps.crds_server.outputs.url }}
+      webbpsf_hash: ${{ steps.webbpsf_hash.outputs.hash }}
+      webbpsf_path: ${{ steps.webbpsf_path.outputs.path }}
     steps:
       # crds:
-      - id: data
-        run: echo "path=/tmp/data" >> $GITHUB_OUTPUT
       - id: crds_context
         run: >
           echo "pmap=$(
@@ -86,7 +80,7 @@ jobs:
       - id: crds_server
         run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT
       # webbpsf:
-      - id: data_hash
+      - id: webbpsf_hash
         run: |
           # use actions/gh-actions-cache to allow filtering by key
           gh extension install actions/gh-actions-cache