Search and upload #32
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Search and upload | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| iid: | |
| required: true | |
| jobs: | |
| parse: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| outputs: | |
| matrix: ${{ steps.parse.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -r workflows/requirements.parsing.txt | |
| - name: Run parse command | |
| id: parse | |
| run: | | |
| echo 'Parsing!' | |
| echo "matrix=$(python workflows/parse-instance-ids.py '${{ github.event.inputs.iid }}')" >> $GITHUB_OUTPUT | |
| echo "${{ env.matrix }}" | |
| upload: | |
| name: upload ${{ matrix.iid }} | |
| needs: parse | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| iid: ${{ fromJson(needs.parse.outputs.matrix) }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -r workflows/requirements.upload.txt | |
| - name: Upload ${{ matrix.iid }} | |
| env: | |
| ESGF_USER: ${{ secrets.ESGF_USER }} | |
| ESGF_PASSWORD: ${{ secrets.ESGF_PASSWORD }} | |
| id: upload | |
| run: | | |
| echo "Uploading!" | |
| echo "iid: ${{ matrix.iid }}" | |
| python workflows/upload-s3.py ${{ matrix.iid }} #'${{ github.event.client_payload.slash_command.args.all }}' | |
| check: | |
| name: check | |
| needs: upload | |
| if: always() # always run, so we never skip the check | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| outputs: | |
| iids: ${{ steps.check.outputs.iids }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -r workflows/requirements.check.txt | |
| - name: Run check command | |
| id: check | |
| env: | |
| PAT: ${{ secrets.PAT }} | |
| run: | | |
| echo 'Checking run_id ${{ github.run_id }}' | |
| iids=$(python workflows/check.py '${{ github.run_id }}') | |
| echo $iids | |
| echo "iids=$iids" >> $GITHUB_OUTPUT | |
| catalog: | |
| name: update catalog | |
| needs: [upload, check] | |
| if: always() # always run, so we never skip the check | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Successful uploads | |
| run: | | |
| echo ${{ needs.check.outputs.iids }} | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -r workflows/requirements.catalog.txt | |
| - name: Update catalog | |
| run: | | |
| python workflows/update-catalog.py "${{ needs.check.outputs.iids }}" |