upload-s3.yaml

name: Search and upload
on:
  workflow_dispatch:
    inputs:
        iid:
          required: true

jobs:

  parse:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    outputs:
      matrix: ${{ steps.parse.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches and tags.
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r workflows/requirements.parsing.txt
      - name: Run parse command
        id: parse
        run: |
          echo 'Parsing!'
          echo "matrix=$(python workflows/parse-instance-ids.py '${{ github.event.inputs.iid }}')" >> $GITHUB_OUTPUT
          echo "${{ env.matrix }}"

  upload:
    name: upload ${{ matrix.iid }}
    needs: parse
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        iid: ${{ fromJson(needs.parse.outputs.matrix) }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches and tags.
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r workflows/requirements.upload.txt
      - name: Upload ${{ matrix.iid }}
        env:
          ESGF_USER: ${{ secrets.ESGF_USER }}
          ESGF_PASSWORD: ${{ secrets.ESGF_PASSWORD }}
        id: upload
        run: |
          echo "Uploading!"
          echo "iid: ${{ matrix.iid }}"
          python workflows/upload-s3.py ${{ matrix.iid }}   #'${{ github.event.client_payload.slash_command.args.all }}'
  check:
    name: check
    needs: upload
    if: always() # always run, so we never skip the check
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    outputs:
      iids: ${{ steps.check.outputs.iids }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches and tags.
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r workflows/requirements.check.txt
      - name: Run check command
        id: check
        env:
          PAT: ${{ secrets.PAT }}
        run: |
          echo 'Checking run_id ${{ github.run_id }}'
          iids=$(python workflows/check.py '${{ github.run_id }}')
          echo $iids
          echo "iids=$iids" >> $GITHUB_OUTPUT

  catalog:
    name: update catalog
    needs: [upload, check]
    if: always() # always run, so we never skip the check
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - name: Successful uploads
        run: |
          echo ${{ needs.check.outputs.iids }}
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches and tags.
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r workflows/requirements.catalog.txt
      - name: Update catalog
        run: |
          python workflows/update-catalog.py "${{ needs.check.outputs.iids }}"