produce_data.yml

name: "[internal] Produce analytic data"

on:
  workflow_call:
  workflow_dispatch:
    inputs:
      test_workflow_run_id:
        description: "Unique GitHub workflow run ID to use for data"
        type: number
      test_workflow_run_attempt:
        description: "Run attempt of the workflow run"
        default: 1
        type: number
  workflow_run:
    workflows: # List workflow that we want to collect data for
      - "Build and Test"
      - "On PR"
      - "On push"
    types:
      - completed

jobs:

  produce-cicd-data:

    runs-on: ubuntu-latest
    env:
        GH_TOKEN: ${{ github.token }}

    steps:

      - uses: actions/checkout@v4

      - name: Output (safe) pipeline values
        run: |
          echo "pipeline_id (id / run #): ${{ github.run_id }}/${{ github.run_attempt }}"
          echo "submissions_ts: "
          echo "start_ts: "
          echo "end_ts: "
          echo "name: ${{ github.workflow }}, but rk recommended name w/out @: ${{ github.workflow_ref }}"
          echo "trigger: ${{ github.event_name }}"
          echo "sha: ${{ github.sha }}"
          echo "(triggering) author/actor: ${{ github.actor }}"
          echo "author/actor: ${{ github.triggering_actor }}"
          echo "orchestrator: github (Static)"
          echo "docker_image: ${{ job.container.image }}"
          echo "build duration is post-process"

      - name: Get workflow run_id attempt number to analyze
        id: get-run-id-and-attempt
        shell: bash
        run: |
          event_name="${{ github.event_name }}"
          if [[ "$event_name" == "workflow_dispatch" ]]; then
            run_id="${{ inputs.test_workflow_run_id }}"
            attempt_number="${{ inputs.test_workflow_run_attempt }}"
          elif [[ "$event_name" == "workflow_run" ]]; then
            run_id="${{ github.event.workflow_run.id }}"
            attempt_number="${{ github.event.workflow_run.run_attempt }}"
            [[ -z "$run_id" ]] && { echo "run_id is empty" ; exit 1; }
            [[ -z "$attempt_number" ]] && { echo "attempt_number is empty" ; exit 1; }
          else
            echo "Unknown event name" && exit 1
          fi

          echo $run_id
          echo $attempt_number
          echo "run-id=$run_id" >> "$GITHUB_OUTPUT"
          echo "attempt-number=$attempt_number" >> "$GITHUB_OUTPUT"

      - name: Collect workflow artifact and job logs
        shell: bash
        run: |
          infra/download_workflow_data.sh ${{ github.repository }} ${{ steps.get-run-id-and-attempt.outputs.run-id }} ${{ steps.get-run-id-and-attempt.outputs.attempt-number }}

      - uses: actions/setup-python@v5
        with:
          python-version: '3.8'
          cache: 'pip'
          cache-dependency-path: 'infra/requirements.txt'

      - name: Install infra dependencies
        run: pip install -r infra/requirements.txt

      - name: Create JSON
        env:
          PYTHONPATH: ${{ github.workspace }}
        run: |
          python3 infra/src/generate_data.py --run_id ${{ steps.get-run-id-and-attempt.outputs.run-id }}
          # Workaround: Copy file to avoid GH upload filename limitations
          cp pipeline_${{ steps.get-run-id-and-attempt.outputs.run-id }}*.json pipelinecopy_${{ steps.get-run-id-and-attempt.outputs.run-id }}.json

      - name: Upload cicd data
        uses: ./.github/actions/upload-data-via-sftp
        if: ${{ github.event_name == 'workflow_run' }}
        with:
          ssh-private-key: ${{ secrets.SFTP_CICD_WRITER_KEY }}
          sftp-batchfile: .github/actions/upload-data-via-sftp/cicd_data_batchfile.txt
          username: ${{ secrets.SFTP_CICD_WRITER_USERNAME }}
          hostname: ${{ secrets.SFTP_CICD_WRITER_HOSTNAME }}

      - name: Upload workflow run data, even on failure
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: workflow-run-data
          path: |
            if-no-files-found: warn
            path: |
              pipelinecopy_${{ steps.get-run-id-and-attempt.outputs.run-id }}.json
              generated/cicd/${{ steps.get-run-id-and-attempt.outputs.run-id }}/workflow.json
              generated/cicd/${{ steps.get-run-id-and-attempt.outputs.run-id }}/workflow_jobs.json