testing pr - Np jw test illumina genotyping arrays #274
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Illumina Genotyping Array | |
| # Controls when the workflow will run | |
| on: | |
| #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING | |
| # push: | |
| # branches: | |
| # - kp_GHA_Terra_auth_PD-2682 | |
| pull_request: | |
| branches: [ "develop", "staging", "master" ] | |
| # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml | |
| #################################### | |
| # SET PIPELINE SPECIFIC PATHS HERE # | |
| #################################### | |
| paths: | |
| - 'pipelines/broad/genotyping/illumina/**' | |
| - 'tasks/**' | |
| - 'verification/**' | |
| - '.github/workflows/test_illumina_genotyping_array.yml' | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| useCallCache: | |
| description: 'Use call cache (default: true)' | |
| required: false | |
| default: "true" | |
| updateTruth: | |
| description: 'Update truth files (default: false)' | |
| required: false | |
| default: "false" | |
| testType: | |
| description: 'Specify the type of test (Plumbing or Scientific)' | |
| required: true | |
| env: | |
| PROJECT_NAME: WARP | |
| # Github repo name | |
| REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| jobs: | |
| run_pipeline: | |
| runs-on: ubuntu-latest | |
| # Add "id-token" with the intended permissions. | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| # actions/checkout MUST come before auth | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.ref }} | |
| - id: 'auth' | |
| name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| token_format: 'access_token' | |
| # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting | |
| # This is provided by the DevOps team - do not change! | |
| workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' | |
| # This is our tester service account | |
| service_account: '[email protected]' | |
| access_token_lifetime: '3600' # seconds, default is 3600 | |
| access_token_scopes: 'profile, email, openid' | |
| # ... further steps are automatically authenticated | |
| - name: Check working directory | |
| run: | | |
| echo "Current directory:" | |
| pwd | |
| ls -lht | |
| - name: Verify checked-out branch | |
| run: | | |
| echo "Branch: ${{ github.head_ref }}" | |
| - name: Submit job, poll status, and get outputs | |
| id: pipeline_run | |
| run: | | |
| # Set common environment variables | |
| TOKEN="${{ steps.auth.outputs.access_token }}" | |
| NAMESPACE="warp-pipelines" | |
| WORKSPACE="WARP Tests" | |
| USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}" | |
| UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}" | |
| TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}" | |
| CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S") | |
| ######################################## | |
| # SET PIPELINE SPECIFIC VARIABLES HERE # | |
| ######################################## | |
| PIPELINE_NAME="TestIlluminaGenotypingArray" | |
| PIPELINE_DIR="pipelines/broad/genotyping/illumina" | |
| # TODO: Need to set the truth and result paths appropriately | |
| # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch | |
| # We may want to keep the truth and resuts buckets separate for TTL reasons | |
| TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master" | |
| RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME" | |
| # Function to call the Firecloud API using the firecloud_api.py script | |
| firecloud_action() { | |
| python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}" | |
| } | |
| # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false) | |
| if [ "$USE_CALL_CACHE" == "true" ]; then | |
| USE_CALL_CACHE_BOOL=true | |
| else | |
| USE_CALL_CACHE_BOOL=false | |
| fi | |
| # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false) | |
| if [ "$UPDATE_TRUTH" = "true" ]; then | |
| UPDATE_TRUTH_BOOL=true | |
| else | |
| UPDATE_TRUTH_BOOL=false | |
| fi | |
| # Create the submission_data.json file which will be the same for all inputs | |
| SUBMISSION_DATA_FILE="submission_data.json" | |
| # Use a heredoc to generate the JSON file content dynamically | |
| cat <<EOF > "$SUBMISSION_DATA_FILE" | |
| { | |
| "methodConfigurationNamespace": "warp-pipelines", | |
| "methodConfigurationName": "$PIPELINE_NAME", | |
| "useCallCache": $USE_CALL_CACHE_BOOL, | |
| "deleteIntermediateOutputFiles": false, | |
| "useReferenceDisks": true, | |
| "memoryRetryMultiplier": 1.2, | |
| "workflowFailureMode": "NoNewCalls", | |
| "userComment": "Automated submission", | |
| "ignoreEmptyOutputs": false | |
| } | |
| EOF | |
| echo "Created submission data file: $SUBMISSION_DATA_FILE" | |
| # Initialize variables to aggregate statuses and outputs | |
| ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---" | |
| ALL_OUTPUTS="" | |
| # Initialize arrays to track submission and workflow statuses | |
| declare -a SUBMISSION_IDS | |
| declare -A WORKFLOW_STATUSES | |
| # Loop through each file in the appropriate test inputs directory | |
| INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE" | |
| for input_file in "$INPUTS_DIR"/*.json; do | |
| echo "Processing input file: $input_file" | |
| test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \ | |
| --results_path "$RESULTS_PATH" \ | |
| --inputs_json "$input_file" \ | |
| --update_truth "$UPDATE_TRUTH_BOOL") | |
| echo "Uploading test input file: $test_input_file" | |
| firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}" | |
| echo "Submitting job for input file: $input_file" | |
| SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE") | |
| if [ -z "$SUBMISSION_ID" ]; then | |
| echo "Submission failed for input file: $input_file" | |
| continue | |
| fi | |
| echo "Submission ID: $SUBMISSION_ID" | |
| SUBMISSION_IDS+=("$SUBMISSION_ID") | |
| done | |
| echo "Monitoring the status of submitted workflows..." | |
| for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do | |
| echo "Polling submission status for Submission ID: $SUBMISSION_ID" | |
| RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") | |
| if [ -z "$RESPONSE" ]; then | |
| echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID" | |
| continue | |
| fi | |
| # Parse and store workflow statuses | |
| WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]') | |
| echo "Statuses for submission $SUBMISSION_ID:" | |
| echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | |
| # Append to aggregate statuses | |
| WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION | |
| # retrieve workflow outputs | |
| echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..." | |
| for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do | |
| WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME") | |
| ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n' | |
| done | |
| done | |
| # Generate final summary tables with hyperlinks for Submission IDs | |
| echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY | |
| for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do | |
| # Generate the Terra URL for the submission | |
| SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID" | |
| # Add the Submission ID as a hyperlink | |
| echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY | |
| # Add the workflows and statuses for this submission | |
| echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY | |
| # Add a blank line for separation | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| done | |
| - name: Print Summary on Success | |
| if: success() | |
| run: | | |
| echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY | |
| - name: Print Summary on Failure | |
| if: failure() | |
| run: | | |
| echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY |