zxcron-extended-test-suite.yaml

##
# Copyright (C) 2023-2025 Hedera Hashgraph, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

name: "ZXCron: Extended Test Suite"
on:
  workflow_dispatch:
  schedule:
    # Runs Extended Test Suite every three hours
    - cron: '0 */3 * * *'

permissions:
  id-token: write
  actions: write
  pull-requests: write
  statuses: write
  checks: write
  contents: read

defaults:
  run:
    shell: bash

env:
  XTS_CANDIDATE_TAG: "xts-candidate"
  XTS_PASS_GREP_PATTERN: "xts-pass-*"
  PROMOTED_GREP_PATTERN: "build-.{5}"

jobs:
  fetch-xts-candidate:
    name: Fetch XTS Candidate Tag
    runs-on: network-node-linux-medium
    outputs:
      xts-tag-exists: ${{ steps.check-tags-exist.outputs.xts-tag-exists }}
      xts-tag-commit: ${{ steps.check-tags-exist.outputs.xts-tag-commit }}
      xts-tag-commit-author: ${{ steps.check-tags-exist.outputs.xts-tag-commit-author }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3
        with:
          egress-policy: audit

      # Checkout the latest from dev
      - name: Checkout Code
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: '0'
          ref: main
          token: ${{ secrets.GH_ACCESS_TOKEN }}

      # Check if the xts-candidate tag exists
      # the command git branch --contains xts-tag-commit | grep --quiet <default branch>
      # will return an exit code of 1 if the tagged commit is not found on the main
      # branch.
      - name: Check for tags
        id: check-tags-exist
        env:
          GH_TOKEN: ${{ github.token }}
          XTS_PASS_PATTERN: ${{ env.XTS_PASS_GREP_PATTERN }}
          BUILD_PROMO_PATTERN: ${{ env.PROMOTED_GREP_PATTERN }}
        run: |
          # Check if the tag exists and if so grab its commit id
          set +e
          XTS_COMMIT=$(git rev-list -n 1 "${XTS_CANDIDATE_TAG}") >/dev/null 2>&1
          XTS_COMMIT_FOUND="${?}"
          set -e

          # Cancel out if the tag does not exist
          if [[ "${XTS_COMMIT_FOUND}" -ne 0 ]]; then
            gh run cancel ${{ github.run_id }}
          fi

          # Check if this commit has already been tagged xts-pass-* or build-*
          set +e
          XTS_PASS_TAGGED=$(git tag --contains "${XTS_COMMIT}" | grep -E "${XTS_PASS_PATTERN}")
          BUILD_PROMOTED_TAGGED=$(git tag --contains "${XTS_COMMIT}" | grep -E "${BUILD_PROMO_PATTERN}")
          set -e

          # Use -n; if the BUILD_PROMOTED_TAGGED/XTS_PASS_TAGGED flags are not empty than the commit has been tagged.
          if [[ -n "${XTS_PASS_TAGGED}" || -n "${BUILD_PROMOTED_TAGGED}" ]]; then
            gh run cancel ${{ github.run_id }}
          fi

          # Check if the tag exists on the main branch
          set +e
          git branch --contains "${XTS_COMMIT}" | grep --quiet main >/dev/null 2>&1
          BRANCH_ON_DEVELOP="${?}"
          set -e

          # Get commit author
          AUTHOR_NAME=$(git log -1 --format='%an' "${XTS_COMMIT}")
          AUTHOR_EMAIL=$(git log -1 --format='%ae' "${XTS_COMMIT}")

          # If the tag exists on the Main Branch set the output variables as appropriate
          # Otherwise cancel out
          if [[ "${BRANCH_ON_DEVELOP}" -eq 0 ]]; then
            echo "xts-tag-exists=true" >> $GITHUB_OUTPUT
            echo "xts-tag-commit=${XTS_COMMIT}" >> $GITHUB_OUTPUT
            echo "xts-tag-commit-author=${AUTHOR_NAME} <${AUTHOR_EMAIL}>" >> $GITHUB_OUTPUT
            echo "### XTS-Candidate commit found" >>  $GITHUB_STEP_SUMMARY
            echo "xts-tag-commit=${XTS_COMMIT}" >> $GITHUB_STEP_SUMMARY

            git push --delete origin "${XTS_CANDIDATE_TAG}"
            git tag -d "${XTS_CANDIDATE_TAG}"
          else
            gh run cancel ${{ github.run_id }}
          fi

  extended-test-suite:
    name: Execute eXtended Test Suite
    uses: ./.github/workflows/node-zxc-compile-application-code.yaml
    needs: fetch-xts-candidate
    if: ${{ needs.fetch-xts-candidate.result == 'success' && needs.fetch-xts-candidate.outputs.xts-tag-exists == 'true' &&
            !github.event.workflow_dispatch.repository.fork }}
    with:
      custom-job-label: Execute eXtended Test Suite
      enable-timing-sensitive-tests: true
      enable-time-consuming-tests: true
      enable-hammer-tests: true
      enable-hapi-tests-time-consuming: true
      enable-network-log-capture: true
      ref: ${{ needs.fetch-xts-candidate.outputs.xts-tag-commit }}
    secrets:
      access-token: ${{ secrets.GITHUB_TOKEN }}
      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}

  abbreviated-panel:
    name: JRS Panel
    uses: ./.github/workflows/zxc-jrs-regression.yaml
    needs: fetch-xts-candidate
    if: ${{ needs.fetch-xts-candidate.result == 'success' && needs.fetch-xts-candidate.outputs.xts-tag-exists == 'true' &&
            !github.event.workflow_dispatch.repository.fork }}
    with:
      custom-job-name: "Platform SDK"
      panel-config: "configs/suites/GCP-PRCheck-Abbrev-4N.json"
      ref: ${{ needs.fetch-xts-candidate.outputs.xts-tag-commit }} # pass the xts-candidate tag to the JRS panel for checkout
      branch-name: ${{ github.head_ref || github.ref_name }}
      base-branch-name: ${{ github.base_ref || '' }}
      slack-results-channel: "regression-test"
      slack-summary-channel: "regression-test"
      use-branch-for-slack-channel: false
    secrets:
      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}
      jrs-ssh-user-name: ${{ secrets.PLATFORM_JRS_SSH_USER_NAME }}
      jrs-ssh-key-file: ${{ secrets.PLATFORM_JRS_SSH_KEY_FILE }}
      gcp-project-number: ${{ secrets.PLATFORM_GCP_PROJECT_NUMBER }}
      gcp-sa-key-contents: ${{ secrets.PLATFORM_GCP_KEY_FILE }}
      slack-api-token: ${{ secrets.PLATFORM_SLACK_API_TOKEN }}
      grafana-agent-username: ${{ secrets.GRAFANA_AGENT_USERNAME }}
      grafana-agent-password: ${{ secrets.GRAFANA_AGENT_PASSWORD }}

  hedera-node-jrs-panel:
    name: Hedera Node JRS Panel
    uses: ./.github/workflows/zxc-jrs-regression.yaml
    needs: fetch-xts-candidate
    if: ${{ needs.fetch-xts-candidate.result == 'success' && needs.fetch-xts-candidate.outputs.xts-tag-exists == 'true' &&
            !github.event.workflow_dispatch.repository.fork }}
    with:
      custom-job-name: "Abbrev Update Test"
      ref: ${{ needs.fetch-xts-candidate.outputs.xts-tag-commit }} # pass the xts-candidate tag to the JRS panel for checkout
      branch-name: ${{ github.head_ref || github.ref_name }}
      hedera-tests-enabled: true
      use-branch-for-slack-channel: false
      panel-config: "configs/services/suites/daily/GCP-Daily-Services-Abbrev-DAB-Update-4N-2C.json"
    secrets:
      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
      jrs-ssh-user-name: ${{ secrets.PLATFORM_JRS_SSH_USER_NAME }}
      jrs-ssh-key-file: ${{ secrets.PLATFORM_JRS_SSH_KEY_FILE }}
      gcp-project-number: ${{ secrets.PLATFORM_GCP_PROJECT_NUMBER }}
      gcp-sa-key-contents: ${{ secrets.PLATFORM_GCP_KEY_FILE }}
      slack-api-token: ${{ secrets.PLATFORM_SLACK_API_TOKEN }}
      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}
      grafana-agent-username: ${{ secrets.GRAFANA_AGENT_USERNAME }}
      grafana-agent-password: ${{ secrets.GRAFANA_AGENT_PASSWORD }}

  tag-for-promotion:
    name: Tag as XTS-Passing
    runs-on: network-node-linux-medium
    needs:
      - abbreviated-panel
      - extended-test-suite
      - fetch-xts-candidate
      - hedera-node-jrs-panel
    if: ${{ needs.abbreviated-panel.result == 'success' || needs.extended-test-suite.result == 'success' || needs.hedera-node-jrs-panel.result == 'success' }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3
        with:
          egress-policy: audit

      - name: Checkout Tagged Code
        id: checkout_tagged_code
        if: ${{ needs.fetch-xts-candidate.outputs.xts-tag-exists == 'true' }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: '0'
          ref: ${{ needs.fetch-xts-candidate.outputs.xts-tag-commit }}
          token: ${{ secrets.GH_ACCESS_TOKEN }}

      - name: Import GPG Key
        id: gpg_importer
        uses: step-security/ghaction-import-gpg@6c8fe4d0126a59d57c21f87c9ae5dd3451fa3cca # v6.1.0
        with:
          git_commit_gpgsign: true
          git_tag_gpgsign: true
          git_user_signingkey: true
          gpg_private_key: ${{ secrets.SVCS_GPG_KEY_CONTENTS }}
          passphrase: ${{ secrets.SVCS_GPG_KEY_PASSPHRASE }}

      # Now that the XTS suite has run we should be able to tag for promotion
      - name: Tag for XTS promotion
        run: |
          EPOCH_TIME=$(date +%s)
          TAG=xts-pass-${EPOCH_TIME}
          git tag --annotate ${TAG} --message "chore: tagging commit for build candidate promotion"
          git push --set-upstream origin --tags
          echo "### Commit Tagged for Promotion" >>  $GITHUB_STEP_SUMMARY
          echo "promotion-tag=${TAG}" >> $GITHUB_STEP_SUMMARY

  report-failure:
    name: Report XTS execution failure
    runs-on: network-node-linux-medium
    needs:
      - abbreviated-panel
      - extended-test-suite
      - fetch-xts-candidate
      - hedera-node-jrs-panel
      - tag-for-promotion

    if: ${{ (needs.abbreviated-panel.result != 'success' ||
            needs.extended-test-suite.result != 'success' ||
            needs.fetch-xts-candidate.result != 'success' ||
            needs.hedera-node-jrs-panel.result != 'success' ||
            needs.tag-for-promotion.result != 'success') &&
            !cancelled() && always() }}

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: '0'
          ref: main
          token: ${{ secrets.GH_ACCESS_TOKEN }}

      - name: Collect run logs in a log file
        env:
          GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
        run: |
          for job_id in $(gh run view ${{ github.run_id }} --json jobs --jq '.jobs | map(.databaseId) | .[0:-1] | .[]'); do
            echo "Fetching logs for job $job_id..."

            current_job_name=$(gh run view ${{ github.run_id }} --json jobs | jq --argjson job_id "$job_id" -r '.jobs[] | select(.databaseId == $job_id) | .name')

            echo "Logs for job $current_job_name :" >> run.log

            gh api \
            -H "Accept: application/vnd.github+json" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            /repos/hashgraph/hedera-services/actions/jobs/$job_id/logs >> run.log
          done

      - name: Upload log as artifact
        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
        with:
          path: run.log

      - name: Report failure (PagerDuty)
        run: |
          curl --request 'POST' \
          --url 'https://events.pagerduty.com/v2/enqueue' \
          --header 'Content-Type: application/json' \
          --data '{
            "payload": {
              "summary": "Hedera Services - eXtended Test Suite Failure",
              "severity": "error",
              "source": "GitHub Actions - eXtended Test Suite",
              "custom_details": {
                "Fetch XTS Candidate Tag": "${{ needs.fetch-xts-candidate.result }}",
                "Execute eXtended Test Suite": "${{ needs.extended-test-suite.result }}",
                "JRS Panel": "${{ needs.abbreviated-panel.result }}",
                "Hedera Node JRS Panel": "${{ needs.hedera-node-jrs-panel.result }}",
                "Tag as XTS-Passing": "${{ needs.tag-for-promotion.result }}",
                "Source Commit": "<${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.fetch-xts-candidate.outputs.xts-tag-commit }}>",
                "Commit Author": "${{ needs.fetch-xts-candidate.outputs.xts-tag-commit-author }}",
                "Workflow Run": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>"
              }
            },
            "routing_key": "${{ secrets.PAGERDUTY_CITR_INTEGRATION_KEY }}",
            "event_action": "trigger"
          }'

      - name: Report failure (slack)
        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
        with:
          webhook: ${{ secrets.SLACK_CITR_FAILURES_WEBHOOK }}
          webhook-type: incoming-webhook
          payload-templated: true
          payload: |
            {
              "attachments": [
                {
                  "color": "#FF0000",
                  "blocks": [
                    {
                      "type": "header",
                      "text": {
                        "type": "plain_text",
                        "text": ":grey_exclamation: Hedera Services - eXtended Test Suite Error Report",
                        "emoji": true
                      }
                    },
                    {
                      "type": "divider"
                    },
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": "*XTS Job Resulted in failure. See status below.*"
                      },
                      "fields": [
                        {
                          "type": "plain_text",
                          "text": "Fetch XTS Candidate Tag"
                        },
                        {
                          "type": "plain_text",
                          "text": "${{ needs.fetch-xts-candidate.result }}"
                        },
                        {
                          "type": "plain_text",
                          "text": "Execute eXtended Test Suite"
                        },
                        {
                          "type": "plain_text",
                          "text": "${{ needs.extended-test-suite.result }}"
                        },
                        {
                          "type": "plain_text",
                          "text": "JRS Panel"
                        },
                        {
                          "type": "plain_text",
                          "text": "${{ needs.abbreviated-panel.result }}"
                        },
                        {
                          "type": "plain_text",
                          "text": "Hedera Node JRS Panel"
                        },
                        {
                          "type": "plain_text",
                          "text": "${{ needs.hedera-node-jrs-panel.result }}"
                        },
                        {
                          "type": "plain_text",
                          "text": "Tag as XTS-Passing"
                        },
                        {
                          "type": "plain_text",
                          "text": "${{ needs.tag-for-promotion.result }}"
                        }
                      ]
                    },
                    {
                      "type": "divider"
                    },
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": "*Workflow and Commit Information*"
                      },
                      "fields": [
                        {
                          "type": "mrkdwn",
                          "text": "*Source Commit*:"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "<${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.fetch-xts-candidate.outputs.xts-tag-commit }}>"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*Commit author*:"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "${{ needs.fetch-xts-candidate.outputs.xts-tag-commit-author }}"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*Workflow run ID*:"
                        },
                        {
                          "type": "mrkdwn",
                          "text": " ${{ github.run_id }}"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*Workflow run URL*:"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>"
                        }
                      ]
                    }
                  ]
                }
              ]
            }