TT-1832 Add slack notification for nightly run with detailed flaky te…

…st summary (#15446) * Update slack notification for flaky test detector * Bump flakeguard and support shuffle flag * bump flakeguard * update notification * Fix * Add separate job for nightly flakeguard workflow * trigger * fix * Fix * Fix codeowners issue * fix * fix * Fix * Fix codeowners issue * revert: fail test * Fix all tests path * fix notification * Fix format * Revert "revert: fail test" This reverts commit 6abd379. * debug * bump * Fix * bump * fix * fix * bump * revert * Fix * bump flakeguard to fix codeowners * bump * fix * Bump flakeguard * fail test on purpose * fix * Fix * Fix * fix * fix lint
smartcontractkit · Dec 4, 2024 · 52676f1 · 52676f1
1 parent c36f30d
commit 52676f1
Show file tree

Hide file tree

Showing 3 changed files with 86 additions and 16 deletions.
diff --git a/.github/workflows/flakeguard-nightly.yml b/.github/workflows/flakeguard-nightly.yml
@@ -12,7 +12,7 @@ jobs:
     uses: ./.github/workflows/flakeguard.yml
     with:
       repoUrl: 'https://github.com/smartcontractkit/chainlink'
-      baseRef: 'origin/develop'
+      headRef: 'develop'
       projectPath: '.'
       maxPassRatio: '1.0'
       runAllTests: true

diff --git a/.github/workflows/flakeguard-on-demand.yml b/.github/workflows/flakeguard-on-demand.yml
@@ -65,7 +65,7 @@ jobs:
       runAllTests: ${{ inputs.runAllTests }}
       findByTestFilesDiff: ${{ inputs.findByTestFilesDiff }}
       findByAffectedPackages: ${{ inputs.findByAffectedPackages }}
-      slackNotificationAfterTestsChannelId: ${{ inputs.slack_notification_after_tests_channel_id }}
+      slackNotificationAfterTestsChannelId: ${{ inputs.slack_notification_after_tests_channel_id || 'C07TRF65CNS' }} #flaky-test-detector-notifications
       extraArgs: ${{ inputs.extraArgs }}
     secrets:
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

diff --git a/.github/workflows/flakeguard.yml b/.github/workflows/flakeguard.yml
@@ -13,7 +13,7 @@ on:
         description: 'The path to the project to run the flaky test detection.'
         default: '.'  
       baseRef:
-        required: true
+        required: false
         type: string
         description: 'The base reference or branch to compare changes for detecting flaky tests.'
       headRef:
@@ -66,6 +66,7 @@ env:
   DEFAULT_RUNNER: 'ubuntu-latest' # The default runner to use for running tests.
   UPLOAD_ALL_TEST_RESULTS: ${{ fromJson(inputs.extraArgs)['upload_all_test_results'] || 'false' }} # Whether to upload all test results as artifacts.
   PRINT_FAILED_TESTS: ${{ fromJson(inputs.extraArgs)['print_failed_tests'] || 'false' }} # Whether to print failed tests in the GitHub console.
+
 
 jobs:
   get-tests:
@@ -100,7 +101,7 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@f03577def97a20a38c0e1de1cbe7fc98d416dd86 # [email protected]
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@9e40f2765df01f20b3bf53f0fb3ead920e3a1f4a # [email protected]
 
       - name: Find new or updated test packages
         if: ${{ inputs.runAllTests == false }}
@@ -259,7 +260,7 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@f03577def97a20a38c0e1de1cbe7fc98d416dd86 # [email protected]
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@9e40f2765df01f20b3bf53f0fb3ead920e3a1f4a # [email protected]
 
       - name: Run tests with flakeguard
         shell: bash
@@ -306,7 +307,7 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@f03577def97a20a38c0e1de1cbe7fc98d416dd86 # [email protected]
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@9e40f2765df01f20b3bf53f0fb3ead920e3a1f4a # [email protected]
 
       - name: Set combined test results
         id: set_test_results
@@ -341,10 +342,27 @@ jobs:
             fi
             echo "Failed tests count: $FAILED_TESTS_COUNT"
             echo "failed_tests_count=$FAILED_TESTS_COUNT" >> "$GITHUB_OUTPUT"
+
+            # Calculate failed ratio (failed / non-failed tests ratio in %)
+            if [ "$ALL_TESTS_COUNT" -gt 0 ]; then
+              NON_FAILED_COUNT=$((ALL_TESTS_COUNT - FAILED_TESTS_COUNT))
+              
+              if [ "$NON_FAILED_COUNT" -gt 0 ]; then
+                FAILED_RATIO=$(awk "BEGIN {printf \"%.2f\", ($FAILED_TESTS_COUNT / $NON_FAILED_COUNT) * 100}")
+              else
+                FAILED_RATIO=0
+              fi
+            else
+              NON_FAILED_COUNT=0
+              FAILED_RATIO=0
+            fi
+            echo "Failed tests ratio: $FAILED_RATIO%"
+            echo "failed_ratio=$FAILED_RATIO" >> "$GITHUB_OUTPUT"
           else
             echo "No test results directory found."
             echo "all_tests_count=0" >> "$GITHUB_OUTPUT"
             echo "failed_tests_count=0" >> "$GITHUB_OUTPUT"
+            echo "failed_ratio=0" >> "$GITHUB_OUTPUT"
           fi   
 
       - name: Tests Summary
@@ -368,47 +386,47 @@ jobs:
         with:
           path: all_tests.md
           name: all-summary.md
-          retention-days: 7
+          retention-days: 90
 
       - name: Upload All Test Results as Artifact
         if: ${{ fromJson(steps.set_test_results.outputs.all_tests_count) > 0 }}
         uses: actions/[email protected]
         with:
           path: all_tests.json
           name: all-test-results.json
-          retention-days: 7
+          retention-days: 90
 
       - name: Upload Failed Tests Summary as Artifact
         if: ${{ fromJson(steps.set_test_results.outputs.all_tests_count) > 0 }}
         uses: actions/[email protected]
         with:
           path: failed_tests.md
           name: failed-summary.md
-          retention-days: 7
+          retention-days: 90
 
       - name: Upload Failed Test Results as Artifact
         if: ${{ fromJson(steps.set_test_results.outputs.failed_tests_count) > 0 }}
         uses: actions/[email protected]
         with:
           path: failed_tests.json
           name: failed-test-results.json
-          retention-days: 7        
+          retention-days: 90        
 
       - name: Upload Failed Test Logs as Artifact
         if: ${{ fromJson(steps.set_test_results.outputs.failed_tests_count) > 0 }}
         uses: actions/[email protected]
         with:
           path: failed_test_logs.json
           name: failed-test-logs.json
-          retention-days: 7             
+          retention-days: 90             
 
       - name: Upload All Test Results as Artifact
         if: ${{ fromJson(steps.set_test_results.outputs.all_tests_count) > 0 && env.UPLOAD_ALL_TEST_RESULTS == 'true' }}
         uses: actions/[email protected]
         with:
           path: all_tests.json
           name: all-test-results.json
-          retention-days: 7
+          retention-days: 90
 
       - name: Post comment on PR if flaky tests found
         if: ${{ fromJson(steps.set_test_results.outputs.failed_tests_count) > 0 && github.event_name == 'pull_request' }}
@@ -427,9 +445,61 @@ jobs:
               body: commentBody
             });
 
-      - name: Send Slack message
+      - name: Send Slack message for failed tests
+        if: ${{ inputs.slackNotificationAfterTestsChannelId != '' &&  fromJson(steps.set_test_results.outputs.failed_tests_count) > 0 }}
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        with:
+          channel-id: ${{ inputs.slackNotificationAfterTestsChannelId }}
+          payload: |
+            {
+              "attachments": [
+                {
+                  "color": "#C62828",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "Flaky Test Detector for `${{ steps.set_project_path_pretty.outputs.path }}` project - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Was cancelled :warning:' || 'Passed :white_check_mark:' }}"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "${{ inputs.runAllTests == true && format('Ran all tests for `{0}` branch.', inputs.headRef) || format('Ran changed tests between `{0}` and `{1}` (`{2}`).', inputs.baseRef, needs.get-tests.outputs.git_head_short_sha, env.GIT_HEAD_REF) }}"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "fields": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "Total Failed Tests: ${{ steps.set_test_results.outputs.failed_tests_count }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "Failed to Non-Failed Ratio: ${{ steps.set_test_results.outputs.failed_ratio }}%"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "${{ format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details> | <{3}/compare/{4}...{5}#files_bucket|Compare Changes>{6}', github.server_url, github.repository, github.run_id, inputs.repoUrl, inputs.baseRef, needs.get-tests.outputs.git_head_sha, github.event_name == 'pull_request' && format(' | <{0}|View PR>', github.event.pull_request.html_url) || '') }}"
+                      }
+                    }
+                  ]
+                }
+              ]
+            }            
+
+      - name: Send general Slack message
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        if: ${{ inputs.slackNotificationAfterTestsChannelId != '' && fromJson(steps.set_test_results.outputs.all_tests_count) > 0 }}
+        if: ${{ inputs.slackNotificationAfterTestsChannelId != '' && fromJson(steps.set_test_results.outputs.failed_tests_count) == 0 && fromJson(steps.set_test_results.outputs.all_tests_count) > 0 }}
         id: slack
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -452,14 +522,14 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "Ran changed tests between `${{ inputs.baseRef }}` and `${{ needs.get-tests.outputs.git_head_short_sha }}` (`${{ env.GIT_HEAD_REF }}`)."
+                        "text": "${{ inputs.runAllTests == true && format('Ran all tests for `{0}` branch.', env.GIT_HEAD_REF) || format('Ran changed tests between `{0}` and `{1}` (`{2}`).', inputs.baseRef, needs.get-tests.outputs.git_head_short_sha, env.GIT_HEAD_REF) }}"
                       }
                     },
                     {
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "${{ format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details> | <{3}/compare/{4}...{5}#files_bucket|Compare Changes>{6}', github.server_url, github.repository, github.run_id, inputs.repoUrl, inputs.baseRef, needs.get-tests.outputs.git_head_sha, github.event_name == 'pull_request' && format(' | <{0}|View PR>', github.event.pull_request.html_url) || '') }}"
+                        "text": "${{ inputs.runAllTests == true && format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details>', github.server_url, github.repository, github.run_id) || format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details> | <{3}/compare/{4}...{5}#files_bucket|Compare Changes>{6}', github.server_url, github.repository, github.run_id, inputs.repoUrl, inputs.baseRef, needs.get-tests.outputs.git_head_sha, github.event_name == 'pull_request' && format(' | <{0}|View PR>', github.event.pull_request.html_url) || '') }}"
                       }
                     }                 
                   ]