From bfb026537706a63c9f4b7e187d68e78f83588e96 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Thu, 30 May 2024 16:23:56 +0200 Subject: [PATCH 01/12] feat: create a test thermometer Signed-off-by: Jaime Silvela --- ...bb7d97458ca95fd421f325f3896ed239d5d3f.json | 2 +- ...76000698c46de8e7f9763a1a0fe63f70fd5f8.json | 2 +- ...dfe622e341f84dd3d1df1cffee843e1fb84a0.json | 2 +- ...bb7d97458ca95fd421f325f3896ed239d5d3f.json | 2 +- ...76000698c46de8e7f9763a1a0fe63f70fd5f8.json | 2 +- ...dfe622e341f84dd3d1df1cffee843e1fb84a0.json | 2 +- summarize_test_results.py | 161 +++++++++++++++--- test_summary.py | 8 +- 8 files changed, 149 insertions(+), 32 deletions(-) diff --git a/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json b/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json index 9ad974f..ab2b262 100644 --- a/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json +++ b/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json @@ -10,7 +10,7 @@ "postgres_kind": "PostgreSQL", "matrix_id": "id1", "postgres_version": "11.1", - "k8s_version": "22", + "k8s_version": "1.22", "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" diff --git a/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json b/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json index 41628dd..fba938d 100644 --- a/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json +++ b/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json @@ -10,7 +10,7 @@ "postgres_kind": "PostgreSQL", "matrix_id": "id1", "postgres_version": "11.1", - "k8s_version": "22", + "k8s_version": "1.22", "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" diff --git a/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json b/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json index d2d2d66..b7eb4d4 100644 --- a/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json +++ b/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json @@ -10,7 +10,7 @@ "postgres_kind": "PostgreSQL", "matrix_id": "id1", "postgres_version": "11.1", - "k8s_version": "22", + "k8s_version": "1.22", "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" diff --git a/few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json b/few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json index 9ad974f..ab2b262 100644 --- a/few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json +++ b/few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json @@ -10,7 +10,7 @@ "postgres_kind": "PostgreSQL", "matrix_id": "id1", "postgres_version": "11.1", - "k8s_version": "22", + "k8s_version": "1.22", "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" diff --git a/few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json b/few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json index 00e9065..0f3beaf 100644 --- a/few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json +++ b/few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json @@ -10,7 +10,7 @@ "postgres_kind": "PostgreSQL", "matrix_id": "id1", "postgres_version": "11.1", - "k8s_version": "22", + "k8s_version": "1.22", "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" diff --git a/few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json b/few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json index d2d2d66..b7eb4d4 100644 --- a/few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json +++ b/few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json @@ -10,7 +10,7 @@ "postgres_kind": "PostgreSQL", "matrix_id": "id1", "postgres_version": "11.1", - "k8s_version": "22", + "k8s_version": "1.22", "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" diff --git a/summarize_test_results.py b/summarize_test_results.py index 81d9570..d98554c 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -133,6 +133,24 @@ def is_test_artifact(test_entry): return True +def compress_kubernetes_version(test_entry): + """ensure the k8s_version field contains only the minor release + of kubernetes, and that the presence or absence of an initial "v" is ignored. + Otherwise, ciclops can over-represent failure percentages and k8s releases tested + """ + k8s = test_entry["k8s_version"] + if k8s[0] == "v": + k8s = k8s[1:] + frags = k8s.split(".") + if len(frags) <= 2: + test_entry["k8s_version"] = k8s + return test_entry + else: + minor = ".".join(frags[0:2]) + test_entry["k8s_version"] = minor + return test_entry + + def combine_postgres_data(test_entry): """combines Postgres kind and version of the test artifact to a single field called `pg_version` @@ -286,7 +304,12 @@ def count_bucketed_by_special_failures(test_results, by_special_failures): if failure not in by_special_failures["total"]: by_special_failures["total"][failure] = 0 - for key in ["tests_failed", "k8s_versions_failed", "pg_versions_failed", "platforms_failed"]: + for key in [ + "tests_failed", + "k8s_versions_failed", + "pg_versions_failed", + "platforms_failed", + ]: if failure not in by_special_failures[key]: by_special_failures[key][failure] = {} @@ -397,6 +420,7 @@ def compute_test_summary(test_dir): # skipping non-artifacts continue test_results = combine_postgres_data(parsed) + test_results = compress_kubernetes_version(test_results) total_runs = 1 + total_runs if is_failed(test_results): @@ -468,12 +492,47 @@ def compile_overview(summary): } +def metric_name(metric): + metric_name = { + "by_test": "Tests", + "by_k8s": "Kubernetes versions", + "by_postgres": "Postgres versions", + "by_platform": "Platforms", + } + return metric_name[metric] + + +def compute_systematic_failures_on_metric(summary, metric): + """tests if there are items within the metric that have systematic failures. + For example, in the "by_test" metric, if there is a test with systematic failures. + Returns a boolean to indicate there are systematic failures, and an output string + with the failures + """ + output = "" + has_systematic_failure_in_metric = False + for bucket_hits in summary[metric]["failed"].items(): + bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs + failures = summary[metric]["failed"][bucket] + runs = summary[metric]["total"][bucket] + if failures == runs and failures > 1: + if not has_systematic_failure_in_metric: + output += f"{metric_name(metric)} with systematic failures:\n\n" + has_systematic_failure_in_metric = True + output += f"- {bucket}: ({failures} out of {runs} tests failed)\n" + if has_systematic_failure_in_metric: + # add a newline after at the end of the list of failures before starting the + # next metric + output += f"\n" + return True, output + return False, "" + + def format_alerts(summary, embed=True, file_out=None): """print Alerts for tests that have failed systematically If the `embed` argument is true, it will produce a fragment of Markdown to be included with the action summary. - Otherwise, it will be output as plain text. + Otherwise, it will be output as plain text intended for stand-alone use. We want to capture: - all test combinations failed (if this happens, no more investigation needed) @@ -496,28 +555,12 @@ def format_alerts(summary, embed=True, file_out=None): print("EOF", file=file_out) return - metric_name = { - "by_test": "Tests", - "by_k8s": "Kubernetes versions", - "by_postgres": "Postgres versions", - "by_platform": "Platforms", - } - output = "" for metric in ["by_test", "by_k8s", "by_postgres", "by_platform"]: - has_failure_in_metric = False - for bucket_hits in summary[metric]["failed"].items(): - bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs - failures = summary[metric]["failed"][bucket] - runs = summary[metric]["total"][bucket] - if failures == runs and failures > 1: - if not has_failure_in_metric: - output += f"{metric_name[metric]} with systematic failures:\n\n" - has_failure_in_metric = True - has_systematic_failures = True - output += f"- {bucket}: ({failures} out of {runs} tests failed)\n" - if has_failure_in_metric: - output += f"\n" + has_alerts, out = compute_systematic_failures_on_metric(summary, metric) + if has_alerts: + has_systematic_failures = True + output += out if not has_systematic_failures: return @@ -531,6 +574,77 @@ def format_alerts(summary, embed=True, file_out=None): print("EOF", file=file_out) +def compute_semaphore(success_percent, embed=True): + """create a semaphore light summarizing the success percent. + If set to `embed`, an emoji will be used. Else, a textual representation + of a slackmoji is used. + """ + if embed: + if success_percent >= 95: + return "🟢" + elif success_percent >= 60: + return "🟡" + else: + return "🔴" + else: + if success_percent >= 95: + return ":large_green_circle:" + elif success_percent >= 60: + return ":large_yellow_circle:" + else: + return ":red_circle:" + + +def compute_thermometer_on_metric(summary, metric, embed=True): + """computes a summary per item in the metric, with the success percentage + and a color coding based on said percentage + """ + + output = "" + output += f"{metric_name(metric)} thermometer:\n\n" + for bucket_hits in summary[metric]["failed"].items(): + bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs + failures = summary[metric]["failed"][bucket] + runs = summary[metric]["total"][bucket] + success_percent = (1 - failures / runs) * 100 + color = compute_semaphore(success_percent, embed) + output += f"- {color} - {bucket}: {round(success_percent,1)}% success.\t({failures} out of {runs} tests failed)\n" + output += f"\n" + return output + + +def format_thermometer(summary, embed=True, file_out=None): + """print thermometer with the percentage of success for a set of metrics. + e.g. per-platform and per-kubernetes + + If the `embed` argument is true, it will produce a fragment of Markdown + to be included with the action summary. + Otherwise, it will be output as plain text intended for stand-alone use. + """ + + if summary["total_run"] == summary["total_failed"]: + if embed: + print(f"## Alerts\n", file=file_out) + print(f"All test combinations failed\n", file=file_out) + else: + print("alerts< Date: Thu, 30 May 2024 16:54:58 +0200 Subject: [PATCH 02/12] chore: version and doc updates Signed-off-by: Jaime Silvela --- .github/workflows/overflow-test.yaml | 4 ++-- .github/workflows/test.yaml | 5 ++++- .github/workflows/unit-test.yaml | 4 ++-- README.md | 20 ++++++++++++++++++-- action.yaml | 2 ++ 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/overflow-test.yaml b/.github/workflows/overflow-test.yaml index bfedf83..00eafa0 100644 --- a/.github/workflows/overflow-test.yaml +++ b/.github/workflows/overflow-test.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest name: Overflow Test steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Generate Test Summary id: generate-summary @@ -23,7 +23,7 @@ jobs: - name: If there is an overflow summary, archive it if: ${{steps.generate-summary.outputs.Overflow}} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{steps.generate-summary.outputs.Overflow}} path: ${{steps.generate-summary.outputs.Overflow}} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7cd87f3..90ca794 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest name: Smoke Test steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Generate Test Summary id: generate-summary @@ -18,3 +18,6 @@ jobs: - name: If there are alerts, echo them if: ${{steps.generate-summary.outputs.alerts}} run: echo "${{steps.generate-summary.outputs.alerts}}" + + - name: Echo the thermometer + run: echo "${{steps.generate-summary.outputs.thermometer}}" diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index bbcceb7..dd4d2f1 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -7,10 +7,10 @@ jobs: runs-on: ubuntu-latest name: Unit test steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' diff --git a/README.md b/README.md index f0ef6e1..d75ef97 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,12 @@ watch over Continuous Integration pipelines for all eternity. ## Outputs -Two outputs might be produced: +Up to three outputs might be produced: + +- `thermometer`: this will contain stand-alone text with a color-coded list + of test metrics that can serve as an overview of the state of the test suite + on CI/CD. This is generated on every execution of Ciclops. + It is *always* generated. - `alerts`: this will contain stand-alone text with systematic failures detected by CIclops. It is meant to enable further steps in the calling @@ -97,7 +102,11 @@ There are two advanced cases we want to call attention to: called `Overflow`. 2. Monitoring with chatops \ - CIclops will create a series of alerts when systematic failures are detected. + Ciclops will generate a "thermometer" on every execution, offering a + color-coded overview of the test health. This thermometer is included in + the GitHub summary, and in addition, is exported as an output in plain + text, which can be sent via chatops. + In addition, Ciclops will create a series of alerts when systematic failures are detected. By "systematic", we mean cases such as: - all test combinations have failed @@ -131,6 +140,13 @@ The following snippet shows how to use these features: path: ${{steps.generate-summary.outputs.Overflow}} retention-days: 7 + - name: Get a slack message with the Ciclops thermometer + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_USERNAME: cnpg-bot + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_MESSAGE: ${{steps.generate-summary.outputs.thermometer}} + - name: If there are alerts, send them over Slack if: ${{steps.generate-summary.outputs.alerts}} uses: rtCamp/action-slack-notify@v2 diff --git a/action.yaml b/action.yaml index abb6486..3127fde 100644 --- a/action.yaml +++ b/action.yaml @@ -22,6 +22,8 @@ inputs: outputs: alerts: description: 'Any systematic failures found by CIclops' + thermometer: + description: 'A color-coded health meter' Overflow: description: 'The name of the file where the full report was written, on oveflow' runs: From 24c28095e4daa9e68f25ca479f7dd77c58f65b03 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Thu, 30 May 2024 17:44:39 +0200 Subject: [PATCH 03/12] chore: cleanup Signed-off-by: Jaime Silvela --- summarize_test_results.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index d98554c..4146fea 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -622,16 +622,6 @@ def format_thermometer(summary, embed=True, file_out=None): Otherwise, it will be output as plain text intended for stand-alone use. """ - if summary["total_run"] == summary["total_failed"]: - if embed: - print(f"## Alerts\n", file=file_out) - print(f"All test combinations failed\n", file=file_out) - else: - print("alerts< Date: Thu, 30 May 2024 18:23:29 +0200 Subject: [PATCH 04/12] chore: cap alerts view for stand-alone Signed-off-by: Jaime Silvela --- summarize_test_results.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index 4146fea..b544cc8 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -502,14 +502,19 @@ def metric_name(metric): return metric_name[metric] -def compute_systematic_failures_on_metric(summary, metric): +def compute_systematic_failures_on_metric(summary, metric, embed=True): """tests if there are items within the metric that have systematic failures. For example, in the "by_test" metric, if there is a test with systematic failures. Returns a boolean to indicate there are systematic failures, and an output string - with the failures + with the failures. + + The `embed` argument controls the output. If True (default) it computes the full list + of alerts for the metric. If False, it will cap at 2 rows with alerts, so as not to + to flood the chatops client. """ output = "" has_systematic_failure_in_metric = False + counter = 0 for bucket_hits in summary[metric]["failed"].items(): bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs failures = summary[metric]["failed"][bucket] @@ -518,7 +523,12 @@ def compute_systematic_failures_on_metric(summary, metric): if not has_systematic_failure_in_metric: output += f"{metric_name(metric)} with systematic failures:\n\n" has_systematic_failure_in_metric = True - output += f"- {bucket}: ({failures} out of {runs} tests failed)\n" + if counter < 2: + output += f"- {bucket}: ({failures} out of {runs} tests failed)\n" + counter += 1 + else: + output += f"- ...and more. See full story in GH Test Summary\n" + break if has_systematic_failure_in_metric: # add a newline after at the end of the list of failures before starting the # next metric @@ -557,7 +567,7 @@ def format_alerts(summary, embed=True, file_out=None): output = "" for metric in ["by_test", "by_k8s", "by_postgres", "by_platform"]: - has_alerts, out = compute_systematic_failures_on_metric(summary, metric) + has_alerts, out = compute_systematic_failures_on_metric(summary, metric, embed) if has_alerts: has_systematic_failures = True output += out From ba10a7bae2faf3556551a2a73640b34fcdcaca13 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Thu, 30 May 2024 18:28:52 +0200 Subject: [PATCH 05/12] fix: cap alerts view only for stand-alone Signed-off-by: Jaime Silvela --- summarize_test_results.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index b544cc8..8e50634 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -523,12 +523,12 @@ def compute_systematic_failures_on_metric(summary, metric, embed=True): if not has_systematic_failure_in_metric: output += f"{metric_name(metric)} with systematic failures:\n\n" has_systematic_failure_in_metric = True - if counter < 2: - output += f"- {bucket}: ({failures} out of {runs} tests failed)\n" - counter += 1 - else: + if counter >= 2 and not embed: output += f"- ...and more. See full story in GH Test Summary\n" break + else: + output += f"- {bucket}: ({failures} out of {runs} tests failed)\n" + counter += 1 if has_systematic_failure_in_metric: # add a newline after at the end of the list of failures before starting the # next metric From 8ddbba84df582f02fa58b94adf7441de5b2d416e Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Thu, 30 May 2024 18:39:16 +0200 Subject: [PATCH 06/12] chore: fix nit Signed-off-by: Jaime Silvela --- summarize_test_results.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index 8e50634..441e8e5 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -610,8 +610,7 @@ def compute_thermometer_on_metric(summary, metric, embed=True): and a color coding based on said percentage """ - output = "" - output += f"{metric_name(metric)} thermometer:\n\n" + output = f"{metric_name(metric)} thermometer:\n\n" for bucket_hits in summary[metric]["failed"].items(): bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs failures = summary[metric]["failed"][bucket] From 1efaaef03039b39bfe86af0e1d6004d200922fc7 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Fri, 31 May 2024 09:59:28 +0200 Subject: [PATCH 07/12] chore: nits Signed-off-by: Jaime Silvela --- README.md | 4 ++-- summarize_test_results.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d75ef97..cd0c191 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,6 @@ Up to three outputs might be produced: - `thermometer`: this will contain stand-alone text with a color-coded list of test metrics that can serve as an overview of the state of the test suite on CI/CD. This is generated on every execution of Ciclops. - It is *always* generated. - `alerts`: this will contain stand-alone text with systematic failures detected by CIclops. It is meant to enable further steps in the calling @@ -106,7 +105,8 @@ There are two advanced cases we want to call attention to: color-coded overview of the test health. This thermometer is included in the GitHub summary, and in addition, is exported as an output in plain text, which can be sent via chatops. - In addition, Ciclops will create a series of alerts when systematic failures are detected. + In addition, Ciclops will create a series of alerts when systematic failures + are detected. By "systematic", we mean cases such as: - all test combinations have failed diff --git a/summarize_test_results.py b/summarize_test_results.py index 441e8e5..cfbd69c 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -632,6 +632,7 @@ def format_thermometer(summary, embed=True, file_out=None): """ output = "" + # we only test the "by_platform" metric for the thermometer, at the moment for metric in ["by_platform"]: output += compute_thermometer_on_metric(summary, metric, embed) From 0c0f66cf3bca211730fbb1bbc0286f48761bd8bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Mon, 3 Jun 2024 14:22:34 +0200 Subject: [PATCH 08/12] chore: nits and PEP8 style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- summarize_test_results.py | 21 +++++++++++---------- test_summary.py | 3 ++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index cfbd69c..04608ed 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -209,7 +209,7 @@ def track_time_taken(test_results, test_times, suite_times): if duration < test_times["min"][name]: test_times["min"][name] = duration - # track suite time. + # Track test suite timings. # For each platform-matrix branch, track the earliest start and the latest end platform = test_results["platform"] if platform not in suite_times["start_time"]: @@ -261,7 +261,7 @@ def count_bucketed_by_code(test_results, by_failing_code): name = test_results["name"] if test_results["error"] == "" or test_results["state"] == "ignoreFailed": return - # it does not make sense to show failing code that is outside of the test + # it does not make sense to show failing code that is outside the test, # so we skip special failures if not is_normal_failure(test_results): return @@ -493,13 +493,13 @@ def compile_overview(summary): def metric_name(metric): - metric_name = { + metric_type = { "by_test": "Tests", "by_k8s": "Kubernetes versions", "by_postgres": "Postgres versions", "by_platform": "Platforms", } - return metric_name[metric] + return metric_type[metric] def compute_systematic_failures_on_metric(summary, metric, embed=True): @@ -510,7 +510,7 @@ def compute_systematic_failures_on_metric(summary, metric, embed=True): The `embed` argument controls the output. If True (default) it computes the full list of alerts for the metric. If False, it will cap at 2 rows with alerts, so as not to - to flood the chatops client. + flood the ChatOps client. """ output = "" has_systematic_failure_in_metric = False @@ -587,7 +587,7 @@ def format_alerts(summary, embed=True, file_out=None): def compute_semaphore(success_percent, embed=True): """create a semaphore light summarizing the success percent. If set to `embed`, an emoji will be used. Else, a textual representation - of a slackmoji is used. + of a Slack emoji is used. """ if embed: if success_percent >= 95: @@ -617,7 +617,8 @@ def compute_thermometer_on_metric(summary, metric, embed=True): runs = summary[metric]["total"][bucket] success_percent = (1 - failures / runs) * 100 color = compute_semaphore(success_percent, embed) - output += f"- {color} - {bucket}: {round(success_percent,1)}% success.\t({failures} out of {runs} tests failed)\n" + output += f"- {color} - {bucket}: {round(success_percent, 1)}% success.\t" + output += f"({failures} out of {runs} tests failed)\n" output += f"\n" return output @@ -787,7 +788,7 @@ def format_by_code(summary, structure, file_out=None): for bucket in sorted_by_code: tests = ", ".join(summary["by_code"]["tests"][bucket].keys()) - # replace newlines and pipes to avoid interference with markdown tables + # replace newlines and pipes to avoid interference with Markdown tables errors = ( summary["by_code"]["errors"][bucket] .replace("\n", "
") @@ -1121,8 +1122,8 @@ def format_short_test_summary(summary, file_out=None): format_test_summary(test_summary, file_out=f) if args.limit: print("with GITHUB_STEP_SUMMARY limit", args.limit) - bytes = os.stat(os.getenv("GITHUB_STEP_SUMMARY")).st_size - if bytes > args.limit: + summary_bytes = os.stat(os.getenv("GITHUB_STEP_SUMMARY")).st_size + if summary_bytes > args.limit: # we re-open the STEP_SUMMARY with "w" to wipe out previous content with open(os.getenv("GITHUB_STEP_SUMMARY"), "w") as f: format_short_test_summary(test_summary, file_out=f) diff --git a/test_summary.py b/test_summary.py index e81bb14..cea0e4e 100644 --- a/test_summary.py +++ b/test_summary.py @@ -35,7 +35,8 @@ def test_compute_summary(self): summary["by_code"]["tests"], { "/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80": { - "InitDB settings - initdb custom post-init SQL scripts -- can find the tables created by the post-init SQL queries": True + "InitDB settings - initdb custom post-init SQL scripts -- can find the" + " tables created by the post-init SQL queries": True } }, "unexpected summary", From f5666d6324f30c390f70f02112d362c1dbaf564c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Tue, 4 Jun 2024 16:08:17 +0200 Subject: [PATCH 09/12] tests: add unit tests for thermometer and systematic failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- test_summary.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/test_summary.py b/test_summary.py index cea0e4e..128dbd3 100644 --- a/test_summary.py +++ b/test_summary.py @@ -20,19 +20,20 @@ class TestIsFailed(unittest.TestCase): + summary = summarize_test_results.compute_test_summary("few-artifacts") + def test_compute_summary(self): self.maxDiff = None - summary = summarize_test_results.compute_test_summary("few-artifacts") - self.assertEqual(summary["total_run"], 3) - self.assertEqual(summary["total_failed"], 1) + self.assertEqual(self.summary["total_run"], 3) + self.assertEqual(self.summary["total_failed"], 1) self.assertEqual( - summary["by_code"]["total"], + self.summary["by_code"]["total"], {"/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80": 1}, "unexpected summary", ) self.assertEqual( - summary["by_code"]["tests"], + self.summary["by_code"]["tests"], { "/Users/myuser/repos/cloudnative-pg/tests/e2e/initdb_test.go:80": { "InitDB settings - initdb custom post-init SQL scripts -- can find the" @@ -42,20 +43,20 @@ def test_compute_summary(self): "unexpected summary", ) self.assertEqual( - summary["by_matrix"], {"total": {"id1": 3}, "failed": {"id1": 1}} + self.summary["by_matrix"], {"total": {"id1": 3}, "failed": {"id1": 1}} ) self.assertEqual( - summary["by_k8s"], {"total": {"1.22": 3}, "failed": {"1.22": 1}} + self.summary["by_k8s"], {"total": {"1.22": 3}, "failed": {"1.22": 1}} ) self.assertEqual( - summary["by_platform"], {"total": {"local": 3}, "failed": {"local": 1}} + self.summary["by_platform"], {"total": {"local": 3}, "failed": {"local": 1}} ) self.assertEqual( - summary["by_postgres"], + self.summary["by_postgres"], {"total": {"PostgreSQL-11.1": 3}, "failed": {"PostgreSQL-11.1": 1}}, ) self.assertEqual( - summary["suite_durations"], + self.summary["suite_durations"], { "end_time": { "local": {"id1": datetime.datetime(2021, 11, 29, 18, 31, 7)} @@ -66,6 +67,24 @@ def test_compute_summary(self): }, ) + def test_compute_thermometer(self): + self.maxDiff = None + thermometer = summarize_test_results.compute_thermometer_on_metric(self.summary, "by_platform") + + self.assertEqual( + thermometer, + "Platforms thermometer:\n\n" + "- 🟡 - local: 66.7% success.\t(1 out of 3 tests failed)\n\n" + ) + + def test_compute_systematic_failures(self): + self.maxDiff = None + + for metric in ["by_test", "by_k8s", "by_postgres", "by_platform"]: + has_alerts, out = summarize_test_results.compute_systematic_failures_on_metric(self.summary, metric) + self.assertEqual(has_alerts, False) + self.assertEqual(out, "") + if __name__ == "__main__": unittest.main() From 27ef09315cc7d5cc5d4787028381fb3daa01df53 Mon Sep 17 00:00:00 2001 From: John Long Date: Tue, 4 Jun 2024 16:25:09 -0400 Subject: [PATCH 10/12] iterating over total and not failed hits Signed-off-by: John Long --- summarize_test_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index 04608ed..de0b70c 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -611,7 +611,7 @@ def compute_thermometer_on_metric(summary, metric, embed=True): """ output = f"{metric_name(metric)} thermometer:\n\n" - for bucket_hits in summary[metric]["failed"].items(): + for bucket_hits in summary[metric]["total"].items(): bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs failures = summary[metric]["failed"][bucket] runs = summary[metric]["total"][bucket] From bc95e163acd51476d740da5800bcd1efee11b59f Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Wed, 5 Jun 2024 08:49:52 +0200 Subject: [PATCH 11/12] fix: dereference when no failures in bucket Signed-off-by: Jaime Silvela --- summarize_test_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/summarize_test_results.py b/summarize_test_results.py index de0b70c..2a1bc4a 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -613,7 +613,9 @@ def compute_thermometer_on_metric(summary, metric, embed=True): output = f"{metric_name(metric)} thermometer:\n\n" for bucket_hits in summary[metric]["total"].items(): bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs - failures = summary[metric]["failed"][bucket] + failures = 0 + if bucket in summary[metric]["failed"]: + failures = summary[metric]["failed"][bucket] runs = summary[metric]["total"][bucket] success_percent = (1 - failures / runs) * 100 color = compute_semaphore(success_percent, embed) From ffd816ed78565407489739645507442a3e864b29 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Wed, 5 Jun 2024 12:52:04 +0200 Subject: [PATCH 12/12] chore: add some documentation Signed-off-by: Jaime Silvela --- DEVELOPERS_DEVELOPERS_DEVELOPERS.md | 35 ++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md index bf36689..014338d 100644 --- a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md +++ b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md @@ -1,8 +1,25 @@ -# Building and testing locally +# Building, testing, releasing The `ciclops` GitHub Action runs using a Docker container that encapsulates the Python script that does the CI test analysis. +## Releasing + +We recommend that users of Ciclops use released versions rather than `main`. +For testing, it may be convenient to [use a full SHA](#testing-within-a-calling-github-workflow). + +The procedure for cutting a release: + +1. Decide on the version number (following semVer) +1. Update the [Release notes file](ReleaseNotes.md), following the convention + in the file, i.e. the version number included in the section, and the release + date in the first line +1. Review and merge the release notes, and create and push a new tag with the + desired version number +1. Cut a new release in [GitHub](https://github.com/cloudnative-pg/ciclops/releases/new), + choosing the recent tag, and pasting the relevant content from the + Release Notes file (no need for the release date line). + ## Developing and testing You can test directly with the Python code on the `example-artifacts` directory, @@ -72,6 +89,22 @@ CIclops has the beginning of a unit test suite. You can run it with: python3 -m unittest ``` +## Testing within a calling GitHub workflow + +Even with unit tests and local tests, it's good to try Ciclops code out from a +client workflow. We can use a full length commit SHA to test out changes, +before cutting out a new release. +See the [GitHub document on using third party actions](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-third-party-actions). + +Example: +``` yaml + - name: Compute the E2E test summary + id: generate-summary + uses: cloudnative-pg/ciclops@ + with: + artifact_directory: test-artifacts/da +``` + ## How it works The files in this repository are needed for the Dockerfile to build and run, of