From df4c81a16e258202fd6a91ede3d1abb23f271771 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 13 Jun 2024 15:53:04 +0000 Subject: [PATCH] Creating Issue Reports for Flaky Test Failures in Gradle Check (#436) Signed-off-by: Prudhvi Godithi (cherry picked from commit 2e95cddcdf42ea02f45025b9ff1941240d7ba82f) Signed-off-by: github-actions[bot] --- build.gradle | 26 +-- src/gradlecheck/CreateMarkDownTable.groovy | 50 +++++ .../FetchPostMergeFailedTestClass.groovy | 81 ++++++++ .../FetchPostMergeFailedTestName.groovy | 130 ++++++++++++ .../FetchPostMergeTestGitReference.groovy | 105 ++++++++++ src/gradlecheck/FetchTestPullRequests.groovy | 105 ++++++++++ src/gradlecheck/OpenSearchMetricsQuery.groovy | 42 ++++ .../CreateMarkDownTableTest.groovy | 50 +++++ .../FetchPostMergeFailedTestClassTest.groovy | 104 ++++++++++ .../FetchPostMergeFailedTestNameTest.groovy | 194 ++++++++++++++++++ .../FetchPostMergeTestGitReferenceTest.groovy | 130 ++++++++++++ .../FetchTestPullRequestsTest.groovy | 130 ++++++++++++ .../TestCreateBuildFailureGithubIssue.groovy | 31 +-- tests/jenkins/TestCreateGithubIssue.groovy | 53 +++-- .../TestUpdateBuildFailuresIssues.groovy | 22 +- ...reGithubExistingIssueCheck_Jenkinsfile.txt | 15 +- ...ateBuildFailureGithubIssue_Jenkinsfile.txt | 22 +- .../jobs/CreateGithubIssue_Jenkinsfile.txt | 7 +- .../jenkins/jobs/EditGithubIssue_Jenkinsfile | 28 +++ .../EditGithubIssue_Jenkinsfile_IssueBody.txt | 15 ++ .../UpdateBuildFailureIssue_Jenkinsfile.txt | 28 +-- .../CreateGithubIssueLibTester.groovy | 20 +- vars/createGithubIssue.groovy | 44 ++-- vars/gradleCheckFlakyTestChecker.groovy | 59 ++++++ 24 files changed, 1365 insertions(+), 126 deletions(-) create mode 100644 src/gradlecheck/CreateMarkDownTable.groovy create mode 100644 src/gradlecheck/FetchPostMergeFailedTestClass.groovy create mode 100644 src/gradlecheck/FetchPostMergeFailedTestName.groovy create mode 100644 src/gradlecheck/FetchPostMergeTestGitReference.groovy create mode 100644 src/gradlecheck/FetchTestPullRequests.groovy create mode 100644 src/gradlecheck/OpenSearchMetricsQuery.groovy create mode 100644 tests/gradlecheck/CreateMarkDownTableTest.groovy create mode 100644 tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy create mode 100644 tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy create mode 100644 tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy create mode 100644 tests/gradlecheck/FetchTestPullRequestsTest.groovy create mode 100644 tests/jenkins/jobs/EditGithubIssue_Jenkinsfile create mode 100644 tests/jenkins/jobs/EditGithubIssue_Jenkinsfile_IssueBody.txt create mode 100644 vars/gradleCheckFlakyTestChecker.groovy diff --git a/build.gradle b/build.gradle index adf37a353..47b2bcb72 100644 --- a/build.gradle +++ b/build.gradle @@ -27,6 +27,7 @@ dependencies { testImplementation group: 'org.yaml', name: 'snakeyaml', version: '2.0' testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.24.2' testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.12.0' } configurations.all { @@ -40,13 +41,13 @@ configurations.all { sourceSets { main { groovy { - srcDirs = ['src/jenkins'] + srcDirs = ['src/jenkins', 'src/gradlecheck'] } } test { groovy { - srcDirs = ['tests/jenkins'] + srcDirs = ['tests/jenkins', 'tests/gradlecheck'] } } @@ -80,8 +81,8 @@ sharedLibrary { test { testLogging { - events "failed" - exceptionFormat "full" + events "failed" + exceptionFormat "full" } if (project.hasProperty('pipeline.stack.write')) { systemProperty 'pipeline.stack.write', project.getProperty('pipeline.stack.write') @@ -105,14 +106,15 @@ jacocoTestReport { afterEvaluate { classDirectories.from = fileTree( - dir: "$buildDir/jacoco/classpathdumps", - includes: [ - '**/*_Jenkinsfile.*', - '**/jenkins/*' - ], - excludes: [ - '**/*\$_get_closure*' - ] + dir: "$buildDir/jacoco/classpathdumps", + includes: [ + '**/*_Jenkinsfile.*', + '**/jenkins/*', + '**/gradlecheck/*' + ], + excludes: [ + '**/*\$_get_closure*' + ] ) } diff --git a/src/gradlecheck/CreateMarkDownTable.groovy b/src/gradlecheck/CreateMarkDownTable.groovy new file mode 100644 index 000000000..2fe61e95e --- /dev/null +++ b/src/gradlecheck/CreateMarkDownTable.groovy @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +class CreateMarkDownTable { + String failedTest + ArrayList tableData + ArrayList additionalPullRequests + + CreateMarkDownTable(String failedTest, List> tableData, List additionalPullRequests) { + this.failedTest = failedTest + this.tableData = tableData + this.additionalPullRequests = additionalPullRequests + } + + def createMarkdownTable() { + + def tableHeader = """ +## Flaky Test Report for `${this.failedTest}` + +Noticed the `${this.failedTest}` has some flaky, failing tests that failed during **post-merge actions**. + +### Details + +| Git Reference | Merged Pull Request | Build Details | Test Name | +|---------------|----------------------|---------------|-----------| +""" + def tableRows = this.tableData.collect { row -> + "| ${row.gitReference} | ${row.pullRequestLink} | ${row.buildDetailLink} | ${row.testNames.join('

')} |" + }.join("\n") + + def additionalPRSection = """ +\nThe other pull requests, besides those involved in post-merge actions, that contain failing tests with the `${this.failedTest}` class are: + +${this.additionalPullRequests.collect { pr -> "- [${pr}](https://github.com/opensearch-project/OpenSearch/pull/${pr})" }.join('\n')} + +For more details on the failed tests refer to [OpenSearch Gradle Check Metrics](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) dashboard. +""" + + return tableHeader + tableRows + additionalPRSection + } + +} \ No newline at end of file diff --git a/src/gradlecheck/FetchPostMergeFailedTestClass.groovy b/src/gradlecheck/FetchPostMergeFailedTestClass.groovy new file mode 100644 index 000000000..f4d6660c6 --- /dev/null +++ b/src/gradlecheck/FetchPostMergeFailedTestClass.groovy @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import groovy.json.JsonOutput +import gradlecheck.OpenSearchMetricsQuery + +class FetchPostMergeFailedTestClass { + String metricsUrl + String awsAccessKey + String awsSecretKey + String awsSessionToken + def script + + FetchPostMergeFailedTestClass(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.script = script + } + + def getQuery() { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR" + ] + ] + ] + ] + ] + ], + aggregations: [ + test_class_keyword_agg: [ + terms: [ + field: "test_class", + size: 500 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + + def getPostMergeFailedTestClass() { + def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery()) + def keys = jsonResponse.aggregations.test_class_keyword_agg.buckets.collect { it.key } + return keys + } +} diff --git a/src/gradlecheck/FetchPostMergeFailedTestName.groovy b/src/gradlecheck/FetchPostMergeFailedTestName.groovy new file mode 100644 index 000000000..1a4d67148 --- /dev/null +++ b/src/gradlecheck/FetchPostMergeFailedTestName.groovy @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import groovy.json.JsonOutput +import gradlecheck.OpenSearchMetricsQuery + +class FetchPostMergeFailedTestName { + String metricsUrl + String awsAccessKey + String awsSecretKey + String awsSessionToken + def script + + FetchPostMergeFailedTestName(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.script = script + } + + def getQuery(testName, gitReference) { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + "git_reference.keyword": [ + query: gitReference, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + test_name_keyword_agg: [ + terms: [ + field: "test_name", + size: 500 + ] + ], + build_number_agg: [ + terms: [ + field: "build_number", + size: 500 + ] + ], + pull_request_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + + } + def getPostMergeFailedTestName(testName, gitReference) { + return new OpenSearchMetricsQuery(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName, gitReference)) + } +} diff --git a/src/gradlecheck/FetchPostMergeTestGitReference.groovy b/src/gradlecheck/FetchPostMergeTestGitReference.groovy new file mode 100644 index 000000000..c8f53acd4 --- /dev/null +++ b/src/gradlecheck/FetchPostMergeTestGitReference.groovy @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import groovy.json.JsonOutput +import gradlecheck.OpenSearchMetricsQuery + +class FetchPostMergeTestGitReference { + String metricsUrl + String awsAccessKey + String awsSecretKey + String awsSessionToken + def script + + FetchPostMergeTestGitReference(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.script = script + } + + def getQuery(testName) { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + git_reference_keyword_agg: [ + terms: [ + field: "git_reference.keyword", + size: 500 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + + def getPostMergeTestGitReference(testName) { + def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName)) + def keys = jsonResponse.aggregations.git_reference_keyword_agg.buckets.collect { it.key } + return keys + } +} \ No newline at end of file diff --git a/src/gradlecheck/FetchTestPullRequests.groovy b/src/gradlecheck/FetchTestPullRequests.groovy new file mode 100644 index 000000000..329ca7bae --- /dev/null +++ b/src/gradlecheck/FetchTestPullRequests.groovy @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import groovy.json.JsonOutput +import gradlecheck.OpenSearchMetricsQuery + +class FetchTestPullRequests { + String metricsUrl + String awsAccessKey + String awsSecretKey + String awsSessionToken + def script + + FetchTestPullRequests(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.script = script + } + + def getQuery(testName) { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Pull Request", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + pull_request_keyword_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + List getTestPullRequests(testName) { + def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName)) + def keys = jsonResponse.aggregations.pull_request_keyword_agg.buckets.collect { it.key } + return keys + } + +} \ No newline at end of file diff --git a/src/gradlecheck/OpenSearchMetricsQuery.groovy b/src/gradlecheck/OpenSearchMetricsQuery.groovy new file mode 100644 index 000000000..72f6ac700 --- /dev/null +++ b/src/gradlecheck/OpenSearchMetricsQuery.groovy @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import groovy.json.JsonSlurper + +class OpenSearchMetricsQuery { + String metricsUrl + String awsAccessKey + String awsSecretKey + String awsSessionToken + def script + + OpenSearchMetricsQuery(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.script = script + } + + def fetchMetrics(String query) { + def response = script.sh( + script: """ + set -e + set +x + MONTH_YEAR=\$(date +"%m-%Y") + INDEX_NAME="gradle-check-\$MONTH_YEAR" + curl -s -XGET "${metricsUrl}/\$INDEX_NAME/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.' + """, + returnStdout: true + ).trim() + return new JsonSlurper().parseText(response) + } +} \ No newline at end of file diff --git a/tests/gradlecheck/CreateMarkDownTableTest.groovy b/tests/gradlecheck/CreateMarkDownTableTest.groovy new file mode 100644 index 000000000..c14c56df8 --- /dev/null +++ b/tests/gradlecheck/CreateMarkDownTableTest.groovy @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import org.junit.* + + +class CreateMarkDownTableTest { + + @Test + void testCreateMarkdownTableWithSampleData() { + def failedTest = "ExampleTest" + def tableData = [ + [gitReference: "abc123", pullRequestLink: "https://github.com/opensearch-project/OpenSearch/pull/1", buildDetailLink: "https://ci.opensearch.org/1", testNames: ["test1", "test2"]], + [gitReference: "def456", pullRequestLink: "https://github.com/opensearch-project/OpenSearch/pull/2", buildDetailLink: "https://ci.opensearch.org/2", testNames: ["test3"]] + ] + def additionalPullRequests = ["3", "4"] + + def createMarkDownTable = new CreateMarkDownTable(failedTest, tableData, additionalPullRequests) + + def result = createMarkDownTable.createMarkdownTable() + + def expectedOutput = """ +## Flaky Test Report for `ExampleTest` + +Noticed the `ExampleTest` has some flaky, failing tests that failed during **post-merge actions**. + +### Details + +| Git Reference | Merged Pull Request | Build Details | Test Name | +|---------------|----------------------|---------------|-----------| +| abc123 | https://github.com/opensearch-project/OpenSearch/pull/1 | https://ci.opensearch.org/1 | test1

test2 | +| def456 | https://github.com/opensearch-project/OpenSearch/pull/2 | https://ci.opensearch.org/2 | test3 | +\nThe other pull requests, besides those involved in post-merge actions, that contain failing tests with the `ExampleTest` class are: + +- [3](https://github.com/opensearch-project/OpenSearch/pull/3) +- [4](https://github.com/opensearch-project/OpenSearch/pull/4) + +For more details on the failed tests refer to [OpenSearch Gradle Check Metrics](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) dashboard. +""" + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy b/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy new file mode 100644 index 000000000..aafbc2eb2 --- /dev/null +++ b/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.mock.interceptor.MockFor + +class FetchPostMergeFailedTestClassTest { + + private FetchPostMergeFailedTestClass fetchPostMergeFailedTestClass + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "test_class_keyword_agg": { + "buckets": [ + {"key": "testClass1"}, + {"key": "testClass2"} + ] + } + } + } + """ + } + return "" + } + fetchPostMergeFailedTestClass = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR" + ] + ] + ] + ] + ] + ], + aggregations: [ + test_class_keyword_agg: [ + terms: [ + field: "test_class", + size: 500 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchPostMergeFailedTestClass.getQuery() + + assert result == expectedOutput + } + + @Test + void testGetPostMergeFailedTestClassReturnsKeys() { + def expectedOutput = ["testClass1", "testClass2"] + + def result = fetchPostMergeFailedTestClass.getPostMergeFailedTestClass() + + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy b/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy new file mode 100644 index 000000000..25824f43e --- /dev/null +++ b/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy @@ -0,0 +1,194 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +class FetchPostMergeFailedTestNameTest { + + private FetchPostMergeFailedTestName fetchPostMergeFailedTestName + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "test_name_keyword_agg": { + "buckets": [ + {"key": "testName1"}, + {"key": "testName2"} + ] + }, + "build_number_agg": { + "buckets": [ + {"key": "buildNumber1"}, + {"key": "buildNumber2"} + ] + }, + "pull_request_agg": { + "buckets": [ + {"key": "pullRequest1"}, + {"key": "pullRequest2"} + ] + } + } + } + """ + } + return "" + } + fetchPostMergeFailedTestName = new FetchPostMergeFailedTestName(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def testName = "ExampleTest" + def gitReference = "abc123" + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + "git_reference.keyword": [ + query: gitReference, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + test_name_keyword_agg: [ + terms: [ + field: "test_name", + size: 500 + ] + ], + build_number_agg: [ + terms: [ + field: "build_number", + size: 500 + ] + ], + pull_request_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchPostMergeFailedTestName.getQuery(testName, gitReference) + + assert result == expectedOutput + } + + @Test + void testGetPostMergeFailedTestNameReturnsMetrics() { + def testName = "ExampleTest" + def gitReference = "abc123" + def expectedOutput = new JsonSlurper().parseText(""" + { + "aggregations": { + "test_name_keyword_agg": { + "buckets": [ + {"key": "testName1"}, + {"key": "testName2"} + ] + }, + "build_number_agg": { + "buckets": [ + {"key": "buildNumber1"}, + {"key": "buildNumber2"} + ] + }, + "pull_request_agg": { + "buckets": [ + {"key": "pullRequest1"}, + {"key": "pullRequest2"} + ] + } + } + } + """) + + def result = fetchPostMergeFailedTestName.getPostMergeFailedTestName(testName, gitReference) + + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy b/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy new file mode 100644 index 000000000..f4882b4ad --- /dev/null +++ b/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +class FetchPostMergeTestGitReferenceTest { + + private FetchPostMergeTestGitReference fetchPostMergeTestGitReference + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "git_reference_keyword_agg": { + "buckets": [ + {"key": "gitReference1"}, + {"key": "gitReference2"} + ] + } + } + } + """ + } + return "" + } + fetchPostMergeTestGitReference = new FetchPostMergeTestGitReference(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def testName = "ExampleTest" + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + git_reference_keyword_agg: [ + terms: [ + field: "git_reference.keyword", + size: 500 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchPostMergeTestGitReference.getQuery(testName) + + assert result == expectedOutput + } + + @Test + void testGetPostMergeTestGitReferenceReturnsKeys() { + def testName = "ExampleTest" + def expectedOutput = ["gitReference1", "gitReference2"] + + def result = fetchPostMergeTestGitReference.getPostMergeTestGitReference(testName) + + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchTestPullRequestsTest.groovy b/tests/gradlecheck/FetchTestPullRequestsTest.groovy new file mode 100644 index 000000000..a2da0e1fd --- /dev/null +++ b/tests/gradlecheck/FetchTestPullRequestsTest.groovy @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +class FetchTestPullRequestsTest { + + private FetchTestPullRequests fetchTestPullRequests + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "pull_request_keyword_agg": { + "buckets": [ + {"key": "PR-1"}, + {"key": "PR-2"} + ] + } + } + } + """ + } + return "" + } + fetchTestPullRequests = new FetchTestPullRequests(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def testName = "ExampleTest" + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Pull Request", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + pull_request_keyword_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchTestPullRequests.getQuery(testName) + + assert result == expectedOutput + } + + @Test + void testGetTestPullRequestsReturnsKeys() { + def testName = "ExampleTest" + def expectedOutput = ["PR-1", "PR-2"] + + def result = fetchTestPullRequests.getTestPullRequests(testName) + + assert result == expectedOutput + } +} diff --git a/tests/jenkins/TestCreateBuildFailureGithubIssue.groovy b/tests/jenkins/TestCreateBuildFailureGithubIssue.groovy index 8a2bb0114..99f4b2123 100644 --- a/tests/jenkins/TestCreateBuildFailureGithubIssue.groovy +++ b/tests/jenkins/TestCreateBuildFailureGithubIssue.groovy @@ -6,6 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ + package jenkins.tests import jenkins.tests.BuildPipelineTest @@ -26,35 +27,19 @@ class TestCreateBuildFailureGithubIssue extends BuildPipelineTest { @Test public void testCreateGithubIssue() { - - helper.addShMock("date -d \"3 days ago\" +'%Y-%m-%d'") { script -> - return [stdout: "2023-10-24", exitValue: 0] - } - - helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number'""") { script -> - return [stdout: "", exitValue: 0] - } - helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number'""") { script -> - return [stdout: "", exitValue: 0] - } super.testPipeline('tests/jenkins/jobs/CreateBuildFailureGithubIssue_Jenkinsfile') - assertThat(getCommands('sh', 'create'), hasItem('{script=gh issue create --title \"[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0\" --body \"***Received Error***: **Error building performance-analyzer, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0ed in the next build. This might have performance impact if it keeps failing. Run the javaToolchains task for more det.yml --component performance-analyzer**.\n The distribution build for performance-analyzer has failed for version: 2.0.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\" --label autocut,v2.0.0 --label \"untriaged\" --repo https://github.com/opensearch-project/performance-analyzer.git, returnStdout=true}')) + assertThat(getCommands('sh', 'script'), hasItem('{script=gh issue comment bbb\n' + + 'ccc --repo https://github.com/opensearch-project/performance-analyzer.git --body "***Received Error***: **Error building performance-analyzer, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0ed in the next build. This might have performance impact if it keeps failing. Run the javaToolchains task for more det.yml --component performance-analyzer**.\n' + + ' The distribution build for performance-analyzer has failed for version: 2.0.0.\n' + + ' Please see build log at www.example.com/job/build_url/32/display/redirect.\n' + + ' The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details", returnStdout=true}')) } @Test - public void testExistingGithubIssue() { - helper.addShMock("date -d \"3 days ago\" +'%Y-%m-%d'") { script -> - return [stdout: "2023-10-24", exitValue: 0] - } - helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number'""") { script -> - return [stdout: "22", exitValue: 0] - } - helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number'""") { script -> - return [stdout: "", exitValue: 0] - } + public void testExistingGithubIssue_TestCreateBuildFailureGithubIssue() { super.testPipeline('tests/jenkins/jobs/CreateBuildFailureGithubIssue_Jenkinsfile', 'tests/jenkins/jobs/CreateBuildFailureGithubExistingIssueCheck_Jenkinsfile') assertThat(getCommands('println', ''), hasItem('Issue already exists, adding a comment')) - assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue comment bbb\nccc --repo https://github.com/opensearch-project/OpenSearch.git --body \"***Received Error***: **Error building OpenSearch, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component OpenSearch --snapshot**.\n The distribution build for OpenSearch has failed for version: 2.0.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\", returnStdout=true}")) + assertThat(getCommands('sh', 'script'), hasItem("""{script=gh issue comment bbb\nccc --repo https://github.com/opensearch-project/OpenSearch.git --body \"***Received Error***: **Error building OpenSearch, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component OpenSearch --snapshot**.\n The distribution build for OpenSearch has failed for version: 2.0.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\", returnStdout=true}""")) } def getCommands(method, text) { diff --git a/tests/jenkins/TestCreateGithubIssue.groovy b/tests/jenkins/TestCreateGithubIssue.groovy index 6d6648e48..d36e93011 100644 --- a/tests/jenkins/TestCreateGithubIssue.groovy +++ b/tests/jenkins/TestCreateGithubIssue.groovy @@ -29,10 +29,10 @@ class TestCreateGithubIssue extends BuildPipelineTest { @Test void testCreateGithubIssueComment() { this.registerLibTester(new CreateGithubIssueLibTester( - "https://github.com/opensearch-project/opensearch-build", - "Test GH issue title", - "Test GH issue body", - "label101" + "https://github.com/opensearch-project/opensearch-build", + "Test GH issue title", + "Test GH issue body", + "label101" )) helper.addShMock("date -d \"5 days ago\" +'%Y-%m-%d'") { script -> return [stdout: "2023-10-24", exitValue: 0] @@ -45,15 +45,15 @@ class TestCreateGithubIssue extends BuildPipelineTest { } super.testPipeline('tests/jenkins/jobs/CreateGithubIssue_Jenkinsfile') assertThat(getCommands('println', ''), hasItem("Issue already exists, adding a comment")) - assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue comment 22 --repo https://github.com/opensearch-project/opensearch-build --body \"Test GH issue body\", returnStdout=true}")) + assertThat(getCommands('sh', 'script'), hasItem("""{script=gh issue comment bbb\nccc --repo https://github.com/opensearch-project/opensearch-build --body \"Test GH issue body\", returnStdout=true}""")) } void testCreateGithubIssueCreate() { this.registerLibTester(new CreateGithubIssueLibTester( - "https://github.com/opensearch-project/opensearch-build", - "Test GH issue title", - "Test GH issue body", - "label101" + "https://github.com/opensearch-project/opensearch-build", + "Test GH issue title", + "Test GH issue body", + "label101" )) helper.addShMock("date -d \"5 days ago\" +'%Y-%m-%d'") { script -> return [stdout: "2023-10-24", exitValue: 0] @@ -71,10 +71,10 @@ class TestCreateGithubIssue extends BuildPipelineTest { void testCreateGithubIssueReOpen() { this.registerLibTester(new CreateGithubIssueLibTester( - "https://github.com/opensearch-project/opensearch-build", - "Test GH issue title", - "Test GH issue body", - "label101" + "https://github.com/opensearch-project/opensearch-build", + "Test GH issue title", + "Test GH issue body", + "label101" )) helper.addShMock("date -d \"5 days ago\" +'%Y-%m-%d'") { script -> return [stdout: "2023-10-24", exitValue: 0] @@ -93,16 +93,32 @@ class TestCreateGithubIssue extends BuildPipelineTest { void testCreateGithubIssueReOpenWithDays() { this.registerLibTester(new CreateGithubIssueLibTester( - "https://github.com/opensearch-project/opensearch-build", - "Test GH issue title", - "Test GH issue body", - "label101", - "5" + "https://github.com/opensearch-project/opensearch-build", + "Test GH issue title", + "Test GH issue body", + "label101", + "5" )) super.testPipeline('tests/jenkins/jobs/CreateGithubIssue_Jenkinsfile') assertThat(getCommands('sh', 'script'), hasItem("""{script=date -d "5 days ago" +'%Y-%m-%d'}""")) } + @Test + void testCreateGithubIssueWithBodyFile() { + helper.addShMock("date -d \"5 days ago\" +'%Y-%m-%d'") { script -> + return [stdout: "2023-10-24", exitValue: 0] + } + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title" --label label101 --json number --jq '.[0].number'""") { script -> + return [stdout: "22", exitValue: 0] + } + helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title is:closed closed:>=2023-10-24" --label label101 --json number --jq '.[0].number'""") { script -> + return [stdout: "", exitValue: 0] + } + super.testPipeline('tests/jenkins/jobs/EditGithubIssue_Jenkinsfile', 'tests/jenkins/jobs/EditGithubIssue_Jenkinsfile_IssueBody') + assertThat(getCommands('println', ''), hasItem("Issue already exists, editing the issue body")) + assertThat(getCommands('sh', 'script'), hasItem("""{script=gh issue edit bbb\nccc --repo https://github.com/opensearch-project/opensearch-build --body-file issueBody.md, returnStdout=true}""")) + } + def getCommands(method, text) { def shCommands = helper.callStack.findAll { call -> call.methodName == method @@ -114,4 +130,3 @@ class TestCreateGithubIssue extends BuildPipelineTest { return shCommands } } - diff --git a/tests/jenkins/TestUpdateBuildFailuresIssues.groovy b/tests/jenkins/TestUpdateBuildFailuresIssues.groovy index f49880c9d..681dea8d4 100644 --- a/tests/jenkins/TestUpdateBuildFailuresIssues.groovy +++ b/tests/jenkins/TestUpdateBuildFailuresIssues.groovy @@ -25,22 +25,6 @@ class TestUpdateBuildFailuresIssues extends BuildPipelineTest { super.setUp() } - @Test - public void testCreateGithubIssue() { - helper.addShMock("date -d \"3 days ago\" +'%Y-%m-%d'") { script -> - return [stdout: "2023-10-24", exitValue: 0] - } - - helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/asynchronous-search.git -S "[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> - return [stdout: "", exitValue: 0] - } - helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/asynchronous-search.git -S "[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number'""") { script -> - return [stdout: "", exitValue: 0] - } - super.testPipeline('tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile') - assertThat(getCommands('sh', 'create'), hasItem('{script=gh issue create --title \"[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0\" --body \"***Received Error***: **Error building asynchronous-search, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component asynchronous-search**.\n asynchronous-search failed during the distribution build for version: 2.2.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\" --label autocut,v2.2.0 --label \"untriaged\" --repo https://github.com/opensearch-project/asynchronous-search.git, returnStdout=true}')) - } - @Test public void testCommentOnExistingGithubIssue() { helper.addShMock("date -d \"3 days ago\" +'%Y-%m-%d'") { script -> @@ -54,7 +38,7 @@ class TestUpdateBuildFailuresIssues extends BuildPipelineTest { } runScript('tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile') assertThat(getCommands('println', ''), hasItem('Issue already exists, adding a comment')) - assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue comment 22 --repo https://github.com/opensearch-project/asynchronous-search.git --body \"***Received Error***: **Error building asynchronous-search, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component asynchronous-search**.\n asynchronous-search failed during the distribution build for version: 2.2.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\", returnStdout=true}")) + assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue comment bbb\nccc --repo https://github.com/opensearch-project/asynchronous-search.git --body \"***Received Error***: **Error building asynchronous-search, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component asynchronous-search**.\n asynchronous-search failed during the distribution build for version: 2.2.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\", returnStdout=true}")) } @Test @@ -79,8 +63,8 @@ class TestUpdateBuildFailuresIssues extends BuildPipelineTest { return [stdout: "20", exitValue: 0] } runScript('tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile') - assertThat(getCommands('sh', 'notifications'), not(hasItem("{script=gh issue close 20 -R opensearch-project/notifications --comment \"Closing the issue as the distribution build for notifications has passed for version: **2.2.0**.\n Please see build log at www.example.com/job/build_url/32/display/redirect\", returnStdout=true}"))) - assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue comment 20 --repo https://github.com/opensearch-project/notifications.git --body \"***Received Error***: **Error building notifications, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component notifications**.\n notifications failed during the distribution build for version: 2.2.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\", returnStdout=true}")) + assertThat(getCommands('sh', 'notifications'), not(hasItem("{script=gh issue close bbb\nccc -R opensearch-project/notifications --comment \"Closing the issue as the distribution build for notifications has passed for version: **2.2.0**.\n Please see build log at www.example.com/job/build_url/32/display/redirect\", returnStdout=true}"))) + assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue comment bbb\nccc --repo https://github.com/opensearch-project/notifications.git --body \"***Received Error***: **Error building notifications, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component notifications**.\n notifications failed during the distribution build for version: 2.2.0.\n Please see build log at www.example.com/job/build_url/32/display/redirect.\n The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details\", returnStdout=true}")) } def getCommands(method, text) { diff --git a/tests/jenkins/jobs/CreateBuildFailureGithubExistingIssueCheck_Jenkinsfile.txt b/tests/jenkins/jobs/CreateBuildFailureGithubExistingIssueCheck_Jenkinsfile.txt index 3f7b5d881..6af5d4812 100644 --- a/tests/jenkins/jobs/CreateBuildFailureGithubExistingIssueCheck_Jenkinsfile.txt +++ b/tests/jenkins/jobs/CreateBuildFailureGithubExistingIssueCheck_Jenkinsfile.txt @@ -13,9 +13,10 @@ The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.0.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title is:closed closed:>=bbb +ccc" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/OpenSearch.git --body "***Received Error***: **Error building OpenSearch, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component OpenSearch --snapshot**. @@ -29,9 +30,10 @@ ccc --repo https://github.com/opensearch-project/OpenSearch.git --body "***Recei The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.0.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title is:closed closed:>=bbb +ccc" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/geospatial.git --body "***Received Error***: **Error building geospatial, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component geospatial**. @@ -45,9 +47,10 @@ ccc --repo https://github.com/opensearch-project/geospatial.git --body "***Recei The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.0.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title is:closed closed:>=bbb +ccc" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/performance-analyzer.git --body "***Received Error***: **Error building performance-analyzer, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0ed in the next build. This might have performance impact if it keeps failing. Run the javaToolchains task for more det.yml --component performance-analyzer**. diff --git a/tests/jenkins/jobs/CreateBuildFailureGithubIssue_Jenkinsfile.txt b/tests/jenkins/jobs/CreateBuildFailureGithubIssue_Jenkinsfile.txt index fd09e38aa..6af5d4812 100644 --- a/tests/jenkins/jobs/CreateBuildFailureGithubIssue_Jenkinsfile.txt +++ b/tests/jenkins/jobs/CreateBuildFailureGithubIssue_Jenkinsfile.txt @@ -13,9 +13,10 @@ The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.0.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Distribution Build Failed for OpenSearch-2.0.0 in:title is:closed closed:>=bbb +ccc" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/OpenSearch.git --body "***Received Error***: **Error building OpenSearch, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component OpenSearch --snapshot**. @@ -29,9 +30,10 @@ ccc --repo https://github.com/opensearch-project/OpenSearch.git --body "***Recei The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.0.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/geospatial.git -S "[AUTOCUT] Distribution Build Failed for geospatial-2.0.0 in:title is:closed closed:>=bbb +ccc" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/geospatial.git --body "***Received Error***: **Error building geospatial, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component geospatial**. @@ -45,12 +47,14 @@ ccc --repo https://github.com/opensearch-project/geospatial.git --body "***Recei The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.0.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.0.0 --json number --jq '.[0].number', returnStdout=true}) - createGithubIssue.println(Creating new issue) - createGithubIssue.sh({script=gh issue create --title "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0" --body "***Received Error***: **Error building performance-analyzer, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0ed in the next build. This might have performance impact if it keeps failing. Run the javaToolchains task for more det.yml --component performance-analyzer**. + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.0.0 in:title is:closed closed:>=bbb +ccc" --label "autocut,v2.0.0" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.println(Issue already exists, adding a comment) + createGithubIssue.sh({script=gh issue comment bbb +ccc --repo https://github.com/opensearch-project/performance-analyzer.git --body "***Received Error***: **Error building performance-analyzer, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0ed in the next build. This might have performance impact if it keeps failing. Run the javaToolchains task for more det.yml --component performance-analyzer**. The distribution build for performance-analyzer has failed for version: 2.0.0. Please see build log at www.example.com/job/build_url/32/display/redirect. - The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details" --label autocut,v2.0.0 --label "untriaged" --repo https://github.com/opensearch-project/performance-analyzer.git, returnStdout=true}) + The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details", returnStdout=true}) createBuildFailureGithubIssue.sleep({time=3, unit=SECONDS}) diff --git a/tests/jenkins/jobs/CreateGithubIssue_Jenkinsfile.txt b/tests/jenkins/jobs/CreateGithubIssue_Jenkinsfile.txt index dc58fe1c5..e0a317c00 100644 --- a/tests/jenkins/jobs/CreateGithubIssue_Jenkinsfile.txt +++ b/tests/jenkins/jobs/CreateGithubIssue_Jenkinsfile.txt @@ -6,8 +6,9 @@ CreateGithubIssue_Jenkinsfile.createGithubIssue({repoUrl=https://github.com/opensearch-project/opensearch-build, issueTitle=Test GH issue title, issueBody=Test GH issue body, label=label101, daysToReOpen=5}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title" --label label101 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title" --label "label101" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "5 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title is:closed closed:>=2023-10-24" --label label101 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title is:closed closed:>=2023-10-24" --label "label101" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) - createGithubIssue.sh({script=gh issue comment 22 --repo https://github.com/opensearch-project/opensearch-build --body "Test GH issue body", returnStdout=true}) + createGithubIssue.sh({script=gh issue comment bbb +ccc --repo https://github.com/opensearch-project/opensearch-build --body "Test GH issue body", returnStdout=true}) diff --git a/tests/jenkins/jobs/EditGithubIssue_Jenkinsfile b/tests/jenkins/jobs/EditGithubIssue_Jenkinsfile new file mode 100644 index 000000000..24096e95a --- /dev/null +++ b/tests/jenkins/jobs/EditGithubIssue_Jenkinsfile @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('createGithubIssue') { + steps { + script { + createGithubIssue( + repoUrl: "https://github.com/opensearch-project/opensearch-build", + issueTitle: "Test GH issue title", + issueBodyFile: "issueBody.md", + issueEdit: true, + label: "label101", + ) + } + } + } + } +} + diff --git a/tests/jenkins/jobs/EditGithubIssue_Jenkinsfile_IssueBody.txt b/tests/jenkins/jobs/EditGithubIssue_Jenkinsfile_IssueBody.txt new file mode 100644 index 000000000..f8632b7a2 --- /dev/null +++ b/tests/jenkins/jobs/EditGithubIssue_Jenkinsfile_IssueBody.txt @@ -0,0 +1,15 @@ + EditGithubIssue_Jenkinsfile.run() + EditGithubIssue_Jenkinsfile.pipeline(groovy.lang.Closure) + EditGithubIssue_Jenkinsfile.echo(Executing on agent [label:none]) + EditGithubIssue_Jenkinsfile.stage(createGithubIssue, groovy.lang.Closure) + EditGithubIssue_Jenkinsfile.script(groovy.lang.Closure) + EditGithubIssue_Jenkinsfile.createGithubIssue({repoUrl=https://github.com/opensearch-project/opensearch-build, issueTitle=Test GH issue title, issueBodyFile=issueBody.md, issueEdit=true, label=label101}) + createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title" --label "label101" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/opensearch-build -S "Test GH issue title in:title is:closed closed:>=bbb +ccc" --label "label101" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.println(Issue already exists, editing the issue body) + createGithubIssue.sh({script=gh issue edit bbb +ccc --repo https://github.com/opensearch-project/opensearch-build --body-file issueBody.md, returnStdout=true}) diff --git a/tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile.txt b/tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile.txt index 923fd196c..db1430d4d 100644 --- a/tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile.txt +++ b/tests/jenkins/jobs/UpdateBuildFailureIssue_Jenkinsfile.txt @@ -14,9 +14,9 @@ The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.2.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/common-utils.git -S "[AUTOCUT] Distribution Build Failed for common-utils-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/common-utils.git -S "[AUTOCUT] Distribution Build Failed for common-utils-2.2.0 in:title" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/common-utils.git -S "[AUTOCUT] Distribution Build Failed for common-utils-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/common-utils.git -S "[AUTOCUT] Distribution Build Failed for common-utils-2.2.0 in:title is:closed closed:>=2023-10-24" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/common-utils.git --body "***Received Error***: **Error building common-utils, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component common-utils**. @@ -32,9 +32,9 @@ ccc --repo https://github.com/opensearch-project/common-utils.git --body "***Rec The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.2.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.2.0 in:title" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/performance-analyzer.git -S "[AUTOCUT] Distribution Build Failed for performance-analyzer-2.2.0 in:title is:closed closed:>=2023-10-24" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/performance-analyzer.git --body "***Received Error***: **Error building performance-analyzer, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0ed in the next build. This might have performance impact if it keeps failing. Run the javaToolchains task for more det.yml --component performance-analyzer**. @@ -53,9 +53,9 @@ ccc --repo https://github.com/opensearch-project/performance-analyzer.git --body The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.2.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/notifications.git -S "[AUTOCUT] Distribution Build Failed for notifications-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/notifications.git -S "[AUTOCUT] Distribution Build Failed for notifications-2.2.0 in:title" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/notifications.git -S "[AUTOCUT] Distribution Build Failed for notifications-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/notifications.git -S "[AUTOCUT] Distribution Build Failed for notifications-2.2.0 in:title is:closed closed:>=2023-10-24" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/notifications.git --body "***Received Error***: **Error building notifications, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component notifications**. @@ -69,9 +69,9 @@ ccc --repo https://github.com/opensearch-project/notifications.git --body "***Re The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.2.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/anomaly-detection.git -S "[AUTOCUT] Distribution Build Failed for anomaly-detection-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/anomaly-detection.git -S "[AUTOCUT] Distribution Build Failed for anomaly-detection-2.2.0 in:title" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/anomaly-detection.git -S "[AUTOCUT] Distribution Build Failed for anomaly-detection-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/anomaly-detection.git -S "[AUTOCUT] Distribution Build Failed for anomaly-detection-2.2.0 in:title is:closed closed:>=2023-10-24" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.println(Issue already exists, adding a comment) createGithubIssue.sh({script=gh issue comment bbb ccc --repo https://github.com/opensearch-project/anomaly-detection.git --body "***Received Error***: **Error building anomaly-detection, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component anomaly-detection**. @@ -85,14 +85,15 @@ ccc --repo https://github.com/opensearch-project/anomaly-detection.git --body "* The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details, label=autocut,v2.2.0}) createGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) createGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/asynchronous-search.git -S "[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0 in:title" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/asynchronous-search.git -S "[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0 in:title" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) createGithubIssue.sh({script=date -d "3 days ago" +'%Y-%m-%d', returnStdout=true}) - createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/asynchronous-search.git -S "[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0 in:title is:closed closed:>=2023-10-24" --label autocut,v2.2.0 --json number --jq '.[0].number', returnStdout=true}) - createGithubIssue.println(Creating new issue) - createGithubIssue.sh({script=gh issue create --title "[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0" --body "***Received Error***: **Error building asynchronous-search, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component asynchronous-search**. + createGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/asynchronous-search.git -S "[AUTOCUT] Distribution Build Failed for asynchronous-search-2.2.0 in:title is:closed closed:>=2023-10-24" --label "autocut,v2.2.0" --json number --jq '.[0].number', returnStdout=true}) + createGithubIssue.println(Issue already exists, adding a comment) + createGithubIssue.sh({script=gh issue comment bbb +ccc --repo https://github.com/opensearch-project/asynchronous-search.git --body "***Received Error***: **Error building asynchronous-search, retry with: ./build.sh manifests/2.2.0/opensearch-2.2.0.yml --component asynchronous-search**. asynchronous-search failed during the distribution build for version: 2.2.0. Please see build log at www.example.com/job/build_url/32/display/redirect. - The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details" --label autocut,v2.2.0 --label "untriaged" --repo https://github.com/opensearch-project/asynchronous-search.git, returnStdout=true}) + The failed build stage will be marked as unstable(!). Please see ./build.sh step for more details", returnStdout=true}) UpdateBuildFailureIssues.sleep({time=3, unit=SECONDS}) UpdateBuildFailureIssues.sleep({time=3, unit=SECONDS}) UpdateBuildFailureIssues.closeGithubIssue({repoUrl=https://github.com/opensearch-project/index-management.git, issueTitle=[AUTOCUT] Distribution Build Failed for index-management-2.2.0, closeComment=Closing the issue as the distribution build for index-management has passed for version: **2.2.0**. @@ -115,3 +116,4 @@ ccc -R opensearch-project/sql --comment "Closing the issue as the distribution b UpdateBuildFailureIssues.sleep({time=3, unit=SECONDS}) UpdateBuildFailureIssues.sleep({time=3, unit=SECONDS}) UpdateBuildFailureIssues.sleep({time=3, unit=SECONDS}) + diff --git a/tests/jenkins/lib-testers/CreateGithubIssueLibTester.groovy b/tests/jenkins/lib-testers/CreateGithubIssueLibTester.groovy index 994332ddc..014110220 100644 --- a/tests/jenkins/lib-testers/CreateGithubIssueLibTester.groovy +++ b/tests/jenkins/lib-testers/CreateGithubIssueLibTester.groovy @@ -53,19 +53,19 @@ class CreateGithubIssueLibTester extends LibFunctionTester { @Override boolean expectedParametersMatcher(Object call) { - if (call.args.label.isEmpty()) { + if (call.args.label.isEmpty()) { return call.args.label.first().equals('autocut') - && call.args.repoUrl.first().equals(this.repoUrl) - && call.args.issueTitle.first().equals(this.issueTitle) - && call.args.issueBody.first().equals(this.issueBody)} - if (call.args.daysToReOpen.isEmpty()) { + && call.args.repoUrl.first().equals(this.repoUrl) + && call.args.issueTitle.first().equals(this.issueTitle) + && call.args.issueBody.first().equals(this.issueBody)} + if (call.args.daysToReOpen.isEmpty()) { return call.args.daysToReOpen.first().equals('3') - && call.args.repoUrl.first().equals(this.repoUrl) - && call.args.issueTitle.first().equals(this.issueTitle) - && call.args.issueBody.first().equals(this.issueBody)} + && call.args.repoUrl.first().equals(this.repoUrl) + && call.args.issueTitle.first().equals(this.issueTitle) + && call.args.issueBody.first().equals(this.issueBody)} return call.args.repoUrl.first().equals(this.repoUrl) - && call.args.issueTitle.first().equals(this.issueTitle) - && call.args.issueBody.first().equals(this.issueBody) + && call.args.issueTitle.first().equals(this.issueTitle) + && call.args.issueBody.first().equals(this.issueBody) } @Override diff --git a/vars/createGithubIssue.groovy b/vars/createGithubIssue.groovy index 3e1740773..70af3557c 100644 --- a/vars/createGithubIssue.groovy +++ b/vars/createGithubIssue.groovy @@ -14,6 +14,8 @@ @param args.issueBody - GitHub issue body @param args.label - GitHub issue label to be attached along with 'untriaged'. Defaults to autocut. @param args.daysToReOpen - Look for a closed Github issues older than `daysToReOpen`. + @param args.issueEdit - Updates the body of the issue, the default if not passed is to add a comment. + @param args.issueBodyFile - GitHub issue body from an `.md` file */ void call(Map args = [:]) { @@ -22,7 +24,7 @@ void call(Map args = [:]) { try { withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { def openIssue = sh( - script: "gh issue list --repo ${args.repoUrl} -S \"${args.issueTitle} in:title\" --label ${label} --json number --jq '.[0].number'", + script: "gh issue list --repo ${args.repoUrl} -S \"${args.issueTitle} in:title\" --label \"${label}\" --json number --jq '.[0].number'", returnStdout: true ).trim() @@ -32,16 +34,26 @@ void call(Map args = [:]) { ).trim() def closedIssue = sh( - script: "gh issue list --repo ${args.repoUrl} -S \"${args.issueTitle} in:title is:closed closed:>=${currentDayMinusDaysToReOpen}\" --label ${label} --json number --jq '.[0].number'", + script: "gh issue list --repo ${args.repoUrl} -S \"${args.issueTitle} in:title is:closed closed:>=${currentDayMinusDaysToReOpen}\" --label \"${label}\" --json number --jq '.[0].number'", returnStdout: true ).trim() + def bodyOption = args.issueBodyFile ? "--body-file ${args.issueBodyFile}" : "--body \"${args.issueBody}\"" + if (openIssue) { - println('Issue already exists, adding a comment') - sh( - script: "gh issue comment ${openIssue} --repo ${args.repoUrl} --body \"${args.issueBody}\"", - returnStdout: true - ) + if (args.issueEdit) { + println('Issue already exists, editing the issue body') + sh( + script: "gh issue edit ${openIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } else { + println('Issue already exists, adding a comment') + sh( + script: "gh issue comment ${openIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } } else if (!openIssue && closedIssue) { println("Re-opening a recently closed issue and commenting on it") @@ -49,15 +61,23 @@ void call(Map args = [:]) { script: "gh issue reopen --repo ${args.repoUrl} ${closedIssue}", returnStdout: true ) - sh( - script: "gh issue comment ${closedIssue} --repo ${args.repoUrl} --body \"${args.issueBody}\"", - returnStdout: true - ) + // Default behavior is comment unless `issueEdit` is passed + if (args.issueEdit) { + sh( + script: "gh issue edit ${closedIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } else { + sh( + script: "gh issue comment ${closedIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } } else { println("Creating new issue") sh( - script: "gh issue create --title \"${args.issueTitle}\" --body \"${args.issueBody}\" --label ${label} --label \"untriaged\" --repo ${args.repoUrl}", + script: "gh issue create --title \"${args.issueTitle}\" ${bodyOption} --label \"${label}\" --label \"untriaged\" --repo ${args.repoUrl}", returnStdout: true ) } diff --git a/vars/gradleCheckFlakyTestChecker.groovy b/vars/gradleCheckFlakyTestChecker.groovy new file mode 100644 index 000000000..68a3f68d1 --- /dev/null +++ b/vars/gradleCheckFlakyTestChecker.groovy @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import gradlecheck.FetchPostMergeFailedTestClass +import gradlecheck.FetchPostMergeTestGitReference +import gradlecheck.FetchPostMergeFailedTestName +import gradlecheck.FetchTestPullRequests +import gradlecheck.CreateMarkDownTable + +void call(Map args = [:]) { + withCredentials([ + string(credentialsId: 'jenkins-health-metrics-account-number', variable: 'METRICS_HOST_ACCOUNT'), + string(credentialsId: 'jenkins-health-metrics-cluster-endpoint', variable: 'METRICS_HOST_URL') + ]) { + withAWS(role: 'OpenSearchJenkinsAccessRole', roleAccount: "${METRICS_HOST_ACCOUNT}", duration: 900, roleSessionName: 'jenkins-session') { + def metricsUrl = env.METRICS_HOST_URL + def awsAccessKey = env.AWS_ACCESS_KEY_ID + def awsSecretKey = env.AWS_SECRET_ACCESS_KEY + def awsSessionToken = env.AWS_SESSION_TOKEN + def postMergeFailedTests = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeFailedTestClass() + postMergeFailedTests.each { failedTest -> + def testData = [] + def allPullRequests = [] + def postMergeTestGitReference = new FetchPostMergeTestGitReference(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeTestGitReference(failedTest) + postMergeTestGitReference.each { gitReference -> + def failedTestNames = new FetchPostMergeFailedTestName(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeFailedTestName(failedTest, gitReference) + def testNames = failedTestNames.aggregations.test_name_keyword_agg.buckets.collect { it.key } + def buildNumber = failedTestNames.aggregations.build_number_agg.buckets.collect { it.key } + def pullRequests = failedTestNames.aggregations.pull_request_agg.buckets.collect { it.key } + allPullRequests.addAll(pullRequests) + def rowData = [ + gitReference: gitReference, + pullRequestLink: pullRequests.collect { pr -> "[${pr}](https://github.com/opensearch-project/OpenSearch/pull/${pr})" }.join('

'), + buildDetailLink: buildNumber.collect { build -> "[${build}](https://build.ci.opensearch.org/job/gradle-check/${build}/testReport/)" }.join('

'), + testNames: testNames.collect { testName -> "`${testName}`" } + ] + testData << rowData + } + def testNameAdditionalPullRequests = new FetchTestPullRequests(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getTestPullRequests(failedTest).findAll { !allPullRequests.contains(it) } + def markdownTable = new CreateMarkDownTable(failedTest, testData, testNameAdditionalPullRequests).createMarkdownTable() + writeFile file: "${failedTest}.md", text: markdownTable + createGithubIssue( + repoUrl: "https://github.com/opensearch-project/OpenSearch", + issueTitle: "[AUTOCUT] Gradle Check Flaky Test Report for ${failedTest}", + issueBodyFile: "${failedTest}.md", + label: 'autocut,>test-failure', + issueEdit: true + ) + } + } + } +} +