Refactor metrics query to use any index and move it under utils (#509)

Signed-off-by: Sayali Gaikawad <[email protected]>
opensearch-project · Oct 2, 2024 · 8a67658 · 8a67658
1 parent 5e17480
commit 8a67658
Show file tree

Hide file tree

Showing 15 changed files with 55 additions and 40 deletions.
diff --git a/build.gradle b/build.gradle
@@ -45,13 +45,13 @@ configurations.all {
 sourceSets {
  main {
  groovy {
- srcDirs = ['src/jenkins', 'src/gradlecheck']
+ srcDirs = ['src/jenkins', 'src/gradlecheck', 'src/utils']
  }
  }
 
  test {
  groovy {
- srcDirs = ['tests/jenkins', 'tests/gradlecheck']
+ srcDirs = ['tests/jenkins', 'tests/gradlecheck', 'tests/utils']
  }
  }
 
@@ -114,7 +114,8 @@ jacocoTestReport {
  includes: [
  '**/*_Jenkinsfile.*',
  '**/jenkins/*',
- '**/gradlecheck/*'
+ '**/gradlecheck/*',
+ '**/utils/*'
  ],
  excludes: [
  '**/*\$_get_closure*'

diff --git a/src/gradlecheck/FetchPostMergeFailedTestClass.groovy b/src/gradlecheck/FetchPostMergeFailedTestClass.groovy
@@ -10,20 +10,22 @@
 package gradlecheck
 
 import groovy.json.JsonOutput
-import gradlecheck.OpenSearchMetricsQuery
+import utils.OpenSearchMetricsQuery
 
 class FetchPostMergeFailedTestClass {
  String metricsUrl
  String awsAccessKey
  String awsSecretKey
  String awsSessionToken
+ String indexName
  def script
 
- FetchPostMergeFailedTestClass(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
+ FetchPostMergeFailedTestClass(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, def script) {
  this.metricsUrl = metricsUrl
  this.awsAccessKey = awsAccessKey
  this.awsSecretKey = awsSecretKey
  this.awsSessionToken = awsSessionToken
+ this.indexName = indexName
  this.script = script
  }
 
@@ -95,7 +97,7 @@ class FetchPostMergeFailedTestClass {
  }
 
  def getPostMergeFailedTestClass(timeFrame) {
- def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(timeFrame))
+ def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(timeFrame))
  def keys = jsonResponse.aggregations.test_class_keyword_agg.buckets.collect { it.key }
  return keys
  }

diff --git a/src/gradlecheck/FetchPostMergeFailedTestName.groovy b/src/gradlecheck/FetchPostMergeFailedTestName.groovy
@@ -10,20 +10,22 @@
 package gradlecheck
 
 import groovy.json.JsonOutput
-import gradlecheck.OpenSearchMetricsQuery
+import utils.OpenSearchMetricsQuery
 
 class FetchPostMergeFailedTestName {
  String metricsUrl
  String awsAccessKey
  String awsSecretKey
  String awsSessionToken
+ String indexName
  def script
 
- FetchPostMergeFailedTestName(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
+ FetchPostMergeFailedTestName(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, def script) {
  this.metricsUrl = metricsUrl
  this.awsAccessKey = awsAccessKey
  this.awsSecretKey = awsSecretKey
  this.awsSessionToken = awsSessionToken
+ this.indexName = indexName
  this.script = script
  }
 
@@ -125,6 +127,6 @@ class FetchPostMergeFailedTestName {
 
  }
  def getPostMergeFailedTestName(testName, gitReference) {
- return new OpenSearchMetricsQuery(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName, gitReference))
+ return new OpenSearchMetricsQuery(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(testName, gitReference))
  }
 }
diff --git a/src/gradlecheck/FetchPostMergeTestGitReference.groovy b/src/gradlecheck/FetchPostMergeTestGitReference.groovy
@@ -10,20 +10,22 @@
 package gradlecheck
 
 import groovy.json.JsonOutput
-import gradlecheck.OpenSearchMetricsQuery
+import utils.OpenSearchMetricsQuery
 
 class FetchPostMergeTestGitReference {
  String metricsUrl
  String awsAccessKey
  String awsSecretKey
  String awsSessionToken
+ String indexName
  def script
 
- FetchPostMergeTestGitReference(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
+ FetchPostMergeTestGitReference(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, def script) {
  this.metricsUrl = metricsUrl
  this.awsAccessKey = awsAccessKey
  this.awsSecretKey = awsSecretKey
  this.awsSessionToken = awsSessionToken
+ this.indexName = indexName
  this.script = script
  }
 
@@ -98,7 +100,7 @@ class FetchPostMergeTestGitReference {
  }
 
  def getPostMergeTestGitReference(testName) {
- def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName))
+ def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(testName))
  def keys = jsonResponse.aggregations.git_reference_keyword_agg.buckets.collect { it.key }
  return keys
  }

diff --git a/src/gradlecheck/FetchTestPullRequests.groovy b/src/gradlecheck/FetchTestPullRequests.groovy
@@ -10,20 +10,22 @@
 package gradlecheck
 
 import groovy.json.JsonOutput
-import gradlecheck.OpenSearchMetricsQuery
+import utils.OpenSearchMetricsQuery
 
 class FetchTestPullRequests {
  String metricsUrl
  String awsAccessKey
  String awsSecretKey
  String awsSessionToken
+ String indexName
  def script
 
- FetchTestPullRequests(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
+ FetchTestPullRequests(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, def script) {
  this.metricsUrl = metricsUrl
  this.awsAccessKey = awsAccessKey
  this.awsSecretKey = awsSecretKey
  this.awsSessionToken = awsSessionToken
+ this.indexName = indexName
  this.script = script
  }
 
@@ -97,7 +99,7 @@ class FetchTestPullRequests {
  return query.replace('"', '\\"')
  }
  List<String> getTestPullRequests(testName) {
- def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName))
+ def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(testName))
  def keys = jsonResponse.aggregations.pull_request_keyword_agg.buckets.collect { it.key }
  return keys
  }

diff --git a/...gradlecheck/OpenSearchMetricsQuery.groovy → src/utils/OpenSearchMetricsQuery.groovy b/...gradlecheck/OpenSearchMetricsQuery.groovy → src/utils/OpenSearchMetricsQuery.groovy
@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-package gradlecheck
+package utils
 
 import groovy.json.JsonSlurper
 
@@ -16,13 +16,15 @@ class OpenSearchMetricsQuery {
  String awsAccessKey
  String awsSecretKey
  String awsSessionToken
+ String indexName
  def script
 
- OpenSearchMetricsQuery(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
+ OpenSearchMetricsQuery(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, def script) {
  this.metricsUrl = metricsUrl
  this.awsAccessKey = awsAccessKey
  this.awsSecretKey = awsSecretKey
  this.awsSessionToken = awsSessionToken
+ this.indexName = indexName
  this.script = script
  }
 
@@ -32,8 +34,7 @@ class OpenSearchMetricsQuery {
  script: """
  set -e
  set +x
- MONTH_YEAR=\$(date +"%m-%Y")
- curl -s -XGET "${metricsUrl}/gradle-check/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.'
+ curl -s -XGET "${metricsUrl}/${indexName}/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.'
  """,
  returnStdout: true
  ).trim()

diff --git a/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy b/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy
@@ -20,6 +20,7 @@ class FetchPostMergeFailedTestClassTest {
  private final String awsAccessKey = "testAccessKey"
  private final String awsSecretKey = "testSecretKey"
  private final String awsSessionToken = "testSessionToken"
+ private final String indexName = "gradle-check-*"
  private def script
 
  @Before
@@ -42,7 +43,7 @@ class FetchPostMergeFailedTestClassTest {
  }
  return ""
  }
- fetchPostMergeFailedTestClass = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script)
+ fetchPostMergeFailedTestClass = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
  }
 
  @Test

diff --git a/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy b/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy
@@ -20,6 +20,7 @@ class FetchPostMergeFailedTestNameTest {
  private final String awsAccessKey = "testAccessKey"
  private final String awsSecretKey = "testSecretKey"
  private final String awsSessionToken = "testSessionToken"
+ private final String indexName = "gradle-check-*"
  private def script
 
  @Before
@@ -54,7 +55,7 @@ class FetchPostMergeFailedTestNameTest {
  }
  return ""
  }
- fetchPostMergeFailedTestName = new FetchPostMergeFailedTestName(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script)
+ fetchPostMergeFailedTestName = new FetchPostMergeFailedTestName(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
  }
 
  @Test

diff --git a/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy b/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy
@@ -20,6 +20,7 @@ class FetchPostMergeTestGitReferenceTest {
  private final String awsAccessKey = "testAccessKey"
  private final String awsSecretKey = "testSecretKey"
  private final String awsSessionToken = "testSessionToken"
+ private final String indexName = "gradle-check-*"
  private def script
 
  @Before
@@ -42,7 +43,7 @@ class FetchPostMergeTestGitReferenceTest {
  }
  return ""
  }
- fetchPostMergeTestGitReference = new FetchPostMergeTestGitReference(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script)
+ fetchPostMergeTestGitReference = new FetchPostMergeTestGitReference(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
  }
 
  @Test

diff --git a/tests/gradlecheck/FetchTestPullRequestsTest.groovy b/tests/gradlecheck/FetchTestPullRequestsTest.groovy
@@ -20,6 +20,7 @@ class FetchTestPullRequestsTest {
  private final String awsAccessKey = "testAccessKey"
  private final String awsSecretKey = "testSecretKey"
  private final String awsSessionToken = "testSessionToken"
+ private final String indexName = "gradle-check-*"
  private def script
 
  @Before
@@ -42,7 +43,7 @@ class FetchTestPullRequestsTest {
  }
  return ""
  }
- fetchTestPullRequests = new FetchTestPullRequests(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script)
+ fetchTestPullRequests = new FetchTestPullRequests(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
  }
 
  @Test

diff --git a/tests/jenkins/TestGradleCheckFlakyTestGitHubIssue.groovy b/tests/jenkins/TestGradleCheckFlakyTestGitHubIssue.groovy
@@ -29,19 +29,19 @@ class TestGradleCheckFlakyTestGitHubIssue extends BuildPipelineTest {
  @Test
  public void testDefaultIssueEdit() {
  super.testPipeline("tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile", "tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile")
- assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue edit bbb\nccc --repo https://github.com/prudhvigodithi/OpenSearch --body-file \"SampleTest.md\", returnStdout=true}"))
+ assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue edit bbb\nccc --repo https://github.com/opensearch-project/OpenSearch --body-file \"SampleTest.md\", returnStdout=true}"))
  }
 
  @Test
  public void testIssueCreate() {
- helper.addShMock("""gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number'""") { script ->
+ helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number'""") { script ->
  return [stdout: "", exitValue: 0]
  }
- helper.addShMock("""gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number'""") { script ->
+ helper.addShMock("""gh issue list --repo https://github.com/opensearch-project/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number'""") { script ->
  return [stdout: "", exitValue: 0]
  }
  super.testPipeline("tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile", "tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile")
- assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue create --title \"[AUTOCUT] Gradle Check Flaky Test Report for SampleTest\" --body-file \"SampleTest.md\" --label \"autocut,>test-failure,flaky-test\" --label \"untriaged\" --repo https://github.com/prudhvigodithi/OpenSearch, returnStdout=true}"))
+ assertThat(getCommands('sh', 'script'), hasItem("{script=gh issue create --title \"[AUTOCUT] Gradle Check Flaky Test Report for SampleTest\" --body-file \"SampleTest.md\" --label \"autocut,>test-failure,flaky-test\" --label \"untriaged\" --repo https://github.com/opensearch-project/OpenSearch, returnStdout=true}"))
  }
 
  def getCommands(method, text) {

diff --git a/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile.txt b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueCreate_Jenkinsfile.txt
@@ -3,10 +3,10 @@
  TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.echo(Executing on agent [label:none])
  TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.stage(Detect Gradle Check Flaky Tests, groovy.lang.Closure)
  TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.script(groovy.lang.Closure)
- TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.gradleCheckFlakyTestGitHubIssue({repoUrl=https://github.com/prudhvigodithi/OpenSearch, issueTitle=[AUTOCUT] Gradle Check Flaky Test Report for SampleTest, issueBodyFile=SampleTest.md, label=autocut,>test-failure,flaky-test, issueEdit=true})
+ TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.gradleCheckFlakyTestGitHubIssue({repoUrl=https://github.com/opensearch-project/OpenSearch, issueTitle=[AUTOCUT] Gradle Check Flaky Test Report for SampleTest, issueBodyFile=SampleTest.md, label=autocut,>test-failure,flaky-test, issueEdit=true})
  gradleCheckFlakyTestGitHubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
  gradleCheckFlakyTestGitHubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
- gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number', returnStdout=true})
- gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number', returnStdout=true})
+ gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number', returnStdout=true})
+ gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number', returnStdout=true})
  gradleCheckFlakyTestGitHubIssue.println(Creating new issue)
- gradleCheckFlakyTestGitHubIssue.sh({script=gh issue create --title "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest" --body-file "SampleTest.md" --label "autocut,>test-failure,flaky-test" --label "untriaged" --repo https://github.com/prudhvigodithi/OpenSearch, returnStdout=true})
+ gradleCheckFlakyTestGitHubIssue.sh({script=gh issue create --title "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest" --body-file "SampleTest.md" --label "autocut,>test-failure,flaky-test" --label "untriaged" --repo https://github.com/opensearch-project/OpenSearch, returnStdout=true})
diff --git a/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile.txt b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssueEdit_Jenkinsfile.txt
@@ -3,11 +3,11 @@
  TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.echo(Executing on agent [label:none])
  TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.stage(Detect Gradle Check Flaky Tests, groovy.lang.Closure)
  TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.script(groovy.lang.Closure)
- TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.gradleCheckFlakyTestGitHubIssue({repoUrl=https://github.com/prudhvigodithi/OpenSearch, issueTitle=[AUTOCUT] Gradle Check Flaky Test Report for SampleTest, issueBodyFile=SampleTest.md, label=autocut,>test-failure,flaky-test, issueEdit=true})
+ TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile.gradleCheckFlakyTestGitHubIssue({repoUrl=https://github.com/opensearch-project/OpenSearch, issueTitle=[AUTOCUT] Gradle Check Flaky Test Report for SampleTest, issueBodyFile=SampleTest.md, label=autocut,>test-failure,flaky-test, issueEdit=true})
  gradleCheckFlakyTestGitHubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
  gradleCheckFlakyTestGitHubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
- gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number', returnStdout=true})
- gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/prudhvigodithi/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number', returnStdout=true})
+ gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title" --json number --jq '.[0].number', returnStdout=true})
+ gradleCheckFlakyTestGitHubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch -S "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest in:title is:closed" --json number --jq '.[0].number', returnStdout=true})
  gradleCheckFlakyTestGitHubIssue.println(Issue already exists, editing the issue body)
  gradleCheckFlakyTestGitHubIssue.sh({script=gh issue edit bbb
-ccc --repo https://github.com/prudhvigodithi/OpenSearch --body-file "SampleTest.md", returnStdout=true})
+ccc --repo https://github.com/opensearch-project/OpenSearch --body-file "SampleTest.md", returnStdout=true})
diff --git a/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile b/tests/jenkins/jobs/TestGradleCheckFlakyTestGitHubIssue_Jenkinsfile
@@ -14,7 +14,7 @@ pipeline {
  steps {
  script {
  gradleCheckFlakyTestGitHubIssue(
- repoUrl: "https://github.com/prudhvigodithi/OpenSearch",
+ repoUrl: "https://github.com/opensearch-project/OpenSearch",
  issueTitle: "[AUTOCUT] Gradle Check Flaky Test Report for SampleTest",
  issueBodyFile: "SampleTest.md",
  label: 'autocut,>test-failure,flaky-test',