Add release notes generation action (#4879)

Signed-off-by: Sayali Gaikawad <[email protected]>

jenkins/release-notes-check/release-notes-check.jenkinsfile

@@ -16,13 +16,25 @@ pipeline {
     options {
             timeout(time: 2, unit: 'HOURS')
     }
-    agent none
+    agent {
+        docker {
+            label 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host'
+            image 'opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-build-v1'
+            registryUrl 'https://public.ecr.aws/'
+            alwaysPull true
+        }
+    }
     parameters {
         string(
-            name: 'INPUT_MANIFEST',
-            description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.',
+            name: 'RELEASE_VERSION',
+            description: 'Release version number to fetch input manifest path',
             trim: true
         )
+        choice(
+                choices: ['check', 'compile'],
+                name: 'ACTION',
+                description: 'check - Checks the release notes for all components. compile - Consolidates all release notes', 
+        )
         string(
             name: 'GIT_LOG_DATE',
             description: 'in format yyyy-mm-dd, example 2022-07-26.',
@@ -44,87 +56,99 @@ pipeline {
             trim: true
         )
     }
-    environment {
-        AGENT_X64 = 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host'
-    }
     stages {
-        stage('detect docker image + args') {
-            agent {
-                docker {
-                    label AGENT_X64
-                    image 'docker/library/alpine:3'
-                    registryUrl 'https://public.ecr.aws/'
-                    alwaysPull true
-                }
-            }
-            steps {
-                script {
-                    dockerAgent = detectDockerAgent()
-                    currentBuild.description = INPUT_MANIFEST
-                }
-            }
-            post {
-                always {
-                    postCleanup()
-                }
-            }
-        }
         stage('Parameters Check') {
             steps {
                 script {
-                    currentBuild.description = "Comment:${COMMENT}, Manifest:${INPUT_MANIFEST}"
-                    if (GIT_LOG_DATE.isEmpty() || INPUT_MANIFEST.isEmpty()) {
-                                currentBuild.result = 'ABORTED'
-                                error('Make sure all the parameters are passed in.')
+                    if (ACTION.isEmpty() || RELEASE_VERSION.isEmpty()) {
+                        currentBuild.result = 'ABORTED'
+                        error('ACTION and RELEASE_VERSION parameters cannot be empty!')
                     }
-                    if (COMMENT == "ADD" && GIT_ISSUE_NUMBER.isEmpty()) {
-                                currentBuild.result = 'ABORTED'
-                                error('Make sure all the parameters are passed in.')
+                    if (ACTION == "check" && (GIT_LOG_DATE.isEmpty() || COMMENT == "ADD" || GIT_ISSUE_NUMBER.isEmpty())) {
+                        currentBuild.result = 'ABORTED'
+                        error('GIT_LOG_DATE, COMMENT or GIT_ISSUE_NUMBER parameters cannot be empty when ACTION = check!')
                     }
-                    if (COMMENT == "UPDATE" && COMMENT_UNIQUE_ID.isEmpty()) {
-                                currentBuild.result = 'ABORTED'
-                                error('Make sure all the parameters are passed in.')
+                    if (ACTION == "check" && (GIT_LOG_DATE.isEmpty() || COMMENT == "UPDATE" || COMMENT_UNIQUE_ID.isEmpty())) {
+                        currentBuild.result = 'ABORTED'
+                        error('GIT_LOG_DATE, COMMENT or COMMENT_UNIQUE_ID parameters cannot be empty when ACTION = check!')
                     }
                 }
             }
         }
-        stage('Generate MarkDown table') {
-            agent {
-                docker {
-                    label AGENT_X64
-                    image dockerAgent.image
-                    registryUrl 'https://public.ecr.aws/'
-                    alwaysPull true
-                }
+        stage('Check release notes') {
+            when {
+                expression { params.ACTION == 'check' }
             }
             steps {
                 script {
                     withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
                         if (params.COMMENT == "ADD") {
-                            sh '''
+                            sh """
                                 #!/bin/bash
                                 set +e
-                                ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE} --output table.md
+                                ./release_notes.sh check manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --date ${GIT_LOG_DATE} --output table.md
                                 echo "Adding Comment on issue $GIT_ISSUE_NUMBER"
                                 gh repo clone https://github.com/opensearch-project/opensearch-build.git; cd opensearch-build
                                 gh issue comment ${GIT_ISSUE_NUMBER} --body-file ../table.md --repo opensearch-project/opensearch-build
-                            '''
+                            """
                         }
                         if (params.COMMENT == "UPDATE") {
-                            sh '''
+                            sh """
                                 #!/bin/bash
                                 set +e
-                                ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE} --output table.md
+                                ./release_notes.sh check manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --date ${GIT_LOG_DATE} --output table.md
                                 echo "Updating Comment on issue $GIT_ISSUE_NUMBER"
                                 gh repo clone https://github.com/opensearch-project/opensearch-build.git; cd opensearch-build
                                 IFS=
-                                gh api --method PATCH -H "Accept: application/vnd.github+json" /repos/opensearch-project/opensearch-build/issues/comments/${COMMENT_UNIQUE_ID} -f body=$(cat ../table.md)
-                            '''
+                                gh api --method PATCH -H "Accept: application/vnd.github+json" /repos/opensearch-project/opensearch-build/issues/comments/${COMMENT_UNIQUE_ID} -f body=\$(cat ../table.md)
+                            """
                         }
                         if (params.COMMENT == "NO_COMMENT") {
-                            sh '''
-                            ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE}
-                            '''
+                            sh """
+                            ./release_notes.sh check manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --date ${GIT_LOG_DATE}
+                            """
+                        }
+                    }
+                }
+            }
+            post {
+                always {
+                    postCleanup()
+                }
+            }
+        }
+        stage('Generate consolidated release notes') {
+            when {
+                expression { params.ACTION == 'compile' }
+            }
+            steps {
+                script {
+                    sh """
+                        #!/bin/bash
+                        set +e
+                        ./release_notes.sh compile manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --output table.md
+                    """
+                    withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
+                        try {
+                            sh """
+                                git remote set-url origin "https://opensearch-ci:${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-build"
+                                git config user.email "[email protected]"
+                                git config user.name "opensearch-ci"
+                                git checkout -b release-notes
+                            """
+                            def status = sh(returnStdout: true, script: 'git status --porcelain')
+                            if (status) {
+                                sh """
+                                    git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add
+                                    git commit -sm "Add consolidated release notes for ${params.RELEASE_VERSION}"
+                                    git push origin release-notes --force
+                                    gh pr create --title 'Add consolidated release notes for ${params.RELEASE_VERSION}' --body 'Add consolidated release notes for ${params.RELEASE_VERSION}' -H release-notes -B main
+                                """
+                            } else {
+                                println 'Nothing to commit!'
+                            }
+                        } catch (e) {
+                            error 'An error occured while creating pull request for consolidated release notes' + e.toString()
                         }
                     }
                 }

tests/jenkins/TestReleaseNotesCheckAndCompile.groovy

@@ -0,0 +1,81 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
+import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.assertj.core.api.Assertions.assertThat
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.CoreMatchers.hasItems
+
+class TestReleaseNotesCheckAndCompile extends BuildPipelineTest {
+
+    String gitLogDate = '2022-10-10'
+    String comment = 'NO_COMMENT'
+    String gitIssueNumber = '123456'
+    String commentUniqueID = '123456'
+    String releaseVersion = '3.0.0'
+
+    @Override
+    @Before
+    void setUp() {
+
+        helper.registerSharedLibrary(
+            library().name('jenkins')
+                .defaultVersion('1.0.4')
+                .allowOverride(true)
+                .implicit(true)
+                .targetPath('vars')
+                .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git'))
+                .build()
+            )
+        super.setUp()
+
+        addParam('RELEASE_VERSION', releaseVersion)
+    }
+
+    @Test
+    public void releaseNotesCheck() {
+        addParam('ACTION', 'check')
+        addParam('GIT_LOG_DATE', gitLogDate)
+        addParam('COMMENT', comment)
+        addParam('GIT_ISSUE_NUMBER', gitIssueNumber)
+        addParam('COMMENT_UNIQUE_ID', commentUniqueID)
+
+        super.testPipeline("jenkins/release-notes-check/release-notes-check.jenkinsfile",
+                "tests/jenkins/jenkinsjob-regression-files/release-notes-check/release-notes-check.jenkinsfile")
+        assertJobStatusSuccess()
+        def callStack = helper.getCallStack()
+        assertCallStack().contains('Check release notes, groovy.lang.Closure')
+        assertCallStack().contains('Skipping stage Generate consolidated release notes')
+        assertThat(helper.callStack.findAll { call ->
+            call.methodName == 'sh'
+        }.any { call ->
+            callArgsToString(call).contains('./release_notes.sh check manifests/3.0.0/opensearch-3.0.0.yml manifests/3.0.0/opensearch-dashboards-3.0.0.yml --date 2022-10-10')
+        }).isTrue()    
+    }
+
+    @Test
+    public void releaseNotesCompile() {
+        addParam('ACTION', 'compile')
+        super.testPipeline("jenkins/release-notes-check/release-notes-check.jenkinsfile",
+                "tests/jenkins/jenkinsjob-regression-files/release-notes-check/release-notes-compile.jenkinsfile")
+        assertJobStatusSuccess()
+        def callStack = helper.getCallStack()
+        assertCallStack().contains('Skipping stage Check release notes')
+        assertCallStack().contains('Generate consolidated release notes, groovy.lang.Closure')
+        assertThat(helper.callStack.findAll { call ->
+            call.methodName == 'sh'
+        }.any { call ->
+            callArgsToString(call).contains('./release_notes.sh compile manifests/3.0.0/opensearch-3.0.0.yml manifests/3.0.0/opensearch-dashboards-3.0.0.yml --output table.md')
+        }).isTrue()
+    }
+}

tests/jenkins/jenkinsjob-regression-files/release-notes-check/release-notes-check.jenkinsfile.txt

@@ -3,33 +3,16 @@
       release-notes-check.library({[email protected], retriever=null})
       release-notes-check.pipeline(groovy.lang.Closure)
          release-notes-check.timeout({time=2, unit=HOURS})
-         release-notes-check.echo(Executing on agent [label:none])
-         release-notes-check.stage(detect docker image + args, groovy.lang.Closure)
-            release-notes-check.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:docker/library/alpine:3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
-            release-notes-check.script(groovy.lang.Closure)
-               release-notes-check.detectDockerAgent()
-                  detectDockerAgent.legacySCM(groovy.lang.Closure)
-                  detectDockerAgent.library({[email protected], retriever=null})
-                  detectDockerAgent.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml})
-                  InputManifest.asBoolean()
-                  detectDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 (-e JAVA_HOME=/opt/java/openjdk-21))
-                  detectDockerAgent.echo(Using java version openjdk-21)
-         release-notes-check.postCleanup()
-            postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
+         release-notes-check.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-build-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          release-notes-check.stage(Parameters Check, groovy.lang.Closure)
             release-notes-check.script(groovy.lang.Closure)
-         release-notes-check.stage(Generate MarkDown table, groovy.lang.Closure)
-            release-notes-check.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         release-notes-check.stage(Check release notes, groovy.lang.Closure)
             release-notes-check.script(groovy.lang.Closure)
                release-notes-check.usernamePassword({credentialsId=jenkins-github-bot-token, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                release-notes-check.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                   release-notes-check.sh(
-                                #!/bin/bash
-                                set +e
-                                ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE} --output table.md
-                                echo "Adding Comment on issue $GIT_ISSUE_NUMBER"
-                                gh repo clone https://github.com/opensearch-project/opensearch-build.git; cd opensearch-build
-                                gh issue comment ${GIT_ISSUE_NUMBER} --body-file ../table.md --repo opensearch-project/opensearch-build
+                            ./release_notes.sh check manifests/3.0.0/opensearch-3.0.0.yml manifests/3.0.0/opensearch-dashboards-3.0.0.yml --date 2022-10-10
                             )
          release-notes-check.postCleanup()
             postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
+         release-notes-check.echo(Skipping stage Generate consolidated release notes)