Merge branch 'opensearch-project:main' into main

opensearch-project · Nov 11, 2024 · 42c8c5d · 42c8c5d
2 parents e55a281 + e6ec7d1
commit 42c8c5d
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 45 deletions.
diff --git a/build.gradle b/build.gradle
@@ -128,7 +128,7 @@ jacocoTestReport {
     }
 }
 
-String version = '7.3.1'
+String version = '7.3.4'
 
 task updateVersion {
     doLast {

diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
@@ -9,37 +9,41 @@
                   uploadMinSnapshotsToS3.library({identifier=jenkins@main, retriever=null})
                   uploadMinSnapshotsToS3.readYaml({file=tests/data/opensearch-1.3.0.yml})
                   InputManifest.asBoolean()
+                  uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
                   uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml)
                   uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
                   BuildManifest.asBoolean()
-                  uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
-                  createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
-                  createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
-                  createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
-                  createSha512Checksums.sh({script=basename ccc, returnStdout=true})
-                  createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb  
+                  uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
+                  uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
+                  uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
+                  uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
+                     uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
+                     uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
+                     uploadMinSnapshotsToS3.createSha512Checksums()
+                     createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
+                     createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
+                     createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
+                     createSha512Checksums.sh({script=basename ccc, returnStdout=true})
+                     createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb  
 bbb
 ccc
 })
-                  createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz)
-                  createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
-                  createSha512Checksums.sh({script=basename ccc, returnStdout=true})
-                  createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb  
+                     createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz)
+                     createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
+                     createSha512Checksums.sh({script=basename ccc, returnStdout=true})
+                     createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb  
 bbb
 ccc
 })
-                  uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
-                  uploadMinSnapshotsToS3.sh(
-        cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
-        cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
-        cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml
-        sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
-    )
-                  uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
-                  uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
-                  uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
-                  uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
+                     uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
+                     uploadMinSnapshotsToS3.sh(
+                cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
+                cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
+                cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml
+                sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
+            )
                      uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        uploadMinSnapshotsToS3.echo(Upload min snapshots)
                         uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz})
                         uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512})
                         uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml})
diff --git a/vars/publishGradleCheckTestResults.groovy b/vars/publishGradleCheckTestResults.groovy
@@ -173,7 +173,7 @@ void indexFailedTestData() {
                 else
                     echo "Index does not exist. Creating..."
                     create_index_response=\$(curl -s -XPUT "${METRICS_HOST_URL}/\${INDEX_NAME}" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" -H 'Content-Type: application/json' -d "\${INDEX_MAPPING}")
-                    if [[ \$create_index_response == *'"acknowledged":true'* ]]; then
+                    if echo "\$create_index_response" | grep -q '"acknowledged":true'; then
                         echo "Index created successfully."
                         echo "Updating alias..."
                         update_alias_response=\$(curl -s -XPOST "${METRICS_HOST_URL}/_aliases" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" -H "Content-Type: application/json" -d '{
@@ -186,7 +186,7 @@ void indexFailedTestData() {
                                 }
                             ]
                         }')
-                        if [[ \$update_alias_response == *'"acknowledged":true'* ]]; then
+                        if echo "\$update_alias_response" | grep -q '"acknowledged":true'; then
                             echo "Alias updated successfully."
                         else
                             echo "Failed to update alias. Error message: \$update_alias_response"

diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy
@@ -19,6 +19,10 @@ void call(Map args = [:]) {
 
     def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest))
     String productName = inputManifest.build.getFilename()
+    String version_plain = inputManifest.build.version
+    String qualifier = inputManifest.build.qualifier ? '-' + inputManifest.build.qualifier : ''
+    String revision = version_plain + qualifier + '-SNAPSHOT'
+    def buildManifestYamlOnly = readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")
     echo("Retreving build manifest from: $WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")
 
     def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml"))
@@ -42,35 +46,71 @@ void call(Map args = [:]) {
     String dstDir = "snapshots/core/${productName}/${version}"
     String baseName = "${productName}-min-${version}-${platform}-${architecture}"
 
-    // Create checksums
-    echo('Create .sha512 for Min Snapshots Artifacts')
-    argsMap = [:]
-    argsMap['artifactPath'] = srcDir
-    for (Closure action : fileActions) { // running createSha512Checksums()
-        action(argsMap)
-    }
-
-    echo("Start copying files: version-${version} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")
-
-    String sedCmd = "sed"
-    if (platform == "darwin") {
-        sedCmd = "gsed"
-    }
-
-    sh """
-        cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
-        cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
-        cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml
-        ${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
-    """
     withCredentials([
         string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'),
         string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),
         string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) {
+
+            // Setup core plugins snapshots with .sha512 and .sig (Tar x64 only)
+            String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
+            boolean corePluginDirExists = fileExists(corePluginDir)
+            if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
+                echo("Create .sha512 and .sig for Core Plugins Snapshots")
+                fileActions = [createSha512Checksums(), createSignatureFiles()]
+                argsMapPlugins = [:]
+                argsMapPlugins['sigtype'] = '.sig'
+                argsMapPlugins['artifactPath'] = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins"
+                for (Closure action : fileActions) {
+                    action(argsMapPlugins)
+                }
+            }
+
+            // Setup min snapshots with .sha512 (All distributions)
+            echo('Create .sha512 for Min Snapshots Artifacts')
+            fileActions = [createSha512Checksums()]
+            argsMapMin = [:]
+            argsMapMin['artifactPath'] = srcDir
+            for (Closure action : fileActions) {
+                action(argsMapMin)
+            }
+
+            echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")
+
+            String sedCmd = "sed"
+            if (platform == "darwin") {
+                sedCmd = "gsed"
+            }
+
+            sh """
+                cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
+                cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
+                cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml
+                ${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
+            """
             withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
+                // min artifacts
+                echo("Upload min snapshots")
                 s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}")
                 s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512")
                 s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml")
+                // core plugins
+                if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
+                    echo("Upload core-plugins snapshots")
+                    List<String> corePluginList = buildManifestYamlOnly.components.artifacts.'core-plugins'[0]
+                    echo("corePluginList: ${corePluginList}")
+                    for (String pluginSubPath : corePluginList) {
+                        String pluginSubFolder = pluginSubPath.split('/')[0]
+                        String pluginNameWithExt = pluginSubPath.split('/')[1]
+                        String pluginName = pluginNameWithExt.replace('-' + revision + '.zip', '')
+                        String pluginFullPath = ['plugins', pluginName, revision].join('/')
+                        s3Upload(
+                            bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}",
+                            path: "snapshots/${pluginFullPath}/",
+                            workingDir: "${corePluginDir}/",
+                            includePathPattern: "**/${pluginName}*"
+                        )
+                    }
+                }
             }
         }
 }