From ac16ac7398e527cbf4ed47221291b0067cefeaa1 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Fri, 1 Nov 2024 13:11:33 -0700 Subject: [PATCH 1/4] Gradle agent `Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host` error `[[: not found` (#557) Signed-off-by: Prudhvi Godithi --- build.gradle | 2 +- vars/publishGradleCheckTestResults.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 3c3126a4..15e3f0e0 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ jacocoTestReport { } } -String version = '7.3.1' +String version = '7.3.2' task updateVersion { doLast { diff --git a/vars/publishGradleCheckTestResults.groovy b/vars/publishGradleCheckTestResults.groovy index d3eb4054..54fb4ea6 100644 --- a/vars/publishGradleCheckTestResults.groovy +++ b/vars/publishGradleCheckTestResults.groovy @@ -173,7 +173,7 @@ void indexFailedTestData() { else echo "Index does not exist. Creating..." create_index_response=\$(curl -s -XPUT "${METRICS_HOST_URL}/\${INDEX_NAME}" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" -H 'Content-Type: application/json' -d "\${INDEX_MAPPING}") - if [[ \$create_index_response == *'"acknowledged":true'* ]]; then + if echo "\$create_index_response" | grep -q '"acknowledged":true'; then echo "Index created successfully." echo "Updating alias..." update_alias_response=\$(curl -s -XPOST "${METRICS_HOST_URL}/_aliases" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" -H "Content-Type: application/json" -d '{ @@ -186,7 +186,7 @@ void indexFailedTestData() { } ] }') - if [[ \$update_alias_response == *'"acknowledged":true'* ]]; then + if echo "\$update_alias_response" | grep -q '"acknowledged":true'; then echo "Alias updated successfully." else echo "Failed to update alias. Error message: \$update_alias_response" From ac5825dbb572bd8c5ccac765662a9a0b75637561 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 4 Nov 2024 17:10:57 -0500 Subject: [PATCH 2/4] Add core plugins snapshots publishing alongside min snapshots (#556) Signed-off-by: Peter Zhu --- .../uploadMinSnapshotsToS3_Jenkinsfile.txt | 5 ++- vars/uploadMinSnapshotsToS3.groovy | 39 ++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index b0e44531..d1672ca5 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -9,9 +9,11 @@ uploadMinSnapshotsToS3.library({identifier=jenkins@main, retriever=null}) uploadMinSnapshotsToS3.readYaml({file=tests/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() + uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml}) uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml) uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml}) BuildManifest.asBoolean() + uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins) uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts) createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true}) createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz) @@ -28,7 +30,7 @@ ccc bbb ccc }) - uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz) + uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz) uploadMinSnapshotsToS3.sh( cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512 @@ -40,6 +42,7 @@ ccc uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadMinSnapshotsToS3.echo(Upload min snapshots) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml}) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index e185a02b..0d02800d 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -19,6 +19,10 @@ void call(Map args = [:]) { def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest)) String productName = inputManifest.build.getFilename() + String version_plain = inputManifest.build.version + String qualifier = inputManifest.build.qualifier ? '-' + inputManifest.build.qualifier : '' + String revision = version_plain + qualifier + '-SNAPSHOT' + def buildManifestYamlOnly = readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml") echo("Retreving build manifest from: $WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml") def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")) @@ -41,16 +45,27 @@ void call(Map args = [:]) { String srcDir = "${WORKSPACE}/${distribution}/builds/${productName}/dist".replace("\\", "/") String dstDir = "snapshots/core/${productName}/${version}" String baseName = "${productName}-min-${version}-${platform}-${architecture}" + argsMap = [:] + + // Setup core plugins (Tar x64 only) + String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/") + boolean corePluginDirExists = fileExists(corePluginDir) + if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) { + echo("Create .sha512 for Core Plugins Snapshots") + argsMap['artifactPath'] = corePluginDir + for (Closure action : fileActions) { // running createSha512Checksums() + action(argsMap) + } + } // Create checksums echo('Create .sha512 for Min Snapshots Artifacts') - argsMap = [:] argsMap['artifactPath'] = srcDir for (Closure action : fileActions) { // running createSha512Checksums() action(argsMap) } - echo("Start copying files: version-${version} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}") + echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}") String sedCmd = "sed" if (platform == "darwin") { @@ -68,9 +83,29 @@ void call(Map args = [:]) { string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) { withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + // min artifacts + echo("Upload min snapshots") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml") + // core plugins + if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) { + echo("Upload core-plugins snapshots") + List corePluginList = buildManifestYamlOnly.components.artifacts.'core-plugins'[0] + echo("corePluginList: ${corePluginList}") + for (String pluginSubPath : corePluginList) { + String pluginSubFolder = pluginSubPath.split('/')[0] + String pluginNameWithExt = pluginSubPath.split('/')[1] + String pluginName = pluginNameWithExt.replace('-' + revision + '.zip', '') + String pluginFullPath = ['plugins', pluginName, revision].join('/') + s3Upload( + bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", + path: "snapshots/${pluginFullPath}/", + workingDir: "${corePluginDir}/", + includePathPattern: "**/${pluginName}*" + ) + } + } } } } From 0196bcb2f34bc40b08f21f6c59435f9f5a2a2858 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 Nov 2024 17:35:53 -0500 Subject: [PATCH 3/4] Upload signature alongside snapshots artifacts and checksums (#561) Signed-off-by: Peter Zhu --- build.gradle | 2 +- .../uploadMinSnapshotsToS3_Jenkinsfile.txt | 46 ++++++------- vars/uploadMinSnapshotsToS3.groovy | 66 ++++++++++--------- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/build.gradle b/build.gradle index 15e3f0e0..f04c75eb 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ jacocoTestReport { } } -String version = '7.3.2' +String version = '7.3.3' task updateVersion { doLast { diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index d1672ca5..e6663137 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -13,36 +13,38 @@ uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml) uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml}) BuildManifest.asBoolean() - uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins) - uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts) - createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true}) - createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz) - createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true}) - createSha512Checksums.sh({script=basename ccc, returnStdout=true}) - createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb + uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins) + uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts) + createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true}) + createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz) + createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true}) + createSha512Checksums.sh({script=basename ccc, returnStdout=true}) + createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb bbb ccc }) - createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz) - createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true}) - createSha512Checksums.sh({script=basename ccc, returnStdout=true}) - createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb + createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz) + createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true}) + createSha512Checksums.sh({script=basename ccc, returnStdout=true}) + createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb bbb ccc }) - uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz) - uploadMinSnapshotsToS3.sh( - cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz - cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512 - cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml - sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512 - ) - uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) - uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) - uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) - uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz) + uploadMinSnapshotsToS3.sh( + cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz + cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512 + cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sig tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig + cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml + sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512 + ) uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadMinSnapshotsToS3.echo(Upload min snapshots) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml}) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index 0d02800d..d66763fd 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -46,47 +46,51 @@ void call(Map args = [:]) { String dstDir = "snapshots/core/${productName}/${version}" String baseName = "${productName}-min-${version}-${platform}-${architecture}" argsMap = [:] + argsMap['sigtype'] = '.sig' - // Setup core plugins (Tar x64 only) - String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/") - boolean corePluginDirExists = fileExists(corePluginDir) - if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) { - echo("Create .sha512 for Core Plugins Snapshots") - argsMap['artifactPath'] = corePluginDir - for (Closure action : fileActions) { // running createSha512Checksums() - action(argsMap) - } - } - - // Create checksums - echo('Create .sha512 for Min Snapshots Artifacts') - argsMap['artifactPath'] = srcDir - for (Closure action : fileActions) { // running createSha512Checksums() - action(argsMap) - } - - echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}") - - String sedCmd = "sed" - if (platform == "darwin") { - sedCmd = "gsed" - } - - sh """ - cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension} - cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512 - cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml - ${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512 - """ withCredentials([ string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'), string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) { + + // Setup core plugins snapshots with .sha512 and .sig (Tar x64 only) + String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/") + boolean corePluginDirExists = fileExists(corePluginDir) + if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) { + echo("Create .sha512 for Core Plugins Snapshots") + argsMap['artifactPath'] = corePluginDir + for (Closure action : fileActions) { + action(argsMap) + } + } + + // Setup min snapshots with .sha512 and .sig (All distributions) + echo('Create .sha512 for Min Snapshots Artifacts') + argsMap['artifactPath'] = srcDir + for (Closure action : fileActions) { + action(argsMap) + } + + echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}") + + String sedCmd = "sed" + if (platform == "darwin") { + sedCmd = "gsed" + } + + sh """ + cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension} + cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512 + cp -v ${srcDir}/${baseName}.${extension}.sig ${srcDir}/${baseName}-latest.${extension}.sig + cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml + ${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512 + """ withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { // min artifacts echo("Upload min snapshots") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512") + s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sig", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sig") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml") // core plugins if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) { From e6ec7d1782d25ee19bb1738dc943fc55d4fb1b2d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 11 Nov 2024 16:02:15 -0500 Subject: [PATCH 4/4] Only sign core plugins and ignore min artifacts (#564) Signed-off-by: Peter Zhu --- build.gradle | 2 +- .../uploadMinSnapshotsToS3_Jenkinsfile.txt | 3 +-- vars/uploadMinSnapshotsToS3.groovy | 21 ++++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/build.gradle b/build.gradle index f04c75eb..756a79fa 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ jacocoTestReport { } } -String version = '7.3.3' +String version = '7.3.4' task updateVersion { doLast { diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index e6663137..de29b682 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -19,6 +19,7 @@ uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins) uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts) + uploadMinSnapshotsToS3.createSha512Checksums() createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true}) createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz) createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true}) @@ -38,7 +39,6 @@ ccc uploadMinSnapshotsToS3.sh( cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512 - cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sig tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512 ) @@ -46,5 +46,4 @@ ccc uploadMinSnapshotsToS3.echo(Upload min snapshots) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml}) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index d66763fd..401faea1 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -45,8 +45,6 @@ void call(Map args = [:]) { String srcDir = "${WORKSPACE}/${distribution}/builds/${productName}/dist".replace("\\", "/") String dstDir = "snapshots/core/${productName}/${version}" String baseName = "${productName}-min-${version}-${platform}-${architecture}" - argsMap = [:] - argsMap['sigtype'] = '.sig' withCredentials([ string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'), @@ -57,18 +55,23 @@ void call(Map args = [:]) { String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/") boolean corePluginDirExists = fileExists(corePluginDir) if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) { - echo("Create .sha512 for Core Plugins Snapshots") - argsMap['artifactPath'] = corePluginDir + echo("Create .sha512 and .sig for Core Plugins Snapshots") + fileActions = [createSha512Checksums(), createSignatureFiles()] + argsMapPlugins = [:] + argsMapPlugins['sigtype'] = '.sig' + argsMapPlugins['artifactPath'] = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins" for (Closure action : fileActions) { - action(argsMap) + action(argsMapPlugins) } } - // Setup min snapshots with .sha512 and .sig (All distributions) + // Setup min snapshots with .sha512 (All distributions) echo('Create .sha512 for Min Snapshots Artifacts') - argsMap['artifactPath'] = srcDir + fileActions = [createSha512Checksums()] + argsMapMin = [:] + argsMapMin['artifactPath'] = srcDir for (Closure action : fileActions) { - action(argsMap) + action(argsMapMin) } echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}") @@ -81,7 +84,6 @@ void call(Map args = [:]) { sh """ cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension} cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512 - cp -v ${srcDir}/${baseName}.${extension}.sig ${srcDir}/${baseName}-latest.${extension}.sig cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml ${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512 """ @@ -90,7 +92,6 @@ void call(Map args = [:]) { echo("Upload min snapshots") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512") - s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sig", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sig") s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml") // core plugins if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {