diff --git a/build.gradle b/build.gradle index 992f59583c..18ac4790d7 100644 --- a/build.gradle +++ b/build.gradle @@ -56,16 +56,6 @@ sourceSets { srcDirs = ['tests/jenkins'] } } - - jobs { - groovy { - srcDirs 'src/jenkins/jobs' - compileClasspath += main.compileClasspath - } - - compileClasspath += sourceSets.main.output - runtimeClasspath += sourceSets.main.output - } } sharedLibrary { @@ -127,3 +117,7 @@ jacocoTestReport { xml.required = true } } + +tasks.test { + maxParallelForks = Runtime.getRuntime().availableProcessors() +} diff --git a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile index 84a91034db..61b94d06c1 100644 --- a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile @@ -3,11 +3,13 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch-dashboards" + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -20,6 +22,12 @@ pipeline { description: 'The build manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0/98/linux/x64/builds/opensearch-dashboards/manifest.yml.', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) string( name: 'AGENT_LABEL', description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host.', @@ -27,6 +35,19 @@ pipeline { ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { node { @@ -34,6 +55,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = TEST_MANIFEST if (AGENT_LABEL == '') { @@ -63,6 +85,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -78,6 +101,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = downloadBuildManifest( url: BUILD_MANIFEST_URL, @@ -99,6 +123,7 @@ pipeline { post { always { script { + unstash(name: 'sourceRepo') uploadTestResults( buildManifestFileName: BUILD_MANIFEST, jobName: JOB_NAME @@ -113,6 +138,7 @@ pipeline { post { success { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( @@ -122,13 +148,13 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } failure { node(AGENT_LABEL) { + unstash 'sourceRepo' script { def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( @@ -138,7 +164,6 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index f6daefa585..1a5896f782 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -3,12 +3,14 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) pipeline { options { timeout(time: 4, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' AGENT_ARM64 = 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' IMAGE_RPM = 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' // required for rpm to create digest sha256 correctly with rpm 4.12+ + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -38,6 +40,12 @@ pipeline { defaultValue: "bwc-test-opensearch-dashboards", trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -50,6 +58,19 @@ pipeline { ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_X64 + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('detect docker image + args') { agent { docker { @@ -59,6 +80,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = "$INPUT_MANIFEST" dockerAgent = detectDockerAgent() @@ -74,6 +96,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -101,6 +124,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -177,6 +201,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -184,7 +209,6 @@ pipeline { "\n${env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -203,6 +227,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -229,6 +254,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -266,6 +292,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -293,6 +320,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -320,6 +348,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -396,6 +425,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -403,7 +433,6 @@ pipeline { "\n${env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -424,6 +453,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -450,6 +480,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -486,6 +517,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -512,6 +544,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { uploadIndexFile( indexFilePath: env.INDEX_FILE_PATH @@ -535,6 +568,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { echo "env.ARTIFACT_URL_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" echo "env.ARTIFACT_URL_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" @@ -553,6 +587,7 @@ pipeline { post { always { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -567,6 +602,7 @@ pipeline { } success { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { def stashed = lib.jenkins.Messages.new(this).get([ @@ -584,13 +620,13 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - postCleanup() } } } failure { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { publishNotification( @@ -600,7 +636,6 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - postCleanup() } } diff --git a/jenkins/opensearch-dashboards/integ-test.jenkinsfile b/jenkins/opensearch-dashboards/integ-test.jenkinsfile index d09c4461c0..280043412b 100644 --- a/jenkins/opensearch-dashboards/integ-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/integ-test.jenkinsfile @@ -3,11 +3,13 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch-dashboards" + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -25,11 +27,31 @@ pipeline { description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { label AGENT_LABEL } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = BUILD_MANIFEST_URL if (AGENT_LABEL == '') { @@ -59,6 +81,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -74,6 +97,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = downloadBuildManifest( url: BUILD_MANIFEST_URL, @@ -94,6 +118,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { uploadTestResults( buildManifestFileName: BUILD_MANIFEST, @@ -109,6 +134,7 @@ pipeline { post { success { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( @@ -118,13 +144,13 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( @@ -134,7 +160,6 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } diff --git a/jenkins/opensearch/bwc-test.jenkinsfile b/jenkins/opensearch/bwc-test.jenkinsfile index bceb4d8ea4..02842e5378 100644 --- a/jenkins/opensearch/bwc-test.jenkinsfile +++ b/jenkins/opensearch/bwc-test.jenkinsfile @@ -3,11 +3,13 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch" + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -25,8 +27,27 @@ pipeline { description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host.', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { node { @@ -34,6 +55,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = TEST_MANIFEST if (AGENT_LABEL == '') { @@ -63,6 +85,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -78,6 +101,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = downloadBuildManifest( url: BUILD_MANIFEST_URL, @@ -99,6 +123,7 @@ pipeline { post { always { script { + unstash(name: 'sourceRepo') uploadTestResults( buildManifestFileName: BUILD_MANIFEST, jobName: JOB_NAME @@ -114,6 +139,7 @@ pipeline { success { node(AGENT_LABEL) { script { + unstash(name: 'sourceRepo') def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( icon: ':white_check_mark:', @@ -122,13 +148,13 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( @@ -138,10 +164,9 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } } -} +} \ No newline at end of file diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index ad2014549a..3c8f3a153d 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -3,12 +3,14 @@ lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm)) pipeline { options { timeout(time: 4, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' AGENT_ARM64 = 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' IMAGE_RPM = 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' // required for rpm to create digest sha256 correctly with rpm 4.12+ + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -38,6 +40,12 @@ pipeline { defaultValue: "bwc-test", trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -55,6 +63,19 @@ pipeline { ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_X64 + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('detect docker image + args') { agent { docker { @@ -64,6 +85,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { dockerAgent = detectDockerAgent() currentBuild.description = INPUT_MANIFEST @@ -79,6 +101,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -109,6 +132,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildManifest( componentName: "${COMPONENT_NAME}", @@ -127,10 +151,10 @@ pipeline { echo "Skipping publishing snapshots, ${mavenPath} does not exist." } echo("Uploading min snapshots to S3") - uploadMinSnapshotsToS3( + /*uploadMinSnapshotsToS3( fileActions: [createSha512Checksums()], distribution: 'tar' - ) + )*/ } } post { @@ -149,6 +173,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildManifest( componentName: "${COMPONENT_NAME}", @@ -157,10 +182,10 @@ pipeline { snapshot: true ) echo("Uploading min snapshots to S3") - uploadMinSnapshotsToS3( + /*uploadMinSnapshotsToS3( fileActions: [createSha512Checksums()], distribution: 'tar' - ) + )*/ } } post { @@ -179,6 +204,7 @@ pipeline { jdk dockerAgent.javaVersion } steps { + unstash(name: 'sourceRepo') script { buildManifest( componentName: "${COMPONENT_NAME}", @@ -187,10 +213,10 @@ pipeline { snapshot: true ) echo("Uploading darwin min snapshots to S3") - uploadMinSnapshotsToS3( + /*uploadMinSnapshotsToS3( fileActions: [createSha512Checksums()], distribution: 'tar' - ) + )*/ } } post { @@ -199,7 +225,7 @@ pipeline { } } } - stage('build-snapshot-windows-x64-zip') { + /*stage('build-snapshot-windows-x64-zip') { agent { node { label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' @@ -209,6 +235,7 @@ pipeline { jdk dockerAgent.javaVersion } steps { + unstash(name: 'sourceRepo') script { buildManifest( componentName: "${COMPONENT_NAME}", @@ -229,7 +256,7 @@ pipeline { postCleanup() } } - } + }*/ stage('build-and-test-x64-tar') { agent { docker { @@ -240,6 +267,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -314,6 +342,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -321,7 +350,6 @@ pipeline { "\n${env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -340,6 +368,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -366,6 +395,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -403,13 +433,13 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + "\n${env.RPM_VALIDATION_X64_RPM_TEST_RESULT}" ) - postCleanup() } } @@ -427,6 +457,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -500,6 +531,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -507,7 +539,6 @@ pipeline { "\n${env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -526,6 +557,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -552,6 +584,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -595,7 +628,6 @@ pipeline { lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + "\n${env.RPM_VALIDATION_ARM64_RPM_TEST_RESULT}" ) - postCleanup() } } @@ -615,6 +647,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { uploadIndexFile( indexFilePath: env.INDEX_FILE_PATH @@ -638,6 +671,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { echo "env.ARTIFACT_URL_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" echo "env.ARTIFACT_URL_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" @@ -656,6 +690,7 @@ pipeline { post { always { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -670,6 +705,7 @@ pipeline { } success { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { def stashed = lib.jenkins.Messages.new(this).get([ @@ -687,13 +723,13 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - postCleanup() } } } failure { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { publishNotification( @@ -708,11 +744,9 @@ pipeline { message: buildFailureMessage() ) } - postCleanup() } } } } } - diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index 9f92bacb55..6f4efb6603 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -3,12 +3,14 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch" ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name') + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -26,11 +28,31 @@ pipeline { description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { label AGENT_LABEL } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = BUILD_MANIFEST_URL if (AGENT_LABEL == '') { @@ -60,6 +82,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -70,6 +93,7 @@ pipeline { // Can only be run in runner that is at least 50GB per container agent { label AGENT_LABEL } steps { + unstash(name: 'sourceRepo') script { downloadBuildManifest( @@ -159,6 +183,7 @@ pipeline { success { node(AGENT_LABEL) { script { + unstash(name: 'sourceRepo') def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( icon: ':white_check_mark:', @@ -174,6 +199,7 @@ pipeline { } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( @@ -189,4 +215,4 @@ pipeline { } } } -} +} \ No newline at end of file diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index b81a71304d..2b992fd11f 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -4,12 +4,14 @@ pipeline { agent none options { timeout(time: 15, unit: 'HOURS') + skipDefaultCheckout() } environment { AGENT_LABEL = 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Perf-Test' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-performance-test-v1' BUNDLE_MANIFEST = 'bundle-manifest.yml' JOB_NAME = 'perf-test' + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } triggers { parameterizedCron ''' @@ -47,9 +49,27 @@ pipeline { description: 'Number of times to run a workload before collecting data for Mensor (internal client).', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } - stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('validate-and-set-parameters') { agent { docker { @@ -59,6 +79,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { if (BUNDLE_MANIFEST_URL == '') { currentBuild.result = 'ABORTED' @@ -100,6 +121,7 @@ pipeline { expression { return env.HAS_SECURITY.toBoolean() } } steps { + unstash(name: 'sourceRepo') script { def bundleManifestObj = downloadBuildManifest( url: BUNDLE_MANIFEST_URL, @@ -125,6 +147,7 @@ pipeline { } post { success { + unstash(name: 'sourceRepo') script { uploadTestResults( buildManifestFileName: BUNDLE_MANIFEST, @@ -151,6 +174,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def bundleManifestObj = downloadBuildManifest( url: BUNDLE_MANIFEST_URL, @@ -177,6 +201,7 @@ pipeline { } post { success { + unstash(name: 'sourceRepo') script { uploadTestResults( buildManifestFileName: BUNDLE_MANIFEST, @@ -201,6 +226,7 @@ pipeline { post { success { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) publishNotification( @@ -215,6 +241,7 @@ pipeline { } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) publishNotification( diff --git a/jenkins/release-notes-check/release-notes-check.jenkinsfile b/jenkins/release-notes-check/release-notes-check.jenkinsfile index bc4dd84697..d169d1cbf6 100644 --- a/jenkins/release-notes-check/release-notes-check.jenkinsfile +++ b/jenkins/release-notes-check/release-notes-check.jenkinsfile @@ -2,7 +2,8 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) pipeline { options { - timeout(time: 2, unit: 'HOURS') + timeout(time: 2, unit: 'HOURS') + skipDefaultCheckout true } agent none parameters { @@ -31,11 +32,31 @@ pipeline { description: 'Use only with COMMENT: UPDATE', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } environment { AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_X64 + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash 'sourceRepo' + } + } stage('detect docker image + args') { agent { docker { @@ -45,19 +66,23 @@ pipeline { } } steps { + unstash 'sourceRepo' script { dockerAgent = detectDockerAgent() currentBuild.description = INPUT_MANIFEST } } - post { - always { - postCleanup() - } - } } stage('Parameters Check') { + agent { + docker { + label AGENT_X64 + image dockerAgent.image + alwaysPull true + } + } steps { + unstash 'sourceRepo' script { currentBuild.description = "Comment:${COMMENT}, Manifest:${INPUT_MANIFEST}" if (GIT_LOG_DATE.isEmpty() || INPUT_MANIFEST.isEmpty()) { @@ -84,6 +109,7 @@ pipeline { } } steps { + unstash 'sourceRepo' script { withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { if (params.COMMENT == "ADD") { @@ -115,10 +141,12 @@ pipeline { } } } - post { - always { - postCleanup() - } + } + } + post { + always { + node(AGENT_X64) { + postCleanup() } } } diff --git a/tests/jenkins/TestOpenSearchBwcTest.groovy b/tests/jenkins/TestOpenSearchBwcTest.groovy index 1df9ab2fb9..5ce2427369 100644 --- a/tests/jenkins/TestOpenSearchBwcTest.groovy +++ b/tests/jenkins/TestOpenSearchBwcTest.groovy @@ -3,11 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ - import jenkins.tests.BuildPipelineTest import org.junit.Before import org.junit.Test import org.yaml.snakeyaml.Yaml +import com.lesfurets.jenkins.unit.global.* class TestOpenSearchBwcTest extends BuildPipelineTest { @@ -19,6 +19,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest { def buildManifest = "tests/jenkins/data/opensearch-1.3.0-build.yml" def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/${buildId}/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz" def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" + def AGENT_X64 = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" this.registerLibTester(new DetectTestDockerAgentLibTester()) this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest)) @@ -38,6 +39,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest { binding.setVariable('AGENT_LABEL', agentLabel) binding.setVariable('BUILD_MANIFEST', buildManifest) binding.setVariable('BUILD_ID', "${buildId}") + binding.setVariable('BUILD_BRANCH', "main") def env = binding.getVariable('env') env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'] @@ -48,6 +50,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest { }) helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy b/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy index db1ca6b5be..bcbe0e238a 100644 --- a/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy +++ b/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy @@ -38,6 +38,7 @@ class TestOpenSearchDashboardsBwcTest extends BuildPipelineTest { binding.setVariable('AGENT_LABEL', agentLabel) binding.setVariable('BUILD_MANIFEST', buildManifest) binding.setVariable('BUILD_ID', "${buildId}") + binding.setVariable('BUILD_BRANCH', "main") def env = binding.getVariable('env') env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'] @@ -48,6 +49,7 @@ class TestOpenSearchDashboardsBwcTest extends BuildPipelineTest { }) helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy index 0f1069a2d2..99ddcee740 100644 --- a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy @@ -48,7 +48,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { }) helper.registerAllowedMethod('findFiles', [Map.class], null) - helper.registerAllowedMethod('unstash', [String.class], null) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/TestOpenSearchIntegTest.groovy b/tests/jenkins/TestOpenSearchIntegTest.groovy index e724fd7b7b..4061e630e2 100644 --- a/tests/jenkins/TestOpenSearchIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchIntegTest.groovy @@ -44,6 +44,8 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { binding.setVariable('BUILD_MANIFEST', buildManifest) binding.setVariable('BUILD_ID', "${buildId}") binding.setVariable('ARTIFACT_BUCKET_NAME', bucketName) + binding.setVariable('BUILD_BRANCH', "main") + binding.setVariable('BUILD_BRANCH', "main") def env = binding.getVariable('env') env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'] @@ -58,6 +60,7 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { helper.registerAllowedMethod('fileExists', [String.class], { args -> return true; }) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt index 0c62a6ff2b..709c9d5fb8 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt @@ -3,13 +3,20 @@ bwc-test.library({identifier=jenkins@20211118, retriever=null}) bwc-test.pipeline(groovy.lang.Closure) bwc-test.timeout({time=3, unit=HOURS}) + bwc-test.skipDefaultCheckout() bwc-test.echo(Executing on agent [label:none]) + bwc-test.stage(checkout the build repo, groovy.lang.Closure) + bwc-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + bwc-test.stash({name=sourceRepo}) bwc-test.stage(verify-parameters, groovy.lang.Closure) bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.fileExists(manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml) bwc-test.stage(detect docker image + args, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -19,6 +26,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) bwc-test.stage(bwc-test, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/215/linux/x64/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -39,6 +47,7 @@ runBwcTestScript.echo(Paths: opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) runBwcTestScript.sh(./test.sh bwc-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --test-run-id 487 --paths opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) bwc-test.script(groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) bwc-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) @@ -60,6 +69,7 @@ bwc-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) bwc-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([bwc-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt index 5d013a9304..aeebc7d546 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt @@ -3,13 +3,20 @@ integ-test.library({identifier=jenkins@20211118, retriever=null}) integ-test.pipeline(groovy.lang.Closure) integ-test.timeout({time=3, unit=HOURS}) + integ-test.skipDefaultCheckout() integ-test.echo(Executing on agent [label:none]) + integ-test.stage(checkout the build repo, groovy.lang.Closure) + integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + integ-test.stash({name=sourceRepo}) integ-test.stage(verify-parameters, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.fileExists(manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml) integ-test.stage(detect docker image + args, groovy.lang.Closure) integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -19,6 +26,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) integ-test.stage(integ-test, groovy.lang.Closure) integ-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/215/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -41,6 +49,7 @@ runIntegTestScript.echo(Paths: opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.0/latest/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) runIntegTestScript.echo(Component: functionalTestDashboards) runIntegTestScript.sh(./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component functionalTestDashboards --test-run-id 487 --paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.0/latest/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -63,6 +72,7 @@ integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([integ-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt index 8c8c41d3ac..cccf7f2130 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt @@ -3,13 +3,20 @@ bwc-test.library({identifier=jenkins@20211118, retriever=null}) bwc-test.pipeline(groovy.lang.Closure) bwc-test.timeout({time=3, unit=HOURS}) + bwc-test.skipDefaultCheckout() bwc-test.echo(Executing on agent [label:none]) + bwc-test.stage(checkout the build repo, groovy.lang.Closure) + bwc-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + bwc-test.stash({name=sourceRepo}) bwc-test.stage(verify-parameters, groovy.lang.Closure) bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) bwc-test.stage(detect docker image + args, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -19,6 +26,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) bwc-test.stage(bwc-test, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -39,6 +47,7 @@ runBwcTestScript.echo(Paths: opensearch=https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar) runBwcTestScript.sh(./test.sh bwc-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --test-run-id 487 --paths opensearch=https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar) bwc-test.script(groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) bwc-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) @@ -61,6 +70,7 @@ postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) bwc-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) bwc-test.script(groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) Messages.asBoolean() Messages.get([bwc-test]) bwc-test.unstash({name=messages-bwc-test}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt index b737b0a793..8bc6c2506c 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt @@ -4,13 +4,20 @@ integ-test.pipeline(groovy.lang.Closure) integ-test.credentials(jenkins-artifact-bucket-name) integ-test.timeout({time=3, unit=HOURS}) + integ-test.skipDefaultCheckout() integ-test.echo(Executing on agent [label:none]) + integ-test.stage(checkout the build repo, groovy.lang.Closure) + integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + integ-test.stash({name=sourceRepo}) integ-test.stage(verify-parameters, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) integ-test.stage(detect docker image + args, groovy.lang.Closure) integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -20,6 +27,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) integ-test.stage(integ-test, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -86,6 +94,7 @@ postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.script(groovy.lang.Closure) + integ-test.unstash({name=sourceRepo}) Messages.asBoolean() Messages.get([integ-test]) integ-test.unstash({name=messages-integ-test}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index 80fe4f9ee1..d15bbca63e 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -3,6 +3,7 @@ perf-test.library({identifier=jenkins@20211118, retriever=null}) perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=15, unit=HOURS}) + perf-test.skipDefaultCheckout() perf-test.echo(Executing on agent [label:none]) perf-test.parameterizedCron( H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.5/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 @@ -14,8 +15,13 @@ H 9 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 H 9 * * 5 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 ) + perf-test.stage(checkout the build repo, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + perf-test.stash({name=sourceRepo}) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -32,6 +38,7 @@ perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.stage(test-with-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -72,6 +79,7 @@ Performance tests with security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text= Performance tests with security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=307}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -95,6 +103,7 @@ Performance tests with security for 1236 completed}) postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) perf-test.stage(test-without-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -135,6 +144,7 @@ Performance tests without security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text= Performance tests without security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=307}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -157,6 +167,7 @@ Performance tests without security for 1236 completed}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) perf-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([perf-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index f845091810..8ce8de2083 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -3,6 +3,7 @@ perf-test.library({identifier=jenkins@20211118, retriever=null}) perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=15, unit=HOURS}) + perf-test.skipDefaultCheckout() perf-test.echo(Executing on agent [label:none]) perf-test.parameterizedCron( H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.5/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 @@ -14,8 +15,13 @@ H 9 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 H 9 * * 5 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 ) + perf-test.stage(checkout the build repo, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + perf-test.stash({name=sourceRepo}) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -33,6 +39,7 @@ perf-test.echo(Skipping stage test-with-security) perf-test.stage(test-without-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -73,6 +80,7 @@ Performance tests without security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text= Performance tests without security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, jobName=perf-test, buildNumber=307}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -95,6 +103,7 @@ Performance tests without security for 1236 completed}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) perf-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([perf-test]) diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy index 963da32209..a0b2477663 100644 --- a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -44,6 +44,7 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { helper.registerAllowedMethod("downloadBuildManifest", [Map], { c -> lib.jenkins.BuildManifest.new(readYaml(file: bundleManifest)) }) + helper.registerAllowedMethod("git", [Map]) helper.registerAllowedMethod('parameterizedCron', [String], null) binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1')