Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automate locking manifests with tags #4868

Merged
merged 1 commit into from
Jul 22, 2024
Merged

Automate locking manifests with tags #4868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 43 additions & 4 deletions jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ pipeline {
trim: true
)
choice(
choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS'],
choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS', 'UPDATE_TO_TAGS'],
name: 'MANIFEST_LOCK_ACTION',
description: 'The manifest lock action to choose.<br>MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.<br>UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.',
description: 'The manifest lock action to choose.<br>MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.<br>UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.<br>UPDATE_TO_TAGS: Will update the manifest with ref tags',
)
string(
name: 'COMPONENTS',
Expand All @@ -56,9 +56,13 @@ pipeline {
steps {
script {
currentBuild.description = """Action: ${MANIFEST_LOCK_ACTION}<br>Release: ${RELEASE_VERSION} OS=${OPENSEARCH_RELEASE_CANDIDATE} OSD=${OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE}"""
if (MANIFEST_LOCK_ACTION.isEmpty() || RELEASE_VERSION.isEmpty() || OPENSEARCH_RELEASE_CANDIDATE.isEmpty() || OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE.isEmpty()) {
if (MANIFEST_LOCK_ACTION.isEmpty() || RELEASE_VERSION.isEmpty()) {
currentBuild.result = 'ABORTED'
error('Make sure all the parameters are passed in.')
error('MANIFEST_LOCK_ACTION and/or RELEASE_VERSION cannot be empty!')
}
if ((params.MANIFEST_LOCK_ACTION == 'MATCH_BUILD_MANIFEST' || params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_RECENT_COMMITS') && (OPENSEARCH_RELEASE_CANDIDATE.isEmpty() || OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE.isEmpty())) {
currentBuild.result = 'ABORTED'
error('OPENSEARCH_RELEASE_CANDIDATE and/or OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE cannot be empty when MANIFEST_LOCK_ACTION is MATCH_BUILD_MANIFEST or UPDATE_TO_RECENT_COMMITS.')
}
}
}
Expand Down Expand Up @@ -137,6 +141,41 @@ pipeline {
}
}
}
stage('UPDATE_TO_TAGS') {
when {
expression { params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_TAGS' }
}
steps {
script {
def updateManifest = { String productName ->
def existingManifest = readYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml"
def selectedComponents = params.COMPONENTS.isEmpty() ? existingManifest.components : existingManifest.components.findAll { component ->
params.COMPONENTS.split(',').any { it.trim() == component.name }
}
selectedComponents.each { componentName ->
def existingComponent = existingManifest.components.find { it.name == componentName.name }
if (existingComponent) {
def releaseTag = params.RELEASE_VERSION + ".0"
if (componentName.name == 'OpenSearch' || componentName.name == 'OpenSearch-Dashboards' || componentName.name == 'functionalTestDashboards') {
releaseTag = params.RELEASE_VERSION
}
def newComponentRef = existingManifest.components.find { it.name == componentName.name }?.ref
if (newComponentRef) {
existingComponent.ref = "tags/" + releaseTag
}
}
gaiksaya marked this conversation as resolved.
Show resolved Hide resolved
}
writeYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml", data: existingManifest, overwrite: true
sh """
yq eval -i '.' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml
sed -i '1s/^/---\\n/' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml
"""
}
updateManifest("opensearch")
updateManifest("opensearch-dashboards")
}
}
}
stage('Create Pull Request') {
steps {
script {
Expand Down
13 changes: 13 additions & 0 deletions tests/jenkins/TestReleaseManifestCommitLock.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,19 @@ class TestReleaseManifestCommitLock extends BuildPipelineTest {
assertCallStack().contains('stage(UPDATE_TO_RECENT_COMMITS, groovy.lang.Closure)')
}

@Test
public void testManifestCommitLock_updateToTags() {
addParam('MANIFEST_LOCK_ACTION', 'UPDATE_TO_TAGS')
super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile',
'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags')
def callStack = helper.getCallStack()
assertCallStack().contains('stage(Parameters Check, groovy.lang.Closure)')
assertCallStack().contains('stage(UPDATE_TO_TAGS, groovy.lang.Closure)')
assertCallStack().contains('Skipping stage MATCH_BUILD_MANIFEST')
assertCallStack().contains('Skipping stage UPDATE_TO_RECENT_COMMITS')
assertCallStack().contains('release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})')
}

@Test
public void testManifestCommitLock_createPullRequest() {
super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile',
Expand Down
1 change: 1 addition & 0 deletions ...egression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ ccc})
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
Expand Down
1 change: 1 addition & 0 deletions ...gression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ ccc})
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
Expand Down
1 change: 1 addition & 0 deletions ...ssion-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr
yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
Expand Down
44 changes: 44 additions & 0 deletions ...job-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
release-manifest-commit-lock.run()
release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
release-manifest-commit-lock.library({[email protected], retriever=null})
release-manifest-commit-lock.pipeline(groovy.lang.Closure)
release-manifest-commit-lock.timeout({time=2, unit=HOURS})
release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.echo(Skipping stage MATCH_BUILD_MANIFEST)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
release-manifest-commit-lock.stage(UPDATE_TO_TAGS, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml})
release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})
release-manifest-commit-lock.sh(
yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml
)
release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml})
release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})
release-manifest-commit-lock.sh(
yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
)
release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
release-manifest-commit-lock.sh(
git remote set-url origin "https://opensearch-ci:[email protected]/opensearch-project/opensearch-build"
git config user.email "[email protected]"
git config user.name "opensearch-ci"
git checkout -b manifest-lock
)
release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain})
release-manifest-commit-lock.sh(
git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add
git commit -sm "Manifest Commit Lock for Release 2.0.0"
git push origin manifest-lock --force
gh pr create --title '[2.0.0] Manifest Commit Lock with action UPDATE_TO_TAGS' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main
)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.postCleanup()
postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
1 change: 1 addition & 0 deletions ...nkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ ccc})
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
Expand Down
1 change: 1 addition & 0 deletions ...ins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr
yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
Expand Down
1 change: 1 addition & 0 deletions ...-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit_excludeFTRepo.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
)
release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
release-manifest-commit-lock.script(groovy.lang.Closure)
release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
Expand Down
Loading