Skip to content

Commit

Permalink
Enable check-for-build to trigger linux-tar integTest during distribu…
Browse files Browse the repository at this point in the history 
…tion build cron (opensearch-project#4975)

Signed-off-by: Peter Zhu <[email protected]>
  • Loading branch information
@peterzhuamazon
peterzhuamazon authored Aug 28, 2024
1 parent 5870c50 commit 08d0460
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 18 deletions.
20 changes: 16 additions & 4 deletions jenkins/check-for-build.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ pipeline {
}
triggers {
parameterizedCron '''
H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-dashboards-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
'''
}
Expand Down Expand Up @@ -56,6 +56,16 @@ pipeline {
description: 'Distribution to build',
trim: true
)
string(
name: 'TEST_PLATFORM',
description: 'Platform to test',
trim: true
)
string(
name: 'TEST_DISTRIBUTION',
description: 'Distribution to test',
trim: true
)
}
stages {
stage('detect docker image + args') {
Expand Down Expand Up @@ -102,6 +112,8 @@ pipeline {
string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"),
string(name: 'BUILD_PLATFORM', value: "${BUILD_PLATFORM}"),
string(name: 'BUILD_DISTRIBUTION', value: "${BUILD_DISTRIBUTION}")
string(name: 'TEST_PLATFORM', value: "${TEST_PLATFORM}"),
string(name: 'TEST_DISTRIBUTION', value: "${TEST_DISTRIBUTION}")
], wait: true

echo "Build succeeded, uploading build SHA for that job"
Expand Down
30 changes: 18 additions & 12 deletions src/manifests_workflow/input_manifests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,6 @@


class InputManifests(Manifests):
BUILD_PLATFORM = {
"opensearch": "linux macos windows",
"opensearch-dashboards": "linux windows"
}
BUILD_DISTRIBUTION = {
"opensearch": "tar rpm deb zip",
"opensearch-dashboards": "tar rpm deb zip"
}

def __init__(self, name: str) -> None:
self.name = name
self.prefix = name.lower().replace(" ", "-")
Expand Down Expand Up @@ -106,6 +97,7 @@ def update(
# check out and build #main, 1.x, etc.
all_branches = sorted(min_klass.branches())
branches = [b for b in all_branches if not any(b == o or b.startswith((f"{o}-", f"{o}/")) for o in legacy_branches)]
# TODO: Remove
logging.info(f"Checking {self.name} {sorted(branches)} branches")
logging.info(f"Ignoring {self.name} {sorted(set(all_branches) - set(branches))} branches as they are legacy")

Expand Down Expand Up @@ -176,13 +168,27 @@ def add_to_cron(self, version: str) -> None:
with open(jenkinsfile, "r") as f:
data = f.read()

build_platform = self.BUILD_PLATFORM.get(self.prefix, "linux")
build_distribution = self.BUILD_DISTRIBUTION.get(self.prefix, "tar")
# Note: default to linux tar for now as integTest is very heavy on resources
build_platform_map = {
"opensearch": "linux windows",
"opensearch-dashboards": "linux windows"
}
build_distribution_map = {
"opensearch": "tar rpm deb zip",
"opensearch-dashboards": "tar rpm deb zip"
}
build_platform = build_platform_map.get(self.prefix, "linux")
build_distribution = build_distribution_map.get(self.prefix, "tar")
test_platform = "linux"
test_distribution = "tar"

cron_entry = f"H 1 * * * %INPUT_MANIFEST={version}/{self.prefix}-{version}.yml;" \
f"TARGET_JOB_NAME=distribution-build-{self.prefix};" \
f"BUILD_PLATFORM={build_platform};" \
f"BUILD_DISTRIBUTION={build_distribution}\n"
f"BUILD_DISTRIBUTION={build_distribution};" \
f"TEST_MANIFEST={version}/{self.prefix}-{version}-test.yml;" \
f"TEST_PLATFORM={test_platform};" \
f"TEST_DISTRIBUTION={test_distribution}\n"

if cron_entry in data:
raise ValueError(f"{jenkinsfile} already contains an entry for {self.prefix} {version}")
Expand Down
5 changes: 3 additions & 2 deletions tests/tests_manifests_workflow/test_input_manifests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,9 @@ def test_add_to_cron(self, mock_open: MagicMock) -> None:
mock_open.assert_has_calls([call(InputManifests.cron_jenkinsfile(), 'w')])
mock_open.assert_has_calls([call(InputManifests.cron_jenkinsfile(), 'r')])
mock_open().write.assert_called_once_with(
f"parameterizedCron '''\n{' ' * 12}H 1 * * * %INPUT_MANIFEST=0.1.2/test-0.1.2.yml;"
f"TARGET_JOB_NAME=distribution-build-test;BUILD_PLATFORM=linux;BUILD_DISTRIBUTION=tar\n"
"parameterizedCron '''\n H 1 * * * %INPUT_MANIFEST=0.1.2/test-0.1.2.yml;"
"TARGET_JOB_NAME=distribution-build-test;BUILD_PLATFORM=linux;BUILD_DISTRIBUTION=tar;"
"TEST_MANIFEST=0.1.2/test-0.1.2-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar\n"
)

def test_os_versionincrement_workflow(self) -> None:
Expand Down

0 comments on commit 08d0460

Please sign in to comment.