diff --git a/.build/check-code.sh b/.build/check-code.sh index d3baec45d94e..9e56c82554f3 100755 --- a/.build/check-code.sh +++ b/.build/check-code.sh @@ -24,5 +24,5 @@ command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1 [ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; } # execute -ant -f "${CASSANDRA_DIR}/build.xml" check dependency-check +ant -f "${CASSANDRA_DIR}/build.xml" check #dependency-check FIXME uncomment when working again exit $? diff --git a/.build/docker/_docker_run.sh b/.build/docker/_docker_run.sh index 659f5bacf3a1..a729f089bd41 100755 --- a/.build/docker/_docker_run.sh +++ b/.build/docker/_docker_run.sh @@ -118,7 +118,7 @@ docker_command="export ANT_OPTS=\"-Dbuild.dir=\${DIST_DIR} ${CASSANDRA_DOCKER_AN # run without the default seccomp profile # re-use the host's maven repository container_id=$(docker run --name ${container_name} -d --security-opt seccomp=unconfined --rm \ - -v "${cassandra_dir}":/home/build/cassandra -v ~/.m2/repository/:/home/build/.m2/repository/ -v "${build_dir}":/dist \ + -v "${cassandra_dir}":/home/build/cassandra -v ${HOME}/.m2/repository/:/home/build/.m2/repository/ -v "${build_dir}":/dist \ ${docker_volume_opt} \ ${image_name} sleep 1h) diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh index 3eb7a1aedb53..9ef0ae7d8907 100755 --- a/.build/run-python-dtests.sh +++ b/.build/run-python-dtests.sh @@ -167,7 +167,7 @@ pushd ${CASSANDRA_DIR}/ >/dev/null # remove wrapping elements. `ant generate-unified-test-report` doesn't like it` sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml > ${TMPDIR}/nosetests.xml cat ${TMPDIR}/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml -ant -quiet -silent generate-unified-test-report +ant -quiet -silent generate-test-report popd >/dev/null ################################ diff --git a/.build/run-tests.sh b/.build/run-tests.sh index c0a1e1eb2e3c..8e5012a06ed3 100755 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@ -186,7 +186,8 @@ _main() { [[ "/" == "${DIST_DIR}" ]] || rm -rf "${DIST_DIR}/test/{html,output,logs}" # cheap trick to ensure dependency libraries are in place. allows us to stash only project specific build artifacts. - ant -quiet -silent resolver-dist-lib + # also recreate some of the non-build files we need + ant -quiet -silent resolver-dist-lib _createVersionPropFile case ${target} in "stress-test") @@ -262,7 +263,7 @@ _main() { esac # merge all unit xml files into one, and print summary test numbers - ant -quiet -silent generate-unified-test-report + ant -quiet -silent generate-test-report popd >/dev/null } diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 831e2ade65ee..3ab7e1a645b8 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -6,767 +6,556 @@ // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// https://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// Se# Licensed to the Apache Software Foundation (ASF) under onee the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. // // -// Jenkins declaration of how to build and test the current codebase. -// Jenkins infrastructure related settings should be kept in +// Jenkins CI declaration. +// +// This file is declarative initially. +// The declarative component describes the pipeline stages and to what CI-agnostic scripts they map to. +// These CI-agnostic scripts are used as an intermediate dockerised layer above the ant build.xml +// The ant build.xml is never invoked directly. +// +// Below the declarative section there is groovy scripting methods for all the logic between +// the 10k ft declarative view and the CI-agnostic scripts. +// +// This Jenkinsfile is expected to work on any ci-cassandra.a.o clone. +// Functionality that depends upon ASF Infra and the canonical ci-cassandra.a.o setup (e.g. post-commit builds) +// is required to quietly fail when run on other environments. +// +// +// Job definitions are still being migrated from (where other CI jobs can also be found): // https://github.com/apache/cassandra-builds/blob/trunk/jenkins-dsl/cassandra_job_dsl_seed.groovy // +// +// Syntax help can be found at https://ci-cassandra.apache.org/pipeline-syntax/ +// // Validate/lint this file using the following command // `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate` pipeline { agent { label 'cassandra' } + + options { + githubProjectProperty('https://github.com/apache/cassandra') + timestamps() + } + environment { + javaVersionDefault = javaVersionDefault() + javaVersionsSupported = javaVersionsSupported() + } stages { - stage('Init') { + stage('init') { steps { - cleanWs() - script { - currentBuild.result='SUCCESS' - } + script { currentBuild.result='SUCCESS' } + cleanWs() } } - stage('Build') { + stage('jar') { steps { - script { - def attempt = 1 - retry(2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - build job: "${env.JOB_NAME}-artifacts" - } - } + buildJob(".build/docker/_docker_run.sh bullseye-build.docker build-jars.sh", "") } } - stage('Test') { + stage('Tests') { parallel { - stage('stress') { + stage('artifacts') { + steps { + buildJob(".build/docker/build-artifacts.sh", "apache-cassandra-*.tar.gz,apache-cassandra-*.jar,apache-cassandra-*.pom") + } + } + stage('lint') { + steps { + buildJob(".build/docker/check-code.sh", "") + } + } + stage('debian') { + steps { + buildJob(".build/docker/build-debian.sh", "cassandra_*,cassandra-tools_*") + } + } + stage('redhat') { + steps { + buildJob(".build/docker/build-redhat.sh rpm", "*.rpm") + } + } + stage('stress-test') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - stress = build job: "${env.JOB_NAME}-stress-test", propagate: false - if (stress.result != 'FAILURE') break - } - if (stress.result != 'SUCCESS') unstable('stress test failures') - if (stress.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "stress-test", 1) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('stress-test', stress.getNumber()) - } - } - } + } + stage('test') { + steps { + testJob(".build/docker/run-tests.sh", "test", 8) } } - stage('fqltool') { + stage('long-test') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - fqltool = build job: "${env.JOB_NAME}-fqltool-test", propagate: false - if (fqltool.result != 'FAILURE') break - } - if (fqltool.result != 'SUCCESS') unstable('fqltool test failures') - if (fqltool.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "long-test", 8) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('fqltool-test', fqltool.getNumber()) - } - } - } + } + stage('test-oa') { + steps { + testJob(".build/docker/run-tests.sh", "test-oa", 8) } } - stage('units') { + stage('test-system-keyspace-directory') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - test = build job: "${env.JOB_NAME}-test", propagate: false - if (test.result != 'FAILURE') break - } - if (test.result != 'SUCCESS') unstable('unit test failures') - if (test.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "test-system-keyspace-directory", 8) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test', test.getNumber()) - } - } - } + } + stage('test-trie') { + steps { + testJob(".build/docker/run-tests.sh", "test-trie", 8) } } - stage('long units') { + stage('test-burn') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - long_test = build job: "${env.JOB_NAME}-long-test", propagate: false - if (long_test.result != 'FAILURE') break - } - if (long_test.result != 'SUCCESS') unstable('long unit test failures') - if (long_test.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "test-burn", 8) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('long-test', long_test.getNumber()) - } - } - } + } + stage('test-cdc') { + steps { + testJob(".build/docker/run-tests.sh", "test-cdc", 8) } } - stage('burn') { + stage('test-compression') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - burn = build job: "${env.JOB_NAME}-test-burn", propagate: false - if (burn.result != 'FAILURE') break - } - if (burn.result != 'SUCCESS') unstable('burn test failures') - if (burn.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "test-compression", 8) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-burn', burn.getNumber()) - } - } - } + } + stage('cqlsh-test') { + steps { + testJob(".build/docker/run-tests.sh", "cqlsh-test", 1) } } - stage('cdc') { + stage('jvm-dtest') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - cdc = build job: "${env.JOB_NAME}-test-cdc", propagate: false - if (cdc.result != 'FAILURE') break - } - if (cdc.result != 'SUCCESS') unstable('cdc failures') - if (cdc.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "jvm-dtest", 8) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-cdc', cdc.getNumber()) - } - } - } + } + stage('jvm-dtest-novnode') { + steps { + testJob(".build/docker/run-tests.sh", "jvm-dtest-novnode", 8) } } - stage('compression') { + stage('jvm-dtest-upgrade') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - compression = build job: "${env.JOB_NAME}-test-compression", propagate: false - if (compression.result != 'FAILURE') break - } - if (compression.result != 'SUCCESS') unstable('compression failures') - if (compression.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "jvm-dtest-upgrade", 8) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-compression', compression.getNumber()) - } - } - } + } + stage('jvm-dtest-upgrade-novnode') { + steps { + testJob(".build/docker/run-tests.sh", "jvm-dtest-upgrade-novnode", 8) } } - stage('oa') { + stage('dtest') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - oa = build job: "${env.JOB_NAME}-test-oa", propagate: false - if (oa.result != 'FAILURE') break - } - if (oa.result != 'SUCCESS') unstable('oa failures') - if (oa.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "dtest", 64) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-oa', oa.getNumber()) - } - } - } + } + stage('dtest-large') { + steps { + testJob(".build/docker/run-tests.sh", "dtest-large", 8) } } - stage('system-keyspace-directory') { + stage('dtest-novnode') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - system_keyspace_directory = build job: "${env.JOB_NAME}-test-system-keyspace-directory", propagate: false - if (system_keyspace_directory.result != 'FAILURE') break - } - if (system_keyspace_directory.result != 'SUCCESS') unstable('system-keyspace-directory failures') - if (system_keyspace_directory.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "dtest-novnode", 64) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-system-keyspace-directory', system_keyspace_directory.getNumber()) - } - } - } + } + stage('dtest-offheap') { + steps { + testJob(".build/docker/run-tests.sh", "dtest-offheap", 64) } } - stage('trie') { + stage('dtest-large-novnode') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - trie = build job: "${env.JOB_NAME}-test-trie", propagate: false - if (trie.result != 'FAILURE') break - } - if (trie.result != 'SUCCESS') unstable('trie failures') - if (trie.result == 'FAILURE') currentBuild.result='FAILURE' - } + testJob(".build/docker/run-tests.sh", "dtest-large-novnode", 8) } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-trie', trie.getNumber()) - } - } - } + } + stage('dtest-upgrade') { + steps { + testJob(".build/docker/run-tests.sh", "dtest-upgrade", 64) } } - stage('cqlsh') { + stage('dtest-upgrade-novnode') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - cqlsh = build job: "${env.JOB_NAME}-cqlsh-tests", propagate: false - if (cqlsh.result != 'FAILURE') break - } - if (cqlsh.result != 'SUCCESS') unstable('cqlsh failures') - if (cqlsh.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('cqlsh-tests', cqlsh.getNumber()) - } - } - } - } + testJob(".build/docker/run-tests.sh", "dtest-upgrade-novnode", 64) + } + } + stage('dtest-upgrade-large') { + steps { + testJob(".build/docker/run-tests.sh", "dtest-upgrade-large", 64) + } + } + stage('dtest-upgrade-novnode-large') { + steps { + testJob(".build/docker/run-tests.sh", "dtest-upgrade-novnode-large", 64) + } } stage('simulator-dtest') { steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - simulator_dtest = build job: "${env.JOB_NAME}-simulator-dtest", propagate: false - if (simulator_dtest.result != 'FAILURE') break - } - if (simulator_dtest.result != 'SUCCESS') unstable('simulator-dtest failures') - if (simulator_dtest.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('simulator-dtest', simulator_dtest.getNumber()) - } - } - } - } + testJob(".build/docker/run-tests.sh", "simulator-dtest", 1) + } } } } - stage('Distributed Test') { - parallel { - stage('jvm-dtest') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest = build job: "${env.JOB_NAME}-jvm-dtest", propagate: false - if (jvm_dtest.result != 'FAILURE') break - } - if (jvm_dtest.result != 'SUCCESS') unstable('jvm-dtest failures') - if (jvm_dtest.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest', jvm_dtest.getNumber()) - } - } - } - } - } - stage('jvm-dtest-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest_novnode = build job: "${env.JOB_NAME}-jvm-dtest-novnode", propagate: false - if (jvm_dtest_novnode.result != 'FAILURE') break - } - if (jvm_dtest_novnode.result != 'SUCCESS') unstable('jvm-dtest-novnode failures') - if (jvm_dtest_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest-novnode', jvm_dtest_novnode.getNumber()) - } - } - } - } - } - stage('jvm-dtest-upgrade') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest_upgrade = build job: "${env.JOB_NAME}-jvm-dtest-upgrade", propagate: false - if (jvm_dtest_upgrade.result != 'FAILURE') break - } - if (jvm_dtest_upgrade.result != 'SUCCESS') unstable('jvm-dtest-upgrade failures') - if (jvm_dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest-upgrade', jvm_dtest_upgrade.getNumber()) - } - } - } - } - } - stage('jvm-dtest-upgrade-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest_upgrade_novnode = build job: "${env.JOB_NAME}-jvm-dtest-upgrade-novnode", propagate: false - if (jvm_dtest_upgrade_novnode.result != 'FAILURE') break - } - if (jvm_dtest_upgrade_novnode.result != 'SUCCESS') unstable('jvm-dtest-upgrade-novnode failures') - if (jvm_dtest_upgrade_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest-upgrade-novnode', jvm_dtest_upgrade_novnode.getNumber()) - } - } - } - } - } - stage('dtest') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest = build job: "${env.JOB_NAME}-dtest", propagate: false - if (dtest.result != 'FAILURE') break - } - if (dtest.result != 'SUCCESS') unstable('dtest failures') - if (dtest.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest', dtest.getNumber()) - } - } - } - } - } - stage('dtest-large') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_large = build job: "${env.JOB_NAME}-dtest-large", propagate: false - if (dtest_large.result != 'FAILURE') break - } - if (dtest_large.result != 'SUCCESS') unstable('dtest-large failures') - if (dtest_large.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-large', dtest_large.getNumber()) - } - } - } - } - } - stage('dtest-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_novnode = build job: "${env.JOB_NAME}-dtest-novnode", propagate: false - if (dtest_novnode.result != 'FAILURE') break - } - if (dtest_novnode.result != 'SUCCESS') unstable('dtest-novnode failures') - if (dtest_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-novnode', dtest_novnode.getNumber()) - } - } - } - } - } - stage('dtest-offheap') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_offheap = build job: "${env.JOB_NAME}-dtest-offheap", propagate: false - if (dtest_offheap.result != 'FAILURE') break - } - if (dtest_offheap.result != 'SUCCESS') unstable('dtest-offheap failures') - if (dtest_offheap.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-offheap', dtest_offheap.getNumber()) - } - } - } - } - } - stage('dtest-large-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_large_novnode = build job: "${env.JOB_NAME}-dtest-large-novnode", propagate: false - if (dtest_large_novnode.result != 'FAILURE') break - } - if (dtest_large_novnode.result != 'SUCCESS') unstable('dtest-large-novnode failures') - if (dtest_large_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-large-novnode', dtest_large_novnode.getNumber()) - } - } - } - } - } - stage('dtest-upgrade') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade = build job: "${env.JOB_NAME}-dtest-upgrade", propagate: false - if (dtest_upgrade.result != 'FAILURE') break - } - if (dtest_upgrade.result != 'SUCCESS') unstable('dtest failures') - if (dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade', dtest_upgrade.getNumber()) - } - } - } - } + stage('Summary') { + steps { + generateUnifiedTestReport() + // FIXME – post always + sendNotifications() + } + } + } +} + +/////////////////////////// +//// scripting support //// +/////////////////////////// + + +/** + * Return the default JDK defined by build.xml + **/ +def javaVersionDefault() { + sh (returnStdout: true, script: 'grep \'property\\s*name=\"java.default\"\' build.xml | sed -ne \'s/.*value=\"\\([^\"]*\\)\".*/\\1/p\'').trim() +} + +/** + * Return the supported JDKs defined by build.xml + **/ +def javaVersionsSupported() { + sh (returnStdout: true, script: 'grep \'property\\s*name=\"java.supported\"\' build.xml | sed -ne \'s/.*value=\"\\([^\"]*\\)\".*/\\1/p\'').trim() +} + +/** + * Is this a post-commit build (or a pre-commit build) + **/ +def isPostCommit() { + // any build of a branch found on github.com/apache/cassandra is considered a post-commit (post-merge) CI run + return "${GIT_URL}".contains("apache/cassandra") +} + +/** + * Are we running on ci-cassandra.apache.org ? + **/ +def isCanonical() { + return "${JENKINS_URL}".contains("ci-cassandra.apache.org") +} + +def isStageIncluded() { + return true +} + +/** + * Executes `${build_script} ${jdk}` for each arch, and jdk. + * + * Stashes the jarfiles for `package-artifacts.sh` + * Copies build artifacts and logfiles to nightlies + * Cleans the agent (docker prune) afterwards + * Stops the pipeline on any failure + **/ +def buildJob(build_script, toCopy) { + if (! isStageIncluded()) { + echo "Skipping ${STAGE_NAME}" + return; + } + //def archs = ['cassandra', 'cassandra-arm64'] // uncomment when ready for arm64 testing + def archs = ['cassandra'] + def jdks = "${javaVersionsSupported}".split(/,/, -1) + def builds_per_arch = [:] + for (a in archs) { + def arch = a + builds_per_arch[arch] = { + def builds_per_jdk = [:] + for (j in jdks) { + def jdk = j + builds_per_jdk[jdk] = { _buildAxis(build_script, toCopy, arch, jdk) } + } + parallel(builds_per_jdk) + } + } + parallel(builds_per_arch) +} + +/** + * Executes `${build_script} ${jdk}` for stated arch, and jdk. + **/ +def _buildAxis(build_script, toCopy, arch, jdk) { + node(arch) { + branchName: "${STAGE_NAME} ${build_script} ${jdk} ${NODE_NAME}" + checkout scm + tool type: 'jdk', name: "jdk_${jdk}_latest" + def statusCode = 0, attempt = 1 + retry(2) { + if (attempt > 1) { sleep(60 * attempt) } + def logfile = "${JOB_NAME}_${BUILD_NUMBER}_${STAGE_NAME}_jdk${jdk}_${arch}_attempt_${attempt}.log" + sh "mkdir -p build" + def build_dir = sh( returnStdout: true, script:"mktemp -d ${WORKSPACE}/build/build_docker_run.XXXXXXXXXX" ).trim() + def build_dir_rel = build_dir - "${WORKSPACE}/" + try { + if ("jar" != STAGE_NAME) { + dir("${build_dir}") { + unstash name: "${arch}_${jdk}" } - stage('dtest-upgrade-large') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade = build job: "${env.JOB_NAME}-dtest-upgrade-large", propagate: false - if (dtest_upgrade.result != 'FAILURE') break - } - if (dtest_upgrade.result != 'SUCCESS') unstable('dtest failures') - if (dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade', dtest_upgrade.getNumber()) - } - } - } - } + } + echo "Executing ${STAGE_NAME} `build_dir=${build_dir} ${build_script} ${jdk} 2>&1 | tee ${logfile}` on ${NODE_NAME}" + statusCode = sh returnStatus:true, script:"build_dir=\"${build_dir}\" ${build_script} ${jdk} 2>&1 | tee ${build_dir}/${logfile}" + + if ("jar" == STAGE_NAME) { + dir("${build_dir}") { + // only stash the project built files. all dependency libraries are restored from the local maven repo using `ant resolver-dist-lib` + stash name: "${arch}_${jdk}"//, includes: "*.jar,classes/**,test/classes/**,tools/**" } - stage('dtest-upgrade-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade_novnode = build job: "${env.JOB_NAME}-dtest-upgrade-novnode", propagate: false - if (dtest_upgrade_novnode.result != 'FAILURE') break - } - if (dtest_upgrade_novnode.result != 'SUCCESS') unstable('dtest-upgrade-novnode failures') - if (dtest_upgrade_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } + } + } finally { + attempt = attempt + 1 + sh "xz -f ${build_dir}/*_attempt_*.log" + dir("${build_dir}") { + archiveArtifacts artifacts: "**/*_attempt_*.log.xz", fingerprint: true + copyToNightlies("**/*_attempt_*.log.xz", "${STAGE_NAME}/jdk${jdk}/${arch}/") + if (0 == statusCode) { + copyToNightlies("${toCopy}", "${STAGE_NAME}/jdk${jdk}/${arch}/") } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade-novnode', dtest_upgrade_novnode.getNumber()) - } + } + sh "rm -rf \"${build_dir}\"" + cleanAgent(build_script) + if (0 != statusCode) { + currentBuild.result = 'FAILURE' + error("FAILED ${STAGE_NAME}'${build_script} ${jdk}' on ${NODE_NAME}") + } else { + echo "Completed ${STAGE_NAME} '${build_script} ${jdk}' on ${NODE_NAME}" + } + } + } + } +} + +/** + * Saves JUnit results, fails if no reports are found + * Copies (single-file unified) junit results and logfiles to nightlies + * Cleans the agent (docker prune) afterwards + * Stops the pipeline on any failure + **/ +def testJob(test_script, test_type, splits=1) { + // we don't actually need the test_script and test_type, but they are useful explicit for readability and intuitive feedback in the declarative section. + assert STAGE_NAME.equals(test_type) : "Stage's name expected to match test_type (" + STAGE_NAME + " != " + test_type + ")" + assert ".build/docker/run-tests.sh".equals(test_script) : "testJob is hardcoded to '.build/docker/run-tests.sh', found " + test_script + + if (! isStageIncluded()) { + echo "Skipping ${STAGE_NAME}" + return; + } + + // XXX – the following matrix over arrays can be better done with multi-arrays and recursion + + def archs = ['cassandra'] + def jdks = STAGE_NAME.contains("dtest-upgrade") || "simulator-dtest".equals(STAGE_NAME) ? ["${javaVersionDefault}"] : "${javaVersionsSupported}".split(/,/, -1) + def pythons = "cqlshX".equals(STAGE_NAME) ? ['3.7', '3.8', '3.11'] : ['3.7'] + def cythons = "cqlshX".equals(STAGE_NAME) ? ['no', 'yes'] : ['no'] + + def tests_per_arch = [:] + for (a in archs) { + def arch = a + tests_per_arch[arch] = { + def tests_per_jdk = [:] + for (j in jdks) { + def jdk = j + tests_per_jdk[jdk] = { + def tests_per_python = [:] + for (p in pythons) { + def python = p + tests_per_python[python] = { + def tests_per_cython = [:] + for (c in cythons) { + def cython = c + tests_per_cython[cython] = { + def tests_per_split = [:] + for (int s = 1; s <= splits; s++) { + def split = s + tests_per_split[split] = { _testAxis(test_script, test_type, arch, jdk, python, cython, "${split}/${splits}") } } - } - } - } - stage('dtest-upgrade-novnode-large') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade_novnode_large = build job: "${env.JOB_NAME}-dtest-upgrade-novnode-large", propagate: false - if (dtest_upgrade_novnode_large.result != 'FAILURE') break + parallel(tests_per_split) } - if (dtest_upgrade_novnode_large.result != 'SUCCESS') unstable('dtest-upgrade-novnode-large failures') - if (dtest_upgrade_novnode_large.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade-novnode-large', dtest_upgrade_novnode_large.getNumber()) - } - } } + parallel(tests_per_cython) } } + parallel(tests_per_python) } + } + parallel(tests_per_jdk) } - stage('Summary') { - steps { - sh "rm -fR cassandra-builds" - sh "git clone --depth 1 --single-branch https://gitbox.apache.org/repos/asf/cassandra-builds.git" - sh "./cassandra-builds/build-scripts/cassandra-test-report.sh" - junit testResults: '**/build/test/**/TEST*.xml,**/cqlshlib.xml,**/nosetests.xml', testDataPublishers: [[$class: 'StabilityTestDataPublisher']] + } + parallel(tests_per_arch) +} - // the following should fail on any installation other than ci-cassandra.apache.org - // TODO: keep jenkins infrastructure related settings in `cassandra_job_dsl_seed.groovy` - warnError('cannot send notifications') { - script { - changes = formatChanges(currentBuild.changeSets) - echo "changes: ${changes}" - } - slackSend channel: '#cassandra-builds', message: ":apache: <${env.BUILD_URL}|${currentBuild.fullDisplayName}> completed: ${currentBuild.result}. \n${changes}" - emailext to: 'builds@cassandra.apache.org', subject: "Build complete: ${currentBuild.fullDisplayName} [${currentBuild.result}] ${env.GIT_COMMIT}", presendScript: '${FILE,path="cassandra-builds/jenkins-dsl/cassandra_email_presend.groovy"}', body: ''' -------------------------------------------------------------------------------- -Build ${ENV,var="JOB_NAME"} #${BUILD_NUMBER} ${BUILD_STATUS} -URL: ${BUILD_URL} -------------------------------------------------------------------------------- -Changes: -${CHANGES} -------------------------------------------------------------------------------- -Failed Tests: -${FAILED_TESTS,maxTests=500,showMessage=false,showStack=false} -------------------------------------------------------------------------------- -For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/ -''' - } - sh "echo \"summary) cassandra-builds: `git -C cassandra-builds log -1 --pretty=format:'%H %an %ad %s'`\" > builds.head" - sh "./cassandra-builds/jenkins-dsl/print-shas.sh" - sh "xz TESTS-TestSuites.xml" - sh "wget --retry-connrefused --waitretry=1 \"\${BUILD_URL}/timestamps/?time=HH:mm:ss&timeZone=UTC&appendLog\" -qO - > console.log || echo wget failed" - sh "xz console.log" - sh "echo \"For test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/\"" - } - post { - always { - sshPublisher(publishers: [sshPublisherDesc(configName: 'Nightlies', transfers: [sshTransfer(remoteDirectory: 'cassandra/${JOB_NAME}/${BUILD_NUMBER}/', sourceFiles: 'console.log.xz,TESTS-TestSuites.xml.xz')])]) - } +/** + * Executes `.build/test-docker.sh ${testDockerImage} ${test_type} ${split}` for the stated arch, jdk, python, cython, and split. + **/ +def _testAxis(test_script, test_type, arch, jdk, python, cython, split) { + echo "Spawning ${STAGE_NAME} `${test_script} ${test_type} ${split} ${jdk}` with arch=${arch} jdk=${jdk} python${python} cython=${cython}" + node(arch) { + branchName: "${STAGE_NAME} ${test_script} ${test_type} ${split} ${jdk} ${NODE_NAME}" + echo "Starting ${STAGE_NAME} `${test_script} ${test_type} ${split} ${jdk}` with arch=${arch} jdk=${jdk} python${python} cython=${cython} on ${NODE_NAME}" + checkout scm + sh "mkdir -p build" + def build_dir = sh( returnStdout: true, script:"mktemp -d ${WORKSPACE}/build/build_docker_run.XXXXXXXXXX" ).trim() + def build_dir_rel = build_dir - "${WORKSPACE}/" + tool type: 'jdk', name: "jdk_${jdk}_latest" + def statusCode = 0, attempt = 1 + retry(2) { + if (attempt > 1) { sleep(60 * attempt) } + def logfile = "${JOB_NAME}_${BUILD_NUMBER}_${test_type}_jdk${jdk}_python_${python}_${cython}_${arch}_attempt_${attempt}.log" + try { + dir("${build_dir}") { + unstash name: "${arch}_${jdk}" + } + echo "Executing ${STAGE_NAME} `${test_script} ${test_type} ${split} ${jdk} 2>&1 | tee ${build_dir}/${logfile}` on ${NODE_NAME} with jdk=${jdk} python=${python} cython=${cython}" + statusCode = sh returnStatus:true, script:"python_version=\"${python}\" cython=\"${cython}\" build_dir=\"${build_dir}\" ${test_script} ${test_type} ${split} ${jdk} 2>&1 | tee ${build_dir}/${logfile}" + } finally { + dir("${build_dir_rel}") { + if (0 == statusCode) { + junit testResults: "test/output/**/TEST*.xml,test/output/cqlshlib.xml,test/output/nosetests.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] + sh "xz -f test/output/**/TEST*.xml || true" + sh "xz -f test/output/cqlshlib.xml || true" + sh "xz -f test/output/nosetests.xml || true" + } + archiveArtifacts artifacts: "**/*_attempt_*.log.xz,test/logs/**,test/output/**/TEST*.xml.xz,test/output/cqlshlib.xml.xz,test/output/nosetests.xml.xz", fingerprint: true + copyToNightlies("*_attempt_*.log.xz,test/logs/**", "${test_type}/${arch}/jdk${jdk}/python${python}/cython_${cython}/" + "split_${split}".replace("/", "_")) + } + sh "rm -rf \"${build_dir}\"" + cleanAgent(test_script) + if (0 != statusCode) { + currentBuild.result = 'FAILURE' + error("FAILED ${STAGE_NAME} `${test_script} ${test_type} ${split} ${jdk}` with arch=${arch} jdk=${jdk} python=${python} cython=${cython} on ${NODE_NAME}") + } else { + echo "Completed ${STAGE_NAME} `${test_script} ${test_type} ${split} ${jdk}` with arch=${arch} jdk=${jdk} python=${python} cython=${cython} on ${NODE_NAME}" + } } } + + if (currentBuild.result != 'SUCCESS') unstable("${STAGE_NAME} failures (arch=${arch} jdk=${jdk} python=${python} cython=${cython} on ${NODE_NAME})") + if (currentBuild.result == 'ABORTED' && currentBuild.result != 'FAILURE') currentBuild.result='ABORTED' + if (currentBuild.result == 'FAILURE') currentBuild.result='FAILURE' + } +} + +def copyToNightlies(sourceFiles, remoteDirectory='') { + if (isCanonical()) { + def remotePath = remoteDirectory.startsWith("cassandra/") ? "${remoteDirectory}" : "cassandra/${JOB_NAME}/${BUILD_NUMBER}/${remoteDirectory}" + def attempt = 1 + retry(9) { + if (attempt > 1) { sleep(60 * attempt) } + sshPublisher( + continueOnError: true, failOnError: false, + publishers: [ + sshPublisherDesc( + configName: "Nightlies", + transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ] + ) + ]) + } + echo "archived to https://nightlies.apache.org/${remotePath}" } } -def copyTestResults(target, build_number) { - step([$class: 'CopyArtifact', - projectName: "${env.JOB_NAME}-${target}", - optional: true, - fingerprintArtifacts: true, - selector: specific("${build_number}"), - target: target]); +def cleanAgent(job_name) { + if (isCanonical()) { + def maxJobHours = 12 + echo "Cleaning project, processes, docker for '${job_name}' on ${NODE_NAME}…" ; + sh """ + git clean -qxdff -e build/test/jmh-result.json || true; + if pgrep -xa docker || pgrep -af "build/docker" || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --all --force --filter "until=${maxJobHours}h" || true ; else docker system prune --force --volumes || true ; fi; + """ + } } -def formatChanges(changeLogSets) { - def result = '' - for (int i = 0; i < changeLogSets.size(); i++) { - def entries = changeLogSets[i].items - for (int j = 0; j < entries.length; j++) { - def entry = entries[j] - result = result + "${entry.commitId} by ${entry.author} on ${new Date(entry.timestamp)}: ${entry.msg}\n" - } +// CASSANDRA-18130 +def saveAgentReport() { + if (isCanonical()) { + // + // echo "Updating disk usage report…"; + // sh """ + // ( echo "----" ; + // echo \$(date) ; + // echo "${JOB_NAME} ${BUILD_NUMBER} ${STAGE_NAME}" ; + // du -xm / 2>/dev/null | sort -rn | head -n 30 ; + // df -h ) | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt + // """ + // copyToNightlies("*-disk-usage-stats.txt", "cassandra/agents/${NODE_NAME}/disk-usage/") + // sh 'rm *-disk-usage-stats.txt' + } +} + +///////////////////////////////////////// +////// scripting support for summary //// +///////////////////////////////////////// + +def generateUnifiedTestReport() { + assert "Summary".equals(STAGE_NAME) + node('cassandra') { + checkout scm + sh "mkdir -p build" + copyArtifacts filter: 'test/output/**/*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/" + sh 'xz -d build/test/output/**/*.xml.xz' + sh "ANT_OPTS='-Xmx5g' ant generate-test-report" + dir('build/') { + sh "xz -f TESTS-TestSuites.xml" + archiveArtifacts artifacts: "TESTS-TestSuites.xml.xz,test/html/**/*", fingerprint: true + copyToNightlies('TESTS-TestSuites.xml.xz') + } + } +} + +def sendNotifications() { + if (isPostCommit() && isCanonical()) { + // the following is expected only to work on ci-cassandra.apache.org + try { + script { + changes = formatChangeLogChanges(currentBuild.changeSets) + echo "changes: ${changes}" + } + slackSend channel: '#cassandra-builds', message: ":apache: <${BUILD_URL}|${currentBuild.fullDisplayName}> completed: ${currentBuild.result}. \n${changes}" + emailext to: 'builds@cassandra.apache.org', subject: "Build complete: ${currentBuild.fullDisplayName} [${currentBuild.result}] ${GIT_COMMIT}", presendScript: 'msg.removeHeader("In-Reply-To"); msg.removeHeader("References")', body: emailContent() + } catch (Exception ex) { + echo 'failed to send notifications ' + ex.toString() + } + } +} + +def formatChangeLogChanges(changeLogSets) { + def result = '' + for (int i = 0; i < changeLogSets.size(); i++) { + def entries = changeLogSets[i].items + for (int j = 0; j < entries.length; j++) { + def entry = entries[j] + result = result + "${entry.commitId} by ${entry.author} on ${new Date(entry.timestamp)}: ${entry.msg}\n" } - return result + } + return result +} + +def emailContent() { + return ''' + ------------------------------------------------------------------------------- + Build ${ENV,var="JOB_NAME"} #${BUILD_NUMBER} ${BUILD_STATUS} + URL: ${BUILD_URL} + ------------------------------------------------------------------------------- + Changes: + ${CHANGES} + ------------------------------------------------------------------------------- + Failed Tests: + ${FAILED_TESTS,maxTests=500,showMessage=false,showStack=false} + ------------------------------------------------------------------------------- + For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/ + ''' } diff --git a/build.xml b/build.xml index a701786f84e7..13542f71a09c 100644 --- a/build.xml +++ b/build.xml @@ -1664,15 +1664,6 @@ - - - - - - - - - @@ -1825,7 +1816,7 @@ - + @@ -1833,10 +1824,10 @@ - + - +