From d3ebe9c280e9065f1499bfd84903591cdf7649a5 Mon Sep 17 00:00:00 2001 From: Felix Moessbauer Date: Wed, 18 Apr 2018 17:22:17 +0200 Subject: [PATCH 1/3] Migrate CI to CircleCI v2, massively reducing CI time --- .circleci/config.yml | 73 ++++++++++++++++++++++ circle.yml | 31 --------- dash/scripts/circleci/collect-artifacts.sh | 18 ++++++ 3 files changed, 91 insertions(+), 31 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml create mode 100644 dash/scripts/circleci/collect-artifacts.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..cb40dd7b8 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,73 @@ +version: 2 +jobs: + gnu_mpich: + docker: + - image: dashproject/ci:mpich + environment: + DASH_MAKE_PROCS: 2 + DASH_MAX_UNITS: 3 + DASH_BUILDEX: OFF + CC: gcc + CXX: g++ + steps: + - checkout + - run: bash dash/scripts/dash-ci.sh Release + - run: bash dash/scripts/dash-ci.sh Minimal + - run: + command: bash dash/scripts/circleci/collect-artifacts.sh + when: always + - store_artifacts: + path: /tmp/build-logs + - store_test_results: + path: /tmp/build-tests + + gnu_openmpi2: + docker: + - image: dashproject/ci:openmpi2 + environment: + DASH_MAKE_PROCS: 2 + DASH_MAX_UNITS: 3 + DASH_BUILDEX: OFF + CC: gcc + CXX: g++ + steps: + - checkout + - run: bash dash/scripts/dash-ci.sh Release + - run: bash dash/scripts/dash-ci.sh Minimal + - run: + command: bash dash/scripts/circleci/collect-artifacts.sh + when: always + - store_artifacts: + path: /tmp/build-logs + - store_test_results: + path: /tmp/build-tests + + clang_mpich: + docker: + - image: dashproject/ci:mpich + environment: + DASH_MAKE_PROCS: 2 + DASH_MAX_UNITS: 3 + DASH_BUILDEX: OFF + CC: clang-3.8 + CXX: clang++-3.8 + steps: + - checkout + - run: bash dash/scripts/dash-ci.sh Release + - run: bash dash/scripts/dash-ci.sh Minimal + - run: + command: bash dash/scripts/circleci/collect-artifacts.sh + when: always + - store_artifacts: + path: /tmp/build-logs + - store_test_results: + path: /tmp/build-tests + +workflows: + version: 2 + build_and_test: + jobs: + - gnu_mpich + - gnu_openmpi2 + - clang_mpich + diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 25fff329f..000000000 --- a/circle.yml +++ /dev/null @@ -1,31 +0,0 @@ -general: - branches: - ignore: - - /^doc-.*$/ # ignore doc branches - -machine: - pre: - - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 - services: - - docker - -dependencies: - override: - - docker info - -test: - override: - - bash ./dash/scripts/circleci/pull-docker.sh: - parallel: true - - bash ./dash/scripts/circleci/run-docker.sh Release gnu: - parallel: true - - bash ./dash/scripts/circleci/run-docker.sh Minimal gnu: - parallel: true -# - bash ./dash/scripts/circleci/run-docker.sh Nasty gnu: -# parallel: true - - bash ./dash/scripts/circleci/run-docker.sh Release clang: - parallel: true - - grep "FAIL" ./dash-ci.log && (echo "Full log:" ; cat ./dash-ci.log ; exit 1) || exit 0: - parallel: true - post: - - cat ./dash-ci.err diff --git a/dash/scripts/circleci/collect-artifacts.sh b/dash/scripts/circleci/collect-artifacts.sh new file mode 100644 index 000000000..cb6f4a2db --- /dev/null +++ b/dash/scripts/circleci/collect-artifacts.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +[[ "$CIRCLECI" == "true" ]] || { echo "Not running on CircleCI"; exit 1; } + +mkdir -p /tmp/build-logs/${CIRCLE_JOB} +mkdir -p /tmp/build-tests/${CIRCLE_JOB} + +cd build-ci + +echo "Collect Logs" +cp -v --parents -R ./*/*/*/*.log /tmp/build-logs/${CIRCLE_JOB} + +echo "Collect Tests" +cp -v --parents -R ./*/*/*/dash-tests-*.xml /tmp/build-tests/${CIRCLE_JOB} + +echo "Artifacts copied to /tmp/build-logs/${CIRCLE_JOB}" +echo "Testlogs copied to /tmp/build-tests/${CIRCLE_JOB}" + From 43ea7a4865bdfd57fe23084958b72659502e1c24 Mon Sep 17 00:00:00 2001 From: Felix Moessbauer Date: Thu, 19 Apr 2018 12:51:43 +0200 Subject: [PATCH 2/3] added openmpi3 ci container, use up to 4 units --- .circleci/config.yml | 29 +++++++++++++++++++--- dash/scripts/circleci/collect-artifacts.sh | 4 +-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb40dd7b8..be577ea0b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ jobs: - image: dashproject/ci:mpich environment: DASH_MAKE_PROCS: 2 - DASH_MAX_UNITS: 3 + DASH_MAX_UNITS: 4 DASH_BUILDEX: OFF CC: gcc CXX: g++ @@ -26,7 +26,7 @@ jobs: - image: dashproject/ci:openmpi2 environment: DASH_MAKE_PROCS: 2 - DASH_MAX_UNITS: 3 + DASH_MAX_UNITS: 4 DASH_BUILDEX: OFF CC: gcc CXX: g++ @@ -42,12 +42,34 @@ jobs: - store_test_results: path: /tmp/build-tests + gnu_openmpi3: + docker: + - image: dashproject/ci:openmpi3 + environment: + DASH_MAKE_PROCS: 2 + DASH_MAX_UNITS: 4 + DASH_BUILDEX: OFF + CC: gcc + CXX: g++ + steps: + - checkout + - run: bash dash/scripts/dash-ci.sh Release + - run: bash dash/scripts/dash-ci.sh Minimal + - run: + command: bash dash/scripts/circleci/collect-artifacts.sh + when: always + - store_artifacts: + path: /tmp/build-logs + - store_test_results: + path: /tmp/build-tests + + clang_mpich: docker: - image: dashproject/ci:mpich environment: DASH_MAKE_PROCS: 2 - DASH_MAX_UNITS: 3 + DASH_MAX_UNITS: 4 DASH_BUILDEX: OFF CC: clang-3.8 CXX: clang++-3.8 @@ -69,5 +91,6 @@ workflows: jobs: - gnu_mpich - gnu_openmpi2 + - gnu_openmpi3 - clang_mpich diff --git a/dash/scripts/circleci/collect-artifacts.sh b/dash/scripts/circleci/collect-artifacts.sh index cb6f4a2db..63c99e863 100644 --- a/dash/scripts/circleci/collect-artifacts.sh +++ b/dash/scripts/circleci/collect-artifacts.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copy Build artifacts to tmp directory to collect them later using circleci [[ "$CIRCLECI" == "true" ]] || { echo "Not running on CircleCI"; exit 1; } @@ -9,10 +10,9 @@ cd build-ci echo "Collect Logs" cp -v --parents -R ./*/*/*/*.log /tmp/build-logs/${CIRCLE_JOB} +echo "Artifacts copied to /tmp/build-logs/${CIRCLE_JOB}" echo "Collect Tests" cp -v --parents -R ./*/*/*/dash-tests-*.xml /tmp/build-tests/${CIRCLE_JOB} - -echo "Artifacts copied to /tmp/build-logs/${CIRCLE_JOB}" echo "Testlogs copied to /tmp/build-tests/${CIRCLE_JOB}" From e74ddb10cf4bf84cb018aca7cacc644dbec631e9 Mon Sep 17 00:00:00 2001 From: Felix Moessbauer Date: Thu, 19 Apr 2018 14:55:26 +0200 Subject: [PATCH 3/3] set cpu binding for openmpi containers, disable TeamlocalityTest --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index be577ea0b..d9ebcfe73 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,7 @@ jobs: docker: - image: dashproject/ci:mpich environment: + GTEST_FILTER: "-TeamLocalityTest*" DASH_MAKE_PROCS: 2 DASH_MAX_UNITS: 4 DASH_BUILDEX: OFF @@ -25,6 +26,8 @@ jobs: docker: - image: dashproject/ci:openmpi2 environment: + MPI_EXEC_FLAGS: "--allow-run-as-root --map-by core" + GTEST_FILTER: "-TeamLocalityTest*" DASH_MAKE_PROCS: 2 DASH_MAX_UNITS: 4 DASH_BUILDEX: OFF @@ -46,6 +49,8 @@ jobs: docker: - image: dashproject/ci:openmpi3 environment: + MPI_EXEC_FLAGS: "--allow-run-as-root --map-by core" + GTEST_FILTER: "-TeamLocalityTest*" DASH_MAKE_PROCS: 2 DASH_MAX_UNITS: 4 DASH_BUILDEX: OFF @@ -63,11 +68,11 @@ jobs: - store_test_results: path: /tmp/build-tests - clang_mpich: docker: - image: dashproject/ci:mpich environment: + GTEST_FILTER: "-TeamLocalityTest*" DASH_MAKE_PROCS: 2 DASH_MAX_UNITS: 4 DASH_BUILDEX: OFF