Merge branch 'bitcoin' into auxpow

.cirrus.yml

@@ -8,9 +8,6 @@ env: # Global defaults
  CCACHE_DIR: "/tmp/ccache_dir"
  CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine
 
-cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
- DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
-
 # https://cirrus-ci.org/guide/persistent-workers/
 #
 # It is possible to select a specific persistent worker by label. Refer to the
@@ -66,25 +63,11 @@ base_template: &BASE_TEMPLATE
 
 main_template: &MAIN_TEMPLATE
  timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out
- ccache_cache:
- folder: "/tmp/ccache_dir"
  ci_script:
  - ./ci/test_run_all.sh
 
-container_depends_template: &CONTAINER_DEPENDS_TEMPLATE
- << : *BASE_TEMPLATE
- container:
- # https://cirrus-ci.org/faq/#are-there-any-limits
- # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
- cpu: 2
- memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
- dockerfile: ci/test_imagefile # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment
- base_depends_built_cache:
- folder: "/ci_container_base/depends/built"
- fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends)
-
 global_task_template: &GLOBAL_TASK_TEMPLATE
- << : *CONTAINER_DEPENDS_TEMPLATE
+ << : *BASE_TEMPLATE
  << : *MAIN_TEMPLATE
 
 compute_credits_template: &CREDITS_TEMPLATE
@@ -93,7 +76,7 @@ compute_credits_template: &CREDITS_TEMPLATE
  use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != ""
 
 task:
- name: 'lint [bookworm]'
+ name: 'lint'
  << : *BASE_TEMPLATE
  container:
  image: debian:bookworm
@@ -108,11 +91,9 @@ task:
  - git fetch --unshallow --no-tags
  lint_script:
  - ./ci/lint_run_all.sh
- env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
- name: 'tidy [lunar]'
+ name: 'tidy'
  << : *GLOBAL_TASK_TEMPLATE
  persistent_worker:
  labels:
@@ -217,7 +198,7 @@ task:
  - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash
 
 task:
- name: 'ARM [unit tests, no functional tests] [bullseye]'
+ name: 'ARM, unit tests, no functional tests'
  << : *GLOBAL_TASK_TEMPLATE
  persistent_worker:
  labels:
@@ -226,33 +207,25 @@ task:
  FILE_ENV: "./ci/test/00_setup_env_arm.sh"
 
 task:
- name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]'
+ name: 'Win64, unit tests, no gui tests, no boost::process, no functional tests'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- docker_arguments:
- CI_IMAGE_NAME_TAG: ubuntu:jammy
- FILE_ENV: "./ci/test/00_setup_env_win64.sh"
- << : *CREDITS_TEMPLATE
+ persistent_worker:
+ labels:
+ type: small
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+ FILE_ENV: "./ci/test/00_setup_env_win64.sh"
 
 task:
- name: '32-bit + dash [gui] [CentOS 9]'
+ name: '32-bit CentOS, dash, gui'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- docker_arguments:
- CI_IMAGE_NAME_TAG: "quay.io/centos/amd64:stream9"
- FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
- # For faster CI feedback, immediately schedule one task that runs all tests
- << : *CREDITS_TEMPLATE
+ persistent_worker:
+ labels:
+ type: small
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
- PACKAGE_MANAGER_INSTALL: "yum install -y"
+ FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
 
 task:
- name: '[previous releases, qt5 dev package and depends packages, DEBUG] [focal]'
- previous_releases_cache:
- folder: "releases"
+ name: 'previous releases, qt5 dev package and depends packages, DEBUG'
  << : *GLOBAL_TASK_TEMPLATE
  persistent_worker:
  labels:
@@ -261,30 +234,26 @@ task:
  FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
 
 task:
- name: '[TSan, depends, gui] [lunar]'
+ name: 'TSan, depends, gui'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- cpu: 4
- memory: 16G # The default memory is too small, so double everything
- docker_arguments:
- CI_IMAGE_NAME_TAG: ubuntu:lunar
- FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
+ persistent_worker:
+ labels:
+ type: medium
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+ FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
 
 task:
- name: '[MSan, depends] [jammy]'
+ name: 'MSan, depends'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- docker_arguments:
- CI_IMAGE_NAME_TAG: ubuntu:jammy
-  FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+ persistent_worker:
+ labels:
+ type: small
+ timeout_in: 300m # Use longer timeout for the *rare* case where a full build (llvm + msan + depends + ...) needs to be done.
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
- MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan
+ FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
 
 task:
- name: '[ASan + LSan + UBSan + integer, no depends, USDT] [lunar]'
+ name: 'ASan + LSan + UBSan + integer, no depends, USDT'
  enable_bpfcc_script:
  # In the image build step, no external environment variables are available,
  # so any settings will need to be written to the settings env file:
@@ -297,46 +266,37 @@ task:
  FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
 
 task:
- name: '[fuzzer,address,undefined,integer, no depends] [lunar]'
+ name: 'fuzzer,address,undefined,integer, no depends'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- cpu: 4 # Increase CPU and memory to avoid timeout
- memory: 16G
- docker_arguments:
- CI_IMAGE_NAME_TAG: ubuntu:lunar
- FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
+ persistent_worker:
+ labels:
+ type: medium
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+ FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 
 task:
- name: '[multiprocess, i686, DEBUG] [focal]'
+ name: 'multiprocess, i686, DEBUG'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- cpu: 4
- memory: 16G # The default memory is too small, so double everything
- docker_arguments:
- CI_IMAGE_NAME_TAG: "docker.io/amd64/ubuntu:focal"
- FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
+ persistent_worker:
+ labels:
+ type: medium
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+ FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
 
 task:
- name: '[no wallet, libbitcoinkernel] [focal]'
+ name: 'no wallet, libbitcoinkernel'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- docker_arguments:
- CI_IMAGE_NAME_TAG: ubuntu:focal
- FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
- << : *CREDITS_TEMPLATE
+ persistent_worker:
+ labels:
+ type: small
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+ FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
 
 task:
- name: 'macOS 11.0 [gui, no tests] [jammy]'
+ name: 'macOS-cross 11.0, gui, no tests'
  << : *GLOBAL_TASK_TEMPLATE
- container:
- docker_arguments:
- CI_IMAGE_NAME_TAG: ubuntu:jammy
- FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+ persistent_worker:
+ labels:
+ type: small
  env:
- << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+ FILE_ENV: "./ci/test/00_setup_env_mac.sh"

.github/workflows/ci.yml

@@ -14,8 +14,8 @@ on:
  - '**'
 
 concurrency:
- group: ${{ github.ref }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+ group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }}
+ cancel-in-progress: true
 
 env:
  DANGER_RUN_CI_ON_HOST: 1
@@ -24,7 +24,7 @@ env:
 
 jobs:
  macos-native-x86_64:
- name: macOS 13 native, x86_64 [no depends, sqlite only, gui]
+ name: 'macOS 13 native, x86_64, no depends, sqlite only, gui'
  # Use latest image, but hardcode version to avoid silent upgrades (and breaks).
  # See: https://github.com/actions/runner-images#available-images.
  runs-on: macos-13 # Use M1 once available https://github.com/github/roadmap/issues/528
@@ -51,12 +51,20 @@ jobs:
  - name: Set Ccache directory
  run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
 
- - name: Ccache cache
- uses: actions/cache@v3
+ - name: Restore Ccache cache
+ uses: actions/cache/restore@v3
  with:
  path: ${{ env.CCACHE_DIR }}
- key: ${{ github.job }}-ccache-cache-${{ github.run_id }}
- restore-keys: ${{ github.job }}-ccache-cache
+ key: ${{ github.job }}-ccache-${{ github.run_id }}
+ restore-keys: ${{ github.job }}-ccache-
 
  - name: CI script
  run: ./ci/test_run_all.sh
+
+ - name: Save Ccache cache
+ uses: actions/cache/save@v3
+ if: github.event_name != 'pull_request'
+ with:
+ path: ${{ env.CCACHE_DIR }}
+ # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+ key: ${{ github.job }}-ccache-${{ github.run_id }}

ci/README.md

@@ -20,10 +20,11 @@ requires `bash`, `docker`, and `python3` to be installed. To install all require
 sudo apt install bash docker.io python3
 ```
 
-To run the test stage with a specific configuration,
+It is recommended to run the ci system in a clean env. To run the test stage
+with a specific configuration,
 
 ```
-FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh'
 ```
 
 ### Configurations
@@ -38,14 +39,11 @@ the system package manager to install build dependencies. This guarantees that
 the tester is using the same versions as the release builds, which also use
 `./depends`.
 
-If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh`
-is used as the default configuration with fallback values.
-
 It is also possible to force a specific configuration without modifying the
 file. For example,
 
 ```
-MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh'
 ```
 
 The files starting with `0n` (`n` greater than 0) are the scripts that are run

ci/test/00_setup_env.sh

@@ -67,7 +67,6 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
 # The folder for previous release binaries.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases}
-export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}

ci/test/00_setup_env_mac.sh

@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2021 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
+export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
+
 export CONTAINER_NAME=ci_macos_cross
-export CI_IMAGE_NAME_TAG=ubuntu:22.04
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
 export HOST=x86_64-apple-darwin
-export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso"
+export PACKAGES="cmake libz-dev python3-setuptools xorriso"
 export XCODE_VERSION=12.2
 export XCODE_BUILD_ID=12B45b
 export RUN_UNIT_TESTS=false

ci/test/00_setup_env_native_asan.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -17,7 +17,7 @@ fi
 
 export CONTAINER_NAME=ci_native_asan
 export PACKAGES="systemtap-sdt-dev clang-16 llvm-16 libclang-rt-16-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
-export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:23.04"  # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
 export NO_DEPENDS=1
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \

ci/test/00_setup_env_native_fuzz.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:23.04"  # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
 export CONTAINER_NAME=ci_native_fuzz
 export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev"
 export NO_DEPENDS=1

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2020-2022 The Bitcoin Core developers
+# Copyright (c) 2020-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
 LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="debian:bookworm"
+export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm"
 export CONTAINER_NAME=ci_native_fuzz_valgrind
 export PACKAGES="clang llvm libclang-rt-dev libevent-dev libboost-dev libsqlite3-dev valgrind"
 export NO_DEPENDS=1