From 88aabd73f4c5c83d2296956eb3a51ba5f6909bca Mon Sep 17 00:00:00 2001 From: David Rubin Date: Thu, 16 Jan 2025 16:37:58 -0800 Subject: [PATCH] ci: cache build objects in between internal runs --- .github/actions/setup/action.yml | 22 +++ .github/workflows/check.yml | 323 ++++++++++++++++--------------- .github/workflows/setup.yml | 25 +++ scripts/kcov_test.sh | 13 +- 4 files changed, 225 insertions(+), 158 deletions(-) create mode 100644 .github/actions/setup/action.yml create mode 100644 .github/workflows/setup.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 000000000..08df56e50 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,22 @@ +name: setup + +inputs: + name: + required: true + type: string + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - uses: mlugg/setup-zig@v1 + with: + version: 0.13.0 + + - uses: actions/download-artifact@v4 + with: + name: ${{ inputs.name }} + path: ${{ inputs.name }}.tar + + - run: tar xf ${{ inputs.name }}.tar diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5a9245e2e..d62ffbc27 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,8 +10,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: setup-zig uses: mlugg/setup-zig@v1 @@ -25,8 +24,7 @@ jobs: check_style: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v3 @@ -39,188 +37,205 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v3 - - - name: setup-zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 - - # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine - # which is annoying to handle and results in them having invalid hardware info configurations - # on the virtual machine. This leads to incorrect feature detection and the CI is flaky. - # - # If we at some point setup a self-hosted MacOS runner, we could work around the issue and - # enable full testing on MacOS! - - name: build - run: | - zig build -Dtarget=x86_64-linux -Dcpu=x86_64_v4 - zig build -Dtarget=aarch64-macos -Dcpu=apple_m3 - - test: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{matrix.os}} - timeout-minutes: 60 - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: setup-zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 - - - name: test - run: | - zig build test -Denable-tsan=true - zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger" - - kcov_test: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{matrix.os}} - timeout-minutes: 60 - steps: - - name: checkout - uses: actions/checkout@v2 - + - uses: actions/checkout@v4 + - name: setup-zig uses: mlugg/setup-zig@v1 with: version: 0.13.0 - - - name: Set up dependencies - run: sudo apt-get update - - - name: install kcov - run: | - wget https://github.com/SimonKagstrom/kcov/releases/download/v42/kcov-amd64.tar.gz - sudo tar xf kcov-amd64.tar.gz -C / - - name: fix kcov dependencies + - name: build run: | - cd /usr/lib/x86_64-linux-gnu/ - sudo ln libopcodes-2.42-system.so libopcodes-2.38-system.so || echo libopcodes not found - sudo ln libbfd-2.42-system.so libbfd-2.38-system.so || echo libbfd not found + zig build -Denable-tsan -p output/ + zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger" -Dno-run -p output/tsan + + # in order to not loose the execution permissions of the binaries, we tar everything together + # before uploading + - name: bundle + run: tar -cvf zig-build.tar output/ + + - uses: actions/upload-artifact@v4 + with: + name: zig-build + path: zig-build.tar + + # # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine + # # which is annoying to handle and results in them having invalid hardware info configurations + # # on the virtual machine. This leads to incorrect feature detection and the CI is flaky. + # # + # # If we at some point setup a self-hosted MacOS runner, we could work around the issue and + # # enable full testing on MacOS! + # build-macos: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 - - name: run kcov - run: | - bash scripts/kcov_test.sh + # - name: setup-zig + # uses: mlugg/setup-zig@v1 + # with: + # version: 0.13.0 - - name: print coverage report - run: | - python scripts/parse_kcov.py kcov-output/test/coverage.json + # - name: build + # run: zig build -Dtarget=aarch64-macos -Dcpu=apple_m3 - gossip: + test: strategy: matrix: os: [ubuntu-latest] runs-on: ${{matrix.os}} + needs: build timeout-minutes: 60 steps: - - name: checkout - uses: actions/checkout@v2 - - - name: setup-zig - uses: mlugg/setup-zig@v1 + - uses: actions/checkout@v4 + - uses: .github/actions/setup with: - version: 0.13.0 + name: zig-build - - name: build release - run: zig build -Doptimize=ReleaseSafe + - name: test + run: | + output/bin/test + output/tsan/bin/test - - name: run gossip - run: bash scripts/gossip_test.sh 120 # in seconds + # kcov_test: + # strategy: + # matrix: + # os: [ubuntu-latest] + # runs-on: ${{matrix.os}} + # needs: build + # timeout-minutes: 60 + # steps: + # - uses: actions/checkout@v4 - gossip_service_fuzz: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{matrix.os}} - timeout-minutes: 60 - steps: - - name: checkout - uses: actions/checkout@v2 + # - name: setup-zig + # uses: mlugg/setup-zig@v1 + # with: + # version: 0.13.0 - - name: setup zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 + # - uses: actions/download-artifact@v4 + # with: + # name: zig-build + # path: output/ - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz + # - name: Set up dependencies + # run: sudo apt-get update - - name: run - run: ./zig-out/bin/fuzz gossip_service 19 10000 + # - name: install kcov + # run: | + # wget https://github.com/SimonKagstrom/kcov/releases/download/v42/kcov-amd64.tar.gz + # sudo tar xf kcov-amd64.tar.gz -C / + + # - name: fix kcov dependencies + # run: | + # cd /usr/lib/x86_64-linux-gnu/ + # sudo ln libopcodes-2.42-system.so libopcodes-2.38-system.so || echo libopcodes not found + # sudo ln libbfd-2.42-system.so libbfd-2.38-system.so || echo libbfd not found + + # - name: run kcov + # run: | + # bash scripts/kcov_test.sh output/zig-out/bin/test + + # - name: print coverage report + # run: | + # python scripts/parse_kcov.py kcov-output/test/coverage.json + + # build-release: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + + # - name: setup-zig + # uses: mlugg/setup-zig@v1 + # with: + # version: 0.13.0 + + # - name: build + # run: | + # zig build sig fuzz -Doptimize=ReleaseSafe -Dno-run + + # - uses: actions/upload-artifact@v4 + # with: + # name: zig-build-release + # path: zig-out/ - gossip_table_fuzz: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{matrix.os}} - timeout-minutes: 60 - steps: - - name: checkout - uses: actions/checkout@v2 + # gossip: + # strategy: + # matrix: + # os: [ubuntu-latest] + # runs-on: ${{matrix.os}} + # needs: build-release + # timeout-minutes: 60 + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/workflows/setup.yml + # with: + # name: zig-build-release - with: - submodules: recursive + # - name: run gossip + # run: bash scripts/gossip_test.sh 120 # in seconds - - name: setup zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 + # gossip_service_fuzz: + # strategy: + # matrix: + # os: [ubuntu-latest] + # runs-on: ${{matrix.os}} + # timeout-minutes: 60 + # needs: build-release + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/workflows/setup.yml + # with: + # name: zig-build-release - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz + # - name: run + # run: ./zig-out/bin/fuzz gossip_service 19 10000 - - name: run - run: ./zig-out/bin/fuzz gossip_table 19 100000 + # gossip_table_fuzz: + # strategy: + # matrix: + # os: [ubuntu-latest] + # runs-on: ${{matrix.os}} + # needs: build-release + # timeout-minutes: 60 + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/workflows/setup.yml + # with: + # name: zig-build-release - allocators_fuzz: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{matrix.os}} - timeout-minutes: 60 - steps: - - name: checkout - uses: actions/checkout@v2 + # - name: run + # run: ./zig-out/bin/fuzz gossip_table 19 100000 - - name: setup zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 + # allocators_fuzz: + # strategy: + # matrix: + # os: [ubuntu-latest] + # runs-on: ${{matrix.os}} + # needs: build-release + # timeout-minutes: 60 + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/workflows/setup.yml + # with: + # name: zig-build-release - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz + # - name: run + # run: ./zig-out/bin/fuzz allocators 19 10000 - - name: run - run: ./zig-out/bin/fuzz allocators 19 10000 + # ledger_fuzz: + # strategy: + # matrix: + # os: [ubuntu-latest] + # runs-on: ${{matrix.os}} + # needs: build-release + # timeout-minutes: 60 + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/workflows/setup.yml + # with: + # name: zig-build-release - ledger_fuzz: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{matrix.os}} - timeout-minutes: 60 - steps: - - name: checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - name: setup zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz - - name: run - run: ./zig-out/bin/fuzz ledger 19 10000 + # - name: run + # run: ./zig-out/bin/fuzz ledger 19 10000 # benchmarks: # if: ${{ github.ref != 'refs/heads/main' }} diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml new file mode 100644 index 000000000..70b05916b --- /dev/null +++ b/.github/workflows/setup.yml @@ -0,0 +1,25 @@ +name: setup + +on: + workflow_call: + inputs: + name: + required: true + type: string + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: mlugg/setup-zig@v1 + with: + version: 0.13.0 + + - uses: actions/download-artifact@v4 + with: + name: ${{ inputs.name }} + path: ${{ inputs.name }}.tar + + - run: tar xf ${{ inputs.name }}.tar diff --git a/scripts/kcov_test.sh b/scripts/kcov_test.sh index 7778a1bd6..04a8fe0ee 100755 --- a/scripts/kcov_test.sh +++ b/scripts/kcov_test.sh @@ -13,21 +13,26 @@ # export PATH=$PATH:/path/to/kcov/build/src # ``` -set -euxo pipefail +set -exo pipefail echo "=> Cleaning up" rm -rf kcov-output mkdir kcov-output -echo "=> Building Sig" -zig build +if [ -z "$1" ]; then + echo "=> Building Sig" + zig build + test_bin="./zig-out/bin/test" +else + test_bin="$1" +fi echo "=> Running kcov on tests" kcov \ --include-pattern=src/ \ --exclude-pattern=$HOME/.cache \ kcov-output \ - ./zig-out/bin/test + $test_bin echo "=> Opening kcov-output/index.html" open kcov-output/index.html || echo "=> Failed to open kcov-output/index.html"