Merge remote-tracking branch 'origin/release/rcnet-v3'

radixdlt · Aug 31, 2023 · c1ca596 · c1ca596
2 parents 4ab3473 + ebbb68b
commit c1ca596
Show file tree

Hide file tree

Showing 809 changed files with 60,619 additions and 30,156 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -22,6 +22,8 @@ jobs:
   check-code-format:
     name: Check code format
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -30,12 +32,16 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Check format
       run: bash ./check.sh
 
   sbor-unit-tests:
     name: Run SBOR unit tests
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -56,6 +62,8 @@ jobs:
   sbor-integration-tests:
     name: Run SBOR integration tests
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -76,6 +84,8 @@ jobs:
   scrypto-unit-tests:
     name: Run Scrypto unit tests
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -99,6 +109,8 @@ jobs:
   scrypto-integration-tests:
     name: Run Scrypto integration tests
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -116,6 +128,8 @@ jobs:
   scrypto-examples:
     name: Run Scrypto examples
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -124,6 +138,8 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Add wasm target
       run: rustup target add wasm32-unknown-unknown
     - name: Add wasm target (nightly)
@@ -143,10 +159,36 @@ jobs:
     - name: Build and test everything
       run: cargo build --target wasm32-unknown-unknown --release && cargo test --release
       working-directory: examples/everything
-
+
+  radix-engine-build:
+    name: Run Radix Engine build (RocksDB)
+    runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
+    strategy:
+      matrix:
+        os: [k8s-linux-runner]
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
+    - name: Install dependency
+      run: sudo apt-get -y update && sudo apt-get -y install pkg-config libfreetype6-dev libfontconfig1-dev
+    - name: Build with resource tracking
+      run: cargo build --features resource_tracker
+    - name: Build with rocksdb
+      run: cargo test -p radix-engine-profiling -p radix-engine-stores --features rocksdb --release --no-run -- --nocapture
+    - name: Build bench with rocksdb
+      run: cargo bench --features rocksdb --no-run
+
   radix-engine:
     name: Run Radix Engine tests
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -155,20 +197,52 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
+    - name: Install nextest
+      uses: taiki-e/install-action@nextest
+    - name: Add wasm target
+      run: rustup target add wasm32-unknown-unknown
+    - name: Run tests
+      run: |
+        cargo nextest run \
+          --features post_run_db_check serde \
+          -p radix-engine-common \
+          -p radix-engine-derive \
+          -p radix-engine-interface \
+          -p radix-engine \
+          -p radix-engine-tests
+        ./check_stack_usage.sh
+
+  radix-engine-release:
+    # Run tests in release variant.
+    # We are particularly interested with the flags:
+    # - debug-assertions,
+    # - overflow-checks
+    # which are false for release variant
+    name: Run Radix Engine tests (release)
+    runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
+    strategy:
+      matrix:
+        os: [k8s-linux-runner]
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Install nextest
       uses: taiki-e/install-action@nextest
-    - name: Install RocksDB metrics dependency
-      run: sudo apt-get -y update && sudo apt-get -y install pkg-config libfreetype6-dev libfontconfig1-dev
     - name: Add wasm target
       run: rustup target add wasm32-unknown-unknown
-    - name: Build with resource tracking
-      run: cargo build --features resource_tracker
-    - name: Build RocksDB metrics test
-      run: cargo test -p radix-engine-profiling -p radix-engine-stores --features rocksdb test_store_db --release --no-run -- --nocapture
     - name: Run tests
       run: |
         cargo nextest run \
-          --features serde \
+          --release \
+          --features post_run_db_check serde \
           -p radix-engine-common \
           -p radix-engine-derive \
           -p radix-engine-interface \
@@ -179,6 +253,8 @@ jobs:
   radix-engine-no-std:
     name: Run Radix Engine tests (no_std)
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -187,6 +263,8 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Install nextest
       uses: taiki-e/install-action@nextest
     - name: Add wasm target
@@ -201,6 +279,8 @@ jobs:
   radix-engine-wasmer:
     name: Run Radix Engine tests (wasmer)
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -209,6 +289,8 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Install nextest
       uses: taiki-e/install-action@nextest
     - name: Add wasm target
@@ -223,6 +305,8 @@ jobs:
   radix-engine-bench:
     name: Run Radix Engine benchmarks
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -231,6 +315,8 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Add wasm target
       run: rustup target add wasm32-unknown-unknown
     - name: Run bench
@@ -240,6 +326,8 @@ jobs:
   radix-engine-common-bench:
     name: Run Radix Engine common benchmarks
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -248,6 +336,8 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Add wasm target
       run: rustup target add wasm32-unknown-unknown
     - name: Run bench
@@ -257,6 +347,8 @@ jobs:
   transaction:
     name: Run transaction tests
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -270,6 +362,8 @@ jobs:
   transaction-scenarios:
     name: Run transaction scenarios
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -283,6 +377,8 @@ jobs:
   cli-resim-rtmc-rtmd:
     name: Run CLI tests (resim & rtmc & rtmd)
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner, windows-latest-16-cores]
@@ -291,6 +387,8 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - uses: radixdlt/rust-cache@allow_registry_src_caching
       with:
         prefix-key: ""
@@ -318,14 +416,18 @@ jobs:
   cli-scrypto:
     name: Run CLI tests (scrypto)
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
-        os: [k8s-linux-runner, windows-latest-16-cores]
+        os: [k8s-linux-runner, windows-latest-16-cores, macos-latest]
     steps:
     - uses: actions/checkout@v3
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - uses: radixdlt/rust-cache@allow_registry_src_caching
       with:
         prefix-key: ""
@@ -347,6 +449,8 @@ jobs:
   fuzzer-check:
     name: Build fuzzer
     runs-on: ${{ matrix.os }}
+    env:
+      CARGO_LOG: info
     strategy:
       matrix:
         os: [k8s-linux-runner]
@@ -355,6 +459,8 @@ jobs:
     - uses: actions-rs/toolchain@v1
       with:
         toolchain: 1.70.0
+    - name: reset registry
+      run: rm -fr ~/.cargo/registry
     - name: Install nextest
       uses: taiki-e/install-action@nextest
     - name: Add wasm target

diff --git a/.github/workflows/cpu_instructions.yml b/.github/workflows/cpu_instructions.yml
@@ -0,0 +1,68 @@
+name: QEMU costing - CPU instructions
+run-name: QEMU costing - CPU instructions, branch = ${{ github.ref_name }}
+
+on:
+  workflow_dispatch:
+  # pull_request: 
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  cpu-instructions:
+    name: QEMU costing - CPU instructions
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: Setup dependencies
+        run: |
+             sudo apt-get update
+             sudo apt-get install build-essential pkg-config ninja-build libglib2.0-dev meson
+             pip3 install lxml tabulate numpy scikit-learn statsmodels
+      - name: Setup QEMU v8.0.3
+        run: |
+             cd ..
+             wget -nv https://download.qemu.org/qemu-8.0.3.tar.xz
+             tar xJf qemu-8.0.3.tar.xz
+             cd qemu-8.0.3
+             ./configure --disable-system --enable-user --target-list=x86_64-linux-user --enable-plugins
+             make
+             cd ..
+      - name: Compile Scrypto QEMU plugin
+        run: |
+             cd radix-engine-profiling/qemu-plugin
+             sed -i 's/^qemu_source_dir.*/qemu_source_dir=\x27\/home\/runner\/work\/radixdlt-scrypto\/qemu-8.0.3\/\x27/' ./meson.build
+             bash ./build.sh
+      - name: Update paths in scripts
+        run: |
+             cd radix-engine-profiling/resources-tracker-macro/scripts
+             sed -i 's/^qemu_app.*/qemu_app=\/home\/runner\/work\/radixdlt-scrypto\/qemu-8.0.3\/build\/qemu-x86_64/' ./run_tests.sh
+             sed -i 's/^qemu_plugin.*/qemu_plugin=\/home\/runner\/work\/radixdlt-scrypto\/radixdlt-scrypto\/radix-engine-profiling\/qemu-plugin\/build\/libscrypto-qemu-plugin.so/' ./run_tests.sh
+      - name: Build radix-engine-tests
+        run: |
+             cd radix-engine-tests
+             cargo test --release --no-run --features resource_tracker
+             rm -f ../target/release/deps/transaction_multi_threaded-*
+      - name: Run radix-engine-tests using QEMU
+        run: |
+             cd radix-engine-tests
+             bash ../radix-engine-profiling/resources-tracker-macro/scripts/run_tests.sh ../target/release/deps/
+      - name: Build other tests
+        run: |
+             rm ./target/release/deps/*
+             cargo test --release --no-run --features resource_tracker --workspace --exclude radix-engine-common --exclude radix-engine-tests
+      - name: Run other tests using QEMU
+        run: bash ./radix-engine-profiling/resources-tracker-macro/scripts/run_tests.sh ./target/release/deps/
+      - name: Generate results
+        run: python3 ./radix-engine-profiling/resources-tracker-macro/scripts/convert.py /tmp/scrypto-resources-usage
+      - name: Results - table summary
+        run: cat /tmp/_out_table.txt
+      - name: Results - detailed table summary
+        run: cat /tmp/_out_table_detailed.txt
+      - name: Results - linear regression coefficients
+        run: cat /tmp/_out_linear_regression_coeff.txt
+      - name: Results - native blueprints CSV
+        run: cat /tmp/native_function_base_costs.csv