perf(dashmate)!: speedup dashmate command

.cargo/config.toml

@@ -1,8 +1,13 @@
+# TODO: We shouldn't go with tokio_unstable to production
+
 [target.aarch64-unknown-linux-musl]
-rustflags = [ "-C", "target-feature=-crt-static" ]
+rustflags = ["-C", "target-feature=-crt-static", "--cfg", "tokio_unstable"]
 
 [target.x86_64-unknown-linux-musl]
-rustflags = [ "-C", "target-feature=-crt-static" ]
+rustflags = ["-C", "target-feature=-crt-static", "--cfg", "tokio_unstable"]
 
 [target.aarch64-unknown-linux-gnu]
 linker = "aarch64-linux-gnu-gcc"
+
+[build]
+rustflags = ["--cfg", "tokio_unstable"]

.dockerignore

@@ -23,6 +23,7 @@ packages/*/dist
 packages/*/wasm
 packages/*/lib/wasm
 packages/*/node_modules
+packages/*/.env
 
 !packages/platform-test-suite/test
 

.github/actions/javy/action.yaml

@@ -0,0 +1,37 @@
+---
+name: "Setup Javy"
+description: "Setup Javy binaries"
+inputs:
+  version:
+    description: Javy version to use
+    required: false
+    default: "1.4.0"
+runs:
+  using: composite
+  steps:
+    - name: Install Javy
+      shell: bash
+      run: |
+        set -e
+
+        case "${{ runner.arch }}" in
+          "ARM64")
+            JAVY_ARCH="arm-linux"
+            ;;
+          "X64")
+            JAVY_ARCH="x86_64-linux"            
+            ;;
+          *)
+            echo "Unsupported architecture: ${{ runner.arch }}"
+            exit 1
+            ;;
+        esac
+
+        curl -Ls "https://github.com/bytecodealliance/javy/releases/download/v${{ inputs.version }}/javy-${JAVY_ARCH}-v${{ inputs.version }}.gz" | gunzip > javy
+
+        chmod +x javy
+        sudo mv javy /usr/local/bin/javy
+
+    - name: Verify Javy installation
+      shell: bash
+      run: javy --version

.github/actions/local-network/action.yaml

@@ -42,7 +42,7 @@ runs:
         key: local-network-volumes/${{ steps.dashmate-fingerprint.outputs.sha }}
 
     - name: Restore dashmate volumes
-      run: ./scripts/dashmate/volumes/restore.sh
+      run: ./packages/dashmate/scripts/volumes/restore.sh
       shell: bash
       if: steps.local-network-data.outputs.cache-hit == 'true'
 
@@ -51,10 +51,10 @@ runs:
         set -ex
 
         # create tenderdash blocks every 10s to speed up test suite
-        yarn dashmate config set --config=local platform.drive.tenderdash.consensus.createEmptyBlocksInterval "10s"
+        ./scripts/dashmate config set --config=local platform.drive.tenderdash.consensus.createEmptyBlocksInterval "10s"
 
         # collect drive logs for bench suite
-        yarn dashmate config set --config=local platform.drive.abci.logs.stdout.level "trace"
+        ./scripts/dashmate config set --config=local platform.drive.abci.logs.stdout.level "trace"
 
         ./scripts/setup_local_network.sh
         ./scripts/configure_test_suite.sh
@@ -63,7 +63,7 @@ runs:
       if: steps.local-network-data.outputs.cache-hit != 'true'
 
     - name: Dump dashmate volumes
-      run: ./scripts/dashmate/volumes/dump.sh
+      run: ./packages/dashmate/scripts/volumes/dump.sh
       shell: bash
       if: steps.local-network-data.outputs.cache-hit != 'true'
 
@@ -97,3 +97,5 @@ runs:
     - name: Start local network
       shell: bash
       run: yarn start
+      env:
+        DEBUG: 1

.github/actions/nodejs/action.yaml

@@ -5,7 +5,7 @@ runs:
   using: composite
   steps:
     - name: Setup Node.JS
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: "20"
 

.github/actions/rust/action.yaml

@@ -5,7 +5,7 @@ inputs:
   toolchain:
     description: Rust toolchain to use, stable / nightly / beta, or exact version
     # The same as in /README.md
-    default: "stable"
+    default: "1.76"
   target:
     description: Target Rust platform
     required: false
@@ -54,7 +54,7 @@ runs:
       shell: bash
       run: |
         curl -Lo /tmp/protoc.zip \
-          "https://github.com/protocolbuffers/protobuf/releases/download/v22.0/protoc-22.0-linux-${{ steps.protoc_arch.outputs.arch }}.zip"
+          "https://github.com/protocolbuffers/protobuf/releases/download/v25.2/protoc-25.2-linux-${{ steps.protoc_arch.outputs.arch }}.zip"
         unzip /tmp/protoc.zip -d ${HOME}/.local
         echo "PROTOC=${HOME}/.local/bin/protoc" >> $GITHUB_ENV
         export PATH="${PATH}:${HOME}/.local/bin"

.github/package-filters/js-packages.yml

@@ -71,7 +71,12 @@ dash: &dash
 dashmate:
   - .github/workflows/tests*
   - packages/dashmate/**
-  - *dash
+  - *dashpay-contract
+  - *masternode-reward-shares-contract
+  - *dpns-contract
+  - *withdrawals-contract
+  - *wallet-lib
+  - *dapi-client
 
 '@dashevo/platform-test-suite':
   - .github/workflows/tests*

.github/package-filters/rs-packages.yml

@@ -14,13 +14,18 @@ dpns-contract: &dpns-contract
   - .github/workflows/tests*
   - packages/dpns-contract/**
 
+json-schema-compatibility-validator: &json-schema-compatibility-validator
+  - .github/workflows/tests*
+  - packages/rs-json-schema-compatibility-validator/**
+
 dpp: &dpp
   - .github/workflows/tests*
   - packages/rs-dpp/**
   - *dashpay-contract
   - *masternode-reward-shares-contract
   - *dpns-contract
   - *withdrawals-contract
+  - *json-schema-compatibility-validator
   - packages/rs-platform-serialization/**
   - packages/rs-platform-serialization-derive/**
   - packages/rs-platform-value/**
@@ -54,9 +59,14 @@ rs-dapi-client: &dapi_client
   - packages/rs-dapi-client/**
   - *dapi_grpc
 
-rs-sdk:
+dash-sdk:
   - .github/workflows/tests*
   - packages/rs-drive-proof-verifier/**
   - packages/rs-sdk/**
   - *dapi_client
   - *drive
+
+random-document:
+  - .github/workflows/tests*
+  - packages/rs-random-document/**
+  - *dpp

.github/workflows/docs.yml

@@ -2,7 +2,7 @@ name: Publish docs via GitHub Pages
 on:
   push:
     branches:
-      - master
+      - v1.0-dev
 
 jobs:
   build:

.github/workflows/tests-build-image.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   build-image:
     name: Build ${{ inputs.name }} image
-    runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform-4x" ]
+    runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

.github/workflows/tests-build-js.yml

@@ -4,7 +4,7 @@ on:
 jobs:
   build-js:
     name: Build JS
-    runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform-4x" ]
+    runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
     steps:
       - name: Configure AWS credentials and bucket region
         uses: aws-actions/configure-aws-credentials@v4

.github/workflows/tests-dashmate.yml

@@ -49,23 +49,25 @@ jobs:
           key: build-js-artifacts/${{ github.sha }}
 
       - name: Unpack JS build artifacts archive
-        run: tar -xf build-js-artifacts-${{ github.sha }}.tar
+        run: tar -xvf build-js-artifacts-${{ github.sha }}.tar
 
       - name: Replace with pre-built images
         run: |
+          set -x
+
           # Login to ECR
           DOCKER_HUB_ORG="${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com"
           aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin $DOCKER_HUB_ORG
 
           SHA_TAG=sha-${{ github.sha }}
 
           # Drive
-          DRIVE_IMAGE_AND_VERSION=$(yarn dashmate config get --config=local platform.drive.abci.docker.image)
+          DRIVE_IMAGE_AND_VERSION=$(./scripts/dashmate config get --config=local platform.drive.abci.docker.image)
           docker pull $DOCKER_HUB_ORG/drive:$SHA_TAG
           docker tag $DOCKER_HUB_ORG/drive:$SHA_TAG $DRIVE_IMAGE_AND_VERSION
 
           # DAPI
-          DAPI_IMAGE_AND_VERSION=$(yarn dashmate config get --config=local platform.dapi.api.docker.image)
+          DAPI_IMAGE_AND_VERSION=$(./scripts/dashmate config get --config=local platform.dapi.api.docker.image)
           docker pull $DOCKER_HUB_ORG/dapi:$SHA_TAG
           docker tag $DOCKER_HUB_ORG/dapi:$SHA_TAG $DAPI_IMAGE_AND_VERSION
 
@@ -93,19 +95,21 @@ jobs:
         if: inputs.restore_local_network_data == true
 
       - name: Restore dashmate volumes
-        run: ./scripts/dashmate/volumes/restore.sh
+        run: ./packages/dashmate/scripts/volumes/restore.sh
         if: steps.local-network-data.outputs.cache-hit == 'true'
 
       - name: Run tests with cache
         run: yarn workspace dashmate run mocha ${{ inputs.test-pattern }}
         env:
+          DEBUG: 1
           DASHMATE_E2E_TESTS_SKIP_IMAGE_BUILD: true
           DASHMATE_E2E_TESTS_LOCAL_HOMEDIR: /home/ubuntu/.dashmate
         if: steps.local-network-data.outputs.cache-hit == 'true'
 
       - name: Run tests without cache
         run: yarn workspace dashmate run mocha ${{ inputs.test-pattern }}
         env:
+          DEBUG: 1
           DASHMATE_E2E_TESTS_SKIP_IMAGE_BUILD: true
         if: steps.local-network-data.outputs.cache-hit != 'true'
 

.github/workflows/tests-rs-package.yml

@@ -41,6 +41,14 @@ jobs:
         with:
           components: clippy
 
+      - name: Setup Node.JS, required by json-schema-faker-rs
+        if: inputs.package == 'random-document'
+        uses: ./.github/actions/nodejs
+
+      - name: Setup Javy, required by json-schema-faker-rs
+        if: inputs.package == 'random-document'
+        uses: ./.github/actions/javy
+
       - uses: clechasseur/rs-clippy-check@v3
         with:
           args: --package ${{ inputs.package }} --all-features --locked -- --no-deps
@@ -65,7 +73,7 @@ jobs:
           cache: false
 
       - name: Check formatting
-        run: exit `cargo fmt --check --package=${{ inputs.package }} | wc -l`
+        run: cargo fmt --check --package=${{ inputs.package }}
 
   unused_deps:
     name: Unused dependencies
@@ -106,6 +114,10 @@ jobs:
   detect_structure_changes:
     name: Detect immutable structure changes
     runs-on: ubuntu-22.04
+    # FIXME: as we use `gh pr view` below, this check can only
+    # run on pull requests. We should find a way to run it
+    # when manual triggers are used.
+    if: github.event_name == 'pull_request'
     steps:
       - name: Checkout base commit
         uses: actions/checkout@v4
@@ -176,6 +188,14 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
+      - name: Setup Node.JS, required by json-schema-faker-rs
+        if: inputs.package == 'random-document'
+        uses: ./.github/actions/nodejs
+
+      - name: Setup Javy, required by json-schema-faker-rs
+        if: inputs.package == 'random-document'
+        uses: ./.github/actions/javy
+
       - name: Run tests
         run: cargo test --package=${{ inputs.package }} --all-features --locked
         env:
@@ -201,7 +221,15 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
-      - name: Get crate ${{ inputs.package }} info
+      - name: Setup Node.JS, required by json-schema-faker-rs
+        if: inputs.package == 'random-document'
+        uses: ./.github/actions/nodejs
+
+      - name: Setup Javy, required by json-schema-faker-rs
+        if: inputs.package == 'random-document'
+        uses: ./.github/actions/javy
+
+      - name: Get crate ${{ runner.arch }} info
         id: crate_info
         uses: ./.github/actions/crate_info
         with:
@@ -218,6 +246,7 @@ jobs:
           set -ex
           features="${{ steps.crate_info.outputs.features }}"
           fails=""
+          RUSTFLAGS="-D warnings"
           cargo check --no-default-features --package "${{ inputs.package }}" --locked
           for feature in $features ;  do
             echo " ============== Verify feature $feature =============="

.github/workflows/tests.yml

@@ -6,7 +6,7 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
     branches:
       - master
-      - v[0-9]+\.[0-9]+-dev
+      - 'v[0-9]+\.[0-9]+-dev'
   push:
     branches:
       - master
@@ -83,11 +83,13 @@ jobs:
     uses: ./.github/workflows/tests-rs-package.yml
     with:
       package: ${{ matrix.rs-package }}
-      # Run drive and drive-abci linter on self-hosted 2x
-      lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-22.04"' }}
+      # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-22.04"' }}
+      # FIXME: Clippy fails on github hosted runners, most likely due to RAM usage. Using self-hosted runners for now.
+      lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
       # Run drive tests on self-hosted 4x
-      test-runner: ${{ contains(fromJSON('["drive-abci"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform-4x" ]' || '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' }}
-      check-each-feature: ${{ contains(fromJSON('["rs-sdk","rs-dapi-client","dapi-grpc","dpp","drive-abci"]'), matrix.rs-package) }}
+      test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      check-each-feature: ${{ contains(fromJSON('["dash-sdk","rs-dapi-client","dapi-grpc","dpp","drive-abci"]'), matrix.rs-package) }}
+
   rs-crates-security:
     name: Rust crates security audit
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
@@ -126,9 +128,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Node.JS
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: "18"
+          node-version: "20"
 
       - name: Enable corepack
         run: corepack enable