[PH] Performance Harness Updates

.github/workflows/performance_harness_run.yaml

@@ -12,6 +12,16 @@ on:
       override-test-params:
         description: 'Override perf harness params'
         type: string
+      override-leap:
+        description: Override leap target
+        type: string
+      override-leap-prerelease:
+        type: choice
+        description: Override leap prelease
+        options:
+        - default
+        - true
+        - false
 
 permissions:
   packages: read
@@ -28,15 +38,25 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       test-params: ${{steps.overrides.outputs.test-params}}
+      leap-target: ${{steps.overrides.outputs.leap-target}}
+      leap-prerelease: ${{steps.overrides.outputs.leap-prerelease}}
     steps:
       - name: Setup Input Params
         id: overrides
         run: |
           echo test-params=findMax testBpOpMode >> $GITHUB_OUTPUT
+          echo leap-target="DEFAULT" >> $GITHUB_OUTPUT
+          echo leap-prerelease="false" >> $GITHUB_OUTPUT
 
           if [[ "${{inputs.override-test-params}}" != "" ]]; then
             echo test-params=${{inputs.override-test-params}} >> $GITHUB_OUTPUT
           fi
+          if [[ "${{inputs.override-leap}}" != "" ]]; then
+            echo leap-target=${{inputs.override-leap}} >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{inputs.override-leap-prerelease}}" == +(true|false) ]]; then
+            echo leap-prerelease=${{inputs.override-leap-prerelease}} >> $GITHUB_OUTPUT
+          fi
 
   platforms:
     name: Run Platforms Workflow
@@ -99,9 +119,31 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: ${{github.event.inputs.platform-choice}}-build
-      - name: Run Performance Test
+      - name: Extract Build Directory
         run: |
           zstdcat build.tar.zst | tar x
+      - if: ${{ needs.v.outputs.leap-target != 'DEFAULT' }}
+        name: Download Prev Leap Version
+        uses: AntelopeIO/asset-artifact-download-action@v3
+        with:
+          owner: AntelopeIO
+          repo: leap
+          target: '${{needs.v.outputs.leap-target}}'
+          prereleases: ${{fromJSON(needs.v.outputs.leap-prerelease)}}
+          file: 'leap.*${{github.event.inputs.platform-choice}}.*(x86_64|amd64).deb'
+      - if: ${{ needs.v.outputs.leap-target != 'DEFAULT' }}
+        name: Install leap & replace binaries for PH use
+        run: |
+          apt-get update
+          apt-get install -y ./leap*.deb
+          rm build/bin/nodeos
+          rm build/bin/cleos
+          cp /usr/bin/nodeos build/bin
+          cp /usr/bin/cleos build/bin
+          ./build/bin/nodeos --full-version
+          ./build/bin/cleos version full
+      - name: Run Performance Test
+        run: |
           cd build
           ./tests/PerformanceHarnessScenarioRunner.py ${{needs.v.outputs.test-params}}
       - name: Prepare results

.github/workflows/ph_backward_compatibility.yaml

@@ -67,7 +67,8 @@ jobs:
           rm build/bin/cleos
           cp /usr/bin/nodeos build/bin
           cp /usr/bin/cleos build/bin
-          ./build/bin/nodeos --version
+          ./build/bin/nodeos --full-version
+          ./build/bin/cleos version full
       - if: ${{ matrix.release == '3.1' || matrix.release == '3.2' }}
         name: Run Performance Tests (<v4.0)
         run: |

tests/CMakeLists.txt

@@ -274,15 +274,15 @@ set_property(TEST gelf_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME performance_test_bp COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 --calc-chain-threads lmax overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_api COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 --calc-chain-threads lmax overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-add_test(NAME performance_test_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --api-nodes-read-only-threads 2 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME performance_test_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --api-nodes-read-only-threads 2 --read-only-write-window-time-us 1000 --read-only-read-window-time-us 165000 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_cpu_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_p2p COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_http COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --target-tps 10 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_transfer_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --user-trx-data-file tests/PerformanceHarness/userTrxDataTransfer.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_new_acct_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --user-trx-data-file tests/PerformanceHarness/userTrxDataNewAccount.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_cpu_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_ram_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "r" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/ramTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-add_test(NAME performance_test_basic_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --api-nodes-read-only-threads 2 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME performance_test_basic_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --api-nodes-read-only-threads 2 --read-only-write-window-time-us 1000 --read-only-read-window-time-us 165000 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST performance_test_bp PROPERTY LABELS long_running_tests)
 set_property(TEST performance_test_api PROPERTY LABELS long_running_tests)
 set_property(TEST performance_test_read_only_trxs PROPERTY LABELS long_running_tests)

tests/PerformanceHarness/README.md

@@ -506,6 +506,8 @@ usage: PerformanceHarnessScenarioRunner.py findMax testBpOpMode overrideBasicTes
        [--disable-subjective-billing DISABLE_SUBJECTIVE_BILLING]
        [--cpu-effort-percent CPU_EFFORT_PERCENT]
        [--producer-threads PRODUCER_THREADS]
+       [--read-only-write-window-time-us READ_ONLY_WRITE_WINDOW_TIME_US]
+       [--read-only-read-window-time-us READ_ONLY_READ_WINDOW_TIME_US]
        [--http-max-in-flight-requests HTTP_MAX_IN_FLIGHT_REQUESTS]
        [--http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS]
        [--http-max-bytes-in-flight-mb HTTP_MAX_BYTES_IN_FLIGHT_MB]
@@ -581,6 +583,10 @@ Performance Test Basic Base:
                         Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80%
   --producer-threads PRODUCER_THREADS
                         Number of worker threads in producer thread pool
+  --read-only-write-window-time-us READ_ONLY_WRITE_WINDOW_TIME_US
+                        Time in microseconds the write window lasts.
+  --read-only-read-window-time-us READ_ONLY_READ_WINDOW_TIME_US
+                        Time in microseconds the read window lasts.
   --http-max-in-flight-requests HTTP_MAX_IN_FLIGHT_REQUESTS
                         Maximum number of requests http_plugin should use for processing http requests. 429 error response when exceeded. -1 for unlimited
   --http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS

tests/PerformanceHarness/performance_test_basic.py

@@ -639,7 +639,9 @@ def setupClusterConfig(args) -> ClusterConfig:
         producerPluginArgs = ProducerPluginArgs(disableSubjectiveApiBilling=args.disable_subjective_billing,
                                                 disableSubjectiveP2pBilling=args.disable_subjective_billing,
                                                 cpuEffortPercent=args.cpu_effort_percent,
-                                                producerThreads=args.producer_threads, maxTransactionTime=-1)
+                                                producerThreads=args.producer_threads, maxTransactionTime=-1,
+                                                readOnlyWriteWindowTimeUs=args.read_only_write_window_time_us,
+                                                readOnlyReadWindowTimeUs=args.read_only_read_window_time_us)
         httpPluginArgs = HttpPluginArgs(httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, httpMaxInFlightRequests=args.http_max_in_flight_requests,
                                         httpMaxResponseTimeMs=args.http_max_response_time_ms, httpThreads=args.http_threads)
         netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0)
@@ -695,6 +697,8 @@ def _createBaseArgumentParser(defEndpointApiDef: str, defProdNodeCnt: int, defVa
         ptbBaseParserGroup.add_argument("--disable-subjective-billing", type=bool, help=argparse.SUPPRESS if suppressHelp else "Disable subjective CPU billing for API/P2P transactions", default=True)
         ptbBaseParserGroup.add_argument("--cpu-effort-percent", type=int, help=argparse.SUPPRESS if suppressHelp else "Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80%%", default=100)
         ptbBaseParserGroup.add_argument("--producer-threads", type=int, help=argparse.SUPPRESS if suppressHelp else "Number of worker threads in producer thread pool", default=2)
+        ptbBaseParserGroup.add_argument("--read-only-write-window-time-us", type=int, help=argparse.SUPPRESS if suppressHelp else "Time in microseconds the write window lasts.", default=200000)
+        ptbBaseParserGroup.add_argument("--read-only-read-window-time-us", type=int, help=argparse.SUPPRESS if suppressHelp else "Time in microseconds the read window lasts.", default=60000)
         ptbBaseParserGroup.add_argument("--http-max-in-flight-requests", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum number of requests http_plugin should use for processing http requests. 429 error response when exceeded. -1 for unlimited", default=-1)
         ptbBaseParserGroup.add_argument("--http-max-response-time-ms", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum time for processing a request, -1 for unlimited", default=-1)
         ptbBaseParserGroup.add_argument("--http-max-bytes-in-flight-mb", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum size in megabytes http_plugin should use for processing http requests. -1 for unlimited. 429\

tests/trx_generator/trx_provider.cpp

@@ -133,7 +133,6 @@ namespace eosio::testing {
           params, std::move(msg_body),
           [this, trx_id = trx.id()](boost::beast::error_code                                      ec,
                                     boost::beast::http::response<boost::beast::http::string_body> response) {
-             ++this->_acknowledged;
              trx_acknowledged(trx_id, fc::time_point::now());
 
              if (this->needs_response_trace_info() && response.result() == boost::beast::http::status::ok) {
@@ -178,6 +177,7 @@ namespace eosio::testing {
                 elog("async_http_request Failed with response http status code: ${status}",
                      ("status", response.result_int()));
              }
+             ++this->_acknowledged;
           });
       ++_sent;
    }