From cb8366b3892106d5142d85a10db64f7c537a658c Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 5 Aug 2024 15:07:30 +0800 Subject: [PATCH 01/11] Problem: integration test don't verify app hash between block-stm and (#1538) * Problem: integration test don't verify app hash between block-stm and sequential Solution: - only enable block-stm on one of the node, try to detect potencial app hash mismatch * Update integration_tests/configs/default.jsonnet Signed-off-by: yihuang * Update integration_tests/configs/default.jsonnet Signed-off-by: yihuang * bypass config validation * fix config --------- Signed-off-by: yihuang Co-authored-by: mmsqe --- integration_tests/configs/default.jsonnet | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index 79017e338f..07c62ae363 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -23,12 +23,12 @@ 'block-range-cap': 10000, 'logs-cap': 10000, }, + evm: { + 'block-executor': 'sequential', + }, mempool: { 'max-txs': 1000, }, - evm: { - 'block-executor': 'block-stm', - }, }, validators: [{ coins: '1000000000000000000stake,10000000000000000000000basetcro', @@ -46,6 +46,10 @@ versiondb: { enable: true, }, + evm: { + 'block-executor': 'block-stm', + 'block-stm-workers': 16, + }, }, }, { coins: '1000000000000000000stake,10000000000000000000000basetcro', From bb516711d8794739ab423a0229f664341507bc03 Mon Sep 17 00:00:00 2001 From: yihuang Date: Tue, 6 Aug 2024 17:18:35 +0800 Subject: [PATCH 02/11] Problem: app hash mismatch in block-stm (#1539) * Problem: app hash mismatch in block-stm Solution: - try to reproduce in integration test first * Update integration_tests/test_basic.py Co-authored-by: mmsqe Signed-off-by: yihuang * fix import * enable memiavl for easier debugging * cleanup * fix * Revert "enable memiavl for easier debugging" This reverts commit 903bc2ba5c373456f3dc01cd0e1aef0538b7e043. * changelog --------- Signed-off-by: yihuang Co-authored-by: mmsqe --- CHANGELOG.md | 1 + go.mod | 4 +-- go.sum | 4 +-- gomod2nix.toml | 4 +-- integration_tests/configs/default.jsonnet | 2 +- integration_tests/test_basic.py | 40 +++++++++++++++++++++++ 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faa9d8a830..e97378ad43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ ### Bug Fixes * [#1520](https://github.com/crypto-org-chain/cronos/pull/1520) Avoid invalid chain id for signer error when rpc call before chain id set in BeginBlock. +* [#1539](https://github.com/crypto-org-chain/cronos/pull/1539) Fix go-block-stm bug that causes app hash mismatch. *Jun 18, 2024* diff --git a/go.mod b/go.mod index e847e005e0..edf01b867d 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/99designs/keyring v1.2.2 github.com/cometbft/cometbft v0.38.10-0.20240709120009-0792c8bdda44 github.com/cosmos/cosmos-db v1.0.3-0.20240408151834-e75f6e4b28d8 - github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/cosmos-sdk v0.50.7 github.com/cosmos/gogoproto v1.4.12 github.com/cosmos/ibc-go/modules/apps/callbacks v0.0.0-20240410121711-d89cb0895b1e @@ -88,6 +87,7 @@ require ( github.com/coinbase/rosetta-sdk-go/types v1.0.0 // indirect github.com/cometbft/cometbft-db v0.9.1 // indirect github.com/cosmos/btcutil v1.0.5 // indirect + github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v1.1.2 // indirect @@ -97,7 +97,7 @@ require ( github.com/creachadair/atomicfile v0.3.1 // indirect github.com/creachadair/tomledit v0.0.24 // indirect github.com/crypto-org-chain/cronos/memiavl v0.0.4 // indirect - github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde // indirect + github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 589e9830fd..85b4a64e14 100644 --- a/go.sum +++ b/go.sum @@ -432,8 +432,8 @@ github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1 h github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY= github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14 h1:ZVIlVCtx5X6sckKyMhc8nPBjXvr+/CzmXYqh7e/287I= github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14/go.mod h1:Xap56y3WoiP7DyNammmB6N7P+Y83QQd6363RHGuaYV0= -github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde h1:sQIHTJfVt5VTrF7po9eZiFkZiPjlHbFvnXtGCOoBjNM= -github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE= +github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883 h1:Oj7VvlK8iXRaugnpGA8CBXGrgkyigji+Ae5weSRD85I= +github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE= github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 h1:V43F3JFcqG4MUThf9W/DytnPblpR6CcaLBw2Wx6zTgE= github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y= github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= diff --git a/gomod2nix.toml b/gomod2nix.toml index e2582f85be..f2e83c291a 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -213,8 +213,8 @@ schema = 3 version = "v0.0.24" hash = "sha256-4vUukHONOjNn0qfQr4esK6TWfPWsIp+rbdz65og84lw=" [mod."github.com/crypto-org-chain/go-block-stm"] - version = "v0.0.0-20240408011717-9f11af197bde" - hash = "sha256-+UTSUoh4DxkWPQEc/j0atak1Oxq3tdP8r6ZMmB0k5KE=" + version = "v0.0.0-20240806075927-09a64748f883" + hash = "sha256-DuqcnTqpUVRxV+I+CaBuDg/1CLmMeXdw4VEFeuCsUbU=" [mod."github.com/danieljoos/wincred"] version = "v1.2.0" hash = "sha256-LHcvTJCc8++bFndbd8ZgMSTe4L5h2C4rN+cSWHCz54Y=" diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index 07c62ae363..ad7c1c3c40 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -48,7 +48,7 @@ }, evm: { 'block-executor': 'block-stm', - 'block-stm-workers': 16, + 'block-stm-workers': 32, }, }, }, { diff --git a/integration_tests/test_basic.py b/integration_tests/test_basic.py index 140c9192be..d10cf89b93 100644 --- a/integration_tests/test_basic.py +++ b/integration_tests/test_basic.py @@ -25,13 +25,16 @@ contract_address, contract_path, deploy_contract, + derive_new_account, eth_to_bech32, get_receipts_by_block, get_sync_info, modify_command_in_supervisor_config, send_transaction, send_txs, + sign_transaction, submit_any_proposal, + w3_wait_for_block, wait_for_block, wait_for_new_blocks, wait_for_port, @@ -934,3 +937,40 @@ def test_submit_send_enabled(cronos, tmp_path): approve_proposal(cronos, rsp["events"]) print("check params have been updated now") assert cli.query_bank_send(*denoms) == send_enable + + +def test_block_stm_delete(cronos): + """ + this test case revealed a bug in block-stm, + see: https://github.com/crypto-org-chain/go-block-stm/pull/11 + """ + w3 = cronos.w3 + cli = cronos.cosmos_cli() + acc = derive_new_account(3) + sender = acc.address + + # fund new sender + fund = 3000000000000000000 + tx = {"to": sender, "value": fund, "gasPrice": w3.eth.gas_price} + send_transaction(w3, tx) + assert w3.eth.get_balance(sender, "latest") == fund + nonce = w3.eth.get_transaction_count(sender) + wait_for_new_blocks(cli, 1) + txhashes = [] + total = 3 + for n in range(total): + tx = { + "to": "0x2956c404227Cc544Ea6c3f4a36702D0FD73d20A2", + "value": fund // total, + "gas": 21000, + "maxFeePerGas": 6556868066901, + "maxPriorityFeePerGas": 1500000000, + "nonce": nonce + n, + } + signed = sign_transaction(w3, tx, acc.key) + txhash = w3.eth.send_raw_transaction(signed.rawTransaction) + txhashes.append(txhash) + for txhash in txhashes[0 : total - 1]: + res = w3.eth.wait_for_transaction_receipt(txhash) + assert res.status == 1 + w3_wait_for_block(w3, w3.eth.block_number + 3, timeout=30) From 5310e8623cfbb3eb1cbba5aea8178625f629924f Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 7 Aug 2024 11:37:43 +0800 Subject: [PATCH 03/11] Problem: disable of create vesting account messages are not complete (#1541) --- CHANGELOG.md | 1 + app/app.go | 2 ++ go.mod | 4 ++-- go.sum | 8 ++++---- gomod2nix.toml | 8 ++++---- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e97378ad43..a3a4842197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * [#1458](https://github.com/crypto-org-chain/cronos/pull/1458) Adjust require gas for recvPacket when ReceiverChainIsSource. * [#1519](https://github.com/crypto-org-chain/cronos/pull/1519) Upgrade ibc-go to 8.3 and remove icaauth module. * [#1518](https://github.com/crypto-org-chain/cronos/pull/1518) Keep versiondb/memiavl compatible with upstream sdk, stop supporting other streaming service. +* [#1541](https://github.com/crypto-org-chain/cronos/pull/1541) Disable MsgCreatePermanentLockedAccount and MsgCreatePeriodicVestingAccount messages. ### Improvements diff --git a/app/app.go b/app/app.go index b1ba42d3eb..b4df0fc360 100644 --- a/app/app.go +++ b/app/app.go @@ -1040,6 +1040,8 @@ func (app *App) setAnteHandler(txConfig client.TxConfig, maxGasWanted uint64, bl DisabledAuthzMsgs: []string{ sdk.MsgTypeURL(&evmtypes.MsgEthereumTx{}), sdk.MsgTypeURL(&vestingtypes.MsgCreateVestingAccount{}), + sdk.MsgTypeURL(&vestingtypes.MsgCreatePermanentLockedAccount{}), + sdk.MsgTypeURL(&vestingtypes.MsgCreatePeriodicVestingAccount{}), }, ExtraDecorators: []sdk.AnteDecorator{blockAddressDecorator}, PendingTxListener: app.onPendingTx, diff --git a/go.mod b/go.mod index edf01b867d..532e0888e3 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( cosmossdk.io/x/upgrade v0.1.1 filippo.io/age v1.1.1 github.com/99designs/keyring v1.2.2 - github.com/cometbft/cometbft v0.38.10-0.20240709120009-0792c8bdda44 + github.com/cometbft/cometbft v0.38.10 github.com/cosmos/cosmos-db v1.0.3-0.20240408151834-e75f6e4b28d8 github.com/cosmos/cosmos-sdk v0.50.7 github.com/cosmos/gogoproto v1.4.12 @@ -276,7 +276,7 @@ replace ( github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2 github.com/ethereum/go-ethereum => github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 // block-stm branch - github.com/evmos/ethermint => github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14 + github.com/evmos/ethermint => github.com/crypto-org-chain/ethermint v0.6.1-0.20240802052216-0e0baa4ea540 // Fix upstream GHSA-h395-qcrw-5vmq and GHSA-3vp4-m3rf-835h vulnerabilities. // TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409 github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.0 diff --git a/go.sum b/go.sum index 85b4a64e14..0029ed4d6a 100644 --- a/go.sum +++ b/go.sum @@ -366,8 +366,8 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coinbase/rosetta-sdk-go/types v1.0.0 h1:jpVIwLcPoOeCR6o1tU+Xv7r5bMONNbHU7MuEHboiFuA= github.com/coinbase/rosetta-sdk-go/types v1.0.0/go.mod h1:eq7W2TMRH22GTW0N0beDnN931DW0/WOI1R2sdHNHG4c= -github.com/cometbft/cometbft v0.38.10-0.20240709120009-0792c8bdda44 h1:Tm6SG54Tj/bGVEZUyFPs2UyG0BYvCO3oqvWNyw99bi4= -github.com/cometbft/cometbft v0.38.10-0.20240709120009-0792c8bdda44/go.mod h1:jHPx9vQpWzPHEAiYI/7EDKaB1NXhK6o3SArrrY8ExKc= +github.com/cometbft/cometbft v0.38.10 h1:2ePuglchT+j0Iao+cfmt/nw5U7K2lnGDzXSUPGVdXaU= +github.com/cometbft/cometbft v0.38.10/go.mod h1:jHPx9vQpWzPHEAiYI/7EDKaB1NXhK6o3SArrrY8ExKc= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -430,8 +430,8 @@ github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1 github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg= github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1 h1:r0ALP31Wnw19FqEmqzsK2SFNqdMetHshnM/X/FeJRIo= github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY= -github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14 h1:ZVIlVCtx5X6sckKyMhc8nPBjXvr+/CzmXYqh7e/287I= -github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14/go.mod h1:Xap56y3WoiP7DyNammmB6N7P+Y83QQd6363RHGuaYV0= +github.com/crypto-org-chain/ethermint v0.6.1-0.20240802052216-0e0baa4ea540 h1:werx5A5sAiqxHy95ARhXGSRRVkedS5y6s0m7oQTASes= +github.com/crypto-org-chain/ethermint v0.6.1-0.20240802052216-0e0baa4ea540/go.mod h1:soxGcxnn/3k0LO9r0IY5CyfKqBK0rRO/HrYcQLFEg3s= github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883 h1:Oj7VvlK8iXRaugnpGA8CBXGrgkyigji+Ae5weSRD85I= github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE= github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 h1:V43F3JFcqG4MUThf9W/DytnPblpR6CcaLBw2Wx6zTgE= diff --git a/gomod2nix.toml b/gomod2nix.toml index f2e83c291a..83e954284a 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -154,8 +154,8 @@ schema = 3 version = "v1.0.0" hash = "sha256-z/0E0NiEGo7zxM7d94ImgUf8P0/KG6hbP9T4Vuym4p0=" [mod."github.com/cometbft/cometbft"] - version = "v0.38.10-0.20240709120009-0792c8bdda44" - hash = "sha256-ZKAAgJ78Z1Mgww2D8BZz9Qug/m3CXWa9MKM8W4CTbGI=" + version = "v0.38.10" + hash = "sha256-zd/vD0uUgawzuuzuNweRK4HctzWmCb3tnLUFmaOT4OE=" [mod."github.com/cometbft/cometbft-db"] version = "v0.0.0-20231011055109-57922ac52a63" hash = "sha256-iLs/FN1be3AcoyhKj2+b5Msqeat9j2ja4acyOs+w+Uk=" @@ -262,8 +262,8 @@ schema = 3 hash = "sha256-lE4G5FaRb3MVi9FFVn+WlwsSTOB4SbjmVboKyQ5yB0A=" replaced = "github.com/crypto-org-chain/go-ethereum" [mod."github.com/evmos/ethermint"] - version = "v0.6.1-0.20240729070824-350955227f14" - hash = "sha256-aZ9Kq7PujK4alMXp1kk7LF7PTap8UdJbt+gK2ljvwyw=" + version = "v0.6.1-0.20240802052216-0e0baa4ea540" + hash = "sha256-qGQPxKz2nWEj8saqR5lU38j0ToV2oIRR7Bhhin5iIfg=" replaced = "github.com/crypto-org-chain/ethermint" [mod."github.com/fatih/color"] version = "v1.16.0" From 867d53c281f2ca535ae1276dc203804607148af0 Mon Sep 17 00:00:00 2001 From: yihuang Date: Wed, 7 Aug 2024 12:41:42 +0800 Subject: [PATCH 04/11] Problem: sim test fails without objstore build tag (#1540) Solution: - set build tags in environment variable --- Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index e24e6609d8..34eb0954d4 100644 --- a/Makefile +++ b/Makefile @@ -166,26 +166,31 @@ test-sim-random-genesis-fast: @go test -tags=objstore -mod=readonly $(SIMAPP) -run TestFullAppSimulation \ -Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h +test-sim-import-export: export GOFLAGS=-tags=objstore test-sim-import-export: runsim @echo "Running application import/export simulation. This may take several minutes..." - @$(BINDIR)/runsim -Tags=objstore -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 50 5 TestAppImportExport + @$(BINDIR)/runsim -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 50 5 TestAppImportExport +test-sim-after-import: export GOFLAGS=-tags=objstore test-sim-after-import: runsim @echo "Running application simulation-after-import. This may take several minutes..." - @$(BINDIR)/runsim -Tags=objstore -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 50 5 TestAppSimulationAfterImport + @$(BINDIR)/runsim -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 50 5 TestAppSimulationAfterImport +test-sim-custom-genesis-multi-seed: export GOFLAGS=-tags=objstore test-sim-custom-genesis-multi-seed: runsim @echo "Running multi-seed custom genesis simulation..." @echo "By default, ${HOME}/.cronosd/config/genesis.json will be used." - @$(BINDIR)/runsim -Tags=objstore -Genesis=${HOME}/.cronosd/config/genesis.json -SimAppPkg=$(SIMAPP) -ExitOnFail 400 5 TestFullAppSimulation + @$(BINDIR)/runsim -Genesis=${HOME}/.cronosd/config/genesis.json -SimAppPkg=$(SIMAPP) -ExitOnFail 400 5 TestFullAppSimulation +test-sim-multi-seed-long: export GOFLAGS=-tags=objstore test-sim-multi-seed-long: runsim @echo "Running long multi-seed application simulation. This may take awhile!" - @$(BINDIR)/runsim -Tags=objstore -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 500 50 TestFullAppSimulation + @$(BINDIR)/runsim -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 500 50 TestFullAppSimulation +test-sim-multi-seed-short: export GOFLAGS=-tags=objstore test-sim-multi-seed-short: runsim @echo "Running short multi-seed application simulation. This may take awhile!" - @$(BINDIR)/runsim -Tags=objstore -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 50 10 TestFullAppSimulation + @$(BINDIR)/runsim -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 50 10 TestFullAppSimulation test-sim-benchmark-invariants: @echo "Running simulation invariant benchmarks..." From d82fbf5f2bc35c93b2228e7e762fc05fe2591d1d Mon Sep 17 00:00:00 2001 From: yihuang Date: Thu, 8 Aug 2024 17:40:22 +0800 Subject: [PATCH 05/11] Problem: benchmark nodes take awhile to be ready in autoscald cluster (#1542) * Problem: benchmark nodes take awhile to be ready in autoscald cluster Solution: - a custom echo server to poke readiness before start the node * close socket * fix close * log * dump block stats * log * fix * Update testground/benchmark/benchmark/utils.py Signed-off-by: yihuang * Update testground/benchmark/benchmark/stateless.py Signed-off-by: yihuang * write to file * increase timeout * bigger timeout * wait with timeout * daemon flag --------- Signed-off-by: yihuang --- testground/benchmark/benchmark/echo.py | 24 +++++++++++ testground/benchmark/benchmark/stateless.py | 47 ++++++++++++++++++--- 2 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 testground/benchmark/benchmark/echo.py diff --git a/testground/benchmark/benchmark/echo.py b/testground/benchmark/benchmark/echo.py new file mode 100644 index 0000000000..5bb752f2db --- /dev/null +++ b/testground/benchmark/benchmark/echo.py @@ -0,0 +1,24 @@ +import asyncio +import threading + + +async def handle_echo(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + data = await reader.read(1024) + if data: + writer.write(data) + await writer.drain() + writer.close() + await writer.wait_closed() + + +async def echo_server(port: int): + server = await asyncio.start_server(handle_echo, "0.0.0.0", port) + async with server: + await server.serve_forever() + + +def run_echo_server(port: int): + """ + run async echo server in a thread. + """ + threading.Thread(target=asyncio.run, args=(echo_server(port),), daemon=True).start() diff --git a/testground/benchmark/benchmark/stateless.py b/testground/benchmark/benchmark/stateless.py index 021552c548..0280ccf936 100644 --- a/testground/benchmark/benchmark/stateless.py +++ b/testground/benchmark/benchmark/stateless.py @@ -11,8 +11,10 @@ import fire import requests +import tomlkit from .cli import ChainCommand +from .echo import run_echo_server from .peer import ( CONTAINER_CRONOSD_PATH, FULLNODE_GROUP, @@ -33,6 +35,7 @@ # the container must be deployed with the prefixed name HOSTNAME_TEMPLATE = "testplan-{index}" LOCAL_RPC = "http://localhost:26657" +ECHO_SERVER_PORT = 26659 class CLI: @@ -118,8 +121,9 @@ def run( cronosd=CONTAINER_CRONOSD_PATH, global_seq=None, ): - datadir = Path(datadir) + run_echo_server(ECHO_SERVER_PORT) + datadir = Path(datadir) cfg = json.loads((datadir / "config.json").read_text()) if global_seq is None: @@ -132,11 +136,14 @@ def run( home = datadir / group / str(group_seq) - # start the node - logfile = home / "node.log" + # wait for persistent peers to be ready + wait_for_peers(home) + + print("start node") + logfile = open(home / "node.log", "ab", buffering=0) proc = subprocess.Popen( [cronosd, "start", "--home", str(home)], - stdout=open(logfile, "ab", buffering=0), + stdout=logfile, ) cli = ChainCommand(cronosd) @@ -151,8 +158,11 @@ def run( wait_for_w3() generate_load(cli, cfg["num_accounts"], cfg["num_txs"], home=home) + with (home / "block_stats.log").open("w") as logfile: + dump_block_stats(logfile) + proc.kill() - proc.wait() + proc.wait(20) # collect outputs output = Path("/data.tar.bz2") @@ -192,9 +202,12 @@ def block_height(): return int(rsp["result"]["sync_info"]["latest_block_height"]) +def block(height): + return requests.get(f"{LOCAL_RPC}/block?height={height}").json() + + def block_txs(height): - rsp = requests.get(f"{LOCAL_RPC}/block?height={height}").json() - return rsp["result"]["block"]["data"]["txs"] + return block(height)["result"]["block"]["data"]["txs"] def init_node_local( @@ -238,6 +251,26 @@ def node_index() -> int: return int(hostname.rsplit("-", 1)[-1]) +def wait_for_peers(home: Path): + cfg = tomlkit.parse((home / "config" / "config.toml").read_text()) + peers = cfg["p2p"]["persistent_peers"] + for peer in peers.split(","): + host = peer.split("@", 1)[1].split(":", 1)[0] + print("wait for peer to be ready:", host) + wait_for_port(ECHO_SERVER_PORT, host=host, timeout=600) + + +def dump_block_stats(fp): + """ + dump simple statistics for blocks for analysis + """ + for i in range(1, block_height() + 1): + blk = block(i) + timestamp = blk["result"]["block"]["header"]["time"] + txs = len(blk["result"]["block"]["data"]["txs"]) + print("block", i, txs, timestamp, file=fp) + + def main(): fire.Fire(CLI) From 0a5bd6d6cb4ceb37dc65ac3747baec2d63fe1781 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Thu, 8 Aug 2024 17:47:58 +0800 Subject: [PATCH 06/11] Problem: no debug files uploaded when timeout in ci (#1544) 30mins for marked, 100mins for all in pytest --- integration_tests/poetry.lock | 16 +++++++++++++++- integration_tests/pyproject.toml | 1 + scripts/run-integration-tests | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/integration_tests/poetry.lock b/integration_tests/poetry.lock index b42209e692..4ebd9bf581 100644 --- a/integration_tests/poetry.lock +++ b/integration_tests/poetry.lock @@ -1559,6 +1559,20 @@ files = [ [package.dependencies] pytest = ">=4.0.0" +[[package]] +name = "pytest-timeout" +version = "2.3.1" +description = "pytest plugin to abort hanging tests" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-timeout-2.3.1.tar.gz", hash = "sha256:12397729125c6ecbdaca01035b9e5239d4db97352320af155b3f5de1ba5165d9"}, + {file = "pytest_timeout-2.3.1-py3-none-any.whl", hash = "sha256:68188cb703edfc6a18fad98dc25a3c61e9f24d644b0b70f33af545219fc7813e"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + [[package]] name = "python-dateutil" version = "2.8.2" @@ -2316,4 +2330,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f4a81d2f69d093b02c4861288d41ab79aa75959de1e69043566ef4cdbb6bd3de" +content-hash = "fe26f7d4aad7c657be0d077c9e9d4db86d142be4b2ebfe267d014ddf5010bedd" diff --git a/integration_tests/pyproject.toml b/integration_tests/pyproject.toml index 1d2fed78fa..0c2c65505d 100644 --- a/integration_tests/pyproject.toml +++ b/integration_tests/pyproject.toml @@ -27,6 +27,7 @@ flaky = "^3.7" eth-bloom = "^3.0" eth-hash = "^0" pyyaml = "^6.0.2rc1" +pytest-timeout = "^2.3.1" [tool.poetry.dev-dependencies] diff --git a/scripts/run-integration-tests b/scripts/run-integration-tests index c2278eb22b..3dd4598e62 100755 --- a/scripts/run-integration-tests +++ b/scripts/run-integration-tests @@ -13,9 +13,9 @@ cd .. TESTS_TO_RUN="${TESTS_TO_RUN:-all}" if [[ "$TESTS_TO_RUN" == "all" ]]; then echo "run all tests" - cmd="pytest -vv -s" + cmd="pytest -vv -s --session-timeout=6000 --timeout=6000" else echo "run tests matching $TESTS_TO_RUN" - cmd="pytest -vv -s -m '$TESTS_TO_RUN'" + cmd="pytest -vv -s --session-timeout=1800 --timeout=1800 -m '$TESTS_TO_RUN'" fi nix-shell --run "$cmd" From d17e654656eca4d8f225d6024c55e4224dfd7a3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:06:24 +0800 Subject: [PATCH 07/11] Bump aiohttp from 3.9.5 to 3.10.2 in /testground/benchmark (#1545) Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.9.5 to 3.10.2. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.9.5...v3.10.2) --- updated-dependencies: - dependency-name: aiohttp dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- testground/benchmark/poetry.lock | 170 +++++++++++++++++-------------- 1 file changed, 91 insertions(+), 79 deletions(-) diff --git a/testground/benchmark/poetry.lock b/testground/benchmark/poetry.lock index 3261ad3ffe..0147734545 100644 --- a/testground/benchmark/poetry.lock +++ b/testground/benchmark/poetry.lock @@ -1,91 +1,103 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "aiohappyeyeballs" +version = "2.3.5" +description = "Happy Eyeballs for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohappyeyeballs-2.3.5-py3-none-any.whl", hash = "sha256:4d6dea59215537dbc746e93e779caea8178c866856a721c9c660d7a5a7b8be03"}, + {file = "aiohappyeyeballs-2.3.5.tar.gz", hash = "sha256:6fa48b9f1317254f122a07a131a86b71ca6946ca989ce6326fff54a99a920105"}, +] [[package]] name = "aiohttp" -version = "3.9.5" +version = "3.10.2" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" files = [ - {file = "aiohttp-3.9.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7"}, - {file = "aiohttp-3.9.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d6b3f1fabe465e819aed2c421a6743d8debbde79b6a8600739300630a01bf2c"}, - {file = "aiohttp-3.9.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ae79c1bc12c34082d92bf9422764f799aee4746fd7a392db46b7fd357d4a17a"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d3ebb9e1316ec74277d19c5f482f98cc65a73ccd5430540d6d11682cd857430"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84dabd95154f43a2ea80deffec9cb44d2e301e38a0c9d331cc4aa0166fe28ae3"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8a02fbeca6f63cb1f0475c799679057fc9268b77075ab7cf3f1c600e81dd46b"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c26959ca7b75ff768e2776d8055bf9582a6267e24556bb7f7bd29e677932be72"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:714d4e5231fed4ba2762ed489b4aec07b2b9953cf4ee31e9871caac895a839c0"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7a6a8354f1b62e15d48e04350f13e726fa08b62c3d7b8401c0a1314f02e3558"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c413016880e03e69d166efb5a1a95d40f83d5a3a648d16486592c49ffb76d0db"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ad7f2919d7dac062f24d6f5fe95d401597fbb015a25771f85e692d043c9d7832"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:702e2c7c187c1a498a4e2b03155d52658fdd6fda882d3d7fbb891a5cf108bb10"}, - {file = "aiohttp-3.9.5-cp310-cp310-win32.whl", hash = "sha256:67c3119f5ddc7261d47163ed86d760ddf0e625cd6246b4ed852e82159617b5fb"}, - {file = "aiohttp-3.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:471f0ef53ccedec9995287f02caf0c068732f026455f07db3f01a46e49d76bbb"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ae53e33ee7476dd3d1132f932eeb39bf6125083820049d06edcdca4381f342"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c088c4d70d21f8ca5c0b8b5403fe84a7bc8e024161febdd4ef04575ef35d474d"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:639d0042b7670222f33b0028de6b4e2fad6451462ce7df2af8aee37dcac55424"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f26383adb94da5e7fb388d441bf09c61e5e35f455a3217bfd790c6b6bc64b2ee"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66331d00fb28dc90aa606d9a54304af76b335ae204d1836f65797d6fe27f1ca2"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ff550491f5492ab5ed3533e76b8567f4b37bd2995e780a1f46bca2024223233"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f22eb3a6c1080d862befa0a89c380b4dafce29dc6cd56083f630073d102eb595"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a81b1143d42b66ffc40a441379387076243ef7b51019204fd3ec36b9f69e77d6"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f64fd07515dad67f24b6ea4a66ae2876c01031de91c93075b8093f07c0a2d93d"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:93e22add827447d2e26d67c9ac0161756007f152fdc5210277d00a85f6c92323"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:55b39c8684a46e56ef8c8d24faf02de4a2b2ac60d26cee93bc595651ff545de9"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4715a9b778f4293b9f8ae7a0a7cef9829f02ff8d6277a39d7f40565c737d3771"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:afc52b8d969eff14e069a710057d15ab9ac17cd4b6753042c407dcea0e40bf75"}, - {file = "aiohttp-3.9.5-cp311-cp311-win32.whl", hash = "sha256:b3df71da99c98534be076196791adca8819761f0bf6e08e07fd7da25127150d6"}, - {file = "aiohttp-3.9.5-cp311-cp311-win_amd64.whl", hash = "sha256:88e311d98cc0bf45b62fc46c66753a83445f5ab20038bcc1b8a1cc05666f428a"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:c7a4b7a6cf5b6eb11e109a9755fd4fda7d57395f8c575e166d363b9fc3ec4678"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0a158704edf0abcac8ac371fbb54044f3270bdbc93e254a82b6c82be1ef08f3c"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d153f652a687a8e95ad367a86a61e8d53d528b0530ef382ec5aaf533140ed00f"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82a6a97d9771cb48ae16979c3a3a9a18b600a8505b1115cfe354dfb2054468b4"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60cdbd56f4cad9f69c35eaac0fbbdf1f77b0ff9456cebd4902f3dd1cf096464c"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8676e8fd73141ded15ea586de0b7cda1542960a7b9ad89b2b06428e97125d4fa"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da00da442a0e31f1c69d26d224e1efd3a1ca5bcbf210978a2ca7426dfcae9f58"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18f634d540dd099c262e9f887c8bbacc959847cfe5da7a0e2e1cf3f14dbf2daf"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:320e8618eda64e19d11bdb3bd04ccc0a816c17eaecb7e4945d01deee2a22f95f"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:2faa61a904b83142747fc6a6d7ad8fccff898c849123030f8e75d5d967fd4a81"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:8c64a6dc3fe5db7b1b4d2b5cb84c4f677768bdc340611eca673afb7cf416ef5a"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:393c7aba2b55559ef7ab791c94b44f7482a07bf7640d17b341b79081f5e5cd1a"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c671dc117c2c21a1ca10c116cfcd6e3e44da7fcde37bf83b2be485ab377b25da"}, - {file = "aiohttp-3.9.5-cp312-cp312-win32.whl", hash = "sha256:5a7ee16aab26e76add4afc45e8f8206c95d1d75540f1039b84a03c3b3800dd59"}, - {file = "aiohttp-3.9.5-cp312-cp312-win_amd64.whl", hash = "sha256:5ca51eadbd67045396bc92a4345d1790b7301c14d1848feaac1d6a6c9289e888"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:694d828b5c41255e54bc2dddb51a9f5150b4eefa9886e38b52605a05d96566e8"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0605cc2c0088fcaae79f01c913a38611ad09ba68ff482402d3410bf59039bfb8"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4558e5012ee03d2638c681e156461d37b7a113fe13970d438d95d10173d25f78"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbc053ac75ccc63dc3a3cc547b98c7258ec35a215a92bd9f983e0aac95d3d5b"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4109adee842b90671f1b689901b948f347325045c15f46b39797ae1bf17019de"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6ea1a5b409a85477fd8e5ee6ad8f0e40bf2844c270955e09360418cfd09abac"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3c2890ca8c59ee683fd09adf32321a40fe1cf164e3387799efb2acebf090c11"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3916c8692dbd9d55c523374a3b8213e628424d19116ac4308e434dbf6d95bbdd"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8d1964eb7617907c792ca00b341b5ec3e01ae8c280825deadbbd678447b127e1"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d5ab8e1f6bee051a4bf6195e38a5c13e5e161cb7bad83d8854524798bd9fcd6e"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:52c27110f3862a1afbcb2af4281fc9fdc40327fa286c4625dfee247c3ba90156"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:7f64cbd44443e80094309875d4f9c71d0401e966d191c3d469cde4642bc2e031"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b4f72fbb66279624bfe83fd5eb6aea0022dad8eec62b71e7bf63ee1caadeafe"}, - {file = "aiohttp-3.9.5-cp38-cp38-win32.whl", hash = "sha256:6380c039ec52866c06d69b5c7aad5478b24ed11696f0e72f6b807cfb261453da"}, - {file = "aiohttp-3.9.5-cp38-cp38-win_amd64.whl", hash = "sha256:da22dab31d7180f8c3ac7c7635f3bcd53808f374f6aa333fe0b0b9e14b01f91a"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1732102949ff6087589408d76cd6dea656b93c896b011ecafff418c9661dc4ed"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6021d296318cb6f9414b48e6a439a7f5d1f665464da507e8ff640848ee2a58a"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:239f975589a944eeb1bad26b8b140a59a3a320067fb3cd10b75c3092405a1372"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b7b30258348082826d274504fbc7c849959f1989d86c29bc355107accec6cfb"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2adf5c87ff6d8b277814a28a535b59e20bfea40a101db6b3bdca7e9926bc24"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a3d838441bebcf5cf442700e3963f58b5c33f015341f9ea86dcd7d503c07e2"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e3a1ae66e3d0c17cf65c08968a5ee3180c5a95920ec2731f53343fac9bad106"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c69e77370cce2d6df5d12b4e12bdcca60c47ba13d1cbbc8645dd005a20b738b"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf56238f4bbf49dab8c2dc2e6b1b68502b1e88d335bea59b3f5b9f4c001475"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d1469f228cd9ffddd396d9948b8c9cd8022b6d1bf1e40c6f25b0fb90b4f893ed"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:45731330e754f5811c314901cebdf19dd776a44b31927fa4b4dbecab9e457b0c"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3fcb4046d2904378e3aeea1df51f697b0467f2aac55d232c87ba162709478c46"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8cf142aa6c1a751fcb364158fd710b8a9be874b81889c2bd13aa8893197455e2"}, - {file = "aiohttp-3.9.5-cp39-cp39-win32.whl", hash = "sha256:7b179eea70833c8dee51ec42f3b4097bd6370892fa93f510f76762105568cf09"}, - {file = "aiohttp-3.9.5-cp39-cp39-win_amd64.whl", hash = "sha256:38d80498e2e169bc61418ff36170e0aad0cd268da8b38a17c4cf29d254a8b3f1"}, - {file = "aiohttp-3.9.5.tar.gz", hash = "sha256:edea7d15772ceeb29db4aff55e482d4bcfb6ae160ce144f2682de02f6d693551"}, + {file = "aiohttp-3.10.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:95213b3d79c7e387144e9cb7b9d2809092d6ff2c044cb59033aedc612f38fb6d"}, + {file = "aiohttp-3.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1aa005f060aff7124cfadaa2493f00a4e28ed41b232add5869e129a2e395935a"}, + {file = "aiohttp-3.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eabe6bf4c199687592f5de4ccd383945f485779c7ffb62a9b9f1f8a3f9756df8"}, + {file = "aiohttp-3.10.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96e010736fc16d21125c7e2dc5c350cd43c528b85085c04bf73a77be328fe944"}, + {file = "aiohttp-3.10.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99f81f9c1529fd8e03be4a7bd7df32d14b4f856e90ef6e9cbad3415dbfa9166c"}, + {file = "aiohttp-3.10.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d611d1a01c25277bcdea06879afbc11472e33ce842322496b211319aa95441bb"}, + {file = "aiohttp-3.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e00191d38156e09e8c81ef3d75c0d70d4f209b8381e71622165f22ef7da6f101"}, + {file = "aiohttp-3.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74c091a5ded6cb81785de2d7a8ab703731f26de910dbe0f3934eabef4ae417cc"}, + {file = "aiohttp-3.10.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:18186a80ec5a701816adbf1d779926e1069392cf18504528d6e52e14b5920525"}, + {file = "aiohttp-3.10.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5a7ceb2a0d2280f23a02c64cd0afdc922079bb950400c3dd13a1ab2988428aac"}, + {file = "aiohttp-3.10.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8bd7be6ff6c162a60cb8fce65ee879a684fbb63d5466aba3fa5b9288eb04aefa"}, + {file = "aiohttp-3.10.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fae962b62944eaebff4f4fddcf1a69de919e7b967136a318533d82d93c3c6bd1"}, + {file = "aiohttp-3.10.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a0fde16d284efcacbe15fb0c1013f0967b6c3e379649239d783868230bf1db42"}, + {file = "aiohttp-3.10.2-cp310-cp310-win32.whl", hash = "sha256:f81cd85a0e76ec7b8e2b6636fe02952d35befda4196b8c88f3cec5b4fb512839"}, + {file = "aiohttp-3.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:54ba10eb5a3481c28282eb6afb5f709aedf53cf9c3a31875ffbdc9fc719ffd67"}, + {file = "aiohttp-3.10.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:87fab7f948e407444c2f57088286e00e2ed0003ceaf3d8f8cc0f60544ba61d91"}, + {file = "aiohttp-3.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ec6ad66ed660d46503243cbec7b2b3d8ddfa020f984209b3b8ef7d98ce69c3f2"}, + {file = "aiohttp-3.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a4be88807283bd96ae7b8e401abde4ca0bab597ba73b5e9a2d98f36d451e9aac"}, + {file = "aiohttp-3.10.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01c98041f90927c2cbd72c22a164bb816fa3010a047d264969cf82e1d4bcf8d1"}, + {file = "aiohttp-3.10.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54e36c67e1a9273ecafab18d6693da0fb5ac48fd48417e4548ac24a918c20998"}, + {file = "aiohttp-3.10.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7de3ddb6f424af54535424082a1b5d1ae8caf8256ebd445be68c31c662354720"}, + {file = "aiohttp-3.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7dd9c7db94b4692b827ce51dcee597d61a0e4f4661162424faf65106775b40e7"}, + {file = "aiohttp-3.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e57e21e1167705f8482ca29cc5d02702208d8bf4aff58f766d94bcd6ead838cd"}, + {file = "aiohttp-3.10.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a1a50e59b720060c29e2951fd9f13c01e1ea9492e5a527b92cfe04dd64453c16"}, + {file = "aiohttp-3.10.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:686c87782481fda5ee6ba572d912a5c26d9f98cc5c243ebd03f95222af3f1b0f"}, + {file = "aiohttp-3.10.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:dafb4abb257c0ed56dc36f4e928a7341b34b1379bd87e5a15ce5d883c2c90574"}, + {file = "aiohttp-3.10.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:494a6f77560e02bd7d1ab579fdf8192390567fc96a603f21370f6e63690b7f3d"}, + {file = "aiohttp-3.10.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6fe8503b1b917508cc68bf44dae28823ac05e9f091021e0c41f806ebbb23f92f"}, + {file = "aiohttp-3.10.2-cp311-cp311-win32.whl", hash = "sha256:4ddb43d06ce786221c0dfd3c91b4892c318eaa36b903f7c4278e7e2fa0dd5102"}, + {file = "aiohttp-3.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:ca2f5abcb0a9a47e56bac173c01e9f6c6e7f27534d91451c5f22e6a35a5a2093"}, + {file = "aiohttp-3.10.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:14eb6b17f6246959fb0b035d4f4ae52caa870c4edfb6170aad14c0de5bfbf478"}, + {file = "aiohttp-3.10.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:465e445ec348d4e4bd349edd8b22db75f025da9d7b6dc1369c48e7935b85581e"}, + {file = "aiohttp-3.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:341f8ece0276a828d95b70cd265d20e257f5132b46bf77d759d7f4e0443f2906"}, + {file = "aiohttp-3.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c01fbb87b5426381cd9418b3ddcf4fc107e296fa2d3446c18ce6c76642f340a3"}, + {file = "aiohttp-3.10.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c474af073e1a6763e1c5522bbb2d85ff8318197e4c6c919b8d7886e16213345"}, + {file = "aiohttp-3.10.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d9076810a5621236e29b2204e67a68e1fe317c8727ee4c9abbfbb1083b442c38"}, + {file = "aiohttp-3.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8f515d6859e673940e08de3922b9c4a2249653b0ac181169313bd6e4b1978ac"}, + {file = "aiohttp-3.10.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:655e583afc639bef06f3b2446972c1726007a21003cd0ef57116a123e44601bc"}, + {file = "aiohttp-3.10.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8da9449a575133828cc99985536552ea2dcd690e848f9d41b48d8853a149a959"}, + {file = "aiohttp-3.10.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:19073d57d0feb1865d12361e2a1f5a49cb764bf81a4024a3b608ab521568093a"}, + {file = "aiohttp-3.10.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8e98e1845805f184d91fda6f9ab93d7c7b0dddf1c07e0255924bfdb151a8d05"}, + {file = "aiohttp-3.10.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:377220a5efde6f9497c5b74649b8c261d3cce8a84cb661be2ed8099a2196400a"}, + {file = "aiohttp-3.10.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:92f7f4a4dc9cdb5980973a74d43cdbb16286dacf8d1896b6c3023b8ba8436f8e"}, + {file = "aiohttp-3.10.2-cp312-cp312-win32.whl", hash = "sha256:9bb2834a6f11d65374ce97d366d6311a9155ef92c4f0cee543b2155d06dc921f"}, + {file = "aiohttp-3.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:518dc3cb37365255708283d1c1c54485bbacccd84f0a0fb87ed8917ba45eda5b"}, + {file = "aiohttp-3.10.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7f98e70bbbf693086efe4b86d381efad8edac040b8ad02821453083d15ec315f"}, + {file = "aiohttp-3.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9f6f0b252a009e98fe84028a4ec48396a948e7a65b8be06ccfc6ef68cf1f614d"}, + {file = "aiohttp-3.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9360e3ffc7b23565600e729e8c639c3c50d5520e05fdf94aa2bd859eef12c407"}, + {file = "aiohttp-3.10.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3988044d1635c7821dd44f0edfbe47e9875427464e59d548aece447f8c22800a"}, + {file = "aiohttp-3.10.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a9d59da1543a6f1478c3436fd49ec59be3868bca561a33778b4391005e499d"}, + {file = "aiohttp-3.10.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f9f49bdb94809ac56e09a310a62f33e5f22973d6fd351aac72a39cd551e98194"}, + {file = "aiohttp-3.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddfd2dca3f11c365d6857a07e7d12985afc59798458a2fdb2ffa4a0332a3fd43"}, + {file = "aiohttp-3.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:685c1508ec97b2cd3e120bfe309a4ff8e852e8a7460f1ef1de00c2c0ed01e33c"}, + {file = "aiohttp-3.10.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:49904f38667c44c041a0b44c474b3ae36948d16a0398a8f8cd84e2bb3c42a069"}, + {file = "aiohttp-3.10.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:352f3a4e5f11f3241a49b6a48bc5b935fabc35d1165fa0d87f3ca99c1fcca98b"}, + {file = "aiohttp-3.10.2-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:fc61f39b534c5d5903490478a0dd349df397d2284a939aa3cbaa2fb7a19b8397"}, + {file = "aiohttp-3.10.2-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:ad2274e707be37420d0b6c3d26a8115295fe9d8e6e530fa6a42487a8ca3ad052"}, + {file = "aiohttp-3.10.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c836bf3c7512100219fe1123743fd8dd9a2b50dd7cfb0c3bb10d041309acab4b"}, + {file = "aiohttp-3.10.2-cp38-cp38-win32.whl", hash = "sha256:53e8898adda402be03ff164b0878abe2d884e3ea03a4701e6ad55399d84b92dc"}, + {file = "aiohttp-3.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:7cc8f65f5b22304693de05a245b6736b14cb5bc9c8a03da6e2ae9ef15f8b458f"}, + {file = "aiohttp-3.10.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9dfc906d656e14004c5bc672399c1cccc10db38df2b62a13fb2b6e165a81c316"}, + {file = "aiohttp-3.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:91b10208b222ddf655c3a3d5b727879d7163db12b634492df41a9182a76edaae"}, + {file = "aiohttp-3.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9fd16b5e1a7bdd14668cd6bde60a2a29b49147a535c74f50d8177d11b38433a7"}, + {file = "aiohttp-3.10.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2bfdda4971bd79201f59adbad24ec2728875237e1c83bba5221284dbbf57bda"}, + {file = "aiohttp-3.10.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69d73f869cf29e8a373127fc378014e2b17bcfbe8d89134bc6fb06a2f67f3cb3"}, + {file = "aiohttp-3.10.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df59f8486507c421c0620a2c3dce81fbf1d54018dc20ff4fecdb2c106d6e6abc"}, + {file = "aiohttp-3.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df930015db36b460aa9badbf35eccbc383f00d52d4b6f3de2ccb57d064a6ade"}, + {file = "aiohttp-3.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:562b1153ab7f766ee6b8b357ec777a302770ad017cf18505d34f1c088fccc448"}, + {file = "aiohttp-3.10.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d984db6d855de58e0fde1ef908d48fe9a634cadb3cf715962722b4da1c40619d"}, + {file = "aiohttp-3.10.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:14dc3fcb0d877911d775d511eb617a486a8c48afca0a887276e63db04d3ee920"}, + {file = "aiohttp-3.10.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b52a27a5c97275e254704e1049f4b96a81e67d6205f52fa37a4777d55b0e98ef"}, + {file = "aiohttp-3.10.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:cd33d9de8cfd006a0d0fe85f49b4183c57e91d18ffb7e9004ce855e81928f704"}, + {file = "aiohttp-3.10.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1238fc979160bc03a92fff9ad021375ff1c8799c6aacb0d8ea1b357ea40932bb"}, + {file = "aiohttp-3.10.2-cp39-cp39-win32.whl", hash = "sha256:e2f43d238eae4f0b04f58d4c0df4615697d4ca3e9f9b1963d49555a94f0f5a04"}, + {file = "aiohttp-3.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:947847f07a8f81d7b39b2d0202fd73e61962ebe17ac2d8566f260679e467da7b"}, + {file = "aiohttp-3.10.2.tar.gz", hash = "sha256:4d1f694b5d6e459352e5e925a42e05bac66655bfde44d81c59992463d2897014"}, ] [package.dependencies] +aiohappyeyeballs = ">=2.3.0" aiosignal = ">=1.1.2" attrs = ">=17.3.0" frozenlist = ">=1.1.1" @@ -93,7 +105,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "brotlicffi"] +speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] [[package]] name = "aiosignal" From b4cb3edc92b5bfb5970e911ff2a9298b5bbf3f12 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 12 Aug 2024 22:06:55 +0800 Subject: [PATCH 08/11] Problem: testground output artifact too large (#1546) Solution: - cleanup some duplicated stuff --- testground/benchmark/benchmark/stateless.py | 27 ++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/testground/benchmark/benchmark/stateless.py b/testground/benchmark/benchmark/stateless.py index 0280ccf936..f5fe43341c 100644 --- a/testground/benchmark/benchmark/stateless.py +++ b/testground/benchmark/benchmark/stateless.py @@ -167,7 +167,7 @@ def run( # collect outputs output = Path("/data.tar.bz2") with tarfile.open(output, "x:bz2") as tar: - tar.add(home, arcname="data") + tar.add(home, arcname="data", filter=output_filter(group, group_seq)) outdir = Path(outdir) if outdir.exists(): assert outdir.is_dir() @@ -176,6 +176,31 @@ def run( shutil.copy(output, filename) +def output_filter(group, group_seq: int): + """ + filter out some big and useless paths to reduce size of output artifacts + """ + + is_validator_leader = group == VALIDATOR_GROUP and group_seq == 0 + is_fullnode_leader = group == FULLNODE_GROUP and group_seq == 0 + + def inner(info: tarfile.TarInfo): + # only keep one copy + if not is_validator_leader and info.name in ( + "data/data/cs.wal", + "data/data/blockstore.db", + "data/data/application.db", + "data/data/memiavl.db", + "data/data/state.db", + ): + return None + if not is_fullnode_leader and info.name == "data/data/tx_index.db": + return None + return info + + return inner + + def detect_idle(idle_blocks: int, interval: int): """ returns if the chain is empty for consecutive idle_blocks From bc5dfc5f5809cedfb66c7dc8126b444b56633b6f Mon Sep 17 00:00:00 2001 From: yihuang Date: Tue, 13 Aug 2024 15:28:47 +0800 Subject: [PATCH 09/11] Problem: memiavl cache not compatible with block-stm (#1547) * Problem: memiavl cache not compatible with block-stm Solution: - disable the memiavl cache if block-stm is enabled * Update CHANGELOG.md Signed-off-by: yihuang * cleanup * fix test * cleanup * cleanup --------- Signed-off-by: yihuang Co-authored-by: mmsqe --- CHANGELOG.md | 1 + app/app.go | 12 +++++++++--- store/setup.go | 12 ++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3a4842197..e3aa703460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ * (block-stm) [#1515](https://github.com/crypto-org-chain/cronos/pull/1515) Improve performance by cache signature verification result between incarnations of same tx. * (store) [#1526](https://github.com/crypto-org-chain/cronos/pull/1526) Cache index/filters in rocksdb application.db to reduce ram usage. * (store)[#1529](https://github.com/crypto-org-chain/cronos/pull/1529) Enable pinL0FilterAndIndexBlocksInCache. +* (store)[#1547](https://github.com/crypto-org-chain/cronos/pull/1547) Disable memiavl cache if block-stm is enabled. ### Bug Fixes diff --git a/app/app.go b/app/app.go index b4df0fc360..80401cee2e 100644 --- a/app/app.go +++ b/app/app.go @@ -382,8 +382,15 @@ func New( app.SetProcessProposal(handler.ProcessProposalHandler()) }) + blockSTMEnabled := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor)) == "block-stm" + homePath := cast.ToString(appOpts.Get(flags.FlagHome)) - baseAppOptions = memiavlstore.SetupMemIAVL(logger, homePath, appOpts, false, false, baseAppOptions) + var cacheSize int + if !blockSTMEnabled { + // only enable memiavl cache if block-stm is not enabled, because it's not concurrency-safe. + cacheSize = cast.ToInt(appOpts.Get(memiavlstore.FlagCacheSize)) + } + baseAppOptions = memiavlstore.SetupMemIAVL(logger, homePath, appOpts, false, false, cacheSize, baseAppOptions) // enable optimistic execution baseAppOptions = append(baseAppOptions, baseapp.SetOptimisticExecution()) @@ -414,8 +421,7 @@ func New( app.SetDisableBlockGasMeter(true) - executor := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor)) - if executor == "block-stm" { + if blockSTMEnabled { sdk.SetAddrCacheEnabled(false) workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers)) app.SetTxExecutor(evmapp.STMTxExecutor(app.GetStoreKeys(), workers)) diff --git a/store/setup.go b/store/setup.go index f17cedf6da..ee4bd9f306 100644 --- a/store/setup.go +++ b/store/setup.go @@ -26,14 +26,22 @@ const ( // SetupMemIAVL insert the memiavl setter in front of baseapp options, so that // the default rootmulti store is replaced by memiavl store, -func SetupMemIAVL(logger log.Logger, homePath string, appOpts servertypes.AppOptions, sdk46Compact bool, supportExportNonSnapshotVersion bool, baseAppOptions []func(*baseapp.BaseApp)) []func(*baseapp.BaseApp) { +func SetupMemIAVL( + logger log.Logger, + homePath string, + appOpts servertypes.AppOptions, + sdk46Compact bool, + supportExportNonSnapshotVersion bool, + cacheSize int, + baseAppOptions []func(*baseapp.BaseApp), +) []func(*baseapp.BaseApp) { if cast.ToBool(appOpts.Get(FlagMemIAVL)) { opts := memiavl.Options{ AsyncCommitBuffer: cast.ToInt(appOpts.Get(FlagAsyncCommitBuffer)), ZeroCopy: cast.ToBool(appOpts.Get(FlagZeroCopy)), SnapshotKeepRecent: cast.ToUint32(appOpts.Get(FlagSnapshotKeepRecent)), SnapshotInterval: cast.ToUint32(appOpts.Get(FlagSnapshotInterval)), - CacheSize: cast.ToInt(appOpts.Get(FlagCacheSize)), + CacheSize: cacheSize, SnapshotWriterLimit: cast.ToInt(appOpts.Get(FlagSnapshotWriterLimit)), } From af15fa3c7aab32f802be77e68e8485574376d046 Mon Sep 17 00:00:00 2001 From: yihuang Date: Wed, 14 Aug 2024 10:00:42 +0800 Subject: [PATCH 10/11] Problem: block-stm executor status not logged (#1548) Solution: - add a log with the number of workers --- app/app.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/app.go b/app/app.go index 80401cee2e..7da2128af5 100644 --- a/app/app.go +++ b/app/app.go @@ -12,6 +12,7 @@ import ( "net/http" "os" "path/filepath" + stdruntime "runtime" "sort" abci "github.com/cometbft/cometbft/abci/types" @@ -424,6 +425,10 @@ func New( if blockSTMEnabled { sdk.SetAddrCacheEnabled(false) workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers)) + if workers == 0 { + workers = stdruntime.NumCPU() + } + logger.Info("block-stm executor enabled", "workers", workers) app.SetTxExecutor(evmapp.STMTxExecutor(app.GetStoreKeys(), workers)) } else { app.SetTxExecutor(evmapp.DefaultTxExecutor) From 8b9b42eb2e0f87dbbe9b459422c5629a25415174 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 14 Aug 2024 17:36:54 +0800 Subject: [PATCH 11/11] Problem: cometbft is outdated (#1550) * Problem: cometbft is outdated * overwrite default cache-size when enable block-stm * Apply suggestions from code review --- go.mod | 2 +- go.sum | 4 ++-- gomod2nix.toml | 4 ++-- integration_tests/configs/default.jsonnet | 1 + store/go.mod | 3 +-- store/go.sum | 6 ++---- testground/benchmark/benchmark/peer.py | 2 ++ versiondb/go.mod | 3 +-- versiondb/go.sum | 6 ++---- 9 files changed, 14 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 532e0888e3..a5f7b973e7 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( cosmossdk.io/x/upgrade v0.1.1 filippo.io/age v1.1.1 github.com/99designs/keyring v1.2.2 - github.com/cometbft/cometbft v0.38.10 + github.com/cometbft/cometbft v0.38.11 github.com/cosmos/cosmos-db v1.0.3-0.20240408151834-e75f6e4b28d8 github.com/cosmos/cosmos-sdk v0.50.7 github.com/cosmos/gogoproto v1.4.12 diff --git a/go.sum b/go.sum index 0029ed4d6a..bb4b2e8ab7 100644 --- a/go.sum +++ b/go.sum @@ -366,8 +366,8 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coinbase/rosetta-sdk-go/types v1.0.0 h1:jpVIwLcPoOeCR6o1tU+Xv7r5bMONNbHU7MuEHboiFuA= github.com/coinbase/rosetta-sdk-go/types v1.0.0/go.mod h1:eq7W2TMRH22GTW0N0beDnN931DW0/WOI1R2sdHNHG4c= -github.com/cometbft/cometbft v0.38.10 h1:2ePuglchT+j0Iao+cfmt/nw5U7K2lnGDzXSUPGVdXaU= -github.com/cometbft/cometbft v0.38.10/go.mod h1:jHPx9vQpWzPHEAiYI/7EDKaB1NXhK6o3SArrrY8ExKc= +github.com/cometbft/cometbft v0.38.11 h1:6bNDUB8/xq4uYonYwIfGc9OqK1ZH4NkdaMmR1LZIJqk= +github.com/cometbft/cometbft v0.38.11/go.mod h1:jHPx9vQpWzPHEAiYI/7EDKaB1NXhK6o3SArrrY8ExKc= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/gomod2nix.toml b/gomod2nix.toml index 83e954284a..a908e9aa95 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -154,8 +154,8 @@ schema = 3 version = "v1.0.0" hash = "sha256-z/0E0NiEGo7zxM7d94ImgUf8P0/KG6hbP9T4Vuym4p0=" [mod."github.com/cometbft/cometbft"] - version = "v0.38.10" - hash = "sha256-zd/vD0uUgawzuuzuNweRK4HctzWmCb3tnLUFmaOT4OE=" + version = "v0.38.11" + hash = "sha256-FMAGFFBlCl5/x9l2rF8gmOy1GmhVxYEmJzTjKfYuGUQ=" [mod."github.com/cometbft/cometbft-db"] version = "v0.0.0-20231011055109-57922ac52a63" hash = "sha256-iLs/FN1be3AcoyhKj2+b5Msqeat9j2ja4acyOs+w+Uk=" diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index ad7c1c3c40..3ec6062093 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -42,6 +42,7 @@ enable: true, 'zero-copy': true, 'snapshot-interval': 5, + 'cache-size': 0, }, versiondb: { enable: true, diff --git a/store/go.mod b/store/go.mod index dda694959a..871ece1302 100644 --- a/store/go.mod +++ b/store/go.mod @@ -8,7 +8,7 @@ require ( cosmossdk.io/errors v1.0.1 cosmossdk.io/log v1.3.1 cosmossdk.io/store v1.1.0 - github.com/cometbft/cometbft v0.38.8 + github.com/cometbft/cometbft v0.38.11 github.com/cosmos/cosmos-db v1.0.2 github.com/cosmos/cosmos-sdk v0.50.4 github.com/cosmos/gogoproto v1.4.12 @@ -97,7 +97,6 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 // indirect - github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/linxGnu/grocksdb v1.8.14 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/store/go.sum b/store/go.sum index b705d2dd4c..b74fb22b3a 100644 --- a/store/go.sum +++ b/store/go.sum @@ -110,8 +110,8 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/cometbft/cometbft v0.38.8 h1:XyJ9Cu3xqap6xtNxiemrO8roXZ+KS2Zlu7qQ0w1trvU= -github.com/cometbft/cometbft v0.38.8/go.mod h1:xOoGZrtUT+A5izWfHSJgl0gYZUE7lu7Z2XIS1vWG/QQ= +github.com/cometbft/cometbft v0.38.11 h1:6bNDUB8/xq4uYonYwIfGc9OqK1ZH4NkdaMmR1LZIJqk= +github.com/cometbft/cometbft v0.38.11/go.mod h1:jHPx9vQpWzPHEAiYI/7EDKaB1NXhK6o3SArrrY8ExKc= github.com/cometbft/cometbft-db v0.9.1 h1:MIhVX5ja5bXNHF8EYrThkG9F7r9kSfv8BX4LWaxWJ4M= github.com/cometbft/cometbft-db v0.9.1/go.mod h1:iliyWaoV0mRwBJoizElCwwRA9Tf7jZJOURcRZF9m60U= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -442,8 +442,6 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= -github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linxGnu/grocksdb v1.8.14 h1:HTgyYalNwBSG/1qCQUIott44wU5b2Y9Kr3z7SK5OfGQ= diff --git a/testground/benchmark/benchmark/peer.py b/testground/benchmark/benchmark/peer.py index 35daf20295..ea25490da7 100644 --- a/testground/benchmark/benchmark/peer.py +++ b/testground/benchmark/benchmark/peer.py @@ -135,6 +135,8 @@ def patch_configs(home: Path, group: str, peers: str, block_executor: str): "mempool.max-txs": MEMPOOL_SIZE, "evm.block-executor": block_executor, } + if block_executor == "block-stm": + app_patch["memiavl.cache-size"] = 0 patch_toml(home / "config" / "config.toml", config_patch) patch_toml(home / "config" / "app.toml", app_patch) diff --git a/versiondb/go.mod b/versiondb/go.mod index 11b3c39674..a0f2a0f73b 100644 --- a/versiondb/go.mod +++ b/versiondb/go.mod @@ -9,7 +9,7 @@ require ( cosmossdk.io/log v1.3.1 cosmossdk.io/store v1.1.0 github.com/alitto/pond v1.8.3 - github.com/cometbft/cometbft v0.38.8 + github.com/cometbft/cometbft v0.38.11 github.com/cosmos/cosmos-db v1.0.2 github.com/cosmos/cosmos-sdk v0.50.4 github.com/cosmos/gogoproto v1.4.12 @@ -103,7 +103,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 // indirect github.com/lib/pq v1.10.7 // indirect - github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/versiondb/go.sum b/versiondb/go.sum index be1310d81c..9def9d3997 100644 --- a/versiondb/go.sum +++ b/versiondb/go.sum @@ -125,8 +125,8 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/cometbft/cometbft v0.38.8 h1:XyJ9Cu3xqap6xtNxiemrO8roXZ+KS2Zlu7qQ0w1trvU= -github.com/cometbft/cometbft v0.38.8/go.mod h1:xOoGZrtUT+A5izWfHSJgl0gYZUE7lu7Z2XIS1vWG/QQ= +github.com/cometbft/cometbft v0.38.11 h1:6bNDUB8/xq4uYonYwIfGc9OqK1ZH4NkdaMmR1LZIJqk= +github.com/cometbft/cometbft v0.38.11/go.mod h1:jHPx9vQpWzPHEAiYI/7EDKaB1NXhK6o3SArrrY8ExKc= github.com/cometbft/cometbft-db v0.9.1 h1:MIhVX5ja5bXNHF8EYrThkG9F7r9kSfv8BX4LWaxWJ4M= github.com/cometbft/cometbft-db v0.9.1/go.mod h1:iliyWaoV0mRwBJoizElCwwRA9Tf7jZJOURcRZF9m60U= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= @@ -464,8 +464,6 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= -github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linxGnu/grocksdb v1.9.2 h1:O3mzvO0wuzQ9mtlHbDrShixyVjVbmuqTjFrzlf43wZ8=