Skip to content

Commit

Permalink
tests(sims): Use temp dir for sims (cosmos#20143)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alpe
alpe authored Apr 24, 2024
1 parent 30383e6 commit ca05e4d
Showing 1 changed file with 54 additions and 89 deletions.
143 changes: 54 additions & 89 deletions simapp/sim_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package simapp
import (
"encoding/json"
"flag"
"fmt"
"math/rand"
"os"
"runtime/debug"
Expand All @@ -14,6 +13,7 @@ import (
cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
dbm "github.com/cosmos/cosmos-db"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"cosmossdk.io/log"
Expand Down Expand Up @@ -58,30 +58,7 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) {
}

func TestFullAppSimulation(t *testing.T) {
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID

db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if skip {
t.Skip("skipping application simulation")
}
require.NoError(t, err, "simulation setup failed")

defer func() {
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(dir))
}()

appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue

app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
require.Equal(t, "SimApp", app.Name())

config, db, _, app := setupSimulationApp(t, "skipping application simulation")
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t,
Expand All @@ -97,7 +74,7 @@ func TestFullAppSimulation(t *testing.T) {
)

// export state and simParams before the simulation error is checked
err = simtestutil.CheckExportSimulation(app, config, simParams)
err := simtestutil.CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)

Expand All @@ -107,29 +84,7 @@ func TestFullAppSimulation(t *testing.T) {
}

func TestAppImportExport(t *testing.T) {
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID

db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if skip {
t.Skip("skipping application import/export simulation")
}
require.NoError(t, err, "simulation setup failed")

defer func() {
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(dir))
}()

appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue

app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
require.Equal(t, "SimApp", app.Name())
config, db, appOptions, app := setupSimulationApp(t, "skipping application import/export simulation")

// Run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
Expand All @@ -146,20 +101,20 @@ func TestAppImportExport(t *testing.T) {
)

// export state and simParams before the simulation error is checked
err = simtestutil.CheckExportSimulation(app, config, simParams)
err := simtestutil.CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)

if config.Commit {
simtestutil.PrintStats(db)
}

fmt.Printf("exporting genesis...\n")
t.Log("exporting genesis...\n")

exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{})
require.NoError(t, err)

fmt.Printf("importing genesis...\n")
t.Log("importing genesis...\n")

newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
require.NoError(t, err, "simulation setup failed")
Expand All @@ -169,6 +124,7 @@ func TestAppImportExport(t *testing.T) {
require.NoError(t, os.RemoveAll(newDir))
}()

appOptions[flags.FlagHome] = newDir // ensure a unique folder for the new app
newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
require.Equal(t, "SimApp", newApp.Name())

Expand All @@ -181,17 +137,17 @@ func TestAppImportExport(t *testing.T) {
_, err = newApp.ModuleManager.InitGenesis(ctxB, genesisState)
if err != nil {
if strings.Contains(err.Error(), "validator set is empty after InitGenesis") {
logger.Info("Skipping simulation as all validators have been unbonded")
logger.Info("err", err, "stacktrace", string(debug.Stack()))
t.Log("Skipping simulation as all validators have been unbonded")
t.Logf("err: %s stacktrace: %s\n", err, string(debug.Stack()))
return
}
}

require.NoError(t, err)
err = newApp.StoreConsensusParams(ctxB, exported.ConsensusParams)
require.NoError(t, err)
fmt.Printf("comparing stores...\n")

t.Log("comparing stores...")
// skip certain prefixes
skipPrefixes := map[string][][]byte{
stakingtypes.StoreKey: {
Expand Down Expand Up @@ -222,36 +178,18 @@ func TestAppImportExport(t *testing.T) {
failedKVAs, failedKVBs := simtestutil.DiffKVStores(storeA, storeB, skipPrefixes[keyName])
require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare %s, key stores %s and %s", keyName, appKeyA, appKeyB)

fmt.Printf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), appKeyA, appKeyB)

require.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(keyName, app.SimulationManager().StoreDecoders, failedKVAs, failedKVBs))
t.Logf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), appKeyA, appKeyB)
if !assert.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(keyName, app.SimulationManager().StoreDecoders, failedKVAs, failedKVBs)) {
for _, v := range failedKVAs {
t.Logf("store missmatch: %q\n", v)
}
t.FailNow()
}
}
}

func TestAppSimulationAfterImport(t *testing.T) {
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID

db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if skip {
t.Skip("skipping application simulation after import")
}
require.NoError(t, err, "simulation setup failed")

defer func() {
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(dir))
}()

appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue

app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
require.Equal(t, "SimApp", app.Name())
config, db, appOptions, app := setupSimulationApp(t, "skipping application simulation after import")

// Run randomized simulation
stopEarly, simParams, simErr := simulation.SimulateFromSeed(
Expand All @@ -266,27 +204,27 @@ func TestAppSimulationAfterImport(t *testing.T) {
app.AppCodec(),
codectestutil.CodecOptions{}.GetAddressCodec(),
)
require.NoError(t, simErr)

// export state and simParams before the simulation error is checked
err = simtestutil.CheckExportSimulation(app, config, simParams)
err := simtestutil.CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)

if config.Commit {
simtestutil.PrintStats(db)
}

if stopEarly {
fmt.Println("can't export or import a zero-validator genesis, exiting test...")
t.Log("can't export or import a zero-validator genesis, exiting test...")
return
}

fmt.Printf("exporting genesis...\n")
t.Logf("exporting genesis...\n")

exported, err := app.ExportAppStateAndValidators(true, []string{}, []string{})
require.NoError(t, err)

fmt.Printf("importing genesis...\n")
t.Logf("importing genesis...\n")

newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
require.NoError(t, err, "simulation setup failed")
Expand Down Expand Up @@ -322,6 +260,33 @@ func TestAppSimulationAfterImport(t *testing.T) {
require.NoError(t, err)
}

func setupSimulationApp(t *testing.T, msg string) (simtypes.Config, dbm.DB, simtestutil.AppOptionsMap, *SimApp) {
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID

db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if skip {
t.Skip(msg)
}
require.NoError(t, err, "simulation setup failed")

t.Cleanup(func() {
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(dir))
})

appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = dir // ensure a unique folder
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue

app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
require.Equal(t, "SimApp", app.Name())
return config, db, appOptions, app
}

// TODO: Make another test for the fuzzer itself, which just has noOp txs
// and doesn't depend on the application.
func TestAppStateDeterminism(t *testing.T) {
Expand Down Expand Up @@ -355,7 +320,6 @@ func TestAppStateDeterminism(t *testing.T) {
appOptions.SetDefault(key, value)
}
}
appOptions.SetDefault(flags.FlagHome, DefaultNodeHome)
appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue)
if simcli.FlagVerboseValue {
appOptions.SetDefault(flags.FlagLogLevel, "debug")
Expand All @@ -366,7 +330,7 @@ func TestAppStateDeterminism(t *testing.T) {
config.Seed = rand.Int63()
}

fmt.Println("config.Seed: ", config.Seed)
t.Log("config.Seed: ", config.Seed)

for j := 0; j < numTimesToRunPerSeed; j++ {
var logger log.Logger
Expand All @@ -376,13 +340,14 @@ func TestAppStateDeterminism(t *testing.T) {
logger = log.NewNopLogger()
}

appOptions.SetDefault(flags.FlagHome, t.TempDir())
db := dbm.NewMemDB()
app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}

fmt.Printf(
t.Logf(
"running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n",
config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed,
)
Expand Down

0 comments on commit ca05e4d

Please sign in to comment.