application interfaces for simulation

CHANGELOG.md

@@ -182,7 +182,7 @@ generalized genesis accounts through the `GenesisAccount` interface.
 * (sdk) [\#4640](https://github.com/cosmos/cosmos-sdk/issues/4640) improve import/export simulation errors by extending `DiffKVStores` to return an array of `KVPairs` that are then compared to check for inconsistencies.
 * (sdk) [\#4717](https://github.com/cosmos/cosmos-sdk/issues/4717) refactor `x/slashing` to match the new module spec
 * (sdk) [\#4758](https://github.com/cosmos/cosmos-sdk/issues/4758) update `x/genaccounts` to match module spec
-* (simulation) [\#4824](https://github.com/cosmos/cosmos-sdk/issues/4824) PrintAllInvariants flag will print all failed invariants
+* (simulation) [\#4824](https://github.com/cosmos/cosmos-sdk/issues/4824) `PrintAllInvariants` flag will print all failed invariants
 * (simulation) [\#4490](https://github.com/cosmos/cosmos-sdk/issues/4490) add `InitialBlockHeight` flag to resume a simulation from a given block
   * Support exporting the simulation stats to a given JSON file
 * (simulation) [\#4847](https://github.com/cosmos/cosmos-sdk/issues/4847), [\#4838](https://github.com/cosmos/cosmos-sdk/pull/4838) and [\#4869](https://github.com/cosmos/cosmos-sdk/pull/4869) `SimApp` and simulation refactors:
@@ -194,9 +194,12 @@ generalized genesis accounts through the `GenesisAccount` interface.
   * Add `WeightedOperations` to the `SimulationManager` that define simulation operations (modules' `Msg`s) with their
   respective weights (i.e chance of being simulated).
   * Add `ProposalContents` to the `SimulationManager` to register each module's governance proposal `Content`s.
-* (simulation) [\#4893](https://github.com/cosmos/cosmos-sdk/issues/4893) Change SimApp keepers to be public and add getter functions for keys and codec
+* (simulation) [\#4893](https://github.com/cosmos/cosmos-sdk/issues/4893) Change `SimApp` keepers to be public and add getter functions for keys and codec
 * (simulation) [\#4906](https://github.com/cosmos/cosmos-sdk/issues/4906) Add simulation `Config` struct that wraps simulation flags
 * (simulation) [\#4935](https://github.com/cosmos/cosmos-sdk/issues/4935) Update simulation to reflect a proper `ABCI` application without bypassing `BaseApp` semantics
+* (simulation) [\#5378](https://github.com/cosmos/cosmos-sdk/pull/5378) Simulation tests refactor:
+  * Add `App` interface for general SDK-based app's methods.
+  * Refactor and cleanup simulation tests into util functions to simplify their implementation for other SDK apps.
 * (store) [\#4792](https://github.com/cosmos/cosmos-sdk/issues/4792) panic on non-registered store
 * (types) [\#4821](https://github.com/cosmos/cosmos-sdk/issues/4821) types/errors package added with support for stacktraces. It is meant as a more feature-rich replacement for sdk.Errors in the mid-term.
 * (store) [\#1947](https://github.com/cosmos/cosmos-sdk/issues/1947) Implement inter-block (persistent)

simapp/app.go

@@ -89,6 +89,9 @@ func MakeCodec() *codec.Codec {
 	return cdc
 }
 
+// Verify app interface at compile time
+var _ App = (*SimApp)(nil)
+
 // SimApp extends an ABCI application, but with most of its parameters exported.
 // They are exported for convenience in creating helper functions, as object
 // capabilities aren't needed for testing.
@@ -295,6 +298,9 @@ func NewSimApp(
 	return app
 }
 
+// Name returns the name of the App
+func (app *SimApp) Name() string { return app.BaseApp.Name() }
+
 // BeginBlocker application updates every begin block
 func (app *SimApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock {
 	return app.mm.BeginBlock(ctx, req)
@@ -366,6 +372,11 @@ func (app *SimApp) GetSubspace(moduleName string) params.Subspace {
 	return app.subspaces[moduleName]
 }
 
+// SimulationManager implements the SimulationApp interface
+func (app *SimApp) SimulationManager() *module.SimulationManager {
+	return app.sm
+}
+
 // GetMaccPerms returns a copy of the module account permissions
 func GetMaccPerms() map[string][]string {
 	dupMaccPerms := make(map[string][]string)

simapp/app_test.go

@@ -18,7 +18,7 @@ func TestSimAppExport(t *testing.T) {
 	app := NewSimApp(log.NewTMLogger(log.NewSyncWriter(os.Stdout)), db, nil, true, 0)
 
 	genesisState := NewDefaultGenesisState()
-	stateBytes, err := codec.MarshalJSONIndent(app.cdc, genesisState)
+	stateBytes, err := codec.MarshalJSONIndent(app.Codec(), genesisState)
 	require.NoError(t, err)
 
 	// Initialize the chain

simapp/config.go

@@ -0,0 +1,75 @@
+package simapp
+
+import (
+	"flag"
+
+	"github.com/cosmos/cosmos-sdk/x/simulation"
+)
+
+// List of available flags for the simulator
+var (
+	FlagGenesisFileValue        string
+	FlagParamsFileValue         string
+	FlagExportParamsPathValue   string
+	FlagExportParamsHeightValue int
+	FlagExportStatePathValue    string
+	FlagExportStatsPathValue    string
+	FlagSeedValue               int64
+	FlagInitialBlockHeightValue int
+	FlagNumBlocksValue          int
+	FlagBlockSizeValue          int
+	FlagLeanValue               bool
+	FlagCommitValue             bool
+	FlagOnOperationValue        bool // TODO: Remove in favor of binary search for invariant violation
+	FlagAllInvariantsValue      bool
+
+	FlagEnabledValue     bool
+	FlagVerboseValue     bool
+	FlagPeriodValue      uint
+	FlagGenesisTimeValue int64
+)
+
+// GetSimulatorFlags gets the values of all the available simulation flags
+func GetSimulatorFlags() {
+	// config fields
+	flag.StringVar(&FlagGenesisFileValue, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
+	flag.StringVar(&FlagParamsFileValue, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
+	flag.StringVar(&FlagExportParamsPathValue, "ExportParamsPath", "", "custom file path to save the exported params JSON")
+	flag.IntVar(&FlagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params")
+	flag.StringVar(&FlagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON")
+	flag.StringVar(&FlagExportStatsPathValue, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON")
+	flag.Int64Var(&FlagSeedValue, "Seed", 42, "simulation random seed")
+	flag.IntVar(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation")
+	flag.IntVar(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
+	flag.IntVar(&FlagBlockSizeValue, "BlockSize", 200, "operations per block")
+	flag.BoolVar(&FlagLeanValue, "Lean", false, "lean simulation log output")
+	flag.BoolVar(&FlagCommitValue, "Commit", false, "have the simulation commit")
+	flag.BoolVar(&FlagOnOperationValue, "SimulateEveryOperation", false, "run slow invariants every operation")
+	flag.BoolVar(&FlagAllInvariantsValue, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
+
+	// simulation flags
+	flag.BoolVar(&FlagEnabledValue, "Enabled", false, "enable the simulation")
+	flag.BoolVar(&FlagVerboseValue, "Verbose", false, "verbose log output")
+	flag.UintVar(&FlagPeriodValue, "Period", 0, "run slow invariants only once every period assertions")
+	flag.Int64Var(&FlagGenesisTimeValue, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time")
+}
+
+// NewConfigFromFlags creates a simulation from the retrieved values of the flags.
+func NewConfigFromFlags() simulation.Config {
+	return simulation.Config{
+		GenesisFile:        FlagGenesisFileValue,
+		ParamsFile:         FlagParamsFileValue,
+		ExportParamsPath:   FlagExportParamsPathValue,
+		ExportParamsHeight: FlagExportParamsHeightValue,
+		ExportStatePath:    FlagExportStatePathValue,
+		ExportStatsPath:    FlagExportStatsPathValue,
+		Seed:               FlagSeedValue,
+		InitialBlockHeight: FlagInitialBlockHeightValue,
+		NumBlocks:          FlagNumBlocksValue,
+		BlockSize:          FlagBlockSizeValue,
+		Lean:               FlagLeanValue,
+		Commit:             FlagCommitValue,
+		OnOperation:        FlagOnOperationValue,
+		AllInvariants:      FlagAllInvariantsValue,
+	}
+}

simapp/params.go → simapp/params/params.go

@@ -1,4 +1,4 @@
-package simapp
+package params
 
 // Simulation parameter constants
 const (

simapp/sim_bench_test.go

@@ -2,113 +2,89 @@ package simapp
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 
-	abci "github.com/tendermint/tendermint/abci/types"
-	"github.com/tendermint/tendermint/libs/log"
-	dbm "github.com/tendermint/tm-db"
-
-	"github.com/cosmos/cosmos-sdk/simapp/helpers"
-	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/simulation"
+	abci "github.com/tendermint/tendermint/abci/types"
 )
 
 // Profile with:
 // /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
 func BenchmarkFullAppSimulation(b *testing.B) {
-	logger := log.NewNopLogger()
-	config := NewConfigFromFlags()
-	config.ChainID = helpers.SimAppChainID
+	config, db, dir, logger, _, err := SetupSimulation("goleveldb-app-sim", "Simulation")
+	if err != nil {
+		b.Fatalf("simulation setup failed: %s", err.Error())
+	}
 
-	var db dbm.DB
-	dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
-	db, _ = sdk.NewLevelDB("Simulation", dir)
 	defer func() {
 		db.Close()
-		os.RemoveAll(dir)
+		err = os.RemoveAll(dir)
+		if err != nil {
+			b.Fatal(err)
+		}
 	}()
 
 	app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
 
-	// Run randomized simulation
-	// TODO: parameterize numbers, save for a later PR
+	// run randomized simulation
 	_, simParams, simErr := simulation.SimulateFromSeed(
-		b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
+		b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
 		SimulationOperations(app, app.Codec(), config),
 		app.ModuleAccountAddrs(), config,
 	)
 
-	// export state and params before the simulation error is checked
-	if config.ExportStatePath != "" {
-		if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
-	}
-
-	if config.ExportParamsPath != "" {
-		if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
+	// export state and simParams before the simulation error is checked
+	if err = CheckExportSimulation(app, config, simParams); err != nil {
+		fmt.Println(err)
+		b.Fatal(err)
 	}
 
 	if simErr != nil {
-		fmt.Println(simErr)
-		b.FailNow()
+		b.Fatal(simErr)
 	}
 
 	if config.Commit {
-		fmt.Println("\nGoLevelDB Stats")
-		fmt.Println(db.Stats()["leveldb.stats"])
-		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
+		PrintStats(db)
 	}
 }
 
 func BenchmarkInvariants(b *testing.B) {
-	logger := log.NewNopLogger()
+	config, db, dir, logger, _, err := SetupSimulation("leveldb-app-invariant-bench", "Simulation")
+	if err != nil {
+		b.Fatalf("simulation setup failed: %s", err.Error())
+	}
 
-	config := NewConfigFromFlags()
 	config.AllInvariants = false
-	config.ChainID = helpers.SimAppChainID
-
-	dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench")
-	db, _ := sdk.NewLevelDB("simulation", dir)
 
 	defer func() {
 		db.Close()
-		os.RemoveAll(dir)
+		err = os.RemoveAll(dir)
+		if err != nil {
+			b.Fatal(err)
+		}
 	}()
 
 	app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
 
-	// 2. Run parameterized simulation (w/o invariants)
+	// run randomized simulation
 	_, simParams, simErr := simulation.SimulateFromSeed(
-		b, ioutil.Discard, app.BaseApp, AppStateFn(app.Codec(), app.sm),
+		b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
 		SimulationOperations(app, app.Codec(), config),
 		app.ModuleAccountAddrs(), config,
 	)
 
-	// export state and params before the simulation error is checked
-	if config.ExportStatePath != "" {
-		if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
+	// export state and simParams before the simulation error is checked
+	if err = CheckExportSimulation(app, config, simParams); err != nil {
+		b.Fatal(err)
 	}
 
-	if config.ExportParamsPath != "" {
-		if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
-			fmt.Println(err)
-			b.Fail()
-		}
+	if simErr != nil {
+		b.Fatal(simErr)
 	}
 
-	if simErr != nil {
-		fmt.Println(simErr)
-		b.FailNow()
+	if config.Commit {
+		PrintStats(db)
 	}
 
 	ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
@@ -121,8 +97,10 @@ func BenchmarkInvariants(b *testing.B) {
 		cr := cr
 		b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
 			if res, stop := cr.Invar(ctx); stop {
-				fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res)
-				b.FailNow()
+				b.Fatalf(
+					"broken invariant at block %d of %d\n%s",
+					ctx.BlockHeight()-1, config.NumBlocks, res,
+				)
 			}
 		})
 	}