Merge PR cosmos#4784: JSON representation of event stats

.clog.yaml

@@ -11,3 +11,4 @@ tags:
   - rest
   - cli
   - modules
+  - simulation

.pending/improvements/simulation/_4670-JSON-sim-stats

@@ -0,0 +1,2 @@
+#4670 Update simulation statistics to JSON format
+  - Support exporting the simulation stats to a given JSON file

simapp/sim_test.go

@@ -36,6 +36,7 @@ func init() {
 	flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON")
 	flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params")
 	flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON")
+	flag.StringVar(&exportStatsPath, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON")
 	flag.Int64Var(&seed, "Seed", 42, "simulation random seed")
 	flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks")
 	flag.IntVar(&blockSize, "BlockSize", 200, "operations per block")
@@ -52,15 +53,15 @@ func init() {
 // helper function for populating input for SimulateFromSeed
 func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) (
 	testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
-	simulation.WeightedOperations, sdk.Invariants, int, int, int,
+	simulation.WeightedOperations, sdk.Invariants, int, int, int, string,
 	bool, bool, bool, bool, bool, map[string]bool) {
 
 	exportParams := exportParamsPath != ""
 
 	return tb, w, app.BaseApp, appStateFn, seed,
 		testAndRunTxs(app), invariants(app),
 		numBlocks, exportParamsHeight, blockSize,
-		exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
+		exportStatsPath, exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
 }
 
 func appStateFn(
@@ -412,7 +413,7 @@ func BenchmarkFullAppSimulation(b *testing.B) {
 	}
 
 	if commit {
-		fmt.Println("GoLevelDB Stats")
+		fmt.Println("\nGoLevelDB Stats")
 		fmt.Println(db.Stats()["leveldb.stats"])
 		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
 	}
@@ -471,7 +472,7 @@ func TestFullAppSimulation(t *testing.T) {
 	if commit {
 		// for memdb:
 		// fmt.Println("Database Size", db.Stats()["database.size"])
-		fmt.Println("GoLevelDB Stats")
+		fmt.Println("\nGoLevelDB Stats")
 		fmt.Println(db.Stats()["leveldb.stats"])
 		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
 	}
@@ -528,7 +529,7 @@ func TestAppImportExport(t *testing.T) {
 	if commit {
 		// for memdb:
 		// fmt.Println("Database Size", db.Stats()["database.size"])
-		fmt.Println("GoLevelDB Stats")
+		fmt.Println("\nGoLevelDB Stats")
 		fmt.Println(db.Stats()["leveldb.stats"])
 		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
 	}
@@ -644,7 +645,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 	if commit {
 		// for memdb:
 		// fmt.Println("Database Size", db.Stats()["database.size"])
-		fmt.Println("GoLevelDB Stats")
+		fmt.Println("\nGoLevelDB Stats")
 		fmt.Println(db.Stats()["leveldb.stats"])
 		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
 	}
@@ -705,7 +706,7 @@ func TestAppStateDeterminism(t *testing.T) {
 			simulation.SimulateFromSeed(
 				t, os.Stdout, app.BaseApp, appStateFn, seed,
 				testAndRunTxs(app), []sdk.Invariant{},
-				50, 100, 0,
+				50, 100, 0, "",
 				false, true, false, false, false, app.ModuleAccountAddrs(),
 			)
 			appHash := app.LastCommitID().Hash
@@ -734,7 +735,7 @@ func BenchmarkInvariants(b *testing.B) {
 	_, params, simErr := simulation.SimulateFromSeed(
 		b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
 		[]sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize,
-		exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
+		exportStatsPath, exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
 	)
 
 	// export state and params before the simulation error is checked

simapp/utils.go

@@ -43,6 +43,7 @@ var (
 	exportParamsPath   string
 	exportParamsHeight int
 	exportStatePath    string
+	exportStatsPath    string
 	seed               int64
 	numBlocks          int
 	blockSize          int

x/distribution/simulation/msgs.go

@@ -13,7 +13,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/x/simulation"
 )
 
-// SimulateMsgSetWithdrawAddress
+// SimulateMsgSetWithdrawAddress generates a MsgSetWithdrawAddress with random values.
 func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
 	handler := distribution.NewHandler(k)
 	return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
@@ -24,7 +24,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper)
 		msg := distribution.NewMsgSetWithdrawAddress(accountOrigin.Address, accountDestination.Address)
 
 		if msg.ValidateBasic() != nil {
-			return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+			return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
 		}
 
 		ctx, write := ctx.CacheContext()
@@ -38,7 +38,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper)
 	}
 }
 
-// SimulateMsgWithdrawDelegatorReward
+// SimulateMsgWithdrawDelegatorReward generates a MsgWithdrawDelegatorReward with random values.
 func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
 	handler := distribution.NewHandler(k)
 	return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
@@ -49,7 +49,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee
 		msg := distribution.NewMsgWithdrawDelegatorReward(delegatorAccount.Address, sdk.ValAddress(validatorAccount.Address))
 
 		if msg.ValidateBasic() != nil {
-			return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+			return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
 		}
 
 		ctx, write := ctx.CacheContext()
@@ -63,7 +63,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee
 	}
 }
 
-// SimulateMsgWithdrawValidatorCommission
+// SimulateMsgWithdrawValidatorCommission generates a MsgWithdrawValidatorCommission with random values.
 func SimulateMsgWithdrawValidatorCommission(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
 	handler := distribution.NewHandler(k)
 	return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
@@ -73,7 +73,7 @@ func SimulateMsgWithdrawValidatorCommission(m auth.AccountKeeper, k distribution
 		msg := distribution.NewMsgWithdrawValidatorCommission(sdk.ValAddress(account.Address))
 
 		if msg.ValidateBasic() != nil {
-			return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+			return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
 		}
 
 		ctx, write := ctx.CacheContext()

x/genutil/types/genesis_state_test.go

@@ -3,11 +3,12 @@ package types
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
-	"github.com/stretchr/testify/require"
-	"github.com/tendermint/tendermint/crypto/ed25519"
 )
 
 var (

x/gov/simulation/msgs.go

@@ -54,7 +54,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, contentSim Con
 		content := contentSim(r, app, ctx, accs)
 		msg, err := simulationCreateMsgSubmitProposal(r, content, sender)
 		if err != nil {
-			return simulation.NoOpMsg(), nil, err
+			return simulation.NoOpMsg(gov.ModuleName), nil, err
 		}
 
 		ok := simulateHandleMsgSubmitProposal(msg, handler, ctx)
@@ -66,7 +66,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, contentSim Con
 
 		proposalID, err := k.GetProposalID(ctx)
 		if err != nil {
-			return simulation.NoOpMsg(), nil, err
+			return simulation.NoOpMsg(gov.ModuleName), nil, err
 		}
 
 		proposalID = uint64(math.Max(float64(proposalID)-1, 0))
@@ -122,20 +122,20 @@ func simulationCreateMsgSubmitProposal(r *rand.Rand, c gov.Content, s simulation
 	return
 }
 
-// SimulateMsgDeposit
+// SimulateMsgDeposit generates a MsgDeposit with random values.
 func SimulateMsgDeposit(k gov.Keeper) simulation.Operation {
 	return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account) (
 		opMsg simulation.OperationMsg, fOps []simulation.FutureOperation, err error) {
 
 		acc := simulation.RandomAcc(r, accs)
 		proposalID, ok := randomProposalID(r, k, ctx)
 		if !ok {
-			return simulation.NoOpMsg(), nil, nil
+			return simulation.NoOpMsg(gov.ModuleName), nil, nil
 		}
 		deposit := randomDeposit(r)
 		msg := gov.NewMsgDeposit(acc.Address, proposalID, deposit)
 		if msg.ValidateBasic() != nil {
-			return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+			return simulation.NoOpMsg(gov.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
 		}
 		ctx, write := ctx.CacheContext()
 		ok = gov.NewHandler(k)(ctx, msg).IsOK()
@@ -148,8 +148,7 @@ func SimulateMsgDeposit(k gov.Keeper) simulation.Operation {
 	}
 }
 
-// SimulateMsgVote
-// nolint: unparam
+// SimulateMsgVote generates a MsgVote with random values.
 func SimulateMsgVote(k gov.Keeper) simulation.Operation {
 	return operationSimulateMsgVote(k, simulation.Account{}, 0)
 }
@@ -167,14 +166,14 @@ func operationSimulateMsgVote(k gov.Keeper, acc simulation.Account, proposalID u
 			var ok bool
 			proposalID, ok = randomProposalID(r, k, ctx)
 			if !ok {
-				return simulation.NoOpMsg(), nil, nil
+				return simulation.NoOpMsg(gov.ModuleName), nil, nil
 			}
 		}
 		option := randomVotingOption(r)
 
 		msg := gov.NewMsgVote(acc.Address, proposalID, option)
 		if msg.ValidateBasic() != nil {
-			return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+			return simulation.NoOpMsg(gov.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
 		}
 
 		ctx, write := ctx.CacheContext()

x/simulation/event_stats.go

@@ -1,33 +1,55 @@
 package simulation
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
-	"sort"
+	"io/ioutil"
 )
 
-type eventStats map[string]uint
+// EventStats defines an object that keeps a tally of each event that has occurred
+// during a simulation.
+type EventStats map[string]map[string]map[string]int
 
-func newEventStats() eventStats {
-	events := make(map[string]uint)
-	return events
+// NewEventStats creates a new empty EventStats object
+func NewEventStats() EventStats {
+	return make(EventStats)
 }
 
-func (es eventStats) tally(eventDesc string) {
-	es[eventDesc]++
+// Tally increases the count of a simulation event.
+func (es EventStats) Tally(route, op, evResult string) {
+	_, ok := es[route]
+	if !ok {
+		es[route] = make(map[string]map[string]int)
+	}
+
+	_, ok = es[route][op]
+	if !ok {
+		es[route][op] = make(map[string]int)
+	}
+
+	es[route][op][evResult]++
 }
 
-// Pretty-print events as a table
-func (es eventStats) Print(w io.Writer) {
-	var keys []string
-	for key := range es {
-		keys = append(keys, key)
+// Print the event stats in JSON format.
+func (es EventStats) Print(w io.Writer) {
+	obj, err := json.MarshalIndent(es, "", " ")
+	if err != nil {
+		panic(err)
 	}
 
-	sort.Strings(keys)
-	fmt.Fprintf(w, "Event statistics: \n")
+	fmt.Fprintf(w, string(obj))
+}
+
+// ExportJSON saves the event stats as a JSON file on a given path
+func (es EventStats) ExportJSON(path string) {
+	bz, err := json.MarshalIndent(es, "", " ")
+	if err != nil {
+		panic(err)
+	}
 
-	for _, key := range keys {
-		fmt.Fprintf(w, "  % 60s => %d\n", key, es[key])
+	err = ioutil.WriteFile(path, bz, 0644)
+	if err != nil {
+		panic(err)
 	}
 }

x/simulation/mock_tendermint.go

@@ -79,7 +79,7 @@ func (vals mockValidators) randomProposer(r *rand.Rand) cmn.HexBytes {
 // nolint: unparam
 func updateValidators(tb testing.TB, r *rand.Rand, params Params,
 	current map[string]mockValidator, updates []abci.ValidatorUpdate,
-	event func(string)) map[string]mockValidator {
+	event func(route, op, evResult string)) map[string]mockValidator {
 
 	for _, update := range updates {
 		str := fmt.Sprintf("%v", update.PubKey)
@@ -88,21 +88,21 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params,
 			if _, ok := current[str]; !ok {
 				tb.Fatalf("tried to delete a nonexistent validator")
 			}
-			event("endblock/validatorupdates/kicked")
+			event("end_block", "validator_updates", "kicked")
 			delete(current, str)
 
 		} else if mVal, ok := current[str]; ok {
 			// validator already exists
 			mVal.val = update
-			event("endblock/validatorupdates/updated")
+			event("end_block", "validator_updates", "updated")
 
 		} else {
 			// Set this new validator
 			current[str] = mockValidator{
 				update,
 				GetMemberOfInitialState(r, params.InitialLivenessWeightings),
 			}
-			event("endblock/validatorupdates/added")
+			event("end_block", "validator_updates", "added")
 		}
 	}
 
@@ -114,7 +114,7 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params,
 func RandomRequestBeginBlock(r *rand.Rand, params Params,
 	validators mockValidators, pastTimes []time.Time,
 	pastVoteInfos [][]abci.VoteInfo,
-	event func(string), header abci.Header) abci.RequestBeginBlock {
+	event func(route, op, evResult string), header abci.Header) abci.RequestBeginBlock {
 
 	if len(validators) == 0 {
 		return abci.RequestBeginBlock{
@@ -139,9 +139,9 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params,
 		}
 
 		if signed {
-			event("beginblock/signing/signed")
+			event("begin_block", "signing", "signed")
 		} else {
-			event("beginblock/signing/missed")
+			event("begin_block", "signing", "missed")
 		}
 
 		pubkey, err := tmtypes.PB2TM.PubKey(mVal.val.PubKey)
@@ -197,7 +197,7 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params,
 				TotalVotingPower: totalVotingPower,
 			},
 		)
-		event("beginblock/evidence")
+		event("begin_block", "evidence", "ok")
 	}
 
 	return abci.RequestBeginBlock{