Add wrapper for load tests to run in different modes (#383)

* Add wrapper for loadtests

* Update run_tests.py

* Update run_tests.py

* return path

* close

* deltee

* test

* refvert

loadtest/config.json

@@ -25,9 +25,10 @@
         }
     },
     "constant": true,
-    "const_load_interval": 600,
+    "loadtest_interval": 600,
     "message_type": "bank,dex,staking,tokenfactory",
     "run_oracle": true,
+    "metrics_port": 9696,
     "contract_distribution": [
             {
               "contract_address":"sei14hj2tavq8fpesdwxxcu44rty3hh90vhujrvcmstl4zr3txmfvw9sh9m79m",

loadtest/main.go

@@ -56,20 +56,20 @@ func init() {
 func run(config Config) {
 	// Start metrics collector in another thread
 	metricsServer := MetricsServer{}
-	go metricsServer.StartMetricsClient()
+	go metricsServer.StartMetricsClient(config)
+	sleepDuration := time.Duration(config.LoadInterval) * time.Second
 
 	if config.Constant {
-		fmt.Printf("Running in constant mode with interval=%d\n", config.ConstLoadInterval)
-		sleepDuration := time.Duration(config.ConstLoadInterval) * time.Second
-
+		fmt.Printf("Running in constant mode with interval=%d\n", config.LoadInterval)
 		for {
 			runOnce(config)
 			fmt.Printf("Sleeping for %f seconds before next run...\n", sleepDuration.Seconds())
 			time.Sleep(sleepDuration)
-
 		}
 	} else {
 		runOnce(config)
+		fmt.Print("Sleeping for 60 seconds for metrics to be scraped...\n")
+		time.Sleep(time.Duration(60))
 	}
 }
 
@@ -368,6 +368,9 @@ func ReadConfig(path string) Config {
 
 func main() {
 	configFilePath := flag.String("config-file", GetDefaultConfigFilePath(), "Path to the config.json file to use for this run")
+	flag.Parse()
+
 	config := ReadConfig(*configFilePath)
+	fmt.Printf("Using config file: %s\n", *configFilePath)
 	run(config)
 }

loadtest/metrics.go

@@ -5,14 +5,18 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"strconv"
 	"time"
 
 	metrics "github.com/armon/go-metrics"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	"github.com/cosmos/cosmos-sdk/types/rest"
 )
 
-const defaultListenAddress = "0.0.0.0:9696"
+const (
+	defaultListenAddress = "0.0.0.0"
+	defaultMetricsPort   = 9696
+)
 
 type MetricsServer struct {
 	metrics *telemetry.Metrics
@@ -30,25 +34,34 @@ func (s *MetricsServer) metricsHandler(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write(gr.Metrics)
 }
 
-func (s *MetricsServer) StartMetricsClient() {
+func (s *MetricsServer) StartMetricsClient(config Config) {
 	m, err := telemetry.New(telemetry.Config{
 		ServiceName:             "loadtest-client",
 		Enabled:                 true,
 		EnableHostnameLabel:     true,
 		EnableServiceLabel:      true,
 		PrometheusRetentionTime: 600,
-		GlobalLabels:            [][]string{},
+		GlobalLabels: [][]string{
+			{"constant_mode", strconv.FormatBool(config.Constant)},
+		},
 	})
 	if err != nil {
 		panic(err)
 	}
 	s.metrics = m
 	http.HandleFunc("/healthz", s.healthzHandler)
 	http.HandleFunc("/metrics", s.metricsHandler)
-	log.Printf("Listening for metrics scrapes on %s", defaultListenAddress)
+
+	metricsPort := config.MetricsPort
+	if config.MetricsPort == 0 {
+		metricsPort = defaultMetricsPort
+	}
+
+	listenAddr := fmt.Sprintf("%s:%d", defaultListenAddress, metricsPort)
+	log.Printf("Listening for metrics scrapes on %s", listenAddr)
 
 	s.server = &http.Server{
-		Addr:              defaultListenAddress,
+		Addr:              listenAddr,
 		ReadHeaderTimeout: 3 * time.Second,
 	}
 	err = s.server.ListenAndServe()

loadtest/scripts/run_tests.py

@@ -0,0 +1,115 @@
+import argparse
+import json
+import os
+import subprocess
+import tempfile
+
+from dataclasses import dataclass
+
+
+# This strategy is used to simulate sudden spikes or bursts of load on the on the chain.
+# Burst load testing is done to ensure that the chain hub is able to handle such sudden
+# spikes in traffic.
+BURST = 'BURST'
+
+#  This is used to simulate a "steady-state" scenario where there is a constant load.
+STEADY = 'STEADY'
+
+
+@dataclass
+class LoadTestConfig:
+    config_file_path: str
+    loadtest_binary_file_path: str
+
+
+def write_to_temp_json_file(data):
+    with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', delete=False) as temp_file:
+        json.dump(data, temp_file, ensure_ascii=False)
+        return temp_file.name
+
+def create_burst_loadtest_config(base_config_json):
+    new_config = base_config_json.copy()
+    new_config["constant"] = True
+    new_config["metrics_port"] = 9697
+    new_config["txs_per_block"] = 2000
+    # Run every 20 mins
+    new_config["loadtest_interval"] = 1200
+    return new_config
+
+
+def create_steady_loadtest_config(base_config_json):
+    new_config = base_config_json.copy()
+    new_config["constant"] = True
+    new_config["metrics_port"] = 9696
+    new_config["txs_per_block"] = 100
+    # Run every min
+    new_config["loadtest_interval"] = 60
+    return new_config
+
+
+def read_config_json(config_json_file_path):
+    # Default path for running on EC2 instances
+    file_path = "/home/ubuntu/sei-chain/loadtest/config.json"
+
+    if config_json_file_path is not None:
+        file_path = config_json_file_path
+
+    with open(file_path, 'r', encoding="utf-8") as file:
+        return json.load(file)
+
+
+def run_go_loadtest_client(config_file_path, binary_path):
+    # Default path for running on EC2 instances
+    cmd = ["/home/ubuntu/sei-chain/build/loadtest", "-config-file", config_file_path]
+    if binary_path is not None:
+       cmd[0] = binary_path
+
+    print(f'Running {" ".join(cmd)}')
+    subprocess.run(cmd, check=True)
+
+def run_test(test_type, loadtest_config):
+    config = base_config_json = read_config_json(loadtest_config.config_file_path)
+    if test_type == BURST:
+        config = create_burst_loadtest_config(base_config_json)
+    elif test_type == STEADY:
+        config = create_steady_loadtest_config(base_config_json)
+
+    temp_file_path = write_to_temp_json_file(config)
+    try:
+        run_go_loadtest_client(temp_file_path, binary_path=loadtest_config.loadtest_binary_file_path)
+    finally:
+        os.remove(temp_file_path)
+
+def run():
+    parser = argparse.ArgumentParser(
+                        prog = 'Loadtest Client',
+                        description = 'Wrapper for the golang client to run loadtests with different configs')
+    parser.add_argument(
+        'type',
+        help='Type of loadtest to run (e.g constant, burst)',
+        type = lambda s : s.upper(),
+        choices=[BURST, STEADY],
+    )
+    parser.add_argument(
+        '--config-file',
+        help='Base config file to modify',
+        required=False,
+    )
+    parser.add_argument(
+        '--loadtest-binary',
+        help='binary of the loadtest client to run',
+        required=False,
+    )
+
+    args = parser.parse_args()
+    test_type = args.type
+    print(f'type={test_type} loadtests')
+
+    run_test(
+        test_type=test_type,
+        loadtest_config=LoadTestConfig(args.config_file, args.loadtest_binary)
+    )
+
+
+if __name__ == '__main__':
+    run()

loadtest/types.go

@@ -25,18 +25,19 @@ const (
 )
 
 type Config struct {
-	ChainID           string                `json:"chain_id"`
-	TxsPerBlock       uint64                `json:"txs_per_block"`
-	MsgsPerTx         uint64                `json:"msgs_per_tx"`
-	Rounds            uint64                `json:"rounds"`
-	MessageType       string                `json:"message_type"`
-	RunOracle         bool                  `json:"run_oracle"`
-	PriceDistr        NumericDistribution   `json:"price_distribution"`
-	QuantityDistr     NumericDistribution   `json:"quantity_distribution"`
-	MsgTypeDistr      MsgTypeDistribution   `json:"message_type_distribution"`
-	ContractDistr     ContractDistributions `json:"contract_distribution"`
-	Constant          bool                  `json:"constant"`
-	ConstLoadInterval int64                 `json:"const_load_interval"`
+	ChainID       string                `json:"chain_id"`
+	TxsPerBlock   uint64                `json:"txs_per_block"`
+	MsgsPerTx     uint64                `json:"msgs_per_tx"`
+	Rounds        uint64                `json:"rounds"`
+	MessageType   string                `json:"message_type"`
+	RunOracle     bool                  `json:"run_oracle"`
+	PriceDistr    NumericDistribution   `json:"price_distribution"`
+	QuantityDistr NumericDistribution   `json:"quantity_distribution"`
+	MsgTypeDistr  MsgTypeDistribution   `json:"message_type_distribution"`
+	ContractDistr ContractDistributions `json:"contract_distribution"`
+	MetricsPort   uint64                `json:"metrics_port"`
+	Constant      bool                  `json:"constant"`
+	LoadInterval  int64                 `json:"loadtest_interval"`
 }
 
 type EncodingConfig struct {

x/oracle/simulation/operations.go

@@ -65,7 +65,7 @@ func WeightedOperations(
 }
 
 // SimulateMsgAggregateExchangeRateVote generates a MsgAggregateExchangeRateVote with random values.
-// nolint: funlen
+//nolint: funlen
 func SimulateMsgAggregateExchangeRateVote(ak types.AccountKeeper, bk types.BankKeeper, k keeper.Keeper) simtypes.Operation {
 	return func(
 		r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string,
@@ -122,7 +122,7 @@ func SimulateMsgAggregateExchangeRateVote(ak types.AccountKeeper, bk types.BankK
 }
 
 // SimulateMsgDelegateFeedConsent generates a MsgDelegateFeedConsent with random values.
-// nolint: funlen
+//nolint: funlen
 func SimulateMsgDelegateFeedConsent(ak types.AccountKeeper, bk types.BankKeeper, k keeper.Keeper) simtypes.Operation {
 	return func(
 		r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string,