test the fixed rollback cmd

- check the rollback cmd works in integration tests

go.mod

@@ -186,6 +186,7 @@ require (
 
 replace (
 	github.com/99designs/keyring => github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76
+	github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.2-0.20220830065254-2ec56aa83350
 
 	// Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409

go.sum

@@ -244,8 +244,8 @@ github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44=
 github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/TvgUaxbU=
 github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0=
 github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw=
-github.com/cosmos/cosmos-sdk v0.46.1 h1:7vUZXMyrmEb4xtBYpz1TobtrcnpgiZTi+tVjc0XWB4o=
-github.com/cosmos/cosmos-sdk v0.46.1/go.mod h1:2+o8Qw8qnE02V+lQVZDJFQ8tri/hsiA5GmWaPERqVa0=
+github.com/cosmos/cosmos-sdk v0.46.2-0.20220830065254-2ec56aa83350 h1:pp5Z8u1B/MmyQZEQfvHMrunLhwRU1Y0UtehtLuoPfbc=
+github.com/cosmos/cosmos-sdk v0.46.2-0.20220830065254-2ec56aa83350/go.mod h1:sCTHeC0fzLy67Jr+XyVhCYUiDHT3IHNtiWtR4WTSlbg=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY=
 github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=

gomod2nix.toml

@@ -90,8 +90,9 @@ schema = 3
     version = "v1.0.0-alpha7"
     hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.46.1"
-    hash = "sha256-AZm3KzGee03chXTi6S/m0AZoGRkx4XLS6Rh2sBAWc30="
+    version = "v0.46.2-0.20220830065254-2ec56aa83350"
+    hash = "sha256-lAD91Jcm5twLF419LFosG5K6UV4G5cgsnqdTwekWP64="
+    replaced = "github.com/cosmos/cosmos-sdk"
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"
     hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="

server/util.go

@@ -50,7 +50,7 @@ func AddCommands(
 		tendermintCmd,
 		sdkserver.ExportCmd(appExport, defaultNodeHome),
 		version.NewVersionCommand(),
-		sdkserver.NewRollbackCmd(defaultNodeHome),
+		sdkserver.NewRollbackCmd(appCreator, defaultNodeHome),
 
 		// custom tx indexer command
 		NewIndexTxCmd(),

tests/integration_tests/configs/broken-ethermintd.nix

@@ -0,0 +1,8 @@
+{ pkgs ? import ../../nix { } }:
+let ethermintd = (pkgs.callPackage ../../. { });
+in
+ethermintd.overrideAttrs (oldAttrs: {
+  patches = oldAttrs.patches or [ ] ++ [
+    ./broken-ethermintd.patch
+  ];
+})

tests/integration_tests/configs/broken-ethermintd.patch

@@ -0,0 +1,15 @@
+diff --git a/app/app.go b/app/app.go
+index 158bf7a3..a3b5718c 100644
+--- a/app/app.go
++++ b/app/app.go
+@@ -681,6 +681,10 @@ func (app *EthermintApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBloc
+
+ // EndBlocker updates every end block
+ func (app *EthermintApp) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock {
++	if ctx.BlockHeight()%10 == 0 {
++		store := ctx.KVStore(app.keys["evm"])
++		store.Set([]byte("hello"), []byte("world"))
++	}
+ 	return app.mm.EndBlock(ctx, req)
+ }
+

tests/integration_tests/configs/rollback-test.jsonnet

@@ -0,0 +1,18 @@
+local config = import 'default.jsonnet';
+
+config {
+  'cronos_777-1'+: {
+    validators: super.validators + [{
+      name: 'fullnode',
+    }],
+    genesis+: {
+      app_state+: {
+        feemarket: {
+          params: {
+            no_base_fee: true,
+          },
+        },
+      },
+    },
+  },
+}

tests/integration_tests/network.py

@@ -96,7 +96,9 @@ def setup_geth(path, base_port):
             proc.wait()
 
 
-def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary=None):
+def setup_custom_ethermint(
+    path, base_port, config, post_init=None, chain_binary=None, wait_port=True
+):
     cmd = [
         "pystarport",
         "init",
@@ -119,8 +121,9 @@ def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary
         preexec_fn=os.setsid,
     )
     try:
-        wait_for_port(ports.evmrpc_port(base_port))
-        wait_for_port(ports.evmrpc_ws_port(base_port))
+        if wait_port:
+            wait_for_port(ports.evmrpc_port(base_port))
+            wait_for_port(ports.evmrpc_ws_port(base_port))
         yield Ethermint(path / "ethermint_9000-1")
     finally:
         os.killpg(os.getpgid(proc.pid), signal.SIGTERM)

tests/integration_tests/test_rollback.py

@@ -0,0 +1,98 @@
+import configparser
+import subprocess
+from pathlib import Path
+
+import pytest
+from pystarport import ports
+from pystarport.cluster import SUPERVISOR_CONFIG_FILE
+
+from .network import setup_custom_ethermint
+from .utils import supervisorctl, wait_for_block, wait_for_port
+
+
+def update_node2_cmd(path, cmd, i):
+    ini_path = path / SUPERVISOR_CONFIG_FILE
+    ini = configparser.RawConfigParser()
+    ini.read(ini_path)
+    for section in ini.sections():
+        if section == f"program:ethermint_9000-1-node{i}":
+            ini[section].update(
+                {
+                    "command": f"{cmd} start --home %(here)s/node{i}",
+                    "autorestart": "false",  # don't restart when stopped
+                }
+            )
+    with ini_path.open("w") as fp:
+        ini.write(fp)
+
+
+def post_init(broken_binary):
+    def inner(path, base_port, config):
+        chain_id = "ethermint_9000-1"
+        update_node2_cmd(path / chain_id, broken_binary, 2)
+
+    return inner
+
+
+@pytest.fixture(scope="module")
+def custom_ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("rollback")
+
+    cmd = [
+        "nix-build",
+        "--no-out-link",
+        Path(__file__).parent / "configs/broken-ethermintd.nix",
+    ]
+    print(*cmd)
+    broken_binary = (
+        Path(subprocess.check_output(cmd).strip().decode()) / "bin/ethermintd"
+    )
+    print(broken_binary)
+
+    # init with genesis binary
+    yield from setup_custom_ethermint(
+        path,
+        26300,
+        Path(__file__).parent / "configs/rollback-test-network.jsonnet",
+        post_init=post_init(broken_binary),
+        wait_port=False,
+    )
+
+
+def test_rollback(custom_ethermint):
+    """
+    test using rollback command to fix app-hash mismatch situation.
+    - the broken node will sync up to block 10 then crash.
+    - use rollback command to rollback the db.
+    - switch to correct binary should make the node syncing again.
+    """
+    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+
+    print("wait for node2 to sync the first 10 blocks")
+    cli2 = custom_ethermint.cosmos_cli(2)
+    wait_for_block(cli2, 10)
+
+    print("wait for a few more blocks on the healthy nodes")
+    cli = custom_ethermint.cosmos_cli(0)
+    wait_for_block(cli, 13)
+
+    # (app hash mismatch happens after the 10th block, detected in the 11th block)
+    print("check node2 get stuck at block 10")
+    assert cli2.block_height() == 10
+
+    print("stop node2")
+    supervisorctl(
+        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2"
+    )
+
+    print("do rollback on node2")
+    cli2.rollback()
+
+    print("switch to normal binary")
+    update_node2_cmd(custom_ethermint.base_dir, "ethermintd", 2)
+    supervisorctl(custom_ethermint.base_dir / "../tasks.ini", "update")
+    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+
+    print("check node2 sync again")
+    cli2 = custom_ethermint.cosmos_cli(2)
+    wait_for_block(cli2, 15)