Problem: integration tests CI spend a lot of time (#1271)

* Problem: integration tests CI spend a lot of time Solution: - run in parallel using markers and job matrix * fix matrix * flock is not nesserary * fix marker * break up ibc test cases * separate gov test cases * split out some slow test cases * split more ibc test * Update .github/workflows/test.yml Signed-off-by: yihuang <huang@crypto.com> * fix pytest lint * more balance --------- Signed-off-by: yihuang <huang@crypto.com>
crypto-org-chain · Dec 20, 2023 · b47e5c1 · b47e5c1
1 parent 12aa3ad
commit b47e5c1
Show file tree

Hide file tree

Showing 20 changed files with 70 additions and 3 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -17,6 +17,11 @@ jobs:
   integration_tests:
     runs-on: ubuntu-latest
     timeout-minutes: 240
+    strategy:
+      matrix:
+        tests: [unmarked, ibc, ibc_rly_evm, ibc_rly_gas, ibc_timeout, ibc_update_client, ica, gov, upgrade, slow]
+    env:
+      TESTS_TO_RUN: ${{ matrix.tests }}
     steps:
       - uses: actions/checkout@v3
         with:

diff --git a/Makefile b/Makefile
@@ -219,6 +219,12 @@ test-sim-profile:
 ###                                Integration Test                         ###
 ###############################################################################
 
+# possible values:
+# - all: run all integration tests
+# - unmarked: run integration tests that are not marked
+# - marker1,marker2: markers separated by comma, run integration tests that are marked with any of the markers
+TESTS_TO_RUN ?= all
+
 run-integration-tests:
 	@make gen-bindings-contracts
 	@nix-shell ./integration_tests/shell.nix --run ./scripts/run-integration-tests

diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
@@ -13,6 +13,20 @@
 def pytest_configure(config):
     config.addinivalue_line("markers", "slow: marks tests as slow")
     config.addinivalue_line("markers", "gravity: gravity bridge test cases")
+    config.addinivalue_line("markers", "ica: marks ica tests")
+    config.addinivalue_line("markers", "upgrade: marks upgrade tests")
+    config.addinivalue_line("markers", "ibc: marks default ibc tests")
+    config.addinivalue_line("markers", "ibc_rly_evm: marks ibc_rly_evm tests")
+    config.addinivalue_line("markers", "ibc_rly_gas: marks ibc relayer gas tests")
+    config.addinivalue_line("markers", "ibc_timeout: marks ibc timeout tests")
+    config.addinivalue_line("markers", "ibc_update_client: marks ibc updateclient test")
+    config.addinivalue_line("markers", "gov: marks gov related tests")
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("unmarked")
 
 
 @pytest.fixture(scope="session")

diff --git a/integration_tests/test_basic.py b/integration_tests/test_basic.py
@@ -857,10 +857,12 @@ def test_replay_protection(cronos):
         w3.eth.send_raw_transaction(HexBytes(raw))
 
 
+@pytest.mark.gov
 def test_submit_any_proposal(cronos, tmp_path):
     submit_any_proposal(cronos, tmp_path)
 
 
+@pytest.mark.gov
 def test_submit_send_enabled(cronos, tmp_path):
     # check bank send enable
     cli = cronos.cosmos_cli()

diff --git a/integration_tests/test_broadcast.py b/integration_tests/test_broadcast.py
@@ -5,6 +5,8 @@
 from .network import setup_custom_cronos
 from .utils import submit_any_proposal
 
+pytestmark = pytest.mark.gov
+
 
 @pytest.fixture(scope="module")
 def custom_cronos(tmp_path_factory):

diff --git a/integration_tests/test_exported_genesis.py b/integration_tests/test_exported_genesis.py
@@ -6,6 +6,8 @@
 from .network import setup_custom_cronos
 from .utils import ADDRS, CONTRACTS
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def custom_cronos(tmp_path_factory):

diff --git a/integration_tests/test_gov_update_params.py b/integration_tests/test_gov_update_params.py
@@ -1,7 +1,11 @@
 import json
 
+import pytest
+
 from .utils import approve_proposal
 
+pytestmark = pytest.mark.gov
+
 
 def test_gov_update_params(cronos, tmp_path):
     cli = cronos.cosmos_cli()

diff --git a/integration_tests/test_ibc.py b/integration_tests/test_ibc.py
@@ -12,6 +12,8 @@
 )
 from .utils import ADDRS, CONTRACTS, deploy_contract, send_transaction, wait_for_fn
 
+pytestmark = pytest.mark.ibc
+
 
 @pytest.fixture(scope="module", params=[True, False])
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_ibc_rly.py b/integration_tests/test_ibc_rly.py
@@ -28,6 +28,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.ibc_rly_evm
+
 CONTRACT = "0x0000000000000000000000000000000000000065"
 contract_info = json.loads(CONTRACT_ABIS["IRelayerModule"].read_text())
 method_map = get_method_map(contract_info)

diff --git a/integration_tests/test_ibc_rly_gas.py b/integration_tests/test_ibc_rly_gas.py
@@ -4,6 +4,8 @@
 from .ibc_utils import log_gas_records, prepare_network, rly_transfer
 from .utils import wait_for_new_blocks
 
+pytestmark = pytest.mark.ibc_rly_gas
+
 
 @pytest.fixture(scope="module", params=["ibc_rly", "ibc_rly_evm"])
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_ibc_timeout.py b/integration_tests/test_ibc_timeout.py
@@ -9,6 +9,8 @@
 )
 from .utils import ADDRS, eth_to_bech32, wait_for_fn
 
+pytestmark = pytest.mark.ibc_timeout
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_ibc_update_client.py b/integration_tests/test_ibc_update_client.py
@@ -6,6 +6,8 @@
 from .ibc_utils import prepare_network
 from .utils import approve_proposal, wait_for_fn
 
+pytestmark = pytest.mark.ibc_update_client
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_ica.py b/integration_tests/test_ica.py
@@ -17,6 +17,8 @@
 )
 from .utils import CONTRACTS, wait_for_fn
 
+pytestmark = pytest.mark.ica
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_ica_precompile.py b/integration_tests/test_ica_precompile.py
@@ -27,6 +27,8 @@
     wait_for_fn,
 )
 
+pytestmark = pytest.mark.ica
+
 CONTRACT = "0x0000000000000000000000000000000000000066"
 connid = "connection-0"
 no_timeout = 300000000000

diff --git a/integration_tests/test_mempool.py b/integration_tests/test_mempool.py
@@ -14,6 +14,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def cronos_mempool(tmp_path_factory):

diff --git a/integration_tests/test_pruned_node.py b/integration_tests/test_pruned_node.py
@@ -16,6 +16,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def cronos(request, tmp_path_factory):

diff --git a/integration_tests/test_replay_block.py b/integration_tests/test_replay_block.py
@@ -16,6 +16,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def custom_cronos(tmp_path_factory):
@@ -25,7 +27,7 @@ def custom_cronos(tmp_path_factory):
     )
 
 
-def test_replay_block(custom_cronos):
+def test_block_overflow(custom_cronos):
     w3: web3.Web3 = custom_cronos.w3
     contract = deploy_contract(
         w3,

diff --git a/integration_tests/test_rollback.py b/integration_tests/test_rollback.py
@@ -9,6 +9,8 @@
 from .network import setup_custom_cronos
 from .utils import supervisorctl, wait_for_block, wait_for_port
 
+pytestmark = pytest.mark.slow
+
 
 def update_node2_cmd(path, cmd, i):
     ini_path = path / SUPERVISOR_CONFIG_FILE

diff --git a/integration_tests/test_upgrade.py b/integration_tests/test_upgrade.py
@@ -24,6 +24,8 @@
     wait_for_port,
 )
 
+pytestmark = pytest.mark.upgrade
+
 
 @pytest.fixture(scope="module")
 def testnet(tmp_path_factory):

diff --git a/scripts/run-integration-tests b/scripts/run-integration-tests
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 cd "$(dirname "$0")"
 
@@ -10,4 +10,12 @@ cd ../integration_tests/contracts
 HUSKY_SKIP_INSTALL=1 npm install
 npm run typechain
 cd ..
-nix-shell --run "pytest -vv -s"
+TESTS_TO_RUN="${TESTS_TO_RUN:-all}"
+if [[ "$TESTS_TO_RUN" == "all" ]]; then
+  echo "run all tests"
+  cmd="pytest -vv -s"
+else
+  echo "run tests matching $TESTS_TO_RUN"
+  cmd="pytest -vv -s -m '$TESTS_TO_RUN'"
+fi
+nix-shell --run "$cmd"