diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e3d9262b05..f865bc9f0a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: cache: true - name: Set Test Matrix id: set-test-matrix - run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine fvm ledger module network/test network/p2p utils + run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine/access engine/collection engine/common engine/consensus engine/execution/ingestion:buildjet-8vcpu-ubuntu-2204 engine/execution/computation engine/execution engine/verification engine:buildjet-4vcpu-ubuntu-2204 fvm ledger module/dkg module:buildjet-4vcpu-ubuntu-2204 network/alsp network/test/cohort1:buildjet-16vcpu-ubuntu-2204 network/test/cohort2:buildjet-4vcpu-ubuntu-2204 network/p2p/connection network/p2p/p2pnode:buildjet-4vcpu-ubuntu-2204 network/p2p/scoring network/p2p network state storage utils unit-test: name: Unit Tests (${{ matrix.targets.name }}) @@ -108,8 +108,8 @@ jobs: fail-fast: false matrix: targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}} - # need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 - runs-on: ubuntu-20.04 + ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 + runs-on: ${{ matrix.targets.runner }} steps: - name: Checkout repo uses: actions/checkout@v3 @@ -123,8 +123,8 @@ jobs: - name: Run tests (${{ matrix.targets.name }}) uses: nick-fields/retry@v2 with: - timeout_minutes: 25 - max_attempts: 3 + timeout_minutes: 35 + max_attempts: 5 command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test # TODO(rbtz): re-enable when we fix exisiting races. @@ -148,17 +148,20 @@ jobs: make2: unittest retries: 1 race: 1 + runner: ubuntu-latest - name: insecure make1: install-tools make2: test - retries: 3 + retries: 5 race: 0 + runner: buildjet-4vcpu-ubuntu-2204 - name: integration make1: install-tools make2: test - retries: 3 + retries: 5 race: 0 - runs-on: ubuntu-latest + runner: buildjet-4vcpu-ubuntu-2204 + runs-on: ${{ matrix.runner }} steps: - name: Checkout repo uses: actions/checkout@v3 @@ -174,7 +177,7 @@ jobs: RACE_DETECTOR: ${{ matrix.race }} uses: nick-fields/retry@v2 with: - timeout_minutes: 25 + timeout_minutes: 35 max_attempts: ${{ matrix.retries }} # run `make2` target inside each module's root command: VERBOSE=1 make -C ${{ matrix.name }} ${{ matrix.make2 }} @@ -185,27 +188,101 @@ jobs: flags: unittests name: codecov-umbrella + docker-build: + name: Docker Build + runs-on: buildjet-16vcpu-ubuntu-2204 + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + # all tags are needed for integration tests + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + - name: Build relic + run: make crypto_setup_gopath + - name: Docker build + run: make docker-build-flow docker-build-flow-corrupt + - name: Save Docker images + run: | + docker save \ + gcr.io/flow-container-registry/access:latest \ + gcr.io/flow-container-registry/collection:latest \ + gcr.io/flow-container-registry/consensus:latest \ + gcr.io/flow-container-registry/execution:latest \ + gcr.io/flow-container-registry/ghost:latest \ + gcr.io/flow-container-registry/observer:latest \ + gcr.io/flow-container-registry/verification:latest \ + gcr.io/flow-container-registry/access-corrupted:latest \ + gcr.io/flow-container-registry/execution-corrupted:latest \ + gcr.io/flow-container-registry/verification-corrupted:latest > flow-docker-images.tar + - name: Cache Docker images + uses: actions/cache@v3 + with: + path: flow-docker-images.tar + # use the workflow run id as part of the cache key to ensure these docker images will only be used for a single workflow run + key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} + integration-test: name: Integration Tests + needs: docker-build strategy: fail-fast: false matrix: - make: - - make -C integration access-tests - - make -C integration bft-framework-tests - - make -C integration bft-protocol-tests - - make -C integration bft-gossipsub-tests - - make -C integration collection-tests - - make -C integration consensus-tests - - make -C integration epochs-cohort1-tests - - make -C integration epochs-cohort2-tests - - make -C integration execution-tests - - make -C integration ghost-tests - - make -C integration mvp-tests - - make -C integration network-tests - - make -C integration verification-tests - - make -C integration upgrades-tests - runs-on: ubuntu-latest + include: + - name: Access Cohort1 Integration Tests + make: make -C integration access-cohort1-tests + runner: buildjet-4vcpu-ubuntu-2204 + - name: Access Cohort2 Integration Tests + make: make -C integration access-cohort2-tests + runner: ubuntu-latest + - name: Access Cohort3 Integration Tests + make: make -C integration access-cohort3-tests + runner: ubuntu-latest + # test suite has single test which is flaky and needs to be fixed - reminder here to put it back when it's fixed +# - name: BFT (Framework) Integration Tests +# make: make -C integration bft-framework-tests +# runner: ubuntu-latest + - name: BFT (Protocol) Integration Tests + make: make -C integration bft-protocol-tests + runner: buildjet-8vcpu-ubuntu-2204 + - name: BFT (Gossipsub) Integration Tests + make: make -C integration bft-gossipsub-tests + runner: ubuntu-latest + - name: Collection Integration Tests + make: make -C integration collection-tests + runner: ubuntu-latest + - name: Consensus Integration Tests + make: make -C integration consensus-tests + runner: ubuntu-latest + - name: Epoch Cohort1 Integration Tests + make: make -C integration epochs-cohort1-tests + runner: buildjet-8vcpu-ubuntu-2204 + - name: Epoch Cohort2 Integration Tests + make: make -C integration epochs-cohort2-tests + runner: buildjet-4vcpu-ubuntu-2204 + - name: Execution Integration Tests + make: make -C integration execution-tests + runner: ubuntu-latest + - name: Ghost Integration Tests + make: make -C integration ghost-tests + runner: ubuntu-latest + - name: MVP Integration Tests + make: make -C integration mvp-tests + runner: ubuntu-latest + - name: Network Integration Tests + make: make -C integration network-tests + runner: ubuntu-latest + - name: Verification Integration Tests + make: make -C integration verification-tests + runner: ubuntu-latest + - name: Upgrade Integration Tests + make: make -C integration upgrades-tests + runner: ubuntu-latest + runs-on: ${{ matrix.runner }} steps: - name: Checkout repo uses: actions/checkout@v3 @@ -219,51 +296,20 @@ jobs: cache: true - name: Build relic run: make crypto_setup_gopath - - name: Docker build - run: make docker-build-flow docker-build-flow-corrupt - - name: Run tests + - name: Load cached Docker images + uses: actions/cache@v3 + with: + path: flow-docker-images.tar + # use the same cache key as the docker-build job + key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} + - name: Load Docker images + run: docker load -i flow-docker-images.tar + - name: Run tests (${{ matrix.name }}) # TODO(rbtz): re-enable when we fix exisiting races. #env: # RACE_DETECTOR: 1 uses: nick-fields/retry@v2 with: - timeout_minutes: 25 - max_attempts: 3 + timeout_minutes: 35 + max_attempts: 5 command: VERBOSE=1 ${{ matrix.make }} - - localnet-test: - name: Localnet Compatibility Tests With Flow-CLI Client and Observer - strategy: - fail-fast: false - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: ${{ env.GO_VERSION }} - cache: true - - name: Build relic and other tools - run: make install-tools - - name: Install Flow Client In Docker - # This proved to be more reliable than installing it locally. - run: cd integration/localnet && sh client/client.sh - - name: Set up Localnet - run: bash -c 'cd integration/localnet/ && make -e OBSERVER=2 bootstrap && make start-flow' - - name: Ensure Observer is started - run: docker ps -f name=localnet-observer_1-1 | grep localnet-observer - - name: Get Client Version ensuring the client is provisioned - run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer version - - name: Wait for a default waiting period until a clean state - # This will not cause flakiness. - # The waiting time is a reasonable time to expect an observer to be responsive - run: sleep 10 - - name: Get Status ensuring the access endpoint is online - run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n access status - - name: Wait for finalized blocks and check them - run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer blocks get latest - - name: Wait for finalized blocks and check them with Observer - run: sleep 5 && docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n access blocks get latest && docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer blocks get latest - - name: Stop localnet - run: bash -c 'cd integration/localnet/ && make stop' diff --git a/.github/workflows/flaky-test-monitor.yml b/.github/workflows/flaky-test-monitor.yml index f1c87d03348..06731f77b9a 100644 --- a/.github/workflows/flaky-test-monitor.yml +++ b/.github/workflows/flaky-test-monitor.yml @@ -8,7 +8,6 @@ on: push: paths: - '.github/workflows/flaky-test-monitor.yml' - - '.github/workflows/ci.yml' env: BIGQUERY_DATASET: production_src_flow_test_metrics diff --git a/integration/Makefile b/integration/Makefile index 8869fd8f56f..b538289ae82 100644 --- a/integration/Makefile +++ b/integration/Makefile @@ -24,9 +24,18 @@ ci-integration-test: access-tests ghost-tests mvp-tests epochs-cohort1-tests epo test: go test $(if $(VERBOSE),-v,) -tags relic -coverprofile=$(COVER_PROFILE) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) `go list ./... | grep -v -e integration/tests` -.PHONY: access-tests -access-tests: - go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/... +.PHONY: access-cohort1-tests +access-cohort1-tests: + go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/cohort1/... + +.PHONY: access-cohort2-tests +access-cohort2-tests: + go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/cohort2/... + +.PHONY: access-cohort3-tests +access-cohort3-tests: + go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/cohort3/... + .PHONY: collection-tests collection-tests: diff --git a/integration/tests/access/access_api_test.go b/integration/tests/access/cohort1/access_api_test.go similarity index 93% rename from integration/tests/access/access_api_test.go rename to integration/tests/access/cohort1/access_api_test.go index b4779c4f56d..cb5a175130d 100644 --- a/integration/tests/access/access_api_test.go +++ b/integration/tests/access/cohort1/access_api_test.go @@ -1,10 +1,12 @@ -package access +package cohort1 import ( "context" "testing" "time" + "github.com/onflow/flow-go/integration/tests/mvp" + "github.com/rs/zerolog" "github.com/stretchr/testify/suite" "google.golang.org/grpc/codes" @@ -18,7 +20,6 @@ import ( "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/integration/testnet" "github.com/onflow/flow-go/integration/tests/lib" - "github.com/onflow/flow-go/integration/tests/mvp" "github.com/onflow/flow-go/integration/utils" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -143,30 +144,44 @@ func (s *AccessAPISuite) SetupTest() { }, 30*time.Second, 1*time.Second) } -// TestScriptExecutionAndGetAccounts test the Access API endpoints for executing scripts and getting -// accounts using both local storage and execution nodes. +// TestScriptExecutionAndGetAccountsAN1 test the Access API endpoints for executing scripts and getting +// accounts using execution nodes. // -// Note: combining all tests together to reduce setup/teardown time. test cases are read-only +// Note: not combining AN1, AN2 tests together because that causes a drastic increase in test run times. test cases are read-only // and should not interfere with each other. -func (s *AccessAPISuite) TestScriptExecutionAndGetAccounts() { +func (s *AccessAPISuite) TestScriptExecutionAndGetAccountsAN1() { // deploy the test contract txResult := s.deployContract() targetHeight := txResult.BlockHeight + 1 s.waitUntilIndexed(targetHeight) - // Run tests against Access 2, which uses local storage - s.testGetAccount(s.an2Client) - s.testExecuteScriptWithSimpleScript(s.an2Client) - s.testExecuteScriptWithSimpleContract(s.an2Client, targetHeight) - // Run tests against Access 1, which uses the execution node s.testGetAccount(s.an1Client) s.testExecuteScriptWithSimpleScript(s.an1Client) s.testExecuteScriptWithSimpleContract(s.an1Client, targetHeight) +} + +// TestScriptExecutionAndGetAccountsAN2 test the Access API endpoints for executing scripts and getting +// accounts using local storage. +// +// Note: not combining AN1, AN2 tests together because that causes a drastic increase in test run times. test cases are read-only +// and should not interfere with each other. +func (s *AccessAPISuite) TestScriptExecutionAndGetAccountsAN2() { + // deploy the test contract + txResult := s.deployContract() + targetHeight := txResult.BlockHeight + 1 + s.waitUntilIndexed(targetHeight) + + // Run tests against Access 2, which uses local storage + s.testGetAccount(s.an2Client) + s.testExecuteScriptWithSimpleScript(s.an2Client) + s.testExecuteScriptWithSimpleContract(s.an2Client, targetHeight) +} +func (s *AccessAPISuite) TestMVPScriptExecutionLocalStorage() { // this is a specialized test that creates accounts, deposits funds, deploys contracts, etc, and // uses the provided access node to handle the Access API calls. there is an existing test that - // covers the default config, so we only need to test with local storage here. + // covers the default config, so we only need to test with local storage. mvp.RunMVPTest(s.T(), s.ctx, s.net, s.accessNode2) } diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/cohort2/observer_test.go similarity index 99% rename from integration/tests/access/observer_test.go rename to integration/tests/access/cohort2/observer_test.go index 771f52029b5..38f0c8bca05 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/cohort2/observer_test.go @@ -1,4 +1,4 @@ -package access +package cohort2 import ( "bytes" diff --git a/integration/tests/access/access_circuit_breaker_test.go b/integration/tests/access/cohort3/access_circuit_breaker_test.go similarity index 99% rename from integration/tests/access/access_circuit_breaker_test.go rename to integration/tests/access/cohort3/access_circuit_breaker_test.go index 569119c8469..5a4f0f25715 100644 --- a/integration/tests/access/access_circuit_breaker_test.go +++ b/integration/tests/access/cohort3/access_circuit_breaker_test.go @@ -1,4 +1,4 @@ -package access +package cohort3 import ( "context" diff --git a/integration/tests/access/access_test.go b/integration/tests/access/cohort3/access_test.go similarity index 99% rename from integration/tests/access/access_test.go rename to integration/tests/access/cohort3/access_test.go index 2bcb5645f13..9b060b2c05c 100644 --- a/integration/tests/access/access_test.go +++ b/integration/tests/access/cohort3/access_test.go @@ -1,4 +1,4 @@ -package access +package cohort3 import ( "context" diff --git a/integration/tests/access/consensus_follower_test.go b/integration/tests/access/cohort3/consensus_follower_test.go similarity index 99% rename from integration/tests/access/consensus_follower_test.go rename to integration/tests/access/cohort3/consensus_follower_test.go index b29a76c67af..d227a09ad2e 100644 --- a/integration/tests/access/consensus_follower_test.go +++ b/integration/tests/access/cohort3/consensus_follower_test.go @@ -1,4 +1,4 @@ -package access +package cohort3 import ( "context" diff --git a/integration/tests/access/execution_state_sync_test.go b/integration/tests/access/cohort3/execution_state_sync_test.go similarity index 99% rename from integration/tests/access/execution_state_sync_test.go rename to integration/tests/access/cohort3/execution_state_sync_test.go index 078441010d0..62126b57623 100644 --- a/integration/tests/access/execution_state_sync_test.go +++ b/integration/tests/access/cohort3/execution_state_sync_test.go @@ -1,4 +1,4 @@ -package access +package cohort3 import ( "context" diff --git a/integration/tests/access/grpc_compression_test.go b/integration/tests/access/cohort3/grpc_compression_test.go similarity index 99% rename from integration/tests/access/grpc_compression_test.go rename to integration/tests/access/cohort3/grpc_compression_test.go index 3f7abc86434..1f8f0de59c0 100644 --- a/integration/tests/access/grpc_compression_test.go +++ b/integration/tests/access/cohort3/grpc_compression_test.go @@ -1,4 +1,4 @@ -package access +package cohort3 import ( "context" diff --git a/integration/tests/access/rest_state_stream_test.go b/integration/tests/access/cohort3/rest_state_stream_test.go similarity index 99% rename from integration/tests/access/rest_state_stream_test.go rename to integration/tests/access/cohort3/rest_state_stream_test.go index 08b08224545..ad6eab9d8b5 100644 --- a/integration/tests/access/rest_state_stream_test.go +++ b/integration/tests/access/cohort3/rest_state_stream_test.go @@ -1,4 +1,4 @@ -package access +package cohort3 import ( "context" diff --git a/integration/tests/bft/framework/passthrough_test.go b/integration/tests/bft/framework/passthrough_test.go index f9a436de782..255f5a4069b 100644 --- a/integration/tests/bft/framework/passthrough_test.go +++ b/integration/tests/bft/framework/passthrough_test.go @@ -3,6 +3,8 @@ package framework import ( "testing" + "github.com/onflow/flow-go/utils/unittest" + "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/integration/tests/common" @@ -27,6 +29,7 @@ func TestPassThrough(t *testing.T) { // affected by the emitted result approvals. // Finally, it evaluates whether critical sealing-and-verification-related events from corrupted nodes are passed through the orchestrator. func (p *PassThroughTestSuite) TestSealingAndVerificationPassThrough() { + unittest.SkipUnless(p.T(), unittest.TEST_TODO, "flaky") receipts, approvals := common.SealingAndVerificationHappyPathTest( p.T(), p.BlockState, diff --git a/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go b/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go index 01f823f3c86..d4b46693bb9 100644 --- a/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go +++ b/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go @@ -6,8 +6,9 @@ import ( "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/integration/tests/epochs" "github.com/onflow/flow-go/model/flow" + + "github.com/onflow/flow-go/integration/tests/epochs" ) func TestEpochJoinAndLeaveVN(t *testing.T) { diff --git a/network/p2p/p2pnode/resourceManager_test.go b/network/p2p/p2pnode/resourceManager_test.go index bcef3cb8f7f..e2b49b1c8b7 100644 --- a/network/p2p/p2pnode/resourceManager_test.go +++ b/network/p2p/p2pnode/resourceManager_test.go @@ -134,12 +134,12 @@ func (t testPeerLimitConfig) maxLimit() int { // baseCreateStreamInboundStreamResourceLimitConfig returns a testPeerLimitConfig with default values. func baseCreateStreamInboundStreamResourceLimitConfig() *testPeerLimitConfig { return &testPeerLimitConfig{ - nodeCount: 5, - maxInboundPeerStream: 10, - maxInboundStreamProtocol: 10, - maxInboundStreamPeerProtocol: 10, - maxInboundStreamTransient: 10, - maxInboundStreamSystem: 10, + nodeCount: 10, + maxInboundPeerStream: 100, + maxInboundStreamProtocol: 100, + maxInboundStreamPeerProtocol: 100, + maxInboundStreamTransient: 100, + maxInboundStreamSystem: 100, } } diff --git a/network/test/meshengine_test.go b/network/test/cohort1/meshengine_test.go similarity index 99% rename from network/test/meshengine_test.go rename to network/test/cohort1/meshengine_test.go index 48800ab609e..02870ea57e0 100644 --- a/network/test/meshengine_test.go +++ b/network/test/cohort1/meshengine_test.go @@ -1,4 +1,4 @@ -package test +package cohort1 import ( "context" @@ -49,7 +49,6 @@ type MeshEngineTestSuite struct { // TestMeshNetTestSuite runs all tests in this test suit func TestMeshNetTestSuite(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "this should be revisited once network/test is running in a separate CI job, runs fine locally") suite.Run(t, new(MeshEngineTestSuite)) } diff --git a/network/test/network_test.go b/network/test/cohort1/network_test.go similarity index 99% rename from network/test/network_test.go rename to network/test/cohort1/network_test.go index a895784a846..8f8716308ba 100644 --- a/network/test/network_test.go +++ b/network/test/cohort1/network_test.go @@ -1,4 +1,4 @@ -package test +package cohort1 import ( "context" @@ -244,6 +244,7 @@ func (suite *NetworkTestSuite) TestUpdateNodeAddresses() { } func (suite *NetworkTestSuite) TestUnicastRateLimit_Messages() { + unittest.SkipUnless(suite.T(), unittest.TEST_FLAKY, "flaky") // limiter limit will be set to 5 events/sec the 6th event per interval will be rate limited limit := rate.Limit(5) diff --git a/network/test/blob_service_test.go b/network/test/cohort2/blob_service_test.go similarity index 99% rename from network/test/blob_service_test.go rename to network/test/cohort2/blob_service_test.go index dbdab7f1985..b3f3d6921ae 100644 --- a/network/test/blob_service_test.go +++ b/network/test/cohort2/blob_service_test.go @@ -1,4 +1,4 @@ -package test +package cohort2 import ( "context" diff --git a/network/test/echoengine.go b/network/test/cohort2/echoengine.go similarity index 99% rename from network/test/echoengine.go rename to network/test/cohort2/echoengine.go index b5ffd72ad54..0401a1fff69 100644 --- a/network/test/echoengine.go +++ b/network/test/cohort2/echoengine.go @@ -1,4 +1,4 @@ -package test +package cohort2 import ( "fmt" diff --git a/network/test/echoengine_test.go b/network/test/cohort2/echoengine_test.go similarity index 99% rename from network/test/echoengine_test.go rename to network/test/cohort2/echoengine_test.go index 18384ce1fdd..81d807be180 100644 --- a/network/test/echoengine_test.go +++ b/network/test/cohort2/echoengine_test.go @@ -1,4 +1,4 @@ -package test +package cohort2 import ( "context" @@ -40,7 +40,6 @@ type EchoEngineTestSuite struct { // TestEchoEngineTestSuite runs all the test methods in this test suit func TestEchoEngineTestSuite(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "this should be revisited once network/test is running in a separate CI job, runs fine locally") suite.Run(t, new(EchoEngineTestSuite)) } diff --git a/network/test/epochtransition_test.go b/network/test/cohort2/epochtransition_test.go similarity index 99% rename from network/test/epochtransition_test.go rename to network/test/cohort2/epochtransition_test.go index b3132d5b398..22972e44557 100644 --- a/network/test/epochtransition_test.go +++ b/network/test/cohort2/epochtransition_test.go @@ -1,4 +1,4 @@ -package test +package cohort2 import ( "context" diff --git a/network/test/unicast_authorization_test.go b/network/test/cohort2/unicast_authorization_test.go similarity index 99% rename from network/test/unicast_authorization_test.go rename to network/test/cohort2/unicast_authorization_test.go index b9cec9ba8b0..faffb5e2307 100644 --- a/network/test/unicast_authorization_test.go +++ b/network/test/cohort2/unicast_authorization_test.go @@ -1,4 +1,4 @@ -package test +package cohort2 import ( "context" diff --git a/utils/test_matrix/test_matrix.go b/utils/test_matrix/test_matrix.go index faa8d0bc075..9a5cb9f9713 100644 --- a/utils/test_matrix/test_matrix.go +++ b/utils/test_matrix/test_matrix.go @@ -11,11 +11,18 @@ import ( const flowPackagePrefix = "github.com/onflow/flow-go/" const ciMatrixName = "dynamicMatrix" +const ciDefaultRunner = "ubuntu-latest" // testMatrix represents a single GitHub Actions test matrix combination that consists of a name and a list of flow-go packages associated with that name. type testMatrix struct { Name string `json:"name"` Packages string `json:"packages"` + Runner string `json:"runner"` +} + +type targets struct { + runners map[string]string + packages map[string][]string } // Generates a list of packages to test that will be passed to GitHub Actions @@ -47,14 +54,14 @@ func main() { fmt.Println(testMatrixStr) } -func generateTestMatrix(targetPackages map[string][]string, otherPackages []string) []testMatrix { - +func generateTestMatrix(targetPackages targets, otherPackages []string) []testMatrix { var testMatrices []testMatrix - for names := range targetPackages { + for packageName := range targetPackages.packages { targetTestMatrix := testMatrix{ - Name: names, - Packages: strings.Join(targetPackages[names], " "), + Name: packageName, + Packages: strings.Join(targetPackages.packages[packageName], " "), + Runner: targetPackages.runners[packageName], } testMatrices = append(testMatrices, targetTestMatrix) } @@ -63,6 +70,7 @@ func generateTestMatrix(targetPackages map[string][]string, otherPackages []stri otherTestMatrix := testMatrix{ Name: "others", Packages: strings.Join(otherPackages, " "), + Runner: ciDefaultRunner, } testMatrices = append(testMatrices, otherTestMatrix) @@ -72,8 +80,9 @@ func generateTestMatrix(targetPackages map[string][]string, otherPackages []stri // listTargetPackages returns a map-list of target packages to run as separate CI jobs, based on a list of target package prefixes. // It also returns a list of the "seen" packages that can then be used to extract the remaining packages to run (in a separate CI job). -func listTargetPackages(targetPackagePrefixes []string, allFlowPackages []string) (map[string][]string, map[string]string) { +func listTargetPackages(targetPackagePrefixes []string, allFlowPackages []string) (targets, map[string]string) { targetPackages := make(map[string][]string) + targetRunners := make(map[string]string) // Stores list of packages already seen / allocated to other lists. Needed for the last package which will // have all the leftover packages that weren't allocated to a separate list (CI job). @@ -84,19 +93,39 @@ func listTargetPackages(targetPackagePrefixes []string, allFlowPackages []string for _, targetPackagePrefix := range targetPackagePrefixes { var targetPackage []string + // assume package name specified without runner + targetPackagePrefixNoRunner := targetPackagePrefix + + // check if specify CI runner to use for the package, otherwise use default + colonIndex := strings.Index(targetPackagePrefix, ":") + if colonIndex != -1 { + targetPackagePrefixNoRunner = targetPackagePrefix[:colonIndex] // strip out runner from package name + targetRunners[targetPackagePrefixNoRunner] = targetPackagePrefix[colonIndex+1:] + } else { + // use default CI runner if didn't specify runner + targetRunners[targetPackagePrefix] = ciDefaultRunner + } + // go through all packages to see which ones to pull out for _, allPackage := range allFlowPackages { - if strings.HasPrefix(allPackage, flowPackagePrefix+targetPackagePrefix) { + if strings.HasPrefix(allPackage, flowPackagePrefix+targetPackagePrefixNoRunner) { + // if the package was already seen, don't append it to the list + // this is to support listing sub-sub packages in a CI job without duplicating those sub-sub packages + // in the parent package job + _, seen := seenPackages[allPackage] + if seen { + continue + } targetPackage = append(targetPackage, allPackage) seenPackages[allPackage] = allPackage } } if len(targetPackage) == 0 { - panic("no packages exist with prefix " + targetPackagePrefix) + panic("no packages exist with prefix " + targetPackagePrefixNoRunner) } - targetPackages[targetPackagePrefix] = targetPackage + targetPackages[targetPackagePrefixNoRunner] = targetPackage } - return targetPackages, seenPackages + return targets{targetRunners, targetPackages}, seenPackages } // listOtherPackages compiles the remaining packages that don't match any of the target packages. diff --git a/utils/test_matrix/test_matrix_test.go b/utils/test_matrix/test_matrix_test.go index d585ba7bba7..db5accb6fc2 100644 --- a/utils/test_matrix/test_matrix_test.go +++ b/utils/test_matrix/test_matrix_test.go @@ -13,6 +13,15 @@ func getAllFlowPackages() []string { flowPackagePrefix + "abc/123", flowPackagePrefix + "abc/def", flowPackagePrefix + "abc/def/ghi", + flowPackagePrefix + "abc/def/ghi/jkl", + flowPackagePrefix + "abc/def/ghi/jkl/mno", + flowPackagePrefix + "abc/def/ghi/jkl/mno/pqr", + flowPackagePrefix + "abc/def/ghi/mno/abc", + flowPackagePrefix + "abc/def/ghi/mno/def", + flowPackagePrefix + "abc/def/ghi/mno/ghi", + flowPackagePrefix + "abc/def/jkl", + flowPackagePrefix + "abc/def/jkl/mno", + flowPackagePrefix + "abc/def/jkl/mno/pqr", flowPackagePrefix + "def", flowPackagePrefix + "def/abc", flowPackagePrefix + "ghi", @@ -26,46 +35,97 @@ func getAllFlowPackages() []string { } } -// TestListTargetPackages tests that the target packages are included in the target packages and seen packages. -func TestListTargetPackages(t *testing.T) { - targetPackages, seenPackages := listTargetPackages([]string{"abc", "ghi"}, getAllFlowPackages()) - require.Equal(t, 2, len(targetPackages)) - - // there should be 4 packages that start with "abc" - require.Equal(t, 4, len(targetPackages["abc"])) - require.Contains(t, targetPackages["abc"], flowPackagePrefix+"abc") - require.Contains(t, targetPackages["abc"], flowPackagePrefix+"abc/123") - require.Contains(t, targetPackages["abc"], flowPackagePrefix+"abc/def") - require.Contains(t, targetPackages["abc"], flowPackagePrefix+"abc/def/ghi") +// TestListTargetPackages_DefaultRunners tests that the target packages are included in the target packages and seen packages. +// All packages use default CI runners. +func TestListTargetPackages_DefaultRunners(t *testing.T) { + target, seenPackages := listTargetPackages([]string{"abc", "ghi"}, getAllFlowPackages()) + require.Equal(t, 2, len(target.packages)) + + // check all TARGET + // these are the expected target packages that start with "abc" + require.Equal(t, 13, len(target.packages["abc"])) + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/123") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/jkl") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/jkl/mno") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/mno/abc") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/mno/def") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/ghi/mno/ghi") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/jkl") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/jkl/mno") + require.Contains(t, target.packages["abc"], flowPackagePrefix+"abc/def/jkl/mno/pqr") // there should be 1 package that starts with "ghi" - require.Equal(t, 1, len(targetPackages["ghi"])) - require.Contains(t, targetPackages["ghi"], flowPackagePrefix+"ghi") + require.Equal(t, 1, len(target.packages["ghi"])) + require.Contains(t, target.packages["ghi"], flowPackagePrefix+"ghi") + + // check all CI RUNNERS for each target package + require.Equal(t, 2, len(target.runners)) + require.Equal(t, target.runners["abc"], ciDefaultRunner) + require.Equal(t, target.runners["ghi"], ciDefaultRunner) - // there should be 5 packages that start with "abc" - require.Equal(t, 5, len(seenPackages)) + // check all SEEN packages + // these are all expected packages that start with "abc" or "ghi" + require.Equal(t, 14, len(seenPackages)) require.Contains(t, seenPackages, flowPackagePrefix+"abc") require.Contains(t, seenPackages, flowPackagePrefix+"abc/123") require.Contains(t, seenPackages, flowPackagePrefix+"abc/def") require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/abc") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/def") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/ghi") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno/pqr") + require.Contains(t, seenPackages, flowPackagePrefix+"ghi") } -// TestListTargetSubPackages tests that if a subpackage is specified as a target package, then the sub package and +// TestListTargetSubPackages_CustomRunners tests that if a subpackage is specified as a target package, then the sub package and // all children of the sub package are also included in the target packages. -func TestListTargetSubPackages(t *testing.T) { - targetPackages, seenPackages := listTargetPackages([]string{"abc/def"}, getAllFlowPackages()) - require.Equal(t, 1, len(targetPackages)) +func TestListTargetSubPackages_CustomRunners(t *testing.T) { + target, seenPackages := listTargetPackages([]string{"abc/def:foo_runner"}, getAllFlowPackages()) + require.Equal(t, 1, len(target.packages)) + // check all TARGET packages // there should be 2 target subpackages that starts with "abc/def" - require.Equal(t, 2, len(targetPackages["abc/def"])) - require.Contains(t, targetPackages["abc/def"], flowPackagePrefix+"abc/def") - require.Contains(t, targetPackages["abc/def"], flowPackagePrefix+"abc/def/ghi") - - // there should be 2 seen subpackages that start with "abc/def" - require.Equal(t, 2, len(seenPackages)) + require.Equal(t, 11, len(target.packages["abc/def"])) + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/jkl") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/jkl/mno") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/mno/abc") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/mno/def") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/ghi/mno/ghi") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/jkl") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/jkl/mno") + require.Contains(t, target.packages["abc/def"], flowPackagePrefix+"abc/def/jkl/mno/pqr") + + // check all CI RUNNERS for each target package + require.Equal(t, 1, len(target.runners)) + require.Equal(t, target.runners["abc/def"], "foo_runner") + + // check all SEEN packages + // there should be 11 seen subpackages that start with "abc/def" + require.Equal(t, 11, len(seenPackages)) require.Contains(t, seenPackages, flowPackagePrefix+"abc/def") require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/abc") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/def") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/ghi/mno/ghi") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno") + require.Contains(t, seenPackages, flowPackagePrefix+"abc/def/jkl/mno/pqr") } // TestListOtherPackages tests that the remaining packages that don't match any of the target packages are included @@ -79,10 +139,19 @@ func TestListOtherPackages(t *testing.T) { otherPackages := listOtherPackages(getAllFlowPackages(), seenPackages) - require.Equal(t, 9, len(otherPackages)) + require.Equal(t, 18, len(otherPackages)) require.Contains(t, otherPackages, flowPackagePrefix+"abc") require.Contains(t, otherPackages, flowPackagePrefix+"abc/123") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/jkl") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/jkl/mno/pqr") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/mno/abc") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/mno/def") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/ghi/mno/ghi") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/jkl") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/jkl/mno") + require.Contains(t, otherPackages, flowPackagePrefix+"abc/def/jkl/mno/pqr") require.Contains(t, otherPackages, flowPackagePrefix+"def") require.Contains(t, otherPackages, flowPackagePrefix+"def/abc") require.Contains(t, otherPackages, flowPackagePrefix+"jkl") @@ -92,34 +161,85 @@ func TestListOtherPackages(t *testing.T) { require.Contains(t, otherPackages, flowPackagePrefix+"yz") } -// TestGenerateTestMatrix tests that the test matrix is generated correctly where the target packages include top level -// packages as well as sub packages. -func TestGenerateTestMatrix(t *testing.T) { - targetPackages, seenPackages := listTargetPackages([]string{"abc/def", "def", "ghi"}, getAllFlowPackages()) - require.Equal(t, 3, len(targetPackages)) - require.Equal(t, 5, len(seenPackages)) +// TestGenerateTestMatrix_CustomRunners tests that the test matrix is generated correctly where the target packages include top level +// packages as well as sub packages. It also tests having 2 different custom CI runners, as well as default runners. +func TestGenerateTestMatrix_CustomRunners(t *testing.T) { + target, seenPackages := listTargetPackages([]string{"abc/def", "def:foo-runner", "ghi", "vwx/ghi:foo-runner2"}, getAllFlowPackages()) + require.Equal(t, 4, len(target.packages)) + require.Equal(t, 4, len(target.runners)) + require.Equal(t, 15, len(seenPackages)) otherPackages := listOtherPackages(getAllFlowPackages(), seenPackages) - matrix := generateTestMatrix(targetPackages, otherPackages) + matrix := generateTestMatrix(target, otherPackages) - // should be 3 groups in test matrix: abc/def, def, ghi, others - require.Equal(t, 4, len(matrix)) + // should be 4 groups in test matrix: abc/def, def, ghi, vwx/ghi, others + require.Equal(t, 5, len(matrix)) require.Contains(t, matrix, testMatrix{ Name: "abc/def", - Packages: "github.com/onflow/flow-go/abc/def github.com/onflow/flow-go/abc/def/ghi"}, + Packages: "github.com/onflow/flow-go/abc/def github.com/onflow/flow-go/abc/def/ghi github.com/onflow/flow-go/abc/def/ghi/jkl github.com/onflow/flow-go/abc/def/ghi/jkl/mno github.com/onflow/flow-go/abc/def/ghi/jkl/mno/pqr github.com/onflow/flow-go/abc/def/ghi/mno/abc github.com/onflow/flow-go/abc/def/ghi/mno/def github.com/onflow/flow-go/abc/def/ghi/mno/ghi github.com/onflow/flow-go/abc/def/jkl github.com/onflow/flow-go/abc/def/jkl/mno github.com/onflow/flow-go/abc/def/jkl/mno/pqr", + Runner: "ubuntu-latest"}, ) require.Contains(t, matrix, testMatrix{ Name: "def", - Packages: "github.com/onflow/flow-go/def github.com/onflow/flow-go/def/abc"}, + Packages: "github.com/onflow/flow-go/def github.com/onflow/flow-go/def/abc", + Runner: "foo-runner"}, ) require.Contains(t, matrix, testMatrix{ Name: "ghi", - Packages: "github.com/onflow/flow-go/ghi"}, + Packages: "github.com/onflow/flow-go/ghi", + Runner: "ubuntu-latest"}, + ) + require.Contains(t, matrix, testMatrix{ + Name: "vwx/ghi", + Packages: "github.com/onflow/flow-go/vwx/ghi", + Runner: "foo-runner2"}, ) require.Contains(t, matrix, testMatrix{ Name: "others", - Packages: "github.com/onflow/flow-go/abc github.com/onflow/flow-go/abc/123 github.com/onflow/flow-go/jkl github.com/onflow/flow-go/mno/abc github.com/onflow/flow-go/pqr github.com/onflow/flow-go/stu github.com/onflow/flow-go/vwx github.com/onflow/flow-go/vwx/ghi github.com/onflow/flow-go/yz"}, + Packages: "github.com/onflow/flow-go/abc github.com/onflow/flow-go/abc/123 github.com/onflow/flow-go/jkl github.com/onflow/flow-go/mno/abc github.com/onflow/flow-go/pqr github.com/onflow/flow-go/stu github.com/onflow/flow-go/vwx github.com/onflow/flow-go/yz", + Runner: "ubuntu-latest"}, + ) +} + +// TestGenerateTestMatrix_SubSubPackages tests that the test matrix is generated correctly where the target packages +// include 2nd and 3rd level sub packages. It also tests having 2 different custom CI runners, as well as default runners. +func TestGenerateTestMatrix_SubSubPackages(t *testing.T) { + target, seenPackages := listTargetPackages([]string{"abc/def/ghi:foo-runner1", "abc/def/jkl:foo-runner2", "abc"}, getAllFlowPackages()) + require.Equal(t, 3, len(target.packages)) + require.Equal(t, 3, len(target.runners)) + require.Equal(t, 13, len(seenPackages)) + + otherPackages := listOtherPackages(getAllFlowPackages(), seenPackages) + + matrix := generateTestMatrix(target, otherPackages) + + // should be 4 groups in test matrix: abc/def/ghi, abc/def/jkl, abc, others + require.Equal(t, 4, len(matrix)) + + require.Contains(t, matrix, testMatrix{ + Name: "abc/def/ghi", + Packages: "github.com/onflow/flow-go/abc/def/ghi github.com/onflow/flow-go/abc/def/ghi/jkl github.com/onflow/flow-go/abc/def/ghi/jkl/mno github.com/onflow/flow-go/abc/def/ghi/jkl/mno/pqr github.com/onflow/flow-go/abc/def/ghi/mno/abc github.com/onflow/flow-go/abc/def/ghi/mno/def github.com/onflow/flow-go/abc/def/ghi/mno/ghi", + Runner: "foo-runner1"}, + ) + + require.Contains(t, matrix, testMatrix{ + Name: "abc/def/jkl", + Packages: "github.com/onflow/flow-go/abc/def/jkl github.com/onflow/flow-go/abc/def/jkl/mno github.com/onflow/flow-go/abc/def/jkl/mno/pqr", + Runner: "foo-runner2"}, + ) + + // parent package should not have any packages from its sub packages because they were already included in the sub package groups + require.Contains(t, matrix, testMatrix{ + Name: "abc", + Packages: "github.com/onflow/flow-go/abc github.com/onflow/flow-go/abc/123 github.com/onflow/flow-go/abc/def", + Runner: "ubuntu-latest"}, + ) + + require.Contains(t, matrix, testMatrix{ + Name: "others", + Packages: "github.com/onflow/flow-go/def github.com/onflow/flow-go/def/abc github.com/onflow/flow-go/ghi github.com/onflow/flow-go/jkl github.com/onflow/flow-go/mno/abc github.com/onflow/flow-go/pqr github.com/onflow/flow-go/stu github.com/onflow/flow-go/vwx github.com/onflow/flow-go/vwx/ghi github.com/onflow/flow-go/yz", + Runner: "ubuntu-latest"}, ) }