Added config option for "test all contracts" (#102)

* Added config option to filter "test all contracts", indicating whether contracts not in the deployment order (dynamically deployed contracts) should be tested
* Changed config to default to only testing DeploymentOrder contracts

fuzzing/config/config.go

@@ -90,6 +90,10 @@ type TestingConfig struct {
 	// to determine which contract a deployed contract is.
 	StopOnFailedContractMatching bool `json:"stopOnFailedContractMatching"`
 
+	// TestAllContracts indicates whether all contracts should be tested (including dynamically deployed ones), rather
+	// than just the contracts specified in the project configuration's deployment order.
+	TestAllContracts bool `json:"testAllContracts"`
+
 	// AssertionTesting describes the configuration used for assertion testing.
 	AssertionTesting AssertionTestingConfig `json:"assertionTesting"`
 

fuzzing/config/config_defaults.go

@@ -53,6 +53,7 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
 			Testing: TestingConfig{
 				StopOnFailedTest:             true,
 				StopOnFailedContractMatching: true,
+				TestAllContracts:             false,
 				AssertionTesting: AssertionTestingConfig{
 					Enabled:         false,
 					TestViewMethods: false,

fuzzing/fuzzer_test.go

@@ -222,6 +222,8 @@ func TestDeploymentsInnerDeployments(t *testing.T) {
 			configUpdates: func(config *config.ProjectConfig) {
 				config.Fuzzing.DeploymentOrder = []string{"InnerDeploymentFactory"}
 				config.Fuzzing.TestLimit = 1_000 // this test should expose a failure quickly.
+				config.Fuzzing.Testing.StopOnFailedContractMatching = true
+				config.Fuzzing.Testing.TestAllContracts = true // test dynamically deployed contracts
 			},
 			method: func(f *fuzzerTestContext) {
 				// Start the fuzzer
@@ -241,6 +243,8 @@ func TestDeploymentsInnerDeployments(t *testing.T) {
 		configUpdates: func(config *config.ProjectConfig) {
 			config.Fuzzing.DeploymentOrder = []string{"InnerDeploymentFactory"}
 			config.Fuzzing.TestLimit = 1_000 // this test should expose a failure quickly.
+			config.Fuzzing.Testing.StopOnFailedContractMatching = true
+			config.Fuzzing.Testing.TestAllContracts = true // test dynamically deployed contracts
 		},
 		method: func(f *fuzzerTestContext) {
 			// Start the fuzzer
@@ -314,6 +318,40 @@ func TestDeploymentsSelfDestruct(t *testing.T) {
 	}
 }
 
+// TestTestingScope runs tests to ensure dynamically deployed contracts are tested when the "test all contracts"
+// config option is specified. It also runs the fuzzer without the option enabled to ensure they are not tested.
+func TestTestingScope(t *testing.T) {
+	for _, testingAllContracts := range []bool{false, true} {
+		runFuzzerTest(t, &fuzzerSolcFileTest{
+			filePath: "testdata/contracts/deployments/testing_scope.sol",
+			configUpdates: func(config *config.ProjectConfig) {
+				config.Fuzzing.DeploymentOrder = []string{"TestContract"}
+				config.Fuzzing.TestLimit = 1_000 // this test should expose a failure quickly.
+				config.Fuzzing.Testing.TestAllContracts = testingAllContracts
+				config.Fuzzing.Testing.StopOnFailedTest = false
+				config.Fuzzing.Testing.AssertionTesting.Enabled = true
+				config.Fuzzing.Testing.PropertyTesting.Enabled = true
+			},
+			method: func(f *fuzzerTestContext) {
+				// Start the fuzzer
+				err := f.fuzzer.Start()
+				assert.NoError(t, err)
+
+				// Define our expected failure count
+				var expectedFailureCount int
+				if testingAllContracts {
+					expectedFailureCount = 4
+				} else {
+					expectedFailureCount = 2
+				}
+
+				// Check for any failed tests and verify coverage was captured
+				assert.EqualValues(t, len(f.fuzzer.TestCasesWithStatus(TestCaseStatusFailed)), expectedFailureCount)
+			},
+		})
+	}
+}
+
 // TestDeploymentsWithArgs runs tests to ensure contracts deployed with config provided constructor arguments are
 // deployed as expected. It expects all properties should fail (indicating values provided were set accordingly).
 func TestDeploymentsWithArgs(t *testing.T) {

fuzzing/fuzzer_test_methods_test.go

@@ -61,17 +61,19 @@ func getFuzzerTestingProjectConfig(t *testing.T, compilationConfig *compilation.
 	projectConfig.Fuzzing.TestLimit = 1_500_000
 	projectConfig.Fuzzing.CallSequenceLength = 100
 	projectConfig.Fuzzing.Testing.StopOnFailedContractMatching = true
+	projectConfig.Fuzzing.Testing.TestAllContracts = false
 	return projectConfig
 }
 
 // assertFailedTestsExpected will check to see whether there are any failed tests. If `expectFailure` is false, then
 // there should be no failed tests
 func assertFailedTestsExpected(f *fuzzerTestContext, expectFailure bool) {
 	// Ensure we captured a failed test, if expected
+	failedTestCount := len(f.fuzzer.TestCasesWithStatus(TestCaseStatusFailed))
 	if expectFailure {
-		assert.Greater(f.t, len(f.fuzzer.TestCasesWithStatus(TestCaseStatusFailed)), 0, "Fuzz test could not be solved before reaching limits")
+		assert.Greater(f.t, failedTestCount, 0, "Fuzz test could not be solved before reaching limits")
 	} else {
-		assert.EqualValues(f.t, 0, len(f.fuzzer.TestCasesWithStatus(TestCaseStatusFailed)), "Fuzz test failed when it should not have")
+		assert.EqualValues(f.t, 0, failedTestCount, "Fuzz test failed when it should not have")
 	}
 }
 

fuzzing/test_case_assertion_provider.go

@@ -3,6 +3,7 @@ package fuzzing
 import (
 	"bytes"
 	"github.com/trailofbits/medusa/fuzzing/calls"
+	"golang.org/x/exp/slices"
 	"math/big"
 	"sync"
 
@@ -118,25 +119,33 @@ func (t *AssertionTestCaseProvider) onFuzzerStarting(event FuzzerStartingEvent)
 
 	// Create a test case for every test method.
 	for _, contract := range t.fuzzer.ContractDefinitions() {
+		// If we're not testing all contracts, verify the current contract is one we specified in our deployment order.
+		if !t.fuzzer.config.Fuzzing.Testing.TestAllContracts && !slices.Contains(t.fuzzer.config.Fuzzing.DeploymentOrder, contract.Name()) {
+			continue
+		}
+
 		for _, method := range contract.CompiledContract().Abi.Methods {
-			if t.isTestableMethod(method) {
-				// Create local variables to avoid pointer types in the loop being overridden.
-				contract := contract
-				method := method
-
-				// Create our test case
-				testCase := &AssertionTestCase{
-					status:         TestCaseStatusNotStarted,
-					targetContract: contract,
-					targetMethod:   method,
-					callSequence:   nil,
-				}
+			// Verify this method is an assertion testable method
+			if !t.isTestableMethod(method) {
+				continue
+			}
+
+			// Create local variables to avoid pointer types in the loop being overridden.
+			contract := contract
+			method := method
 
-				// Add to our test cases and register them with the fuzzer
-				methodId := contracts.GetContractMethodID(contract, &method)
-				t.testCases[methodId] = testCase
-				t.fuzzer.RegisterTestCase(testCase)
+			// Create our test case
+			testCase := &AssertionTestCase{
+				status:         TestCaseStatusNotStarted,
+				targetContract: contract,
+				targetMethod:   method,
+				callSequence:   nil,
 			}
+
+			// Add to our test cases and register them with the fuzzer
+			methodId := contracts.GetContractMethodID(contract, &method)
+			t.testCases[methodId] = testCase
+			t.fuzzer.RegisterTestCase(testCase)
 		}
 	}
 	return nil
@@ -206,9 +215,14 @@ func (t *AssertionTestCaseProvider) callSequencePostCallTest(worker *FuzzerWorke
 
 	// Obtain the test case for this method we're targeting for assertion testing.
 	t.testCasesLock.Lock()
-	testCase := t.testCases[*methodId]
+	testCase, testCaseExists := t.testCases[*methodId]
 	t.testCasesLock.Unlock()
 
+	// Verify a test case exists for this method called (if we're not assertion testing this method, stop)
+	if !testCaseExists {
+		return shrinkRequests, nil
+	}
+
 	// If the test case already failed, skip it
 	if testCase.Status() == TestCaseStatusFailed {
 		return shrinkRequests, nil

fuzzing/test_case_property_provider.go

@@ -5,6 +5,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/trailofbits/medusa/fuzzing/calls"
 	"github.com/trailofbits/medusa/fuzzing/contracts"
+	"golang.org/x/exp/slices"
 	"math/big"
 	"strings"
 	"sync"
@@ -127,25 +128,33 @@ func (t *PropertyTestCaseProvider) onFuzzerStarting(event FuzzerStartingEvent) e
 
 	// Create a test case for every property test method.
 	for _, contract := range t.fuzzer.ContractDefinitions() {
+		// If we're not testing all contracts, verify the current contract is one we specified in our deployment order.
+		if !t.fuzzer.config.Fuzzing.Testing.TestAllContracts && !slices.Contains(t.fuzzer.config.Fuzzing.DeploymentOrder, contract.Name()) {
+			continue
+		}
+
 		for _, method := range contract.CompiledContract().Abi.Methods {
-			if t.isPropertyTest(method) {
-				// Create local variables to avoid pointer types in the loop being overridden.
-				contract := contract
-				method := method
-
-				// Create our property test case
-				propertyTestCase := &PropertyTestCase{
-					status:         TestCaseStatusNotStarted,
-					targetContract: contract,
-					targetMethod:   method,
-					callSequence:   nil,
-				}
+			// Verify this method is a property test method
+			if !t.isPropertyTest(method) {
+				continue
+			}
+
+			// Create local variables to avoid pointer types in the loop being overridden.
+			contract := contract
+			method := method
 
-				// Add to our test cases and register them with the fuzzer
-				methodId := contracts.GetContractMethodID(contract, &method)
-				t.testCases[methodId] = propertyTestCase
-				t.fuzzer.RegisterTestCase(propertyTestCase)
+			// Create our property test case
+			propertyTestCase := &PropertyTestCase{
+				status:         TestCaseStatusNotStarted,
+				targetContract: contract,
+				targetMethod:   method,
+				callSequence:   nil,
 			}
+
+			// Add to our test cases and register them with the fuzzer
+			methodId := contracts.GetContractMethodID(contract, &method)
+			t.testCases[methodId] = propertyTestCase
+			t.fuzzer.RegisterTestCase(propertyTestCase)
 		}
 	}
 	return nil

fuzzing/testdata/contracts/deployments/testing_scope.sol

@@ -0,0 +1,28 @@
+// TestContract deploys a TestContractChild on construction, both containing failing assertion and property tests,
+// to ensure that the project configuration surrounding the "test all contracts" feature works as expected for
+// dynamically deployed contracts.
+contract TestContractChild {
+    function failing_assertion_method_child(uint x) public {
+        assert(false);
+    }
+
+    function fuzz_failing_property_test_method_child() public view returns (bool) {
+        return false;
+    }
+}
+
+contract TestContract {
+    address a;
+
+    constructor() public {
+        a = address(new TestContractChild());
+    }
+
+    function failing_assertion_method(uint x) public {
+        assert(false);
+    }
+
+    function fuzz_failing_property_test_method() public view returns (bool) {
+        return false;
+    }
+}