Add --skip option to CLI

Disassociates hardcoded tests to skip when running via CLI. Tests still skipped when running `go test`
zeriontech · Jun 19, 2015 · 0743243 · 0743243
1 parent a9659e6
commit 0743243
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 79 deletions.
diff --git a/cmd/ethtest/main.go b/cmd/ethtest/main.go
@@ -28,6 +28,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/codegangsta/cli"
 	"github.com/ethereum/go-ethereum/logger/glog"
@@ -40,6 +41,7 @@ var (
 	defaultTest     = "all"
 	defaultDir      = "."
 	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests"}
+	skipTests       = []string{}
 
 	TestFlag = cli.StringFlag{
 		Name:  "test",
@@ -60,20 +62,24 @@ var (
 		Name:  "stdin",
 		Usage: "Accept input from stdin instead of reading from file",
 	}
+	SkipTestsFlag = cli.StringFlag{
+		Name:  "skip",
+		Usage: "Tests names to skip",
+	}
 )
 
 func runTestWithReader(test string, r io.Reader) error {
 	glog.Infoln("runTest", test)
 	var err error
 	switch test {
 	case "bt", "BlockTest", "BlockTests", "BlockChainTest":
-		err = tests.RunBlockTestWithReader(r)
+		err = tests.RunBlockTestWithReader(r, skipTests)
 	case "st", "state", "StateTest", "StateTests":
-		err = tests.RunStateTestWithReader(r)
+		err = tests.RunStateTestWithReader(r, skipTests)
 	case "tx", "TransactionTest", "TransactionTests":
-		err = tests.RunTransactionTestsWithReader(r)
+		err = tests.RunTransactionTestsWithReader(r, skipTests)
 	case "vm", "VMTest", "VMTests":
-		err = tests.RunVmTestWithReader(r)
+		err = tests.RunVmTestWithReader(r, skipTests)
 	default:
 		err = fmt.Errorf("Invalid test type specified: %v", test)
 	}
@@ -174,6 +180,7 @@ func setupApp(c *cli.Context) {
 	flagFile := c.GlobalString(FileFlag.Name)
 	continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name)
 	useStdIn := c.GlobalBool(ReadStdInFlag.Name)
+	skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ")
 
 	if !useStdIn {
 		runSuite(flagTest, flagFile)
@@ -200,6 +207,7 @@ func main() {
 		FileFlag,
 		ContinueOnErrorFlag,
 		ReadStdInFlag,
+		SkipTestsFlag,
 	}
 
 	if err := app.Run(os.Args); err != nil {

diff --git a/tests/block_test.go b/tests/block_test.go
@@ -6,67 +6,67 @@ import (
 )
 
 func TestBcValidBlockTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcUncleTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json"))
+	err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcUncleHeaderValidityTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcInvalidHeaderTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcInvalidRLPTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcRPCAPITests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcForkBlockTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcTotalDifficulty(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcWallet(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}

diff --git a/tests/block_test_util.go b/tests/block_test_util.go
@@ -86,7 +86,7 @@ type btTransaction struct {
 	Value    string
 }
 
-func RunBlockTestWithReader(r io.Reader) error {
+func RunBlockTestWithReader(r io.Reader, skipTests []string) error {
 	btjs := make(map[string]*btJSON)
 	if err := readJson(r, &btjs); err != nil {
 		return err
@@ -97,13 +97,13 @@ func RunBlockTestWithReader(r io.Reader) error {
 		return err
 	}
 
-	if err := runBlockTests(bt); err != nil {
+	if err := runBlockTests(bt, skipTests); err != nil {
 		return err
 	}
 	return nil
 }
 
-func RunBlockTest(file string) error {
+func RunBlockTest(file string, skipTests []string) error {
 	btjs := make(map[string]*btJSON)
 	if err := readJsonFile(file, &btjs); err != nil {
 		return err
@@ -113,15 +113,15 @@ func RunBlockTest(file string) error {
 	if err != nil {
 		return err
 	}
-	if err := runBlockTests(bt); err != nil {
+	if err := runBlockTests(bt, skipTests); err != nil {
 		return err
 	}
 	return nil
 }
 
-func runBlockTests(bt map[string]*BlockTest) error {
-	skipTest := make(map[string]bool, len(BlockSkipTests))
-	for _, name := range BlockSkipTests {
+func runBlockTests(bt map[string]*BlockTest, skipTests []string) error {
+	skipTest := make(map[string]bool, len(skipTests))
+	for _, name := range skipTests {
 		skipTest[name] = true
 	}
 

diff --git a/tests/state_test.go b/tests/state_test.go
@@ -8,84 +8,84 @@ import (
 
 func TestStateSystemOperations(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSystemOperationsTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateExample(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stExample.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStatePreCompiledContracts(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stPreCompiledContracts.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateRecursiveCreate(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stRecursiveCreate.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateSpecial(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSpecialTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateRefund(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stRefundTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateBlockHash(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stBlockHashTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateInitCode(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stInitCodeTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateLog(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stLogTests.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateTransaction(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stTransactionTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestCallCreateCallCode(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stCallCreateCallCodeTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestMemory(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stMemoryTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
@@ -95,7 +95,7 @@ func TestMemoryStress(t *testing.T) {
 		t.Skip()
 	}
 	fn := filepath.Join(stateTestDir, "stMemoryStressTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
@@ -105,29 +105,29 @@ func TestQuadraticComplexity(t *testing.T) {
 		t.Skip()
 	}
 	fn := filepath.Join(stateTestDir, "stQuadraticComplexityTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestSolidity(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSolidityTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestWallet(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stWalletTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateTestsRandom(t *testing.T) {
 	fns, _ := filepath.Glob("./files/StateTests/RandomTests/*")
 	for _, fn := range fns {
-		if err := RunStateTest(fn); err != nil {
+		if err := RunStateTest(fn, StateSkipTests); err != nil {
 			t.Error(err)
 		}
 	}

diff --git a/tests/state_test_util.go b/tests/state_test_util.go
@@ -16,36 +16,36 @@ import (
 	"github.com/ethereum/go-ethereum/logger/glog"
 )
 
-func RunStateTestWithReader(r io.Reader) error {
+func RunStateTestWithReader(r io.Reader, skipTests []string) error {
 	tests := make(map[string]VmTest)
 	if err := readJson(r, &tests); err != nil {
 		return err
 	}
 
-	if err := runStateTests(tests); err != nil {
+	if err := runStateTests(tests, skipTests); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func RunStateTest(p string) error {
+func RunStateTest(p string, skipTests []string) error {
 	tests := make(map[string]VmTest)
 	if err := readJsonFile(p, &tests); err != nil {
 		return err
 	}
 
-	if err := runStateTests(tests); err != nil {
+	if err := runStateTests(tests, skipTests); err != nil {
 		return err
 	}
 
 	return nil
 
 }
 
-func runStateTests(tests map[string]VmTest) error {
-	skipTest := make(map[string]bool, len(StateSkipTests))
-	for _, name := range StateSkipTests {
+func runStateTests(tests map[string]VmTest, skipTests []string) error {
+	skipTest := make(map[string]bool, len(skipTests))
+	for _, name := range skipTests {
 		skipTest[name] = true
 	}
 

diff --git a/tests/transaction_test.go b/tests/transaction_test.go
@@ -6,21 +6,21 @@ import (
 )
 
 func TestTransactions(t *testing.T) {
-	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"))
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"), TransSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestWrongRLPTransactions(t *testing.T) {
-	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"))
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"), TransSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func Test10MBtx(t *testing.T) {
-	err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"))
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"), TransSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}