Add test-runner script for integration tests in Docker.

This is an alternative to 'make integration_test',
with the following advantages:

* Tests run in Docker, so no bootstrap is necessary.
* Tests are hermetic and can run in parallel.
* Test against different flavors just by setting a flag.
* Failing tests are retried to see if they are flaky.
* A failed test will be recorded for later inspection, while the script
  continues to run other tests.
* A test that takes too long will be considered stuck and retried.

There's plenty of room for improvement, but now that we have something
in a more readable language than Makefile, we can iterate.

.dockerignore

@@ -1,2 +1,3 @@
 Godeps/_workspace/pkg
 Godeps/_workspace/bin
+_test

.gitignore

@@ -32,3 +32,6 @@ third_party/acolyte
 
 ## vitess.io preview site
 preview-vitess.io/
+
+# test.go output files
+_test/

Makefile

@@ -4,7 +4,7 @@
 
 MAKEFLAGS = -s
 
-.PHONY: all build test clean unit_test unit_test_cover unit_test_race queryservice_test integration_test bson proto site_test site_integration_test docker_bootstrap docker_test
+.PHONY: all build test clean unit_test unit_test_cover unit_test_race queryservice_test integration_test bson proto site_test site_integration_test docker_bootstrap docker_test docker_unit_test
 
 all: build test
 
@@ -191,3 +191,6 @@ docker_bootstrap:
 # Example: $ make docker_test flavor=mariadb
 docker_test:
 	docker/test/run.sh $(flavor) 'make test'
+
+docker_unit_test:
+	docker/test/run.sh $(flavor) 'make unit_test'

docker/test/run.sh

@@ -2,6 +2,7 @@
 
 flavor=$1
 cmd=$2
+args=
 
 if [[ -z "$flavor" ]]; then
   echo "Flavor must be specified as first argument."
@@ -20,7 +21,7 @@ fi
 # To avoid AUFS permission issues, files must allow access by "other"
 chmod -R o=g *
 
-args="-ti --rm -e USER=vitess -v /dev/log:/dev/log"
+args="$args --rm -e USER=vitess -v /dev/log:/dev/log"
 args="$args -v $PWD:/tmp/src"
 
 # Mount in host VTDATAROOT if one exists, since it might be a RAM disk or SSD.
@@ -33,15 +34,29 @@ if [[ -n "$VTDATAROOT" ]]; then
   echo "Mounting host dir $hostdir as VTDATAROOT"
   args="$args -v $hostdir:/vt/vtdataroot --name=$testid -h $testid"
 else
-  args="$args -h test"
+  testid=test-$$
+  args="$args --name=$testid -h $testid"
 fi
 
 # Run tests
 echo "Running tests in vitess/bootstrap:$flavor image..."
-docker run $args vitess/bootstrap:$flavor \
-  bash -c "rm -rf * && cp -R /tmp/src/* . && rm -rf Godeps/_workspace/pkg && $cmd"
+bashcmd="rm -rf * && cp -R /tmp/src/* . && rm -rf Godeps/_workspace/pkg && $cmd"
+
+if tty -s; then
+  # interactive shell
+  docker run -ti $args vitess/bootstrap:$flavor bash -c "$bashcmd"
+  exitcode=$?
+else
+  # non-interactive shell (kill child on signal)
+  trap 'docker rm -f $testid 2>/dev/null' SIGTERM SIGINT
+  docker run $args vitess/bootstrap:$flavor bash -c "$bashcmd" &
+  wait $!
+  exitcode=$?
+fi
 
 # Clean up host dir mounted VTDATAROOT
 if [[ -n "$hostdir" ]]; then
   rm -rf $hostdir
 fi
+
+exit $exitcode

test.go

@@ -0,0 +1,164 @@
+// Copyright 2015, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+test.go is a "Go script" for running Vitess tests. It runs each test in its own
+Docker container for hermeticity and (potentially) parallelism. If a test fails,
+this script will save the output in _test/ and continue with other tests.
+
+Before using it, you should have Docker 1.5+ installed, and have your user in
+the group that lets you run the docker command without sudo. The first time you
+run against a given flavor, it may take some time for the corresponding
+bootstrap image (vitess/bootstrap:<flavor>) to be downloaded.
+
+It is meant to be run from the Vitess root, like so:
+  ~/src/github.com/youtube/vitess$ go run test.go [args]
+
+For a list of options, run:
+  $ go run test.go --help
+*/
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path"
+	"strings"
+	"syscall"
+	"time"
+)
+
+var (
+	flavor   = flag.String("flavor", "mariadb", "bootstrap flavor to run against")
+	retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests")
+	logPass  = flag.Bool("log-pass", false, "log test output even if it passes")
+	timeout  = flag.Duration("timeout", 10*time.Minute, "timeout for each test")
+)
+
+// Config is the overall object serialized in test/config.json.
+type Config struct {
+	Tests []Test
+}
+
+// Test is an entry from the test/config.json file.
+type Test struct {
+	Name, File, Args string
+}
+
+// run executes a single try.
+func (t Test) run() error {
+	testCmd := fmt.Sprintf("make build && test/%s %s", t.File, t.Args)
+	dockerCmd := exec.Command("docker/test/run.sh", *flavor, testCmd)
+
+	// Kill child process if we get a signal.
+	sigchan := make(chan os.Signal)
+	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		if _, ok := <-sigchan; ok {
+			if dockerCmd.Process != nil {
+				dockerCmd.Process.Signal(syscall.SIGTERM)
+			}
+			log.Fatalf("received signal, quitting")
+		}
+	}()
+
+	// Stop the test if it takes too long.
+	done := make(chan struct{})
+	timer := time.NewTimer(*timeout)
+	defer timer.Stop()
+	go func() {
+		select {
+		case <-done:
+		case <-timer.C:
+			t.logf("timeout exceeded")
+			if dockerCmd.Process != nil {
+				dockerCmd.Process.Signal(syscall.SIGTERM)
+			}
+		}
+	}()
+
+	output, err := dockerCmd.CombinedOutput()
+	close(done)
+	signal.Stop(sigchan)
+	close(sigchan)
+
+	if err != nil || *logPass {
+		outFile := path.Join("_test", t.Name+".log")
+		t.logf("saving test output to %v", outFile)
+		if dirErr := os.MkdirAll("_test", os.FileMode(0755)); dirErr != nil {
+			t.logf("Mkdir error: %v", dirErr)
+		}
+		if fileErr := ioutil.WriteFile(outFile, output, os.FileMode(0644)); fileErr != nil {
+			t.logf("WriteFile error: %v", fileErr)
+		}
+	}
+	return err
+}
+
+func (t Test) logf(format string, v ...interface{}) {
+	log.Printf("%v: %v", t.Name, fmt.Sprintf(format, v...))
+}
+
+func main() {
+	flag.Parse()
+
+	// Get test configs.
+	configData, err := ioutil.ReadFile("test/config.json")
+	if err != nil {
+		log.Fatalf("Can't read config file: %v", err)
+	}
+	var config Config
+	if err := json.Unmarshal(configData, &config); err != nil {
+		log.Fatalf("Can't parse config file: %v", err)
+	}
+
+	// Keep stats.
+	failed := 0
+	passed := 0
+	flaky := 0
+
+	// Run tests.
+	for _, test := range config.Tests {
+		if test.Name == "" {
+			test.Name = strings.TrimSuffix(test.File, ".py")
+		}
+
+		for try := 1; ; try++ {
+			if try > *retryMax {
+				// Every try failed.
+				test.logf("retry limit exceeded")
+				failed++
+				break
+			}
+
+			test.logf("running (try %v/%v)...", try, *retryMax)
+			start := time.Now()
+			if err := test.run(); err != nil {
+				// This try failed.
+				test.logf("FAILED (try %v/%v): %v", try, *retryMax, err)
+				continue
+			}
+
+			if try == 1 {
+				// Passed on the first try.
+				test.logf("PASSED in %v", time.Since(start))
+				passed++
+			} else {
+				// Passed, but not on the first try.
+				test.logf("FLAKY (1/%v passed)", try)
+				flaky++
+			}
+			break
+		}
+	}
+
+	// Print stats.
+	log.Printf("%v PASSED, %v FLAKY, %v FAILED", passed, flaky, failed)
+}

test/config.json

@@ -0,0 +1,80 @@
+{
+  "Tests": [
+    {
+      "Name": "queryservice_vtocc",
+      "File": "queryservice_test.py",
+      "Args": "-m -e vtocc"
+    },
+    {
+      "Name": "queryservice_vttablet",
+      "File": "queryservice_test.py",
+      "Args": "-m -e vttablet"
+    },
+    {
+      "File": "vertical_split.py"
+    },
+    {
+      "File": "vertical_split_vtgate.py"
+    },
+    {
+      "File": "schema.py"
+    },
+    {
+      "File": "keyspace_test.py"
+    },
+    {
+      "File": "keyrange_test.py"
+    },
+    {
+      "File": "mysqlctl.py"
+    },
+    {
+      "File": "sharded.py"
+    },
+    {
+      "File": "secure.py"
+    },
+    {
+      "File": "binlog.py"
+    },
+    {
+      "File": "clone.py"
+    },
+    {
+      "File": "update_stream.py"
+    },
+    {
+      "File": "tabletmanager.py"
+    },
+    {
+      "File": "reparent.py"
+    },
+    {
+      "File": "vtdb_test.py"
+    },
+    {
+      "File": "vtgate_utils_test.py"
+    },
+    {
+      "File": "rowcache_invalidator.py"
+    },
+    {
+      "File": "vtgatev2_test.py"
+    },
+    {
+      "File": "zkocc_test.py"
+    },
+    {
+      "File": "initial_sharding_bytes.py"
+    },
+    {
+      "File": "initial_sharding.py"
+    },
+    {
+      "File": "resharding_bytes.py"
+    },
+    {
+      "File": "resharding.py"
+    }
+  ]
+}