diff --git a/.dockerignore b/.dockerignore index 5f0a90b4041..ccf74e48c96 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ Godeps/_workspace/pkg Godeps/_workspace/bin +_test diff --git a/.gitignore b/.gitignore index 627ad4ce350..910ab8293ec 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ third_party/acolyte ## vitess.io preview site preview-vitess.io/ + +# test.go output files +_test/ diff --git a/Makefile b/Makefile index 7e10943950c..ec5a3dd695d 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ MAKEFLAGS = -s -.PHONY: all build test clean unit_test unit_test_cover unit_test_race queryservice_test integration_test bson proto site_test site_integration_test docker_bootstrap docker_test +.PHONY: all build test clean unit_test unit_test_cover unit_test_race queryservice_test integration_test bson proto site_test site_integration_test docker_bootstrap docker_test docker_unit_test all: build test @@ -191,3 +191,6 @@ docker_bootstrap: # Example: $ make docker_test flavor=mariadb docker_test: docker/test/run.sh $(flavor) 'make test' + +docker_unit_test: + docker/test/run.sh $(flavor) 'make unit_test' diff --git a/docker/test/run.sh b/docker/test/run.sh index e37ed05db60..e9d4e81eac7 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -2,6 +2,7 @@ flavor=$1 cmd=$2 +args= if [[ -z "$flavor" ]]; then echo "Flavor must be specified as first argument." @@ -20,7 +21,7 @@ fi # To avoid AUFS permission issues, files must allow access by "other" chmod -R o=g * -args="-ti --rm -e USER=vitess -v /dev/log:/dev/log" +args="$args --rm -e USER=vitess -v /dev/log:/dev/log" args="$args -v $PWD:/tmp/src" # Mount in host VTDATAROOT if one exists, since it might be a RAM disk or SSD. @@ -33,15 +34,29 @@ if [[ -n "$VTDATAROOT" ]]; then echo "Mounting host dir $hostdir as VTDATAROOT" args="$args -v $hostdir:/vt/vtdataroot --name=$testid -h $testid" else - args="$args -h test" + testid=test-$$ + args="$args --name=$testid -h $testid" fi # Run tests echo "Running tests in vitess/bootstrap:$flavor image..." -docker run $args vitess/bootstrap:$flavor \ - bash -c "rm -rf * && cp -R /tmp/src/* . && rm -rf Godeps/_workspace/pkg && $cmd" +bashcmd="rm -rf * && cp -R /tmp/src/* . && rm -rf Godeps/_workspace/pkg && $cmd" + +if tty -s; then + # interactive shell + docker run -ti $args vitess/bootstrap:$flavor bash -c "$bashcmd" + exitcode=$? +else + # non-interactive shell (kill child on signal) + trap 'docker rm -f $testid 2>/dev/null' SIGTERM SIGINT + docker run $args vitess/bootstrap:$flavor bash -c "$bashcmd" & + wait $! + exitcode=$? +fi # Clean up host dir mounted VTDATAROOT if [[ -n "$hostdir" ]]; then rm -rf $hostdir fi + +exit $exitcode diff --git a/test.go b/test.go new file mode 100644 index 00000000000..b4e4b6f186d --- /dev/null +++ b/test.go @@ -0,0 +1,164 @@ +// Copyright 2015, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +test.go is a "Go script" for running Vitess tests. It runs each test in its own +Docker container for hermeticity and (potentially) parallelism. If a test fails, +this script will save the output in _test/ and continue with other tests. + +Before using it, you should have Docker 1.5+ installed, and have your user in +the group that lets you run the docker command without sudo. The first time you +run against a given flavor, it may take some time for the corresponding +bootstrap image (vitess/bootstrap:) to be downloaded. + +It is meant to be run from the Vitess root, like so: + ~/src/github.com/youtube/vitess$ go run test.go [args] + +For a list of options, run: + $ go run test.go --help +*/ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "os/signal" + "path" + "strings" + "syscall" + "time" +) + +var ( + flavor = flag.String("flavor", "mariadb", "bootstrap flavor to run against") + retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") + logPass = flag.Bool("log-pass", false, "log test output even if it passes") + timeout = flag.Duration("timeout", 10*time.Minute, "timeout for each test") +) + +// Config is the overall object serialized in test/config.json. +type Config struct { + Tests []Test +} + +// Test is an entry from the test/config.json file. +type Test struct { + Name, File, Args string +} + +// run executes a single try. +func (t Test) run() error { + testCmd := fmt.Sprintf("make build && test/%s %s", t.File, t.Args) + dockerCmd := exec.Command("docker/test/run.sh", *flavor, testCmd) + + // Kill child process if we get a signal. + sigchan := make(chan os.Signal) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + go func() { + if _, ok := <-sigchan; ok { + if dockerCmd.Process != nil { + dockerCmd.Process.Signal(syscall.SIGTERM) + } + log.Fatalf("received signal, quitting") + } + }() + + // Stop the test if it takes too long. + done := make(chan struct{}) + timer := time.NewTimer(*timeout) + defer timer.Stop() + go func() { + select { + case <-done: + case <-timer.C: + t.logf("timeout exceeded") + if dockerCmd.Process != nil { + dockerCmd.Process.Signal(syscall.SIGTERM) + } + } + }() + + output, err := dockerCmd.CombinedOutput() + close(done) + signal.Stop(sigchan) + close(sigchan) + + if err != nil || *logPass { + outFile := path.Join("_test", t.Name+".log") + t.logf("saving test output to %v", outFile) + if dirErr := os.MkdirAll("_test", os.FileMode(0755)); dirErr != nil { + t.logf("Mkdir error: %v", dirErr) + } + if fileErr := ioutil.WriteFile(outFile, output, os.FileMode(0644)); fileErr != nil { + t.logf("WriteFile error: %v", fileErr) + } + } + return err +} + +func (t Test) logf(format string, v ...interface{}) { + log.Printf("%v: %v", t.Name, fmt.Sprintf(format, v...)) +} + +func main() { + flag.Parse() + + // Get test configs. + configData, err := ioutil.ReadFile("test/config.json") + if err != nil { + log.Fatalf("Can't read config file: %v", err) + } + var config Config + if err := json.Unmarshal(configData, &config); err != nil { + log.Fatalf("Can't parse config file: %v", err) + } + + // Keep stats. + failed := 0 + passed := 0 + flaky := 0 + + // Run tests. + for _, test := range config.Tests { + if test.Name == "" { + test.Name = strings.TrimSuffix(test.File, ".py") + } + + for try := 1; ; try++ { + if try > *retryMax { + // Every try failed. + test.logf("retry limit exceeded") + failed++ + break + } + + test.logf("running (try %v/%v)...", try, *retryMax) + start := time.Now() + if err := test.run(); err != nil { + // This try failed. + test.logf("FAILED (try %v/%v): %v", try, *retryMax, err) + continue + } + + if try == 1 { + // Passed on the first try. + test.logf("PASSED in %v", time.Since(start)) + passed++ + } else { + // Passed, but not on the first try. + test.logf("FLAKY (1/%v passed)", try) + flaky++ + } + break + } + } + + // Print stats. + log.Printf("%v PASSED, %v FLAKY, %v FAILED", passed, flaky, failed) +} diff --git a/test/config.json b/test/config.json new file mode 100644 index 00000000000..ad0ac12a9ec --- /dev/null +++ b/test/config.json @@ -0,0 +1,80 @@ +{ + "Tests": [ + { + "Name": "queryservice_vtocc", + "File": "queryservice_test.py", + "Args": "-m -e vtocc" + }, + { + "Name": "queryservice_vttablet", + "File": "queryservice_test.py", + "Args": "-m -e vttablet" + }, + { + "File": "vertical_split.py" + }, + { + "File": "vertical_split_vtgate.py" + }, + { + "File": "schema.py" + }, + { + "File": "keyspace_test.py" + }, + { + "File": "keyrange_test.py" + }, + { + "File": "mysqlctl.py" + }, + { + "File": "sharded.py" + }, + { + "File": "secure.py" + }, + { + "File": "binlog.py" + }, + { + "File": "clone.py" + }, + { + "File": "update_stream.py" + }, + { + "File": "tabletmanager.py" + }, + { + "File": "reparent.py" + }, + { + "File": "vtdb_test.py" + }, + { + "File": "vtgate_utils_test.py" + }, + { + "File": "rowcache_invalidator.py" + }, + { + "File": "vtgatev2_test.py" + }, + { + "File": "zkocc_test.py" + }, + { + "File": "initial_sharding_bytes.py" + }, + { + "File": "initial_sharding.py" + }, + { + "File": "resharding_bytes.py" + }, + { + "File": "resharding.py" + } + ] +}