diff --git a/hack/e2e-node-test.sh b/hack/e2e-node-test.sh index f964f16d5bccc..10b1d311ce8cc 100755 --- a/hack/e2e-node-test.sh +++ b/hack/e2e-node-test.sh @@ -19,6 +19,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh" focus=${FOCUS:-""} skip=${SKIP:-""} +report=${REPORT:-"/tmp/"} ginkgo=$(kube::util::find-binary "ginkgo") if [[ -z "${ginkgo}" ]]; then @@ -27,6 +28,6 @@ if [[ -z "${ginkgo}" ]]; then fi # Provided for backwards compatibility -"${ginkgo}" --focus=$focus --skip=$skip "${KUBE_ROOT}/test/e2e_node/" -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true +"${ginkgo}" --focus=$focus --skip=$skip "${KUBE_ROOT}/test/e2e_node/" --report-dir=${report} -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true exit $? diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 633f20a86e17d..f533e1f80dccb 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -201,6 +201,7 @@ ir-password ir-user jenkins-host jenkins-jobs +junit-file-number k8s-bin-dir k8s-build-output keep-gogoproto @@ -366,6 +367,7 @@ resolv-conf resource-container resource-quota-sync-period resource-version +results-dir retry_time rkt-api-endpoint rkt-path diff --git a/test/e2e_node/container_list.go b/test/e2e_node/container_list.go index f05a18e8b3d2b..d252ae526b930 100644 --- a/test/e2e_node/container_list.go +++ b/test/e2e_node/container_list.go @@ -53,7 +53,7 @@ func PrePullAllImages() error { for _, image := range ImageRegistry { output, err := exec.Command("docker", "pull", image).CombinedOutput() if err != nil { - glog.Warning("Could not pre-pull image %s %v output: %s", image, err, output) + glog.Warningf("Could not pre-pull image %s %v output: %s", image, err, output) return err } } diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index c3a86a4b45f1a..b450db8131829 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -24,27 +24,41 @@ import ( "fmt" "io/ioutil" "math/rand" + "os" "os/exec" + "path" "strings" "testing" "time" "github.com/golang/glog" . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/types" + more_reporters "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" ) var e2es *e2eService var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.") +var junitFileNumber = flag.Int("junit-file-number", 1, "Used to create junit filename - e.g. junit_01.xml.") func TestE2eNode(t *testing.T) { flag.Parse() + rand.Seed(time.Now().UTC().UnixNano()) RegisterFailHandler(Fail) - reporters := []Reporter{&LogReporter{}} + reporters := []Reporter{} + if *reportDir != "" { + // Create the directory if it doesn't already exists + if err := os.MkdirAll(*reportDir, 0755); err != nil { + glog.Errorf("Failed creating report directory: %v", err) + } else { + // Configure a junit reporter to write to the directory + junitFile := fmt.Sprintf("junit_%02d.xml", *junitFileNumber) + junitPath := path.Join(*reportDir, junitFile) + reporters = append(reporters, more_reporters.NewJUnitReporter(junitPath)) + } + } RunSpecsWithDefaultAndCustomReporters(t, "E2eNode Suite", reporters) } @@ -94,40 +108,6 @@ var _ = AfterSuite(func() { glog.Infof("Tests Finished") }) -var _ Reporter = &LogReporter{} - -type LogReporter struct{} - -func (lr *LogReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { - b := &bytes.Buffer{} - b.WriteString("******************************************************\n") - glog.Infof(b.String()) -} - -func (lr *LogReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {} - -func (lr *LogReporter) SpecWillRun(specSummary *types.SpecSummary) {} - -func (lr *LogReporter) SpecDidComplete(specSummary *types.SpecSummary) {} - -func (lr *LogReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {} - -func (lr *LogReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - // Only log the binary output if the suite failed. - b := &bytes.Buffer{} - if e2es != nil && !summary.SuiteSucceeded { - b.WriteString(fmt.Sprintf("Process Log For Failed Suite On %s\n", *nodeName)) - b.WriteString("-------------------------------------------------------------\n") - b.WriteString(fmt.Sprintf("kubelet output:\n%s\n", e2es.kubeletCombinedOut.String())) - b.WriteString("-------------------------------------------------------------\n") - b.WriteString(fmt.Sprintf("apiserver output:\n%s\n", e2es.apiServerCombinedOut.String())) - b.WriteString("-------------------------------------------------------------\n") - b.WriteString(fmt.Sprintf("etcd output:\n%s\n", e2es.etcdCombinedOut.String())) - } - b.WriteString("******************************************************\n") - glog.Infof(b.String()) -} - func maskLocksmithdOnCoreos() { data, err := ioutil.ReadFile("/etc/os-release") if err != nil { diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go index d6c3faa53a4c3..118543db62d8d 100644 --- a/test/e2e_node/e2e_remote.go +++ b/test/e2e_node/e2e_remote.go @@ -28,11 +28,13 @@ import ( "strings" "github.com/golang/glog" + utilerrors "k8s.io/kubernetes/pkg/util/errors" ) var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce") var testTimeoutSeconds = flag.Int("test-timeout", 45*60, "How long (in seconds) to wait for ginkgo tests to complete.") +var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.") var sshOptionsMap map[string]string @@ -119,7 +121,7 @@ func CreateTestArchive() string { } // RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test -func RunRemote(archive string, host string, cleanup bool) (string, error) { +func RunRemote(archive string, host string, cleanup bool, junitFileNumber int) (string, error) { // Create the temp staging directory glog.Infof("Staging test binaries on %s", host) tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31()) @@ -158,17 +160,39 @@ func RunRemote(archive string, host string, cleanup bool) (string, error) { cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), fmt.Sprintf("tar -xzvf ./%s", archiveName), - fmt.Sprintf("timeout -k 30s %ds ./e2e_node.test --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s", *testTimeoutSeconds, cleanup, host), + fmt.Sprintf("timeout -k 30s %ds ./e2e_node.test --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --junit-file-number=%d", *testTimeoutSeconds, cleanup, host, tmp, junitFileNumber), ) glog.Infof("Starting tests on %s", host) output, err := RunSshCommand("ssh", host, "--", "sh", "-c", cmd) + if err != nil { + scpErr := getTestArtifacts(host, tmp) + + // Return both the testing and scp error + if scpErr != nil { + return "", utilerrors.NewAggregate([]error{err, scpErr}) + } return "", err } + err = getTestArtifacts(host, tmp) return output, nil } +func getTestArtifacts(host, testDir string) error { + _, err := RunSshCommand("scp", "-r", fmt.Sprintf("%s:%s/results/", host, testDir), fmt.Sprintf("%s/%s", *resultsDir, host)) + if err != nil { + return err + } + + // Copy junit to the top of artifacts + _, err = RunSshCommand("scp", fmt.Sprintf("%s:%s/results/junit*", host, testDir), fmt.Sprintf("%s/", *resultsDir)) + if err != nil { + return err + } + return nil +} + // getSshCommand handles proper quoting so that multiple commands are executed in the same shell over ssh func getSshCommand(sep string, args ...string) string { return fmt.Sprintf("'%s'", strings.Join(args, sep)) diff --git a/test/e2e_node/e2e_service.go b/test/e2e_node/e2e_service.go index de7a7bcdf915b..8b8f629730a97 100644 --- a/test/e2e_node/e2e_service.go +++ b/test/e2e_node/e2e_service.go @@ -17,13 +17,13 @@ limitations under the License. package e2e_node import ( - "bytes" "flag" "fmt" "io/ioutil" "net/http" "os" "os/exec" + "path" "strings" "time" @@ -31,17 +31,15 @@ import ( ) var serverStartTimeout = flag.Duration("server-start-timeout", time.Second*120, "Time to wait for each server to become healthy.") +var reportDir = flag.String("report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") type e2eService struct { - etcdCmd *exec.Cmd - etcdCombinedOut bytes.Buffer - etcdDataDir string - apiServerCmd *exec.Cmd - apiServerCombinedOut bytes.Buffer - kubeletCmd *exec.Cmd - kubeletCombinedOut bytes.Buffer - kubeletStaticPodDir string - nodeName string + etcdCmd *exec.Cmd + etcdDataDir string + apiServerCmd *exec.Cmd + kubeletCmd *exec.Cmd + kubeletStaticPodDir string + nodeName string } func newE2eService(nodeName string) *e2eService { @@ -123,23 +121,23 @@ func (es *e2eService) startEtcd() (*exec.Cmd, error) { hcc := newHealthCheckCommand( "http://127.0.0.1:4001/v2/keys/", // Trailing slash is required, cmd, - &es.etcdCombinedOut) + "etcd.log") return cmd, es.startServer(hcc) } func (es *e2eService) startApiServer() (*exec.Cmd, error) { cmd := exec.Command("sudo", getApiServerBin(), - "--v", "2", "--logtostderr", "--log_dir", "./", "--etcd-servers", "http://127.0.0.1:4001", "--insecure-bind-address", "0.0.0.0", "--service-cluster-ip-range", "10.0.0.1/24", "--kubelet-port", "10250", "--allow-privileged", "true", + "--v", "8", "--logtostderr", ) hcc := newHealthCheckCommand( "http://127.0.0.1:8080/healthz", cmd, - &es.apiServerCombinedOut) + "kube-apiserver.log") return cmd, es.startServer(hcc) } @@ -150,7 +148,6 @@ func (es *e2eService) startKubeletServer() (*exec.Cmd, error) { } es.kubeletStaticPodDir = dataDir cmd := exec.Command("sudo", getKubeletServerBin(), - "--v", "2", "--logtostderr", "--log_dir", "./", "--api-servers", "http://127.0.0.1:8080", "--address", "0.0.0.0", "--port", "10250", @@ -160,22 +157,40 @@ func (es *e2eService) startKubeletServer() (*exec.Cmd, error) { "--serialize-image-pulls", "false", "--config", es.kubeletStaticPodDir, "--file-check-frequency", "10s", // Check file frequently so tests won't wait too long + "--v", "8", "--logtostderr", ) hcc := newHealthCheckCommand( "http://127.0.0.1:10255/healthz", cmd, - &es.kubeletCombinedOut) + "kubelet.log") return cmd, es.startServer(hcc) } func (es *e2eService) startServer(cmd *healthCheckCommand) error { cmdErrorChan := make(chan error) go func() { - err := cmd.Run() + defer close(cmdErrorChan) + + // Create the output filename + outPath := path.Join(*reportDir, cmd.outputFilename) + outfile, err := os.Create(outPath) + if err != nil { + cmdErrorChan <- fmt.Errorf("Failed to create file %s for `%s` %v.", outPath, cmd, err) + return + } + defer outfile.Close() + defer outfile.Sync() + + // Set the command to write the output file + cmd.Cmd.Stdout = outfile + cmd.Cmd.Stderr = outfile + + // Run the command + err = cmd.Run() if err != nil { - cmdErrorChan <- fmt.Errorf("%s Failed with error \"%v\". Command output:\n%s", cmd, err, cmd.OutputBuffer) + cmdErrorChan <- fmt.Errorf("%s Failed with error \"%v\". Output written to: %s", cmd, err, outPath) + return } - close(cmdErrorChan) }() endTime := time.Now().Add(*serverStartTimeout) @@ -196,16 +211,14 @@ func (es *e2eService) startServer(cmd *healthCheckCommand) error { type healthCheckCommand struct { *exec.Cmd HealthCheckUrl string - OutputBuffer *bytes.Buffer + outputFilename string } -func newHealthCheckCommand(healthCheckUrl string, cmd *exec.Cmd, combinedOutput *bytes.Buffer) *healthCheckCommand { - cmd.Stdout = combinedOutput - cmd.Stderr = combinedOutput +func newHealthCheckCommand(healthCheckUrl string, cmd *exec.Cmd, filename string) *healthCheckCommand { return &healthCheckCommand{ HealthCheckUrl: healthCheckUrl, Cmd: cmd, - OutputBuffer: combinedOutput, + outputFilename: filename, } } diff --git a/test/e2e_node/jenkins/copy-e2e-image.sh b/test/e2e_node/jenkins/copy-e2e-image.sh new file mode 100644 index 0000000000000..481d4d7636a3a --- /dev/null +++ b/test/e2e_node/jenkins/copy-e2e-image.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: copy-e2e-image.sh + +set -e +set -x + +echo "Copying image $1 from project $2 to project $3..." +gcloud compute --project $3 disks create $1 --image=https://www.googleapis.com/compute/v1/projects/$2/global/images/$1 +gcloud compute --project $3 images create $1 --source-disk=$1 +gcloud compute --project $3 disks delete $1 diff --git a/test/e2e_node/jenkins/e2e-node-jenkins.sh b/test/e2e_node/jenkins/e2e-node-jenkins.sh index 9af3a89e0bed1..5f321385606f2 100755 --- a/test/e2e_node/jenkins/e2e-node-jenkins.sh +++ b/test/e2e_node/jenkins/e2e-node-jenkins.sh @@ -35,6 +35,9 @@ if [ "$INSTALL_GODEP" = true ] ; then fi go build test/e2e_node/environment/conformance.go +ARTIFACTS=${WORKSPACE}/_artifacts +mkdir -p ${ARTIFACTS} go run test/e2e_node/runner/run_e2e.go --logtostderr --vmodule=*=2 --ssh-env="gce" \ --zone="$GCE_ZONE" --project="$GCE_PROJECT" \ - --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" + --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" \ + --results-dir="$ARTIFACTS" diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index 4c8a06bd9ca52..574ced4fd7208 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -114,16 +114,16 @@ func main() { for _, image := range strings.Split(*images, ",") { running++ fmt.Printf("Initializing e2e tests using image %s.\n", image) - go func(image string) { results <- testImage(image, archive) }(image) + go func(image string, junitFileNum int) { results <- testImage(image, archive, junitFileNum) }(image, running) } } if *hosts != "" { for _, host := range strings.Split(*hosts, ",") { fmt.Printf("Initializing e2e tests using host %s.\n", host) running++ - go func(host string) { - results <- testHost(host, archive, *cleanup) - }(host) + go func(host string, junitFileNum int) { + results <- testHost(host, archive, *cleanup, junitFileNum) + }(host, running) } } @@ -150,8 +150,8 @@ func main() { } // Run tests in archive against host -func testHost(host, archive string, deleteFiles bool) *TestResult { - output, err := e2e_node.RunRemote(archive, host, deleteFiles) +func testHost(host, archive string, deleteFiles bool, junitFileNum int) *TestResult { + output, err := e2e_node.RunRemote(archive, host, deleteFiles, junitFileNum) return &TestResult{ output: output, err: err, @@ -161,7 +161,7 @@ func testHost(host, archive string, deleteFiles bool) *TestResult { // Provision a gce instance using image and run the tests in archive against the instance. // Delete the instance afterward. -func testImage(image, archive string) *TestResult { +func testImage(image, archive string, junitFileNum int) *TestResult { host, err := createInstance(image) if *cleanup { defer deleteInstance(image) @@ -171,7 +171,7 @@ func testImage(image, archive string) *TestResult { err: fmt.Errorf("Unable to create gce instance with running docker daemon for image %s. %v", image, err), } } - return testHost(host, archive, false) + return testHost(host, archive, false, junitFileNum) } // Provision a gce instance using image