Add End-to-end test bed (#54)

Problem
=======

We need a way to run E2E tests with real agent and a load generator. Particularly
important class of E2E tests are performance tests in controlled environment.

Solution
========

The test bed allows to easily set up a test that requires running the agent
and a load generator, measure and define resource consumption expectations
for the agent, fail tests automatically when expectations are exceeded.

Each test case requires a agent configuration file and (optionally) load
generator spec file. Test cases are defined as regular Go tests, see examples
in perf_test.go.

To run E2E tests go to testbed directory and run "make runtests" or run "make perf-tests"
from top-level directory.

Note that E2E tests are skipped when running regular "make test" target. The reason
is that because E2E tests take a lot of time and require non-concurrent execution
while other tests are typical run with -race flag.

See also design doc here:
https://docs.google.com/document/d/1omU06mBYGY0slT1yojttn9BCyp18pHaRjkkYZrY8H4Q/edit#

TODO
=====

We plan to add the following functionality later:

- Ability for load generator to record what it generated and then use that
  information to validate data received by mock backend from the agent.

- Enforce resource consumption limits on the agent to allow testing scenarios under
  specific available resource situations (as opposed to measuring and failing tests
  which is already implemented and which is useful in different scenarios).

Makefile

@@ -1,6 +1,7 @@
 # More exclusions can be added similar with: -not -path './vendor/*'
 ALL_SRC := $(shell find . -name '*.go' \
                                 -not -path './vendor/*' \
+                                -not -path './testbed/*' \
                                 -not -path './tools/*' \
                                 -type f | sort)
 
@@ -34,6 +35,10 @@ all-srcs:
 .PHONY: fmt-vet-lint-test
 fmt-vet-lint-test: fmt vet lint test
 
+.PHONY: perf-test
+perf-test: otelsvc
+	$(MAKE) -C testbed runtests
+
 .PHONY: test
 test:
 	$(GOTEST) $(GOTEST_OPT) $(ALL_PKGS)

go.sum

@@ -191,6 +191,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3 h1:/UewZcckqhvnnS0C6r3Sher2hSEbVmM6Ogpcjen08+Y=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v0.0.0-20150905172533-109e267447e9/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=

testbed/Makefile

@@ -0,0 +1,52 @@
+GOTEST_OPT?=-v -race -timeout 30s
+GOTEST_OPT_WITH_COVERAGE = $(GOTEST_OPT) -coverprofile=coverage.txt -covermode=atomic
+GOTEST=go test
+GOFMT=gofmt
+GOLINT=golint
+GOVET=go vet
+GOOS=$(shell go env GOOS)
+
+.DEFAULT_GOAL := fmt-vet-lint-test
+
+.PHONY: fmt-vet-lint-test
+fmt-vet-lint-test: fmt vet lint test
+
+.PHONY: test
+test:
+	$(GOTEST) $(GOTEST_OPT) ./...
+
+.PHONY: runtests
+runtests: test
+	cd tests && TESTBED_CONFIG=local.yaml go test -v 2>&1 | tee results/testoutput.log
+	cd tests && mkdir -p results/junit && go-junit-report < results/testoutput.log > results/junit/results.xml
+
+.PHONY: fmt
+fmt:
+	@FMTOUT=`$(GOFMT) -s -l ./.. 2>&1`; \
+	if [ "$$FMTOUT" ]; then \
+		echo "$(GOFMT) FAILED => gofmt the following files:\n"; \
+		echo "$$FMTOUT\n"; \
+		exit 1; \
+	else \
+	    echo "Fmt finished successfully"; \
+	fi
+
+.PHONY: lint
+lint:
+	@LINTOUT=`$(GOLINT) $(ALL_PKGS) 2>&1`; \
+	if [ "$$LINTOUT" ]; then \
+		echo "$(GOLINT) FAILED => clean the following lint errors:\n"; \
+		echo "$$LINTOUT\n"; \
+		exit 1; \
+	else \
+	    echo "Lint finished successfully"; \
+	fi
+
+.PHONY: vet
+vet:
+	$(GOVET) ./...
+
+.PHONY: install-tools
+install-tools:
+	go install golang.org/x/lint/golint
+	go install github.com/jstemmer/go-junit-report

testbed/README.md

@@ -0,0 +1,3 @@
+# OpenTelemetry Service Testbed
+
+Testbed is a controlled environment and tools for conducting performance tests for the Agent, including reproducible short-term benchmarks,long-running stability tests and maximum load stress tests.

testbed/go.mod

@@ -0,0 +1,13 @@
+module github.com/open-telemetry/opentelemetry-service/testbed
+
+go 1.12
+
+require (
+	contrib.go.opencensus.io/exporter/jaeger v0.1.1-0.20190430175949-e8b55949d948
+	github.com/open-telemetry/opentelemetry-service v0.0.0-20190625135304-4bd705a25a35
+	github.com/shirou/gopsutil v2.18.12+incompatible
+	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
+	github.com/spf13/viper v1.4.0
+	github.com/stretchr/testify v1.3.0
+	go.opencensus.io v0.22.0
+)