Tooling for integration tests (#321)

This adds make and circleci config needed for running separate integration tests. These tests are intended to still be run from go test so that they can output coverage data but they may be longer running and talk to external services like docker.
open-telemetry · Jun 17, 2020 · cb734dc · cb734dc
1 parent df7ec20
commit cb734dc
Show file tree

Hide file tree

Showing 6 changed files with 135 additions and 44 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,10 +1,10 @@
 version: 2.1
 parameters:
   run-build-publish:
-    type: boolean 
-    default: true 
+    type: boolean
+    default: true
   run-stability-tests:
-    type: boolean 
+    type: boolean
     default: false
   collector-sha:
     type: string
@@ -14,8 +14,38 @@ executors:
   golang:
     docker:
       - image: cimg/go:1.14
+  machine:
+    machine:
+      image: ubuntu-1604:201903-01
 
 commands:
+  setup:
+    steps:
+      - checkout
+      - restore_module_cache
+      - run:
+          name: Install deps
+          command: make -j8 for-all-target TARGET=dep
+      - run:
+          name: Install tools
+          command: make install-tools
+      - run:
+          name: Install testbed tools
+          command: make -C testbed install-tools
+      - save_module_cache
+
+  setup_go:
+    steps:
+      - run:
+          name: Install Go 1.14
+          command: |
+            sudo rm -rf /usr/local/go
+            curl -L https://dl.google.com/go/go1.14.4.linux-amd64.tar.gz | sudo tar xz -C /usr/local
+      - run:
+          name: Add ~/go/bin to PATH
+          command: |
+            echo 'export PATH=$HOME/go/bin:$PATH' >> $BASH_ENV
+
   restore_workspace:
     steps:
       - attach_to_workspace
@@ -81,7 +111,7 @@ workflows:
             - build
       - publish-dev:
           requires:
-            - run-stability-tests 
+            - run-stability-tests
 
   build-publish:
     when: << pipeline.parameters.run-build-publish >>
@@ -120,7 +150,7 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - test:
+      - unit-tests:
           requires:
             - setup
           filters:
@@ -129,7 +159,8 @@ workflows:
       - publish-stable:
           requires:
             - lint
-            - test
+            - unit-tests
+            - integration-tests
             - build
             - cross-compile
           filters:
@@ -140,30 +171,24 @@ workflows:
       - spawn-stability-tests-job:
           requires:
             - lint
-            - test
+            - unit-tests
+            - integration-tests
             - build
           filters:
             branches:
               only: /master|release\/.+/
             tags:
               ignore: /.*/
+      - integration-tests:
+          filters:
+            tags:
+              only: /.*/
 
 jobs:
   setup:
     executor: golang
     steps:
-      - checkout
-      - restore_module_cache
-      - run:
-          name: Install deps
-          command: make -j8 for-all-target TARGET='dep'
-      - run:
-          name: Install tools
-          command: make install-tools
-      - run:
-          name: Install testbed tools
-          command: make -C testbed install-tools
-      - save_module_cache
+      - setup
       - persist_to_workspace:
           root: ~/
           paths:
@@ -175,7 +200,7 @@ jobs:
       - restore_workspace
       - run:
           name: Lint
-          command: make -j8 for-all-target TARGET="lint"
+          command: make -j8 for-all-target TARGET=lint
       - run:
           name: Checks
           command: make -j4 checklicense impi misspell
@@ -213,16 +238,16 @@ jobs:
           root: ~/
           paths: project/bin
 
-  test:
+  unit-tests:
     executor: golang
     steps:
       - restore_workspace
       - run:
-          name: Coverage tests
-          command: make test-with-cover
+          name: Unit test coverage
+          command: make unit-tests-with-cover
       - run:
-          name: Code coverage
-          command: bash <(curl -s https://codecov.io/bash)
+          name: Upload unit test coverage
+          command: bash <(curl -s https://codecov.io/bash) -F unit
 
   loadtest:
     executor: golang
@@ -247,7 +272,7 @@ jobs:
           repo: opentelemetry-collector-contrib
           tag: ${CIRCLE_TAG:1}
       - run:
-          name: Calculate checksums 
+          name: Calculate checksums
           command: cd bin && shasum -a 256 * > checksums.txt
       - run:
           name: Create Github release and upload artifacts
@@ -256,7 +281,7 @@ jobs:
   publish-dev:
     executor: golang
     steps:
-      - restore_workspace 
+      - restore_workspace
       - setup_remote_docker
       - publish_docker_images:
           repo: opentelemetry-collector-contrib-dev
@@ -286,7 +311,7 @@ jobs:
   run-stability-tests:
     executor: golang
     steps:
-      - restore_workspace 
+      - restore_workspace
       - run:
           name: Run stability tests
           command: make stability-tests
@@ -302,3 +327,24 @@ jobs:
                 "body": "Link to failed job: '"${CIRCLE_BUILD_URL}"'."
                 }'
           when: on_fail
+
+  integration-tests:
+    executor: machine
+    environment:
+      GOPATH: /home/circleci/go
+    steps:
+      - setup_go
+      - setup
+      - run:
+          name: Integration tests with coverage
+          command: |
+            mkdir -p test-results/junit
+            trap "go-junit-report -set-exit-code < test-results/go-integration-tests.out > test-results/junit/results.xml" EXIT
+            make integration-tests-with-cover | tee test-results/go-integration-tests.out
+      - run:
+          name: Upload integration test coverage
+          command: bash <(curl -s https://codecov.io/bash) -F integration
+      - store_test_results:
+          path: test-results/junit
+      - store_artifacts:
+          path: test-results
diff --git a/.codecov.yml b/.codecov.yml
@@ -1,6 +1,8 @@
 codecov:
   notify:
     require_ci_to_pass: yes
+    # wait for unit and integration test builds.
+    after_n_builds: 2
   strict_yaml_branch: master  # only use the latest copy on master branch
 
 coverage:
@@ -15,4 +17,4 @@ coverage:
     patch:
       default:
         enabled: yes
-        target: 95%
+        target: 95%
diff --git a/.gitignore b/.gitignore
@@ -25,4 +25,6 @@ bin/
 
 # Coverage
 coverage.txt
+integration-coverage.txt
 coverage.html
+integration-coverage.html
diff --git a/Makefile b/Makefile
@@ -14,6 +14,12 @@ BUILD_INFO=-ldflags "${BUILD_X1} ${BUILD_X2} ${BUILD_X3}"
 STATIC_CHECK=staticcheck
 OTEL_VERSION=master
 
+# Modules to run integration tests on.
+# XXX: Find a way to automatically populate this. Too slow to run across all modules when there are just a few.
+INTEGRATION_TEST_MODULES := \
+	receiver/redisreceiver \
+	internal/common
+
 .DEFAULT_GOAL := all
 
 .PHONY: all
@@ -23,26 +29,20 @@ all: common otelcontribcol
 e2e-test: otelcontribcol
 	$(MAKE) -C testbed runtests
 
-.PHONY: precommit
-precommit:
-	$(MAKE) gotidy
-	$(MAKE) ci
-
 .PHONY: test-with-cover
-test-with-cover:
+unit-tests-with-cover:
 	@echo Verifying that all packages have test files to count in coverage
 	@scripts/check-test-files.sh $(subst github.com/open-telemetry/opentelemetry-collector-contrib/,./,$(ALL_PKGS))
-	@echo pre-compiling tests
-	set -e; for dir in $(ALL_MODULES); do \
-	  echo "go test ./... + coverage in $${dir}"; \
-	  (cd "$${dir}" && \
-	    $(GOTEST) $(GOTEST_OPT_WITH_COVERAGE) ./... && \
-	 	go tool cover -html=coverage.txt -o coverage.html ); \
-	done
+	@$(MAKE) for-all CMD="make do-unit-tests-with-cover"
+
+.PHONY: integration-tests-with-cover
+integration-tests-with-cover:
+	@echo $(INTEGRATION_TEST_MODULES)
+	@$(MAKE) for-all CMD="make do-integration-tests-with-cover" ALL_MODULES="$(INTEGRATION_TEST_MODULES)"
 
 .PHONY: stability-tests
 stability-tests:
-	@echo Stability tests have not been implemented yet 
+	@echo Stability tests have not been implemented yet
 
 .PHONY: gotidy
 gotidy:
@@ -93,9 +93,10 @@ install-tools:
 	go install github.com/client9/misspell/cmd/misspell
 	go install github.com/golangci/golangci-lint/cmd/golangci-lint
 	go install github.com/google/addlicense
-	go install honnef.co/go/tools/cmd/staticcheck
+	go install github.com/jstemmer/go-junit-report
 	go install github.com/pavius/impi/cmd/impi
 	go install github.com/tcnksm/ghr
+	go install honnef.co/go/tools/cmd/staticcheck
 
 .PHONY: run
 run:

diff --git a/Makefile.Common b/Makefile.Common
@@ -15,6 +15,7 @@ ALL_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort
 
 GOTEST_OPT?= -race -timeout 30s
 GOTEST_OPT_WITH_COVERAGE = $(GOTEST_OPT) -coverprofile=coverage.txt -covermode=atomic
+GOTEST_OPT_WITH_INTEGRATION=$(GOTEST_OPT) -v -tags=integration -run=Integration -coverprofile=integration-coverage.txt -covermode=atomic
 GOTEST=go test
 GOOS=$(shell go env GOOS)
 GOARCH=$(shell go env GOARCH)
@@ -46,6 +47,20 @@ test:
 	    $(GOTEST) ./... ); \
 	done
 
+.PHONY: do-unit-tests-with-cover
+do-unit-tests-with-cover:
+	@echo "running go unit test ./... + coverage in `pwd`"
+	@$(GOTEST) $(GOTEST_OPT_WITH_COVERAGE) ./...
+	go tool cover -html=coverage.txt -o coverage.html
+
+.PHONY: run-integration-tests-with-cover
+do-integration-tests-with-cover:
+	@echo "running go integration test ./... + coverage in `pwd`"
+	@$(GOTEST) $(GOTEST_OPT_WITH_INTEGRATION) ./...
+	@if [ -e integration-coverage.txt ]; then \
+  		go tool cover -html=integration-coverage.txt -o integration-coverage.html; \
+  	fi
+
 .PHONY: benchmark
 benchmark:
 	$(GOTEST) -bench=. -run=notests $(ALL_PKGS)

diff --git a/internal/common/integration_test.go b/internal/common/integration_test.go
@@ -0,0 +1,25 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build integration
+
+package common
+
+import (
+	"testing"
+)
+
+func TestIntegration(t *testing.T) {
+	t.Log("just for exercising integration coverage, remove in next PR")
+}