Skip to content

Commit

Permalink
[chore] Re-run failed unit tests automatically (open-telemetry#31253)
Browse files Browse the repository at this point in the history
**Description:** 

Re-runs failed unit tests automatically. Follow up to open-telemetry#31163
This re-runs the tests once if there are less than 10 total test
failures.

This should speed up development, but it comes with the risk of missing
real issues.
I think given the current situation our CI is in this is acceptable, but
I assume this PR is going to be controversial :)

One improvement would be to keep this but auto-generate Github issues
when a test fails and then passes on main's CI.

**Link to tracking Issue:** Relates to open-telemetry#30880 (does not speed up
individual tests but reduces the number of attempts to be made)
  • Loading branch information
mx-psi authored and XinRanZhAWS committed Mar 13, 2024
1 parent b822b02 commit fdc523f
Showing 1 changed file with 8 additions and 6 deletions.
14 changes: 8 additions & 6 deletions Makefile.Common
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ GOVULNCHECK := $(TOOLS_BIN_DIR)/govulncheck
GCI := $(TOOLS_BIN_DIR)/gci
GOTESTSUM := $(TOOLS_BIN_DIR)/gotestsum

GOTESTSUM_OPT?= --rerun-fails=1

# BUILD_TYPE should be one of (dev, release).
BUILD_TYPE?=release

Expand Down Expand Up @@ -121,38 +123,38 @@ common: lint test

.PHONY: test
test: $(GOTESTSUM)
$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT)
$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="./..." -- $(GOTEST_OPT)

.PHONY: test-with-cover
test-with-cover: $(GOTESTSUM)
mkdir -p $(PWD)/coverage/unit
$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT) -cover -covermode=atomic -args -test.gocoverdir="$(PWD)/coverage/unit"
$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="./..." -- $(GOTEST_OPT) -cover -covermode=atomic -args -test.gocoverdir="$(PWD)/coverage/unit"

.PHONY: do-unit-tests-with-cover
do-unit-tests-with-cover: $(GOTESTSUM)
@echo "running $(GOCMD) unit test ./... + coverage in `pwd`"
$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT_WITH_COVERAGE)
$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="./..." -- $(GOTEST_OPT_WITH_COVERAGE)
$(GOCMD) tool cover -html=coverage.txt -o coverage.html

.PHONY: mod-integration-test
mod-integration-test: $(GOTESTSUM)
@echo "running $(GOCMD) integration test ./... in `pwd`"
$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT_WITH_INTEGRATION)
$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="./..." -- $(GOTEST_OPT_WITH_INTEGRATION)
@if [ -e integration-coverage.txt ]; then \
$(GOCMD) tool cover -html=integration-coverage.txt -o integration-coverage.html; \
fi

.PHONY: do-integration-tests-with-cover
do-integration-tests-with-cover: $(GOTESTSUM)
@echo "running $(GOCMD) integration test ./... + coverage in `pwd`"
$(GOTESTSUM) --packages="./..." -- $(GOTEST_OPT_WITH_INTEGRATION_COVERAGE)
$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="./..." -- $(GOTEST_OPT_WITH_INTEGRATION_COVERAGE)
@if [ -e integration-coverage.txt ]; then \
$(GOCMD) tool cover -html=integration-coverage.txt -o integration-coverage.html; \
fi

.PHONY: benchmark
benchmark: $(GOTESTSUM)
$(GOTESTSUM) --packages="$(ALL_PKGS)" -- -bench=. -run=notests --tags=$(GO_BUILD_TAGS)
$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="$(ALL_PKGS)" -- -bench=. -run=notests --tags=$(GO_BUILD_TAGS)

.PHONY: addlicense
addlicense: $(ADDLICENSE)
Expand Down

0 comments on commit fdc523f

Please sign in to comment.