Merge pull request #2391 from Brian-McM/bm-pick-commits

[PICK MULTIPLE] Fix flaky tests

.semaphore/semaphore.yml

@@ -48,6 +48,8 @@ blocks:
   task:
     prologue:
       commands:
+      # Free up space on the build machine.
+      - sudo rm -rf ~/.kiex ~/.phpbrew ~/.rbenv ~/.nvm ~/.kerl
       - checkout
       - cache restore go-pkg-cache
       - cache restore go-mod-cache

Makefile

@@ -94,6 +94,7 @@ FV_NUM_BATCHES?=1
 # (with FV_NUM_BATCHES=1) to check that it's not a flake.
 FV_BATCHES_TO_RUN?=$(shell seq $(FV_NUM_BATCHES))
 FV_SLOW_SPEC_THRESH=90
+FV_RACE_DETECTOR_ENABLED?=true
 
 # Linker flags for building Felix.
 #
@@ -246,8 +247,14 @@ ifeq ($(ARCH),amd64)
 	docker tag $(BUILD_IMAGE):latest-$(ARCH) $(BUILD_IMAGE):latest
 endif
 
-image-test: image fv/Dockerfile.test.amd64 bin/pktgen bin/test-workload bin/test-connection bin/calico-felix-race-amd64 image-wgtool
-	docker build -t $(BUILD_IMAGE)-test:latest-$(ARCH) --build-arg QEMU_IMAGE=$(CALICO_BUILD) --file ./fv/Dockerfile.test.$(ARCH) bin;
+ifeq ($(FV_RACE_DETECTOR_ENABLED),true)
+FV_BINARY=calico-felix-race-amd64
+else
+FV_BINARY=calico-felix-amd64
+endif
+
+image-test: image fv/Dockerfile.test.amd64 bin/pktgen bin/test-workload bin/test-connection bin/$(FV_BINARY) image-wgtool
+	docker build -t $(BUILD_IMAGE)-test:latest-$(ARCH) --build-arg FV_BINARY=$(FV_BINARY) --file ./fv/Dockerfile.test.$(ARCH) bin;
 ifeq ($(ARCH),amd64)
 	docker tag $(BUILD_IMAGE)-test:latest-$(ARCH) $(BUILD_IMAGE)-test:latest
 endif
@@ -405,6 +412,7 @@ fv fv/latency.log fv/data-races.log: $(REMOTE_DEPS) image-test bin/iptables-lock
 	  GINKGO_ARGS='$(GINKGO_ARGS)' \
 	  GINKGO_FOCUS="$(GINKGO_FOCUS)" \
 	  FELIX_FV_ENABLE_BPF="$(FELIX_FV_ENABLE_BPF)" \
+	  FV_RACE_DETECTOR_ENABLED=$(FV_RACE_DETECTOR_ENABLED) \
 	  ./run-batches
 	@if [ -e fv/latency.log ]; then \
 	   echo; \

fv/Dockerfile.test.amd64

@@ -30,5 +30,7 @@ RUN apt-get update && apt-get install -y \
     ethtool \
     tcpdump
 
+ARG FV_BINARY=calico-felix-amd64
+
 ADD test-connection test-workload pktgen  /
-ADD calico-felix-race-amd64 /code/calico-felix
+ADD $FV_BINARY /code/calico-felix

fv/config_update_test.go

@@ -162,6 +162,10 @@ var _ = Context("Config update tests, after starting felix", func() {
 				Eventually(felix.GetFelixPIDs, "5s", "100ms").ShouldNot(ContainElement(felixPID))
 				felixPID = felix.GetSinglePID("calico-felix")
 
+				// Wait for felix to come in to sync; otherwise we may manage to remove the config before
+				// felix loads it.
+				waitForFelixInSync(felix)
+
 				// Then remove the config that we added.
 				ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 				defer cancel()

fv/fv_infra_test.go

@@ -203,9 +203,15 @@ var _ = infrastructure.DatastoreDescribe("Container self tests",
 				options.ExtraEnvVars["FELIX_DebugSimulateDataRace"] = "true"
 			})
 
-			It("should detect a race", func() {
-				Eventually(felixes[0].DataRaces).ShouldNot(BeEmpty())
-			})
+			if os.Getenv("FV_RACE_DETECTOR_ENABLED") == "true" {
+				It("should detect a race", func() {
+					Eventually(felixes[0].DataRaces).ShouldNot(BeEmpty())
+				})
+			} else {
+				It("should not detect a race because race detector is disabled", func() {
+					Consistently(felixes[0].DataRaces).Should(BeEmpty())
+				})
+			}
 		})
 	},
 )