Add code coverage for flow aggregator

Add code coverage for flow aggregator binary when running end-to-end tests on different platforms (kind, jenkins, VMC etc.)
antrea-io · Feb 23, 2021 · 43766ca · 43766ca
1 parent 3de66cc
commit 43766ca
Show file tree

Hide file tree

Showing 8 changed files with 119 additions and 30 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -44,26 +44,26 @@ jobs:
         path: antrea-ubuntu.tar
         retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
 
-  build-flow-aggregator-image:
+  build-flow-aggregator-coverage-image:
     name: Build Flow Aggregator image to be used for Kind e2e tests
     needs: check-changes
     if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
     runs-on: [ ubuntu-latest ]
     steps:
     - uses: actions/checkout@v2
-    - run: make flow-aggregator-ubuntu
+    - run: make flow-aggregator-ubuntu-coverage
     - name: Save Flow Aggregator image to tarball
-      run: docker save -o flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator:latest
+      run: docker save -o flow-aggregator.tar antrea/flow-aggregator-coverage:latest
     - name: Upload Flow Aggregator image for subsequent jobs
       uses: actions/upload-artifact@v2
       with:
-        name: flow-aggregator
+        name: flow-aggregator-cov
         path: flow-aggregator.tar
         retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
 
   test-e2e-encap:
     name: E2e tests on a Kind cluster on Linux
-    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -84,11 +84,11 @@ jobs:
     - name: Download Flow Aggregator image from previous job
       uses: actions/download-artifact@v1
       with:
-        name: flow-aggregator
+        name: flow-aggregator-cov
     - name: Load Antrea image
       run:  |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator/flow-aggregator.tar
+        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -128,7 +128,7 @@ jobs:
 
   test-e2e-encap-no-proxy:
     name: E2e tests on a Kind cluster on Linux with AntreaProxy disabled
-    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -147,11 +147,11 @@ jobs:
     - name: Download Flow Aggregator image from previous job
       uses: actions/download-artifact@v1
       with:
-        name: flow-aggregator
+        name: flow-aggregator-cov
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator/flow-aggregator.tar
+        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -191,7 +191,7 @@ jobs:
 
   test-e2e-noencap:
     name: E2e tests on a Kind cluster on Linux (noEncap)
-    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -210,11 +210,11 @@ jobs:
     - name: Download Flow Aggregator image from previous job
       uses: actions/download-artifact@v1
       with:
-        name: flow-aggregator
+        name: flow-aggregator-cov
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator/flow-aggregator.tar
+        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -254,7 +254,7 @@ jobs:
 
   test-e2e-hybrid:
     name: E2e tests on a Kind cluster on Linux (hybrid)
-    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -273,11 +273,11 @@ jobs:
     - name: Download Flow Aggregator image from previous job
       uses: actions/download-artifact@v1
       with:
-        name: flow-aggregator
+        name: flow-aggregator-cov
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator/flow-aggregator.tar
+        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -317,7 +317,7 @@ jobs:
 
   test-e2e-encap-np:
     name: E2e tests on a Kind cluster on Linux with Antrea NetworkPolicies enabled
-    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
     - name: Free disk space
@@ -336,11 +336,11 @@ jobs:
     - name: Download Flow Aggregator image from previous job
       uses: actions/download-artifact@v1
       with:
-        name: flow-aggregator
+        name: flow-aggregator-cov
     - name: Load Antrea image
       run: |
         docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-        docker load -i flow-aggregator/flow-aggregator.tar
+        docker load -i flow-aggregator-cov/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -462,7 +462,7 @@ jobs:
   # yet.
   artifact-cleanup:
     name: Delete uploaded images
-    needs: [build-antrea-coverage-image, build-flow-aggregator-image, build-antrea-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-np, test-netpol-tmp, validate-prometheus-metrics-doc]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image, build-antrea-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-np, test-netpol-tmp, validate-prometheus-metrics-doc]
     if: ${{ always() }}
     runs-on: [ubuntu-latest]
     steps:
@@ -472,10 +472,10 @@ jobs:
       with:
         name: antrea-ubuntu-cov
     - name: Delete flow-aggregator
-      if: ${{ needs.build-flow-aggregator-image.result == 'success' }}
+      if: ${{ needs.build-flow-aggregator-coverage-image.result == 'success' }}
       uses: geekyeggo/delete-artifact@v1
       with:
-        name: flow-aggregator
+        name: flow-aggregator-cov
     - name: Delete antrea-ubuntu
       if: ${{ needs.build-antrea-image.result == 'success' }}
       uses: geekyeggo/delete-artifact@v1

diff --git a/Makefile b/Makefile
@@ -80,6 +80,11 @@ flow-aggregator:
 	@mkdir -p $(BINDIR)
 	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' github.com/vmware-tanzu/antrea/cmd/flow-aggregator
 
+.PHONY: flow-aggregator-instr-binary
+flow-aggregator-instr-binary:
+	@mkdir -p $(BINDIR)
+	GOOS=linux $(GO) test -tags testbincover -covermode count -coverpkg=github.com/vmware-tanzu/antrea/pkg/... -c -o $(BINDIR)/flow-aggregator-coverage $(GOFLAGS) -ldflags '$(LDFLAGS)' github.com/vmware-tanzu/antrea/cmd/flow-aggregator
+
 .PHONY: test-unit test-integration
 ifeq ($(UNAME_S),Linux)
 test-unit: .linux-test-unit
@@ -332,6 +337,7 @@ manifest-scale:
 manifest-coverage:
 	$(CURDIR)/hack/generate-manifest.sh --mode dev --coverage > build/yamls/antrea-coverage.yml
 	$(CURDIR)/hack/generate-manifest.sh --mode dev --ipsec --coverage > build/yamls/antrea-ipsec-coverage.yml
+	$(CURDIR)/hack/generate-manifest-flow-aggregator.sh --mode dev --coverage > build/yamls/flow-aggregator-coverage.yml
 
 .PHONY: octant-antrea-ubuntu
 octant-antrea-ubuntu:
@@ -353,6 +359,16 @@ endif
 	docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-aggregator
 	docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-aggregator:$(DOCKER_IMG_VERSION)
 
+.PHONY: flow-aggregator-ubuntu-coverage
+flow-aggregator-ubuntu-coverage:
+	@echo "===> Building antrea/flow-aggregator-coverage Docker image <==="
+ifneq ($(DOCKER_REGISTRY),"")
+	docker build -t antrea/flow-aggregator-coverage:$(DOCKER_IMG_VERSION) -f build/images/flow-aggregator/Dockerfile.coverage .
+else
+	docker build --pull -t antrea/flow-aggregator-coverage:$(DOCKER_IMG_VERSION) -f build/images/flow-aggregator/Dockerfile.coverage .
+endif
+	docker tag antrea/flow-aggregator-coverage:$(DOCKER_IMG_VERSION) antrea/flow-aggregator-coverage
+
 .PHONY: verify
 verify:
 	@echo "===> Verifying spellings <==="

diff --git a/build/images/flow-aggregator/Dockerfile.coverage b/build/images/flow-aggregator/Dockerfile.coverage
@@ -0,0 +1,16 @@
+FROM golang:1.15 as flow-aggregator-build
+
+WORKDIR /antrea
+
+COPY . /antrea
+
+RUN make flow-aggregator flow-aggregator-instr-binary
+
+FROM antrea/base-ubuntu:2.14.0
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="The docker image for the flow aggregator with code coverage measurement enabled for testing purposes."
+
+USER root
+
+COPY --from=flow-aggregator-build /antrea/bin/flow-aggregator* /usr/local/bin/
diff --git a/build/yamls/flow-aggregator/patches/coverage/startFlowAggCov.yml b/build/yamls/flow-aggregator/patches/coverage/startFlowAggCov.yml
@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flow-aggregator
+  namespace: flow-aggregator
+spec:
+  template:
+    spec:
+      containers:
+      - args:
+        - -test.run=TestBincoverRunMain
+        - -test.coverprofile=flow-aggregator.cov.out
+        - --config
+        - /etc/flow-aggregator/flow-aggregator.conf
+        - --logtostderr=false
+        - --log_dir=/var/log/flowaggregator
+        - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
+        - --v=0
+        command:
+          - flow-aggregator-coverage
+        name: flow-aggregator
+        image: antrea/flow-aggregator-coverage:latest
diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh
@@ -298,7 +298,11 @@ function deliver_antrea {
             VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make && break
         fi
     done
-    VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make flow-aggregator-ubuntu
+    if [[ "$COVERAGE" == true ]]; then
+      VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make flow-aggregator-ubuntu
+    else
+      VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make flow-aggregator-ubuntu-coverage
+    fi
     cd ci/jenkins
 
     if [ "$?" -ne "0" ]; then
@@ -332,10 +336,12 @@ function deliver_antrea {
 
     if [[ "$COVERAGE" == true ]]; then
         docker save -o antrea-ubuntu-coverage.tar antrea/antrea-ubuntu-coverage:${DOCKER_IMG_VERSION}
+        docker save -o flow-aggregator-coverage.tar antrea/flow-aggregator-coverage:${DOCKER_IMG_VERSION}
     else
         docker save -o antrea-ubuntu.tar projects.registry.vmware.com/antrea/antrea-ubuntu:${DOCKER_IMG_VERSION}
+        docker save -o flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator:${DOCKER_IMG_VERSION}
     fi
-    docker save -o flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator:${DOCKER_IMG_VERSION}
+
 
     kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}' | while read control_plane_ip; do
         scp -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key $GIT_CHECKOUT_DIR/build/yamls/*.yml capv@${control_plane_ip}:~
@@ -347,10 +353,11 @@ function deliver_antrea {
         ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R ${IPs[$i]}
         if [[ "$COVERAGE" == true ]]; then
             copy_image antrea-ubuntu-coverage.tar docker.io/antrea/antrea-ubuntu-coverage ${IPs[$i]} ${DOCKER_IMG_VERSION} true
+            copy_image flow-aggregator-coverage.tar docker.io/antrea/flow-aggregator-coverage ${IPs[$i]} ${DOCKER_IMG_VERSION} true
         else
             copy_image antrea-ubuntu.tar projects.registry.vmware.com/antrea/antrea-ubuntu ${IPs[$i]} ${DOCKER_IMG_VERSION} true
+            copy_image flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator ${IPs[$i]} ${DOCKER_IMG_VERSION} true
         fi
-        copy_image flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator ${IPs[$i]} ${DOCKER_IMG_VERSION} true
     done
 
     if [[ -z $OLD_ANTREA_VERSION ]]; then

diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -108,10 +108,12 @@ done
 if $coverage; then
     manifest_args="$manifest_args --coverage"
     COMMON_IMAGES_LIST+=("antrea/antrea-ubuntu-coverage:latest")
+    COMMON_IMAGES_LIST+=("antrea/flow-aggregator-coverage:latest")
 else
     COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/antrea-ubuntu:latest")
+    COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/flow-aggregator:latest")
 fi
-COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/flow-aggregator:latest")
+
 printf -v COMMON_IMAGES "%s " "${COMMON_IMAGES_LIST[@]}"
 
 function run_test {
@@ -122,11 +124,12 @@ function run_test {
   eval "timeout 600 $TESTBED_CMD create kind --antrea-cni false $args"
 
   if $coverage; then
-      $YML_CMD --kind --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-coverage.yml 
+      $YML_CMD --kind --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-coverage.yml
+      $FLOWAGGREGATOR_YML_CMD --coverage | docker exec -i kind-control-plane dd of=/root/flow-aggregator-coverage.yml
   else
       $YML_CMD --kind --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml
+      $FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml
   fi
-  $FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml
   sleep 1
   if $coverage; then
       go test -v -timeout=40m github.com/vmware-tanzu/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --coverage --coverage-dir $ANTREA_COV_DIR

diff --git a/hack/generate-manifest-flow-aggregator.sh b/hack/generate-manifest-flow-aggregator.sh
@@ -26,6 +26,7 @@ Generate a YAML manifest for flow aggregator, using Kustomize, and print it to s
         --flow-collector      Flow collector is the externalFlowCollectorAddr configMap parameter
                               It should be given in format IP:port:proto. Example: 192.168.1.100:4739:udp
         --keep                Debug flag which will preserve the generated kustomization.yml
+        --coverage            Generates a manifest which supports measuring code coverage of Flow Aggregator binaries.
         --help, -h            Print this message and exit
 
 In 'release' mode, environment variables IMG_NAME and IMG_TAG must be set.
@@ -46,6 +47,8 @@ function print_help {
 MODE="dev"
 KEEP=false
 FLOW_COLLECTOR=""
+COVERAGE=false
+
 while [[ $# -gt 0 ]]
 do
 key="$1"
@@ -63,6 +66,10 @@ case $key in
     KEEP=true
     shift
     ;;
+    --coverage)
+    COVERAGE=true
+    shift
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -128,6 +135,17 @@ $KUSTOMIZE edit add base $BASE
 BASE=../configMap
 cd ..
 
+if $COVERAGE; then
+    mkdir coverage && cd coverage
+    cp $KUSTOMIZATION_DIR/patches/coverage/*.yml .
+    touch kustomization.yml
+    $KUSTOMIZE edit add base $BASE
+    # this runs flow-aggregator via the instrumented binary.
+    $KUSTOMIZE edit add patch --path startFlowAggCov.yml
+    BASE=../coverage
+    cd ..
+fi
+
 mkdir $MODE && cd $MODE
 touch kustomization.yml
 $KUSTOMIZE edit add base $BASE

diff --git a/test/e2e/framework.go b/test/e2e/framework.go
@@ -76,6 +76,7 @@ const (
 	antreaCovYML               string = "antrea-coverage.yml"
 	antreaIPSecCovYML          string = "antrea-ipsec-coverage.yml"
 	flowaggregatorYML          string = "flow-aggregator.yml"
+	flowaggregatorCovYML       string = "flow-aggregator-coverage.yml"
 	defaultBridgeName          string = "br-int"
 	monitoringNamespace        string = "monitoring"
 
@@ -491,9 +492,13 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error {
 
 // deployFlowAggregator deploys flow aggregator with ipfix collector address.
 func (data *TestData) deployFlowAggregator(ipfixCollector string) (string, error) {
-	rc, _, _, err := provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl apply -f %s", flowaggregatorYML))
+	flowAggYaml := flowaggregatorYML
+	if testOptions.enableCoverage {
+		flowAggYaml = flowaggregatorCovYML
+	}
+	rc, _, _, err := provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl apply -f %s", flowAggYaml))
 	if err != nil || rc != 0 {
-		return "", fmt.Errorf("error when deploying flow aggregator; %s not available on the control-plane Node", flowaggregatorYML)
+		return "", fmt.Errorf("error when deploying flow aggregator; %s not available on the control-plane Node", flowAggYaml)
 	}
 	if err = data.mutateFlowAggregatorConfigMap(ipfixCollector); err != nil {
 		return "", err