Implementation of the FlowAggregator (#1671)

- Add the build system, scripts for manifest generation and corresponding workflow changes for Flow Aggregator. - The main logic implementation of the flow aggregator that uses the go-ipfix library v0.4.2 with required unit tests. - Agent side changes in Flow Exporter for integration with Flow Aggregator using DNS name resolution. - Add e2e tests for flow aggregator and remove flow exporter tests. Co-authored-by: dyongming@vmware.com Co-authored-by: zyiou@vmware.com Co-authored-by: stati@vmware.com
antrea-io · Dec 23, 2020 · ba78f3a · ba78f3a
1 parent da4b6a0
commit ba78f3a
Show file tree

Hide file tree

Showing 54 changed files with 2,494 additions and 636 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -96,3 +96,20 @@ jobs:
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         make push
         make push-release
+
+  build-flow-aggregator:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-18.04]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build flow-aggregator Docker image
+      run: make flow-aggregator-ubuntu
+    - name: Push flow-aggregator Docker image to registry
+      if: ${{ github.repository == 'vmware-tanzu/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/flow-aggregator:latest
diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
@@ -49,3 +49,17 @@ jobs:
           VERSION="${TAG:10}" make octant-antrea-ubuntu
           echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
           docker push antrea/octant-antrea-ubuntu:"${TAG:10}"
+
+  build-flow-aggregator:
+    runs-on: [ubuntu-18.04]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build flow-aggregator Docker image and push to registry
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        TAG: ${{ github.ref }}
+      run: |
+        VERSION="${TAG:10}" make flow-aggregator-ubuntu
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/flow-aggregator:"${TAG:10}"
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -44,9 +44,26 @@ jobs:
         path: antrea-ubuntu.tar
         retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
 
+  build-flow-aggregator-image:
+    name: Build Flow Aggregator image to be used for Kind e2e tests
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
+    runs-on: [ ubuntu-18.04 ]
+    steps:
+    - uses: actions/checkout@v2
+    - run: make flow-aggregator-ubuntu
+    - name: Save Flow Aggregator image to tarball
+      run: docker save -o flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator:latest
+    - name: Upload Flow Aggregator image for subsequent jobs
+      uses: actions/upload-artifact@v2
+      with:
+        name: flow-aggregator
+        path: flow-aggregator.tar
+        retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
+
   test-e2e-encap:
     name: E2e tests on a Kind cluster on Linux
-    needs: build-antrea-coverage-image
+    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
     runs-on: [ubuntu-18.04]
     steps:
     - name: Free disk space
@@ -62,8 +79,16 @@ jobs:
       uses: actions/download-artifact@v1
       with:
         name: antrea-ubuntu-cov
+      # TODO: Create path to two image artifacts when uploading artifacts, so multiple
+      # artifacts can be downloaded at once.
+    - name: Download Flow Aggregator image from previous job
+      uses: actions/download-artifact@v1
+      with:
+        name: flow-aggregator
     - name: Load Antrea image
-      run:  docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+      run:  |
+        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+        docker load -i flow-aggregator/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -103,7 +128,7 @@ jobs:
 
   test-e2e-encap-no-proxy:
     name: E2e tests on a Kind cluster on Linux with AntreaProxy disabled
-    needs: build-antrea-coverage-image
+    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
     runs-on: [ubuntu-18.04]
     steps:
     - name: Free disk space
@@ -119,8 +144,14 @@ jobs:
       uses: actions/download-artifact@v1
       with:
         name: antrea-ubuntu-cov
+    - name: Download Flow Aggregator image from previous job
+      uses: actions/download-artifact@v1
+      with:
+        name: flow-aggregator
     - name: Load Antrea image
-      run:  docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+      run: |
+        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+        docker load -i flow-aggregator/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -160,7 +191,7 @@ jobs:
 
   test-e2e-noencap:
     name: E2e tests on a Kind cluster on Linux (noEncap)
-    needs: build-antrea-coverage-image
+    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
     runs-on: [ubuntu-18.04]
     steps:
     - name: Free disk space
@@ -176,8 +207,14 @@ jobs:
       uses: actions/download-artifact@v1
       with:
         name: antrea-ubuntu-cov
+    - name: Download Flow Aggregator image from previous job
+      uses: actions/download-artifact@v1
+      with:
+        name: flow-aggregator
     - name: Load Antrea image
-      run:  docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+      run: |
+        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+        docker load -i flow-aggregator/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -217,7 +254,7 @@ jobs:
 
   test-e2e-hybrid:
     name: E2e tests on a Kind cluster on Linux (hybrid)
-    needs: build-antrea-coverage-image
+    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
     runs-on: [ubuntu-18.04]
     steps:
     - name: Free disk space
@@ -233,8 +270,14 @@ jobs:
       uses: actions/download-artifact@v1
       with:
         name: antrea-ubuntu-cov
+    - name: Download Flow Aggregator image from previous job
+      uses: actions/download-artifact@v1
+      with:
+        name: flow-aggregator
     - name: Load Antrea image
-      run:  docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+      run: |
+        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+        docker load -i flow-aggregator/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -274,7 +317,7 @@ jobs:
 
   test-e2e-encap-np:
     name: E2e tests on a Kind cluster on Linux with Antrea NetworkPolicies enabled
-    needs: build-antrea-coverage-image
+    needs: [build-antrea-coverage-image, build-flow-aggregator-image]
     runs-on: [ubuntu-18.04]
     steps:
     - name: Free disk space
@@ -290,8 +333,14 @@ jobs:
       uses: actions/download-artifact@v1
       with:
         name: antrea-ubuntu-cov
+    - name: Download Flow Aggregator image from previous job
+      uses: actions/download-artifact@v1
+      with:
+        name: flow-aggregator
     - name: Load Antrea image
-      run:  docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+      run: |
+        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
+        docker load -i flow-aggregator/flow-aggregator.tar
     - name: Install Kind
       run: |
         curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -413,7 +462,7 @@ jobs:
   # yet.
   artifact-cleanup:
     name: Delete uploaded images
-    needs: [build-antrea-coverage-image, build-antrea-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-np, test-netpol-tmp, validate-prometheus-metrics-doc]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-image, build-antrea-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-np, test-netpol-tmp, validate-prometheus-metrics-doc]
     if: ${{ always() }}
     runs-on: [ubuntu-18.04]
     steps:
@@ -422,6 +471,11 @@ jobs:
       uses: geekyeggo/delete-artifact@v1
       with:
         name: antrea-ubuntu-cov
+    - name: Delete flow-aggregator
+      if: ${{ needs.build-flow-aggregator-image.result == 'success' }}
+      uses: geekyeggo/delete-artifact@v1
+      with:
+        name: flow-aggregator
     - name: Delete antrea-ubuntu
       if: ${{ needs.build-antrea-image.result == 'success' }}
       uses: geekyeggo/delete-artifact@v1

diff --git a/.github/workflows/upload_release_assets.yml b/.github/workflows/upload_release_assets.yml
@@ -178,6 +178,15 @@ jobs:
         asset_path: ./assets/antrea-windows.yml
         asset_name: antrea-windows.yml
         asset_content_type: application/octet-stream
+    - name: Upload flow-aggregator.yml
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: ./assets/flow-aggregator.yml
+        asset_name: flow-aggregator.yml
+        asset_content_type: application/octet-stream
     - name: Upload antrea-agent-windows-x86_64.exe
       uses: actions/upload-release-asset@v1
       env:

diff --git a/Makefile b/Makefile
@@ -70,6 +70,11 @@ windows-bin:
 	GOOS=windows $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' github.com/vmware-tanzu/antrea/cmd/antrea-cni \
 		github.com/vmware-tanzu/antrea/cmd/antrea-agent
 
+.PHONY: flow-aggregator
+flow-aggregator:
+	@mkdir -p $(BINDIR)
+	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' github.com/vmware-tanzu/antrea/cmd/flow-aggregator
+
 .PHONY: test-unit test-integration
 ifeq ($(UNAME_S),Linux)
 test-unit: .linux-test-unit
@@ -300,6 +305,7 @@ manifest:
 	$(CURDIR)/hack/generate-manifest.sh --mode dev --cloud AKS --encap-mode networkPolicyOnly > build/yamls/antrea-aks.yml
 	$(CURDIR)/hack/generate-manifest-octant.sh --mode dev > build/yamls/antrea-octant.yml
 	$(CURDIR)/hack/generate-manifest-windows.sh --mode dev > build/yamls/antrea-windows.yml
+	$(CURDIR)/hack/generate-manifest-flow-aggregator.sh --mode dev > build/yamls/flow-aggregator.yml
 
 .PHONY: manifest-coverage
 manifest-coverage:
@@ -314,6 +320,18 @@ octant-antrea-ubuntu:
 	docker tag antrea/octant-antrea-ubuntu:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/octant-antrea-ubuntu
 	docker tag antrea/octant-antrea-ubuntu:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/octant-antrea-ubuntu:$(DOCKER_IMG_VERSION)
 
+.PHONY: flow-aggregator-ubuntu
+flow-aggregator-ubuntu:
+	@echo "===> Building antrea/flow-aggregator Docker image <==="
+ifneq ($(DOCKER_REGISTRY),"")
+	docker build -t antrea/flow-aggregator:$(DOCKER_IMG_VERSION) -f build/images/flow-aggregator/Dockerfile .
+else
+	docker build --pull -t antrea/flow-aggregator:$(DOCKER_IMG_VERSION) -f build/images/flow-aggregator/Dockerfile .
+endif
+	docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) antrea/flow-aggregator
+	docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-aggregator
+	docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-aggregator:$(DOCKER_IMG_VERSION)
+
 .PHONY: verify
 verify:
 	@echo "===> Verifying spellings <==="

diff --git a/build/images/flow-aggregator/Dockerfile b/build/images/flow-aggregator/Dockerfile
@@ -0,0 +1,16 @@
+FROM golang:1.15 as flow-aggregator-build
+
+WORKDIR /antrea
+
+COPY . /antrea
+
+RUN make flow-aggregator
+
+FROM antrea/base-ubuntu:2.14.0
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="The docker image for the flow aggregator"
+
+USER root
+
+COPY --from=flow-aggregator-build /antrea/bin/flow-aggregator /usr/local/bin/
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -1241,11 +1241,15 @@ data:
     # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
     #enablePrometheusMetrics: true
 
-    # Provide flow collector address as string with format <IP>:<port>[:<proto>], where proto is tcp or udp.
-    # IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with [].
-    # This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge.
-    # If no L4 transport proto is given, we consider tcp as default.
-    #flowCollectorAddr: ""
+    # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
+    # HOST can either be the DNS name or the IP of the Flow Collector. For example,
+    # "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect
+    # to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6.
+    # However, IPv6 address should be wrapped with [].
+    # If PORT is empty, we default to 4739, the standard IPFIX port.
+    # If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp"
+    # L4 transport protocols.
+    #flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp"
 
     # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module.
     # Flow poll interval should be greater than or equal to 1s (one second).
@@ -1312,7 +1316,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mdmtkcfh59
+  name: antrea-config-gm7dcbm584
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1423,7 +1427,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mdmtkcfh59
+          name: antrea-config-gm7dcbm584
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1645,6 +1649,7 @@ spec:
         - mountPath: /var/log/openvswitch
           name: host-var-log-antrea
           subPath: openvswitch
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       initContainers:
       - command:
@@ -1687,7 +1692,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mdmtkcfh59
+          name: antrea-config-gm7dcbm584
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -1241,11 +1241,15 @@ data:
     # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
     #enablePrometheusMetrics: true
 
-    # Provide flow collector address as string with format <IP>:<port>[:<proto>], where proto is tcp or udp.
-    # IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with [].
-    # This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge.
-    # If no L4 transport proto is given, we consider tcp as default.
-    #flowCollectorAddr: ""
+    # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
+    # HOST can either be the DNS name or the IP of the Flow Collector. For example,
+    # "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect
+    # to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6.
+    # However, IPv6 address should be wrapped with [].
+    # If PORT is empty, we default to 4739, the standard IPFIX port.
+    # If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp"
+    # L4 transport protocols.
+    #flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp"
 
     # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module.
     # Flow poll interval should be greater than or equal to 1s (one second).
@@ -1312,7 +1316,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mdmtkcfh59
+  name: antrea-config-gm7dcbm584
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1423,7 +1427,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mdmtkcfh59
+          name: antrea-config-gm7dcbm584
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1647,6 +1651,7 @@ spec:
         - mountPath: /var/log/openvswitch
           name: host-var-log-antrea
           subPath: openvswitch
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       initContainers:
       - command:
@@ -1689,7 +1694,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mdmtkcfh59
+          name: antrea-config-gm7dcbm584
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d