Skip to content
This repository was archived by the owner on Oct 10, 2023. It is now read-only.

Add antrea-interworking package #4510

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 7 additions & 32 deletions addons/controllers/antrea/antreaconfig_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,13 +202,6 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfigNormal(
return err
}

if antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef != nil && antreaConfig.Spec.AntreaNsx.BootstrapFrom.Inline != nil {
err := fmt.Errorf("providerRef and inline should not be both set in AntreaConfig.spec.antreaNsx.bootstrapFrom")
antreaConfig.Status.Message = err.Error()
} else {
// clear the message here.
antreaConfig.Status.Message = ""
}
// update status.secretRef
dataValueSecretName := util.GenerateDataValueSecretName(cluster.Name, constants.AntreaAddonName)
antreaConfig.Status.SecretRef = dataValueSecretName
Expand All @@ -228,18 +221,10 @@ func getClusterName(antreaConfig *cniv1alpha2.AntreaConfig) (name string, exists
return
}

func (r *AntreaConfigReconciler) getProviderServiceAccountName(clusterName string) string {
return fmt.Sprintf("%s-antrea", clusterName)
}

func (r *AntreaConfigReconciler) getNSXServiceAccountName(clusterName string) string {
return fmt.Sprintf("%s-antrea", clusterName)
}

func (r *AntreaConfigReconciler) ensureNsxServiceAccount(ctx context.Context, antreaConfig *cniv1alpha2.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error {
account := &nsxoperatorapi.NSXServiceAccount{}

account.Name = r.getNSXServiceAccountName(cluster.Name)
account.Name = getNSXServiceAccountName(cluster.Name)
account.Namespace = antreaConfig.Namespace
account.OwnerReferences = []metav1.OwnerReference{
{
Expand Down Expand Up @@ -304,7 +289,7 @@ func (r *AntreaConfigReconciler) ensureProviderServiceAccount(ctx context.Contex
r.Log.Error(err, "Error creating or patching cluster role", "name", vsphereAntreaConfigProviderServiceAccountAggregatedClusterRole)
return err
}
provider.Name = r.getProviderServiceAccountName(clusterName)
provider.Name = getProviderServiceAccountName(clusterName)
provider.Namespace = antreaConfig.Namespace
provider.Spec = vsphere.ProviderServiceAccountSpec{
Ref: &corev1.ObjectReference{
Expand All @@ -329,21 +314,11 @@ func (r *AntreaConfigReconciler) ensureProviderServiceAccount(ctx context.Contex
}

func (r *AntreaConfigReconciler) registerAntreaNSX(ctx context.Context, antreaConfig *cniv1alpha2.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error {
if !antreaConfig.Spec.AntreaNsx.Enable || antreaConfig.Spec.AntreaNsx.BootstrapFrom.Inline != nil {
r.Log.Info("antreaNsx is not enabled or inline is set, there is no ProviderServiceAccount or NsxServiceAccount to be created")
if !antreaConfig.Spec.AntreaNsx.Enable || antreaConfig.Spec.AntreaInterworking.Config.BootstrapFrom == bootstrapFromInline {
r.Log.Info("antrea_nsx is not enabled or inline is set, there is no ProviderServiceAccount or NsxServiceAccount to be created")
r.deregisterAntreaNSX(ctx, antreaConfig, cluster)
return nil
}
if antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef != nil {
if strings.ToLower(antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.Kind) != nsxServiceAccountKind ||
strings.ToLower(antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.ApiGroup) != nsxServiceAccountAPIGroup {
err := fmt.Errorf("either ProviderRef.Kind(%s) or ProviderRef.ApiGroup(%s) is invalid, expcted:ProviderRef.Kind(%s) ProviderRef.ApiGroup(%s)",
antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.Kind, antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.ApiGroup,
nsxServiceAccountKind, nsxServiceAccountAPIGroup)
antreaConfig.Status.Message = err.Error()
return err
}
}
antreaConfig.Status.Message = ""
err := r.ensureProviderServiceAccount(ctx, antreaConfig, cluster)
if err != nil {
Expand All @@ -355,7 +330,7 @@ func (r *AntreaConfigReconciler) registerAntreaNSX(ctx context.Context, antreaCo

func (r *AntreaConfigReconciler) deregisterAntreaNSX(ctx context.Context, antreaConfig *cniv1alpha2.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error {
if !antreaConfig.Spec.AntreaNsx.Enable {
r.Log.Info("antreaNsx is not enabled, there is no ProviderServiceAccount or NsxServiceAccount to be deleted")
r.Log.Info("antrea_nsx is not enabled, there is no ProviderServiceAccount or NsxServiceAccount to be deleted")
return nil
}
vsphereCluster, err := cutil.VSphereClusterParavirtualForCAPICluster(ctx, r.Client, cluster)
Expand All @@ -371,7 +346,7 @@ func (r *AntreaConfigReconciler) deregisterAntreaNSX(ctx context.Context, antrea
}
account := &nsxoperatorapi.NSXServiceAccount{}

account.Name = r.getNSXServiceAccountName(clusterName)
account.Name = getNSXServiceAccountName(clusterName)
account.Namespace = antreaConfig.Namespace
err = r.Client.Delete(ctx, account)
if err != nil && !apierrors.IsNotFound(err) {
Expand All @@ -380,7 +355,7 @@ func (r *AntreaConfigReconciler) deregisterAntreaNSX(ctx context.Context, antrea
}

provider := &vsphere.ProviderServiceAccount{}
provider.Name = r.getProviderServiceAccountName(vsphereCluster.Name)
provider.Name = getProviderServiceAccountName(vsphereCluster.Name)
provider.Namespace = vsphereCluster.Namespace
err = r.Client.Delete(ctx, provider)
if err != nil && !apierrors.IsNotFound(err) {
Expand Down
138 changes: 107 additions & 31 deletions addons/controllers/antrea/antreaconfig_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ package controllers
import (
"context"
"fmt"
"reflect"
"strings"

"github.com/pkg/errors"

"golang.org/x/mod/semver"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clusterv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
Expand All @@ -23,54 +23,84 @@ import (
cniv1alpha2 "github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs/cni/v1alpha2"
)

const (
bootstrapFromInline = "Inline"
bootstrapFromSupervisorCluster = "SupervisorCluster"
)

// AntreaConfigSpec defines the desired state of AntreaConfig
type AntreaConfigSpec struct {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AntreaConfigSpec should be aligned with Antrea package schema.yaml.

The AntreaNSX member in this struct needs to be renamed as antrea_nsx, and yaml:"antreaNsx,omitempty" needs to be changed to yaml:"antrea_nsx,omitempty".

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

InfraProvider string `yaml:"infraProvider"`
Antrea antrea `yaml:"antrea,omitempty"`
AntreaNsx antreaNsx `yaml:"antreaNsx,omitempty"`
InfraProvider string `yaml:"infraProvider"`
Antrea antrea `yaml:"antrea,omitempty"`
AntreaNsx antreaNsx `yaml:"antrea_nsx,omitempty"`
AntreaInterworking antreaInterworking `yaml:"antrea_interworking,omitempty"`
}

type antrea struct {
AntreaConfigDataValue antreaConfigDataValue `yaml:"config,omitempty"`
}

type antreaNsx struct {
Enable bool `yaml:"enable,omitempty"`
BootstrapFrom antreaNsxBootstrapFrom `yaml:"bootstrapFrom,omitempty"`
AntreaNsxConfig antreaNsxConfig `yaml:"config,omitempty"`
type antreaInterworking struct {
Config antreaInterworkingConfig `yaml:"config,omitempty"`
}

type antreaNsxBootstrapFrom struct {
// ProviderRef is used with uTKG, which will be filled by NCP operator
ProviderRef *antreaNsxProvider `yaml:"providerRef,omitempty"`
// Inline is used with TKGm, user need to fill in manually
Inline *antreaNsxInline `yaml:"inline,omitempty"`
type antreaNsx struct {
Enable bool `yaml:"enable,omitempty"`
}

type antreaNsxProvider struct {
// Api version for nsxServiceAccount, its value is "nsx.vmware.com/v1alpha1" now
ApiVersion string `yaml:"apiVersion,omitempty"`
// Its value is NsxServiceAccount
Kind string `yaml:"kind,omitempty"`
// Name is the name for NsxServiceAccount
Name string `yaml:"name,omitempty"`
type antreaInterworkingConfig struct {
InfraType string `yaml:"infraType,omitempty"`
BootstrapFrom string `yaml:"bootstrapFrom,omitempty"`
BootstrapSupervisorResourceName string `yaml:"bootstrapSupervisorResourceName,omitempty"`
NSXCert string `yaml:"nsxCert,omitempty"`
NSXKey string `yaml:"nsxKey,omitempty"`
ClusterName string `yaml:"clusterName,omitempty"`
NSXManagers []string `yaml:"NSXManagers,omitempty"`
VPCPath []string `yaml:"vpcPath,omitempty"`
ProxyEndpoints proxyEndpoints `yaml:"proxyEndpoints,omitempty"`
MpAdapterConf mpAdapterConf `yaml:"mp_adapter_conf,omitempty"`
CcpAdapterConf ccpAdapterConf `yaml:"ccp_adapter_conf,omitempty"`
}

type nsxCertRef struct {
// TLSCert is cert file to access nsx manager
TLSCert string `yaml:"tls.crt,omitempty"`
// TLSKey is key file to access nsx manager
TLSKey string `yaml:"tls.key,omitempty"`
type proxyEndpoints struct {
RestApi []string `yaml:"rest_api,omitempty"`
NSXRpcFwdProxy []string `yaml:"nsx_rpc_fwd_proxy,omitempty"`
}

type antreaNsxInline struct {
NsxManagers []string `yaml:"nsxManagers,omitempty"`
ClusterName string `yaml:"clusterName,omitempty"`
NsxCertRef nsxCertRef `yaml:"NsxCert,omitempty"`
type mpAdapterConf struct {
NSXClientAuthCertFile string `yaml:"NSXClientAuthCertFile,omitempty"`
NSXClientAuthKeyFile string `yaml:"NSXClientAuthKeyFile,omitempty"`
NSXRemoteAuth bool `yaml:"NSXRemoteAuth,omitempty"`
NSXCAFile string `yaml:"NSXCAFile,omitempty"`
NSXInsecure bool `yaml:"NSXInsecure,omitempty"`
NSXRPCConnType string `yaml:"NSXRPCConnType,omitempty"`
ClusterType string `yaml:"clusterType,omitempty"`
NSXClientTimeout int `yaml:"NSXClientTimeout,omitempty"`
InventoryBatchSize int `yaml:"InventoryBatchSize,omitempty"`
InventoryBatchPeriod int `yaml:"InventoryBatchPeriod,omitempty"`
EnableDebugServer bool `yaml:"EnableDebugServer,omitempty"`
APIServerPort int `yaml:"APIServerPort,omitempty"`
DebugServerPort int `yaml:"DebugServerPort,omitempty"`
NSXRPCDebug bool `yaml:"NSXRPCDebug,omitempty"`
ConditionTimeout int `yaml:"ConditionTimeout,omitempty"`
}

type antreaNsxConfig struct {
InfraType string `yaml:"infraType,omitempty"`
type ccpAdapterConf struct {
EnableDebugServer bool `yaml:"EnableDebugServer,omitempty"`
APIServerPort int `yaml:"APIServerPort,omitempty"`
DebugServerPort int `yaml:"DebugServerPort,omitempty"`
NSXRPCDebug bool `yaml:"NSXRPCDebug,omitempty"`
// Time to wait for realization
RealizeTimeoutSeconds int `yaml:"RealizeTimeoutSeconds,omitempty"`
// An interval for regularly report latest realization error in background
RealizeErrorSyncIntervalSeconds int `yaml:"RealizeErrorSyncIntervalSeconds,omitempty"`
ReconcilerWorkerCount int `yaml:"ReconcilerWorkerCount,omitempty"`
// Average QPS = ReconcilerWorkerCount * ReconcilerQPS
ReconcilerQPS int `yaml:"ReconcilerQPS,omitempty"`
// Peak QPS = ReconcilerWorkerCount * ReconcilerBurst
ReconcilerBurst int `yaml:"ReconcilerBurst,omitempty"`
// #! 24 Hours
ReconcilerResyncSeconds int `yaml:"ReconcilerResyncSeconds,omitempty"`
}

type antreaEgress struct {
Expand Down Expand Up @@ -300,6 +330,52 @@ func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha2.An
if semver.Compare(version, "v1.9.0") >= 0 {
configSpec.Antrea.AntreaConfigDataValue.FeatureGates.TopologyAwareHints = &config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.TopologyAwareHints
}
// NSX related
if semver.Compare(version, "1.9.0") >= 0 && config.Spec.AntreaNsx.Enable {
configSpec.AntreaNsx.Enable = config.Spec.AntreaNsx.Enable
if config.Spec.AntreaInterworking.Config.BootstrapFrom == bootstrapFromInline {
configSpec.AntreaInterworking.Config.NSXManagers = config.Spec.AntreaInterworking.Config.NSXManagers
configSpec.AntreaInterworking.Config.ClusterName = config.Spec.AntreaInterworking.Config.ClusterName
configSpec.AntreaInterworking.Config.NSXCert = config.Spec.AntreaInterworking.Config.NSXCert
configSpec.AntreaInterworking.Config.NSXKey = config.Spec.AntreaInterworking.Config.NSXKey
configSpec.AntreaInterworking.Config.VPCPath = config.Spec.AntreaInterworking.Config.VPCPath
configSpec.AntreaInterworking.Config.ProxyEndpoints.NSXRpcFwdProxy = config.Spec.AntreaInterworking.Config.ProxyEndpoints.NSXRpcFwdProxy
configSpec.AntreaInterworking.Config.ProxyEndpoints.RestApi = config.Spec.AntreaInterworking.Config.ProxyEndpoints.RestApi
} else {
configSpec.AntreaInterworking.Config.BootstrapFrom = bootstrapFromSupervisorCluster
configSpec.AntreaInterworking.Config.BootstrapSupervisorResourceName = getNSXServiceAccountName(cluster.Name)
}
Copy link
Contributor

@wjun wjun Mar 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any else branch required here?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if this pr #4219 can be merged into v0.28, this should also be included in v0.28


ccpConf := config.Spec.AntreaInterworking.Config.CcpAdapterConf
if err := copyStructAtoB(ccpConf, &configSpec.AntreaInterworking.Config.CcpAdapterConf); err != nil {
return configSpec, err
}
mpConf := config.Spec.AntreaInterworking.Config.MpAdapterConf
if err := copyStructAtoB(mpConf, &configSpec.AntreaInterworking.Config.MpAdapterConf); err != nil {
return configSpec, err
}
}

return configSpec, nil
}

func copyStructAtoB(a interface{}, b interface{}) error {
va := reflect.ValueOf(a)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reflect can make it more feasible, but at same time it can introduce performance cost, maybe consider if the implementation is OK.

vb := reflect.ValueOf(b).Elem()
for i := 0; i < va.NumField(); i++ {
fieldA := va.Field(i)
fieldB := vb.FieldByName(va.Type().Field(i).Name)
if fieldB.IsValid() && fieldA.Type() == fieldB.Type() {
fieldB.Set(fieldA)
}
}
return nil
}

func getProviderServiceAccountName(clusterName string) string {
return fmt.Sprintf("%s-antrea", clusterName)
}

func getNSXServiceAccountName(clusterName string) string {
return fmt.Sprintf("%s-antrea", clusterName)
}
38 changes: 38 additions & 0 deletions addons/controllers/antrea/antreaconfig_util_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package controllers

import (
"testing"

"github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs/cni/v1alpha2"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestCopyStruct(t *testing.T) {
ccpConf := v1alpha2.CcpAdapterConf{
EnableDebugServer: true,
APIServerPort: 1234,
}
descCcpAdapterConf := ccpAdapterConf{
EnableDebugServer: false,
APIServerPort: 0,
}
err := copyStructAtoB(ccpConf, &descCcpAdapterConf)
require.NoError(t, err, "copy CcpAdapterConf values error")
assert.Equal(t, 1234, descCcpAdapterConf.APIServerPort)
assert.Equal(t, true, descCcpAdapterConf.EnableDebugServer)

mpConf := v1alpha2.MpAdapterConf{
NSXClientAuthCertFile: "fake-cert-file",
ConditionTimeout: 150,
}
descMpAdapterConf := mpAdapterConf{
NSXClientAuthCertFile: "",
ConditionTimeout: 0,
}
err = copyStructAtoB(mpConf, &descMpAdapterConf)
require.NoError(t, err, "copy MpAdapterConf values error")
assert.Equal(t, "fake-cert-file", descMpAdapterConf.NSXClientAuthCertFile)
assert.Equal(t, 150, descMpAdapterConf.ConditionTimeout)
}
4 changes: 2 additions & 2 deletions addons/controllers/antreaconfig_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ var _ = Describe("AntreaConfig Reconciler and Webhooks", func() {
)

const (
waitTimeout = waitTimeout //use this to change test speed when debugging
waitTimeout = waitTimeout // use this to change test speed when debugging
antreaManifestsTestFile1 = "testdata/antrea-test-1.yaml"
antreamanifestsTestFile2 = "testdata/antrea-test-2.yaml"
antreaTemplateConfigManifestsTestFile1 = "testdata/antrea-test-template-config-1.yaml"
Expand Down Expand Up @@ -224,7 +224,7 @@ var _ = Describe("AntreaConfig Reconciler and Webhooks", func() {

})

Context("Reconcile AntreaConfig with antreaNsx enabled for management cluster", func() {
Context("Reconcile AntreaConfig with antrea_nsx enabled for management cluster", func() {

BeforeEach(func() {
clusterName = antreaTestCluster2
Expand Down
2 changes: 1 addition & 1 deletion addons/controllers/testdata/antrea-test-2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -94,5 +94,5 @@ spec:
AntreaIPAM: false
ServiceExternalIP: false
Multicast: false
antreaNsx:
antrea_nsx:
enable: true
2 changes: 2 additions & 0 deletions addons/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
github.com/onsi/gomega v1.20.2
github.com/oracle/cluster-api-provider-oci v0.6.0
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.8.0
github.com/vmware-tanzu/carvel-kapp-controller v0.35.0
github.com/vmware-tanzu/carvel-secretgen-controller v0.5.0
github.com/vmware-tanzu/carvel-vendir v0.26.0
Expand Down Expand Up @@ -91,6 +92,7 @@ require (
github.com/nxadm/tail v1.4.8 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.18.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.13.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions addons/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -856,6 +856,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
Expand All @@ -865,6 +866,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
Expand Down
Loading