From 6571a8a4d9aadae5014c9243a5acdb3667e4758f Mon Sep 17 00:00:00 2001 From: Jian Zhu Date: Mon, 20 Nov 2023 22:09:38 +0800 Subject: [PATCH] :sparkles: Support setting manifest work config by addon template (#308) * :sparkles: Support setting manifest work config by addon template Signed-off-by: zhujian * add unit tests Signed-off-by: zhujian --------- Signed-off-by: zhujian --- go.mod | 2 +- go.sum | 4 +- pkg/addon/templateagent/registration.go | 26 ++- pkg/addon/templateagent/registration_test.go | 44 +++++ pkg/addon/templateagent/template_agent.go | 35 +++- .../templateagent/template_agent_test.go | 166 ++++++++++++++++++ test/e2e/addonmanagement_test.go | 24 +++ test/e2e/manifests/addon/addon_template.yaml | 8 + vendor/modules.txt | 2 +- .../pkg/addonfactory/addondeploymentconfig.go | 7 + .../pkg/addonfactory/addonfactory.go | 30 +++- .../pkg/addonfactory/helm_agentaddon.go | 56 +++--- .../pkg/addonfactory/template_agentaddon.go | 26 ++- .../agentdeploy/healthcheck_sync.go | 4 + .../controllers/agentdeploy/utils.go | 77 ++++++++ .../controllers/registration/controller.go | 1 + .../addon-framework/pkg/agent/inteface.go | 9 + .../addon-framework/pkg/utils/probe_helper.go | 3 +- 18 files changed, 472 insertions(+), 52 deletions(-) diff --git a/go.mod b/go.mod index e82c4ac46..888ec2cb8 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( k8s.io/klog/v2 v2.100.1 k8s.io/kube-aggregator v0.28.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b - open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c + open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2 open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6 sigs.k8s.io/controller-runtime v0.15.0 sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 diff --git a/go.sum b/go.sum index 570ed7e64..6137381c7 100644 --- a/go.sum +++ b/go.sum @@ -738,8 +738,8 @@ k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5Ohx k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c h1:9Rvj3UTjVwJWOItlIYx6shFF72f8L3t91T9IwZ8sx6Q= -open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c/go.mod h1:r4sQGR9YgLC4hXC695sfinun2WhuigWrEPk2IeIl800= +open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2 h1:38vY9paEGugvXfYGJ0oFabL4/8Jxrg+GnxxjUO2DMio= +open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2/go.mod h1:aj97pgpGJ0/LpQzBVtU2oDFqqIiZLOPnsjLKG/sVkFw= open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6 h1:QwQ3dTtbCGuZ7iGqINuawGh4c3NmGe2qhcdHvEXRsOs= open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6/go.mod h1:/I/nFccB0tmF+dZg7pHuzY3SaXOX86MI4vcFtidJ0OM= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/pkg/addon/templateagent/registration.go b/pkg/addon/templateagent/registration.go index 94fa74b77..23f37177b 100644 --- a/pkg/addon/templateagent/registration.go +++ b/pkg/addon/templateagent/registration.go @@ -52,16 +52,34 @@ func AddonManagerNamespace() string { func (a *CRDTemplateAgentAddon) GetDesiredAddOnTemplate(addon *addonapiv1alpha1.ManagedClusterAddOn, clusterName, addonName string) (*addonapiv1alpha1.AddOnTemplate, error) { + if addon != nil { + return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences) + } - if addon == nil { - var err error - addon, err = a.addonLister.ManagedClusterAddOns(clusterName).Get(addonName) + if len(clusterName) != 0 { + addon, err := a.addonLister.ManagedClusterAddOns(clusterName).Get(addonName) if err != nil { return nil, err } + + return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences) } - return a.GetDesiredAddOnTemplateByAddon(addon) + // clusterName and addon are both empty, backoff to get the template from the clusterManagementAddOn + cma, err := a.cmaLister.Get(addonName) + if err != nil { + return nil, err + } + + // convert the DefaultConfigReference to ConfigReference + var configReferences []addonapiv1alpha1.ConfigReference + for _, configReference := range cma.Status.DefaultConfigReferences { + configReferences = append(configReferences, addonapiv1alpha1.ConfigReference{ + ConfigGroupResource: configReference.ConfigGroupResource, + DesiredConfig: configReference.DesiredConfig, + }) + } + return a.getDesiredAddOnTemplateInner(cma.Name, configReferences) } func (a *CRDTemplateAgentAddon) TemplateCSRConfigurationsFunc() func(cluster *clusterv1.ManagedCluster) []addonapiv1alpha1.RegistrationConfig { diff --git a/pkg/addon/templateagent/registration_test.go b/pkg/addon/templateagent/registration_test.go index 29e9b2bdd..0b518cd81 100644 --- a/pkg/addon/templateagent/registration_test.go +++ b/pkg/addon/templateagent/registration_test.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "fmt" + "os" "strings" "testing" "time" + "github.com/stretchr/testify/assert" certificatesv1 "k8s.io/api/certificates/v1" certificates "k8s.io/api/certificates/v1beta1" rbacv1 "k8s.io/api/rbac/v1" @@ -665,3 +667,45 @@ func TestTemplatePermissionConfigFunc(t *testing.T) { } } } + +func TestAddonManagerNamespace(t *testing.T) { + cases := []struct { + name string + podNamespace string + envNs string + expected string + }{ + { + name: "pod namespace is not empty", + podNamespace: "test", + envNs: "", + expected: "test", + }, + { + name: "pod namespace is empty, env is not empty", + podNamespace: "", + envNs: "test-env", + expected: "test-env", + }, + { + name: "default namespace", + podNamespace: "", + envNs: "", + expected: "open-cluster-management-hub", + }, + } + + for _, c := range cases { + if c.podNamespace != "" { + podNamespace = c.podNamespace + } + if c.envNs != "" { + os.Setenv("POD_NAMESPACE", c.envNs) + } + ns := AddonManagerNamespace() + assert.Equal(t, c.expected, ns) + // reset podNamespace and env + podNamespace = "" + os.Setenv("POD_NAMESPACE", "") + } +} diff --git a/pkg/addon/templateagent/template_agent.go b/pkg/addon/templateagent/template_agent.go index df339a6de..e844aaf5e 100644 --- a/pkg/addon/templateagent/template_agent.go +++ b/pkg/addon/templateagent/template_agent.go @@ -8,6 +8,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" rbacv1lister "k8s.io/client-go/listers/rbac/v1" "k8s.io/klog/v2" @@ -54,6 +55,7 @@ type CRDTemplateAgentAddon struct { addonClient addonv1alpha1client.Interface addonLister addonlisterv1alpha1.ManagedClusterAddOnLister addonTemplateLister addonlisterv1alpha1.AddOnTemplateLister + cmaLister addonlisterv1alpha1.ClusterManagementAddOnLister rolebindingLister rbacv1lister.RoleBindingLister addonName string agentName string @@ -79,6 +81,7 @@ func NewCRDTemplateAgentAddon( addonClient: addonClient, addonLister: addonInformers.Addon().V1alpha1().ManagedClusterAddOns().Lister(), addonTemplateLister: addonInformers.Addon().V1alpha1().AddOnTemplates().Lister(), + cmaLister: addonInformers.Addon().V1alpha1().ClusterManagementAddOns().Lister(), rolebindingLister: rolebindingLister, addonName: addonName, agentName: agentName, @@ -91,7 +94,7 @@ func (a *CRDTemplateAgentAddon) Manifests( cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) { - template, err := a.GetDesiredAddOnTemplateByAddon(addon) + template, err := a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences) if err != nil { return nil, err } @@ -107,7 +110,7 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions { for gvr := range utils.BuiltInAddOnConfigGVRs { supportedConfigGVRs = append(supportedConfigGVRs, gvr) } - return agent.AgentAddonOptions{ + agentAddonOptions := agent.AgentAddonOptions{ AddonName: a.addonName, InstallStrategy: nil, HealthProber: &agent.HealthProber{ @@ -124,6 +127,19 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions { }, AgentDeployTriggerClusterFilter: utils.ClusterImageRegistriesAnnotationChanged, } + + template, err := a.GetDesiredAddOnTemplate(nil, "", a.addonName) + if err != nil { + utilruntime.HandleError(fmt.Errorf("failed to get addon %s template: %v", a.addonName, err)) + return agentAddonOptions + } + if template == nil { + utilruntime.HandleError(fmt.Errorf("addon %s template is nil", a.addonName)) + return agentAddonOptions + } + agentAddonOptions.ManifestConfigs = template.Spec.AgentSpec.ManifestConfigs + + return agentAddonOptions } func (a *CRDTemplateAgentAddon) renderObjects( @@ -190,19 +206,20 @@ func (a *CRDTemplateAgentAddon) decorateObjects( return objects, nil } -// GetDesiredAddOnTemplateByAddon returns the desired template of the addon -func (a *CRDTemplateAgentAddon) GetDesiredAddOnTemplateByAddon( - addon *addonapiv1alpha1.ManagedClusterAddOn) (*addonapiv1alpha1.AddOnTemplate, error) { - ok, templateRef := AddonTemplateConfigRef(addon.Status.ConfigReferences) +// getDesiredAddOnTemplateInner returns the desired template of the addon +func (a *CRDTemplateAgentAddon) getDesiredAddOnTemplateInner( + addonName string, configReferences []addonapiv1alpha1.ConfigReference, +) (*addonapiv1alpha1.AddOnTemplate, error) { + ok, templateRef := AddonTemplateConfigRef(configReferences) if !ok { - a.logger.V(4).Info("Addon template config in status is empty", "addonName", addon.Name) + a.logger.V(4).Info("Addon template config in status is empty", "addonName", addonName) return nil, nil } desiredTemplate := templateRef.DesiredConfig if desiredTemplate == nil || desiredTemplate.SpecHash == "" { - a.logger.Info("Addon template spec hash is empty", "addonName", addon.Name) - return nil, fmt.Errorf("addon %s template desired spec hash is empty", addon.Name) + a.logger.Info("Addon template spec hash is empty", "addonName", addonName) + return nil, fmt.Errorf("addon %s template desired spec hash is empty", addonName) } template, err := a.addonTemplateLister.Get(desiredTemplate.Name) diff --git a/pkg/addon/templateagent/template_agent_test.go b/pkg/addon/templateagent/template_agent_test.go index 3522a7fe8..c8761b905 100644 --- a/pkg/addon/templateagent/template_agent_test.go +++ b/pkg/addon/templateagent/template_agent_test.go @@ -19,12 +19,14 @@ import ( "k8s.io/klog/v2/ktesting" "open-cluster-management.io/addon-framework/pkg/addonfactory" + "open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting" "open-cluster-management.io/addon-framework/pkg/utils" addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake" addoninformers "open-cluster-management.io/api/client/addon/informers/externalversions" clusterv1 "open-cluster-management.io/api/cluster/v1" clusterv1apha1 "open-cluster-management.io/api/cluster/v1alpha1" + workapiv1 "open-cluster-management.io/api/work/v1" ) func TestAddonTemplateAgentManifests(t *testing.T) { @@ -385,6 +387,170 @@ func TestAgentInstallNamespace(t *testing.T) { } } +func TestAgentManifestConfigs(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + addonName := "hello" + + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = clusterv1apha1.Install(s) + _ = addonapiv1alpha1.Install(s) + + cases := []struct { + name string + addonTemplate *addonapiv1alpha1.AddOnTemplate + clusterManagementAddOn *addonapiv1alpha1.ClusterManagementAddOn + expected []workapiv1.ManifestConfigOption + }{ + { + name: "no addontemplate in cma status", + clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").Build(), + expected: nil, + }, + { + name: "no addontemplate", + clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", ""). + WithDefaultConfigReferences( + addonapiv1alpha1.DefaultConfigReference{ + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: utils.AddOnTemplateGVR.Group, + Resource: utils.AddOnTemplateGVR.Resource, + }, + DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{ + ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"}, + SpecHash: "hash", + }, + }, + ).Build(), + expected: nil, + }, + { + name: "addontemplate does not have manifestconfigs", + clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", ""). + WithDefaultConfigReferences( + addonapiv1alpha1.DefaultConfigReference{ + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: utils.AddOnTemplateGVR.Group, + Resource: utils.AddOnTemplateGVR.Resource, + }, + DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{ + ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"}, + SpecHash: "hash", + }, + }, + ).Build(), + addonTemplate: &addonapiv1alpha1.AddOnTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hello-template", + }, + }, + expected: nil, + }, + { + name: "addontemplate has manifestconfigs", + clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", ""). + WithDefaultConfigReferences( + addonapiv1alpha1.DefaultConfigReference{ + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: utils.AddOnTemplateGVR.Group, + Resource: utils.AddOnTemplateGVR.Resource, + }, + DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{ + ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"}, + SpecHash: "hash", + }, + }, + ).Build(), + addonTemplate: &addonapiv1alpha1.AddOnTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hello-template", + }, + Spec: addonapiv1alpha1.AddOnTemplateSpec{ + AddonName: "hello", + AgentSpec: workapiv1.ManifestWorkSpec{ + ManifestConfigs: []workapiv1.ManifestConfigOption{ + { + ResourceIdentifier: workapiv1.ResourceIdentifier{ + Group: "apps", + Resource: "deployment", + Name: "hello", + Namespace: "default", + }, + FeedbackRules: []workapiv1.FeedbackRule{ + { + Type: workapiv1.WellKnownStatusType, + }, + }, + }, + }, + }, + }, + }, + expected: []workapiv1.ManifestConfigOption{ + { + ResourceIdentifier: workapiv1.ResourceIdentifier{ + Group: "apps", + Resource: "deployment", + Name: "hello", + Namespace: "default", + }, + FeedbackRules: []workapiv1.FeedbackRule{ + { + Type: workapiv1.WellKnownStatusType, + }, + }, + }, + }, + }, + } + + for _, tc := range cases { + var objs []runtime.Object + if tc.clusterManagementAddOn != nil { + objs = append(objs, tc.clusterManagementAddOn) + } + if tc.addonTemplate != nil { + objs = append(objs, tc.addonTemplate) + } + hubKubeClient := fakekube.NewSimpleClientset() + addonClient := fakeaddon.NewSimpleClientset(objs...) + + addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute) + if tc.clusterManagementAddOn != nil { + cmaStore := addonInformerFactory.Addon().V1alpha1().ClusterManagementAddOns().Informer().GetStore() + if err := cmaStore.Add(tc.clusterManagementAddOn); err != nil { + t.Fatal(err) + } + } + if tc.addonTemplate != nil { + atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore() + if err := atStore.Add(tc.addonTemplate); err != nil { + t.Fatal(err) + } + } + kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute) + + agentAddon := NewCRDTemplateAgentAddon( + ctx, + addonName, + "test-agent", + hubKubeClient, + addonClient, + addonInformerFactory, + kubeInformers.Rbac().V1().RoleBindings().Lister(), + addonfactory.GetAddOnDeploymentConfigValues( + addonfactory.NewAddOnDeploymentConfigGetter(addonClient), + addonfactory.ToAddOnCustomizedVariableValues, + ToAddOnNodePlacementPrivateValues, + ToAddOnRegistriesPrivateValues, + ), + ) + + agentManifestConfigs := agentAddon.GetAgentAddonOptions().ManifestConfigs + assert.Equal(t, tc.expected, agentManifestConfigs) + } +} + type testManagedClusterAddOnBuilder struct { managedClusterAddOn *addonapiv1alpha1.ManagedClusterAddOn addonTemplate *addonapiv1alpha1.AddOnTemplate diff --git a/test/e2e/addonmanagement_test.go b/test/e2e/addonmanagement_test.go index deb502937..829707e2c 100644 --- a/test/e2e/addonmanagement_test.go +++ b/test/e2e/addonmanagement_test.go @@ -19,6 +19,7 @@ import ( addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" clusterv1 "open-cluster-management.io/api/cluster/v1" + workapiv1 "open-cluster-management.io/api/work/v1" "open-cluster-management.io/ocm/pkg/addon/templateagent" "open-cluster-management.io/ocm/test/e2e/manifests" @@ -209,6 +210,29 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered, return nil }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + ginkgo.By("Make sure manifestwork config is configured") + manifestWork, err := t.HubWorkClient.WorkV1().ManifestWorks(clusterName).Get(context.Background(), + fmt.Sprintf("addon-%s-deploy-0", addOnName), metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + foundConfig := false + expectedResourceIdentifier := workapiv1.ResourceIdentifier{ + Group: "apps", + Resource: "deployments", + Name: "hello-template-agent", + Namespace: addonInstallNamespace, + } + for _, mc := range manifestWork.Spec.ManifestConfigs { + if mc.ResourceIdentifier == expectedResourceIdentifier { + foundConfig = true + gomega.Expect(mc.UpdateStrategy.Type).To(gomega.Equal(workapiv1.UpdateStrategyTypeServerSideApply)) + break + } + } + if !foundConfig { + gomega.Expect(fmt.Errorf("expected manifestwork is not correct, %v", + manifestWork.Spec.ManifestConfigs)).ToNot(gomega.HaveOccurred()) + } + ginkgo.By(fmt.Sprintf("delete the addon %v on the managed cluster namespace %v", addOnName, clusterName)) err = t.AddOnClinet.AddonV1alpha1().ManagedClusterAddOns(clusterName).Delete( context.TODO(), addOnName, metav1.DeleteOptions{}) diff --git a/test/e2e/manifests/addon/addon_template.yaml b/test/e2e/manifests/addon/addon_template.yaml index 556a2456f..941d57677 100644 --- a/test/e2e/manifests/addon/addon_template.yaml +++ b/test/e2e/manifests/addon/addon_template.yaml @@ -5,6 +5,14 @@ metadata: spec: addonName: hello-template agentSpec: + manifestConfigs: + - resourceIdentifier: + group: apps + name: hello-template-agent + namespace: << AddonInstallNamespace >> + resource: deployments + updateStrategy: + type: ServerSideApply workload: manifests: - kind: Deployment diff --git a/vendor/modules.txt b/vendor/modules.txt index eeb8eb363..ef56a2d5c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1423,7 +1423,7 @@ k8s.io/utils/pointer k8s.io/utils/ptr k8s.io/utils/strings/slices k8s.io/utils/trace -# open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c +# open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2 ## explicit; go 1.20 open-cluster-management.io/addon-framework/pkg/addonfactory open-cluster-management.io/addon-framework/pkg/addonmanager diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addondeploymentconfig.go b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addondeploymentconfig.go index 9a608efd1..d436a977b 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addondeploymentconfig.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addondeploymentconfig.go @@ -154,6 +154,10 @@ func ToAddOnProxyConfigValues(config addonapiv1alpha1.AddOnDeploymentConfig) (Va proxyConfig["NO_PROXY"] = config.Spec.ProxyConfig.NoProxy } + if len(config.Spec.ProxyConfig.CABundle) > 0 { + proxyConfig["PROXY_CA_BUNDLE"] = string(config.Spec.ProxyConfig.CABundle) + } + type global struct { ProxyConfig map[string]string `json:"proxyConfig"` } @@ -264,6 +268,9 @@ func ToAddOnDeploymentConfigValues(config addonapiv1alpha1.AddOnDeploymentConfig if len(config.Spec.ProxyConfig.NoProxy) > 0 { values["NoProxy"] = config.Spec.ProxyConfig.NoProxy } + if len(config.Spec.ProxyConfig.CABundle) > 0 { + values["ProxyCABundle"] = string(config.Spec.ProxyConfig.CABundle) + } return values, nil } diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addonfactory.go b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addonfactory.go index 21d6de39f..23923be4e 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addonfactory.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/addonfactory.go @@ -34,8 +34,9 @@ type AgentAddonFactory struct { getValuesFuncs []GetValuesFunc agentAddonOptions agent.AgentAddonOptions // trimCRDDescription flag is used to trim the description of CRDs in manifestWork. disabled by default. - trimCRDDescription bool - hostingCluster *clusterv1.ManagedCluster + trimCRDDescription bool + hostingCluster *clusterv1.ManagedCluster + agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) string } // NewAgentAddonFactory builds an addonAgentFactory instance with addon name and fs. @@ -78,6 +79,8 @@ func (f *AgentAddonFactory) WithGetValuesFuncs(getValuesFuncs ...GetValuesFunc) } // WithInstallStrategy defines the installation strategy of the manifests prescribed by Manifests(..). +// Deprecated: add annotation "addon.open-cluster-management.io/lifecycle: addon-manager" to ClusterManagementAddon +// and define install strategy in ClusterManagementAddon spec.installStrategy instead. func (f *AgentAddonFactory) WithInstallStrategy(strategy *agent.InstallStrategy) *AgentAddonFactory { if strategy.InstallNamespace == "" { strategy.InstallNamespace = AddonDefaultInstallNamespace @@ -142,8 +145,29 @@ func (f *AgentAddonFactory) WithAgentDeployTriggerClusterFilter( return f } +// WithAgentInstallNamespace defines the namespace where the agent resources will be deployed, this will +// override the default built-in namespace value; And if the registrationOption is not nil but the +// registrationOption.AgentInstallNamespace is nil, this will also set it to this. +func (f *AgentAddonFactory) WithAgentInstallNamespace( + nsFunc func(addon *addonapiv1alpha1.ManagedClusterAddOn) string, +) *AgentAddonFactory { + f.agentInstallNamespace = nsFunc + return f +} + +// preBuildAddon sets the default values for the agentAddonOptions. +func (f *AgentAddonFactory) preBuildAddon() { + if f.agentInstallNamespace != nil { + if f.agentAddonOptions.Registration != nil && f.agentAddonOptions.Registration.AgentInstallNamespace == nil { + f.agentAddonOptions.Registration.AgentInstallNamespace = f.agentInstallNamespace + } + } +} + // BuildHelmAgentAddon builds a helm agentAddon instance. func (f *AgentAddonFactory) BuildHelmAgentAddon() (agent.AgentAddon, error) { + f.preBuildAddon() + if err := validateSupportedConfigGVRs(f.agentAddonOptions.SupportedConfigGVRs); err != nil { return nil, err } @@ -160,6 +184,8 @@ func (f *AgentAddonFactory) BuildHelmAgentAddon() (agent.AgentAddon, error) { // BuildTemplateAgentAddon builds a template agentAddon instance. func (f *AgentAddonFactory) BuildTemplateAgentAddon() (agent.AgentAddon, error) { + f.preBuildAddon() + if err := validateSupportedConfigGVRs(f.agentAddonOptions.SupportedConfigGVRs); err != nil { return nil, err } diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/helm_agentaddon.go b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/helm_agentaddon.go index b3d3f2abe..708ef3efe 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/helm_agentaddon.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/helm_agentaddon.go @@ -42,22 +42,24 @@ type helmDefaultValues struct { } type HelmAgentAddon struct { - decoder runtime.Decoder - chart *chart.Chart - getValuesFuncs []GetValuesFunc - agentAddonOptions agent.AgentAddonOptions - trimCRDDescription bool - hostingCluster *clusterv1.ManagedCluster + decoder runtime.Decoder + chart *chart.Chart + getValuesFuncs []GetValuesFunc + agentAddonOptions agent.AgentAddonOptions + trimCRDDescription bool + hostingCluster *clusterv1.ManagedCluster + agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) string } func newHelmAgentAddon(factory *AgentAddonFactory, chart *chart.Chart) *HelmAgentAddon { return &HelmAgentAddon{ - decoder: serializer.NewCodecFactory(factory.scheme).UniversalDeserializer(), - chart: chart, - getValuesFuncs: factory.getValuesFuncs, - agentAddonOptions: factory.agentAddonOptions, - trimCRDDescription: factory.trimCRDDescription, - hostingCluster: factory.hostingCluster, + decoder: serializer.NewCodecFactory(factory.scheme).UniversalDeserializer(), + chart: chart, + getValuesFuncs: factory.getValuesFuncs, + agentAddonOptions: factory.agentAddonOptions, + trimCRDDescription: factory.trimCRDDescription, + hostingCluster: factory.hostingCluster, + agentInstallNamespace: factory.agentInstallNamespace, } } @@ -176,7 +178,7 @@ func (a *HelmAgentAddon) getValues( overrideValues = MergeValues(overrideValues, builtinValues) values, err := chartutil.ToRenderValues(a.chart, overrideValues, - a.releaseOptions(cluster, addon), a.capabilities(cluster, addon)) + a.releaseOptions(addon), a.capabilities(cluster, addon)) if err != nil { klog.Errorf("failed to render helm chart with values %v. err:%v", overrideValues, err) return values, err @@ -185,17 +187,27 @@ func (a *HelmAgentAddon) getValues( return values, nil } +func (a *HelmAgentAddon) getValueAgentInstallNamespace(addon *addonapiv1alpha1.ManagedClusterAddOn) string { + installNamespace := addon.Spec.InstallNamespace + if len(installNamespace) == 0 { + installNamespace = AddonDefaultInstallNamespace + } + if a.agentInstallNamespace != nil { + ns := a.agentInstallNamespace(addon) + if len(ns) > 0 { + installNamespace = ns + } + } + return installNamespace +} + func (a *HelmAgentAddon) getBuiltinValues( cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) { builtinValues := helmBuiltinValues{} builtinValues.ClusterName = cluster.GetName() - installNamespace := addon.Spec.InstallNamespace - if len(installNamespace) == 0 { - installNamespace = AddonDefaultInstallNamespace - } - builtinValues.AddonInstallNamespace = installNamespace + builtinValues.AddonInstallNamespace = a.getValueAgentInstallNamespace(addon) builtinValues.InstallMode, _ = constants.GetHostedModeInfo(addon.GetAnnotations()) @@ -242,11 +254,9 @@ func (a *HelmAgentAddon) capabilities( // only support Release.Name, Release.Namespace func (a *HelmAgentAddon) releaseOptions( - cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) chartutil.ReleaseOptions { - installNamespace := addon.Spec.InstallNamespace - if len(installNamespace) == 0 { - installNamespace = AddonDefaultInstallNamespace + return chartutil.ReleaseOptions{ + Name: a.agentAddonOptions.AddonName, + Namespace: a.getValueAgentInstallNamespace(addon), } - return chartutil.ReleaseOptions{Name: a.agentAddonOptions.AddonName, Namespace: installNamespace} } diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/template_agentaddon.go b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/template_agentaddon.go index 5cc864c04..57584ac19 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/template_agentaddon.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/addonfactory/template_agentaddon.go @@ -37,19 +37,21 @@ type templateFile struct { } type TemplateAgentAddon struct { - decoder runtime.Decoder - templateFiles []templateFile - getValuesFuncs []GetValuesFunc - agentAddonOptions agent.AgentAddonOptions - trimCRDDescription bool + decoder runtime.Decoder + templateFiles []templateFile + getValuesFuncs []GetValuesFunc + agentAddonOptions agent.AgentAddonOptions + trimCRDDescription bool + agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) string } func newTemplateAgentAddon(factory *AgentAddonFactory) *TemplateAgentAddon { return &TemplateAgentAddon{ - decoder: serializer.NewCodecFactory(factory.scheme).UniversalDeserializer(), - getValuesFuncs: factory.getValuesFuncs, - agentAddonOptions: factory.agentAddonOptions, - trimCRDDescription: factory.trimCRDDescription, + decoder: serializer.NewCodecFactory(factory.scheme).UniversalDeserializer(), + getValuesFuncs: factory.getValuesFuncs, + agentAddonOptions: factory.agentAddonOptions, + trimCRDDescription: factory.trimCRDDescription, + agentInstallNamespace: factory.agentInstallNamespace, } } @@ -123,6 +125,12 @@ func (a *TemplateAgentAddon) getBuiltinValues( if len(installNamespace) == 0 { installNamespace = AddonDefaultInstallNamespace } + if a.agentInstallNamespace != nil { + ns := a.agentInstallNamespace(addon) + if len(ns) > 0 { + installNamespace = ns + } + } builtinValues.AddonInstallNamespace = installNamespace builtinValues.InstallMode, _ = constants.GetHostedModeInfo(addon.GetAnnotations()) diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/healthcheck_sync.go b/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/healthcheck_sync.go index e65e7c454..86a3c091a 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/healthcheck_sync.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/healthcheck_sync.go @@ -221,6 +221,10 @@ func (s *healthCheckSyncer) analyzeDeploymentWorkProber( deployments := utils.FilterDeployments(manifests) for _, deployment := range deployments { manifestConfig := utils.DeploymentWellKnowManifestConfig(deployment.Namespace, deployment.Name) + // only probe the deployment with non-zero replicas + if deployment.Spec.Replicas != nil && *deployment.Spec.Replicas == 0 { + continue + } probeFields = append(probeFields, agent.ProbeField{ ResourceIdentifier: manifestConfig.ResourceIdentifier, ProbeRules: manifestConfig.FeedbackRules, diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/utils.go b/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/utils.go index 3bc50dd5d..ad830fdef 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/utils.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy/utils.go @@ -517,9 +517,86 @@ func getManifestConfigOption(agentAddon agent.AgentAddon, } } + userConfiguredManifestConfigs := agentAddon.GetAgentAddonOptions().ManifestConfigs + for _, mc := range userConfiguredManifestConfigs { + index := containsResourceIdentifier(manifestConfigs, mc.ResourceIdentifier) + if index == -1 { + manifestConfigs = append(manifestConfigs, mc) + continue + } + + // merge the feedback rules generated by the addon manager and the feedback rules configured by users + for _, rule := range mc.FeedbackRules { + manifestConfigs[index].FeedbackRules = mergeFeedbackRule(manifestConfigs[index].FeedbackRules, rule) + } + + if mc.UpdateStrategy != nil { + manifestConfigs[index].UpdateStrategy = mc.UpdateStrategy + } + } return manifestConfigs } +func containsResourceIdentifier(mcs []workapiv1.ManifestConfigOption, ri workapiv1.ResourceIdentifier) int { + for index, mc := range mcs { + if mc.ResourceIdentifier == ri { + return index + } + } + return -1 +} + +// mergeFeedbackRule merges the new rule into the existing rules. +// If the new rule is json path type, will ignore the json path which +// is already in the existing rules, compare by the path name. +func mergeFeedbackRule(rules []workapiv1.FeedbackRule, rule workapiv1.FeedbackRule) []workapiv1.FeedbackRule { + rrules := rules + var existJsonPaths []workapiv1.JsonPath + var existWellKnownStatus bool = false + for _, rule := range rules { + if rule.Type == workapiv1.WellKnownStatusType { + existWellKnownStatus = true + continue + } + if rule.Type == workapiv1.JSONPathsType { + existJsonPaths = append(existJsonPaths, rule.JsonPaths...) + } + } + + if rule.Type == workapiv1.WellKnownStatusType { + if !existWellKnownStatus { + rrules = append(rrules, rule) + } + return rrules + } + + if rule.Type == workapiv1.JSONPathsType { + var jsonPaths []workapiv1.JsonPath + for _, path := range rule.JsonPaths { + found := false + for _, rpath := range existJsonPaths { + if path.Name == rpath.Name { + found = true + break + } + } + if !found { + jsonPaths = append(jsonPaths, path) + } + } + if len(jsonPaths) != 0 { + rrules = append(rrules, workapiv1.FeedbackRule{ + Type: workapiv1.JSONPathsType, + JsonPaths: jsonPaths, + }) + } + return rrules + } + + rrules = append(rrules, rule) + return rrules +} + func getDeletionOrphaningRule(obj runtime.Object) (*workapiv1.OrphaningRule, error) { accessor, err := meta.Accessor(obj) if err != nil { diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/registration/controller.go b/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/registration/controller.go index f1888e3ec..28a0c6016 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/registration/controller.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/registration/controller.go @@ -99,6 +99,7 @@ func (c *addonRegistrationController) sync(ctx context.Context, syncCtx factory. // wait until the mca's ownerref is set. if !utils.IsOwnedByCMA(managedClusterAddonCopy) { + klog.Warningf("OwnerReferences is not set for %q", key) return nil } diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/agent/inteface.go b/vendor/open-cluster-management.io/addon-framework/pkg/agent/inteface.go index 32514684f..406f9cc57 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/agent/inteface.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/agent/inteface.go @@ -88,7 +88,16 @@ type AgentAddonOptions struct { // AgentDeployTriggerClusterFilter: func(old, new *clusterv1.ManagedCluster) bool { // return !equality.Semantic.DeepEqual(old.Annotations, new.Annotations) // } + // +optional AgentDeployTriggerClusterFilter func(old, new *clusterv1.ManagedCluster) bool + + // ManifestConfigs represents the configurations of manifests defined in workload field. It will: + // - override the update strategy set by the "Updaters" field + // - merge the feedback rules set by the "HealthProber" field if they have the same resource identifier, + // when merging the feedback rules, if the rule configured here is json path type, will ignore the + // json path which is already in the existing rules, compare by the path name. + // +optional + ManifestConfigs []workapiv1.ManifestConfigOption } type CSRSignerFunc func(csr *certificatesv1.CertificateSigningRequest) []byte diff --git a/vendor/open-cluster-management.io/addon-framework/pkg/utils/probe_helper.go b/vendor/open-cluster-management.io/addon-framework/pkg/utils/probe_helper.go index 5f0a52837..4d0387942 100644 --- a/vendor/open-cluster-management.io/addon-framework/pkg/utils/probe_helper.go +++ b/vendor/open-cluster-management.io/addon-framework/pkg/utils/probe_helper.go @@ -76,7 +76,8 @@ func DeploymentAvailabilityHealthCheck(identifier workapiv1.ResourceIdentifier, return nil } - return fmt.Errorf("readyReplica is %d for deployment %s/%s", *value.Value.Integer, identifier.Namespace, identifier.Name) + return fmt.Errorf("readyReplica is %d for deployment %s/%s", + *value.Value.Integer, identifier.Namespace, identifier.Name) } return fmt.Errorf("readyReplica is not probed") }