Skip to content

Commit

Permalink
✨ Support setting manifest work config by addon template (#308)
Browse files Browse the repository at this point in the history 
* ✨ Support setting manifest work config by addon template

Signed-off-by: zhujian <jiazhu@redhat.com>

* add unit tests

Signed-off-by: zhujian <jiazhu@redhat.com>

---------

Signed-off-by: zhujian <jiazhu@redhat.com>
  • Loading branch information
@zhujian7
zhujian7 authored Nov 20, 2023
1 parent 615f5a4 commit 6571a8a
Show file tree
Hide file tree
Showing 18 changed files with 472 additions and 52 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ require (
k8s.io/klog/v2 v2.100.1
k8s.io/kube-aggregator v0.28.1
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c
open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2
open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6
sigs.k8s.io/controller-runtime v0.15.0
sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,8 +738,8 @@ k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5Ohx
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c h1:9Rvj3UTjVwJWOItlIYx6shFF72f8L3t91T9IwZ8sx6Q=
open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c/go.mod h1:r4sQGR9YgLC4hXC695sfinun2WhuigWrEPk2IeIl800=
open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2 h1:38vY9paEGugvXfYGJ0oFabL4/8Jxrg+GnxxjUO2DMio=
open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2/go.mod h1:aj97pgpGJ0/LpQzBVtU2oDFqqIiZLOPnsjLKG/sVkFw=
open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6 h1:QwQ3dTtbCGuZ7iGqINuawGh4c3NmGe2qhcdHvEXRsOs=
open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6/go.mod h1:/I/nFccB0tmF+dZg7pHuzY3SaXOX86MI4vcFtidJ0OM=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
Expand Down
26 changes: 22 additions & 4 deletions pkg/addon/templateagent/registration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,34 @@ func AddonManagerNamespace() string {

func (a *CRDTemplateAgentAddon) GetDesiredAddOnTemplate(addon *addonapiv1alpha1.ManagedClusterAddOn,
clusterName, addonName string) (*addonapiv1alpha1.AddOnTemplate, error) {
if addon != nil {
return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences)
}

if addon == nil {
var err error
addon, err = a.addonLister.ManagedClusterAddOns(clusterName).Get(addonName)
if len(clusterName) != 0 {
addon, err := a.addonLister.ManagedClusterAddOns(clusterName).Get(addonName)
if err != nil {
return nil, err
}

return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences)
}

return a.GetDesiredAddOnTemplateByAddon(addon)
// clusterName and addon are both empty, backoff to get the template from the clusterManagementAddOn
cma, err := a.cmaLister.Get(addonName)
if err != nil {
return nil, err
}

// convert the DefaultConfigReference to ConfigReference
var configReferences []addonapiv1alpha1.ConfigReference
for _, configReference := range cma.Status.DefaultConfigReferences {
configReferences = append(configReferences, addonapiv1alpha1.ConfigReference{
ConfigGroupResource: configReference.ConfigGroupResource,
DesiredConfig: configReference.DesiredConfig,
})
}
return a.getDesiredAddOnTemplateInner(cma.Name, configReferences)
}

func (a *CRDTemplateAgentAddon) TemplateCSRConfigurationsFunc() func(cluster *clusterv1.ManagedCluster) []addonapiv1alpha1.RegistrationConfig {
Expand Down
44 changes: 44 additions & 0 deletions pkg/addon/templateagent/registration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ import (
"bytes"
"context"
"fmt"
"os"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"
certificatesv1 "k8s.io/api/certificates/v1"
certificates "k8s.io/api/certificates/v1beta1"
rbacv1 "k8s.io/api/rbac/v1"
Expand Down Expand Up @@ -665,3 +667,45 @@ func TestTemplatePermissionConfigFunc(t *testing.T) {
}
}
}

func TestAddonManagerNamespace(t *testing.T) {
cases := []struct {
name string
podNamespace string
envNs string
expected string
}{
{
name: "pod namespace is not empty",
podNamespace: "test",
envNs: "",
expected: "test",
},
{
name: "pod namespace is empty, env is not empty",
podNamespace: "",
envNs: "test-env",
expected: "test-env",
},
{
name: "default namespace",
podNamespace: "",
envNs: "",
expected: "open-cluster-management-hub",
},
}

for _, c := range cases {
if c.podNamespace != "" {
podNamespace = c.podNamespace
}
if c.envNs != "" {
os.Setenv("POD_NAMESPACE", c.envNs)
}
ns := AddonManagerNamespace()
assert.Equal(t, c.expected, ns)
// reset podNamespace and env
podNamespace = ""
os.Setenv("POD_NAMESPACE", "")
}
}
35 changes: 26 additions & 9 deletions pkg/addon/templateagent/template_agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
rbacv1lister "k8s.io/client-go/listers/rbac/v1"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -54,6 +55,7 @@ type CRDTemplateAgentAddon struct {
addonClient addonv1alpha1client.Interface
addonLister addonlisterv1alpha1.ManagedClusterAddOnLister
addonTemplateLister addonlisterv1alpha1.AddOnTemplateLister
cmaLister addonlisterv1alpha1.ClusterManagementAddOnLister
rolebindingLister rbacv1lister.RoleBindingLister
addonName string
agentName string
Expand All @@ -79,6 +81,7 @@ func NewCRDTemplateAgentAddon(
addonClient: addonClient,
addonLister: addonInformers.Addon().V1alpha1().ManagedClusterAddOns().Lister(),
addonTemplateLister: addonInformers.Addon().V1alpha1().AddOnTemplates().Lister(),
cmaLister: addonInformers.Addon().V1alpha1().ClusterManagementAddOns().Lister(),
rolebindingLister: rolebindingLister,
addonName: addonName,
agentName: agentName,
Expand All @@ -91,7 +94,7 @@ func (a *CRDTemplateAgentAddon) Manifests(
cluster *clusterv1.ManagedCluster,
addon *addonapiv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) {

template, err := a.GetDesiredAddOnTemplateByAddon(addon)
template, err := a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences)
if err != nil {
return nil, err
}
Expand All @@ -107,7 +110,7 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions {
for gvr := range utils.BuiltInAddOnConfigGVRs {
supportedConfigGVRs = append(supportedConfigGVRs, gvr)
}
return agent.AgentAddonOptions{
agentAddonOptions := agent.AgentAddonOptions{
AddonName: a.addonName,
InstallStrategy: nil,
HealthProber: &agent.HealthProber{
Expand All @@ -124,6 +127,19 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions {
},
AgentDeployTriggerClusterFilter: utils.ClusterImageRegistriesAnnotationChanged,
}

template, err := a.GetDesiredAddOnTemplate(nil, "", a.addonName)
if err != nil {
utilruntime.HandleError(fmt.Errorf("failed to get addon %s template: %v", a.addonName, err))
return agentAddonOptions
}
if template == nil {
utilruntime.HandleError(fmt.Errorf("addon %s template is nil", a.addonName))
return agentAddonOptions
}
agentAddonOptions.ManifestConfigs = template.Spec.AgentSpec.ManifestConfigs

return agentAddonOptions
}

func (a *CRDTemplateAgentAddon) renderObjects(
Expand Down Expand Up @@ -190,19 +206,20 @@ func (a *CRDTemplateAgentAddon) decorateObjects(
return objects, nil
}

// GetDesiredAddOnTemplateByAddon returns the desired template of the addon
func (a *CRDTemplateAgentAddon) GetDesiredAddOnTemplateByAddon(
addon *addonapiv1alpha1.ManagedClusterAddOn) (*addonapiv1alpha1.AddOnTemplate, error) {
ok, templateRef := AddonTemplateConfigRef(addon.Status.ConfigReferences)
// getDesiredAddOnTemplateInner returns the desired template of the addon
func (a *CRDTemplateAgentAddon) getDesiredAddOnTemplateInner(
addonName string, configReferences []addonapiv1alpha1.ConfigReference,
) (*addonapiv1alpha1.AddOnTemplate, error) {
ok, templateRef := AddonTemplateConfigRef(configReferences)
if !ok {
a.logger.V(4).Info("Addon template config in status is empty", "addonName", addon.Name)
a.logger.V(4).Info("Addon template config in status is empty", "addonName", addonName)
return nil, nil
}

desiredTemplate := templateRef.DesiredConfig
if desiredTemplate == nil || desiredTemplate.SpecHash == "" {
a.logger.Info("Addon template spec hash is empty", "addonName", addon.Name)
return nil, fmt.Errorf("addon %s template desired spec hash is empty", addon.Name)
a.logger.Info("Addon template spec hash is empty", "addonName", addonName)
return nil, fmt.Errorf("addon %s template desired spec hash is empty", addonName)
}

template, err := a.addonTemplateLister.Get(desiredTemplate.Name)
Expand Down
166 changes: 166 additions & 0 deletions pkg/addon/templateagent/template_agent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ import (
"k8s.io/klog/v2/ktesting"

"open-cluster-management.io/addon-framework/pkg/addonfactory"
"open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting"
"open-cluster-management.io/addon-framework/pkg/utils"
addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake"
addoninformers "open-cluster-management.io/api/client/addon/informers/externalversions"
clusterv1 "open-cluster-management.io/api/cluster/v1"
clusterv1apha1 "open-cluster-management.io/api/cluster/v1alpha1"
workapiv1 "open-cluster-management.io/api/work/v1"
)

func TestAddonTemplateAgentManifests(t *testing.T) {
Expand Down Expand Up @@ -385,6 +387,170 @@ func TestAgentInstallNamespace(t *testing.T) {
}
}

func TestAgentManifestConfigs(t *testing.T) {
_, ctx := ktesting.NewTestContext(t)
addonName := "hello"

s := runtime.NewScheme()
_ = scheme.AddToScheme(s)
_ = clusterv1apha1.Install(s)
_ = addonapiv1alpha1.Install(s)

cases := []struct {
name string
addonTemplate *addonapiv1alpha1.AddOnTemplate
clusterManagementAddOn *addonapiv1alpha1.ClusterManagementAddOn
expected []workapiv1.ManifestConfigOption
}{
{
name: "no addontemplate in cma status",
clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").Build(),
expected: nil,
},
{
name: "no addontemplate",
clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").
WithDefaultConfigReferences(
addonapiv1alpha1.DefaultConfigReference{
ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
Group: utils.AddOnTemplateGVR.Group,
Resource: utils.AddOnTemplateGVR.Resource,
},
DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"},
SpecHash: "hash",
},
},
).Build(),
expected: nil,
},
{
name: "addontemplate does not have manifestconfigs",
clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").
WithDefaultConfigReferences(
addonapiv1alpha1.DefaultConfigReference{
ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
Group: utils.AddOnTemplateGVR.Group,
Resource: utils.AddOnTemplateGVR.Resource,
},
DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"},
SpecHash: "hash",
},
},
).Build(),
addonTemplate: &addonapiv1alpha1.AddOnTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "hello-template",
},
},
expected: nil,
},
{
name: "addontemplate has manifestconfigs",
clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").
WithDefaultConfigReferences(
addonapiv1alpha1.DefaultConfigReference{
ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
Group: utils.AddOnTemplateGVR.Group,
Resource: utils.AddOnTemplateGVR.Resource,
},
DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"},
SpecHash: "hash",
},
},
).Build(),
addonTemplate: &addonapiv1alpha1.AddOnTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "hello-template",
},
Spec: addonapiv1alpha1.AddOnTemplateSpec{
AddonName: "hello",
AgentSpec: workapiv1.ManifestWorkSpec{
ManifestConfigs: []workapiv1.ManifestConfigOption{
{
ResourceIdentifier: workapiv1.ResourceIdentifier{
Group: "apps",
Resource: "deployment",
Name: "hello",
Namespace: "default",
},
FeedbackRules: []workapiv1.FeedbackRule{
{
Type: workapiv1.WellKnownStatusType,
},
},
},
},
},
},
},
expected: []workapiv1.ManifestConfigOption{
{
ResourceIdentifier: workapiv1.ResourceIdentifier{
Group: "apps",
Resource: "deployment",
Name: "hello",
Namespace: "default",
},
FeedbackRules: []workapiv1.FeedbackRule{
{
Type: workapiv1.WellKnownStatusType,
},
},
},
},
},
}

for _, tc := range cases {
var objs []runtime.Object
if tc.clusterManagementAddOn != nil {
objs = append(objs, tc.clusterManagementAddOn)
}
if tc.addonTemplate != nil {
objs = append(objs, tc.addonTemplate)
}
hubKubeClient := fakekube.NewSimpleClientset()
addonClient := fakeaddon.NewSimpleClientset(objs...)

addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute)
if tc.clusterManagementAddOn != nil {
cmaStore := addonInformerFactory.Addon().V1alpha1().ClusterManagementAddOns().Informer().GetStore()
if err := cmaStore.Add(tc.clusterManagementAddOn); err != nil {
t.Fatal(err)
}
}
if tc.addonTemplate != nil {
atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore()
if err := atStore.Add(tc.addonTemplate); err != nil {
t.Fatal(err)
}
}
kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute)

agentAddon := NewCRDTemplateAgentAddon(
ctx,
addonName,
"test-agent",
hubKubeClient,
addonClient,
addonInformerFactory,
kubeInformers.Rbac().V1().RoleBindings().Lister(),
addonfactory.GetAddOnDeploymentConfigValues(
addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
addonfactory.ToAddOnCustomizedVariableValues,
ToAddOnNodePlacementPrivateValues,
ToAddOnRegistriesPrivateValues,
),
)

agentManifestConfigs := agentAddon.GetAgentAddonOptions().ManifestConfigs
assert.Equal(t, tc.expected, agentManifestConfigs)
}
}

type testManagedClusterAddOnBuilder struct {
managedClusterAddOn *addonapiv1alpha1.ManagedClusterAddOn
addonTemplate *addonapiv1alpha1.AddOnTemplate
Expand Down
Loading

0 comments on commit 6571a8a

Please sign in to comment.