Skip to content

Commit

Permalink
implementation of Policy LazyReconcile feature
Browse files Browse the repository at this point in the history 
Signed-off-by: chaosi-zju <chaosi@zju.edu.cn>
  • Loading branch information
@chaosi-zju
chaosi-zju committed Jan 21, 2024
1 parent 0e501b6 commit c245c93
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 36 deletions.
109 changes: 73 additions & 36 deletions pkg/detector/detector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1127,7 +1127,10 @@ func (d *ResourceDetector) ReconcileClusterPropagationPolicy(key util.QueueKey)
// After a policy is removed, the label marked on relevant resource template will be removed(which gives
// the resource template a change to match another policy).
//
// Note: The relevant ResourceBinding will continue to exist until the resource template is gone.
// Notes:
// 1. The relevant ResourceBinding will continue to exist until the resource template is gone.
// 2. If PolicyLazyReconcile feature enabled, the marked label will also be removed, but the label change is by Karmada self,
// which will be ignored by Detector, so Resource will not execute reconcile and keep status unchanged after label removed.
func (d *ResourceDetector) HandlePropagationPolicyDeletion(policyNS string, policyName string) error {
labelSet := labels.Set{
policyv1alpha1.PropagationPolicyNamespaceLabel: policyNS,
Expand Down Expand Up @@ -1164,7 +1167,10 @@ func (d *ResourceDetector) HandlePropagationPolicyDeletion(policyNS string, poli
// After a policy is removed, the label marked on relevant resource template will be removed(which gives
// the resource template a change to match another policy).
//
// Note: The relevant ClusterResourceBinding or ResourceBinding will continue to exist until the resource template is gone.
// Notes:
// 1. The relevant ClusterResourceBinding or ResourceBinding will continue to exist until the resource template is gone.
// 2. If PolicyLazyReconcile feature enabled, the marked label will also be removed, but the label change is by Karmada self,
// which will be ignored by Detector, so Resource will not execute reconcile and keep status unchanged after label removed.
func (d *ResourceDetector) HandleClusterPropagationPolicyDeletion(policyName string) error {
var errs []error
labelSet := labels.Set{
Expand Down Expand Up @@ -1234,26 +1240,34 @@ func (d *ResourceDetector) HandleClusterPropagationPolicyDeletion(policyName str
// from waiting list and throw the object to it's reconcile queue. If not, do nothing.
// Finally, handle the propagation policy preemption process if preemption is enabled.
func (d *ResourceDetector) HandlePropagationPolicyCreationOrUpdate(policy *policyv1alpha1.PropagationPolicy) error {
// If ResourceSelectors of Policy changed and thus some Resource no longer match to the Policy,
// the policy label marked on Resource and Binding will be deleted, and thus Resource will watch the label change and execute the reconcile.
// However, if PolicyLazyReconcile feature enabled, the marked label will also be removed, but the label change is by Karmada self,
// thus the event will be ignored by Detector, so the resource will not execute reconcile and keep status unchanged after label removed.
err := d.cleanPPUnmatchedResourceBindings(policy.Namespace, policy.Name, policy.Spec.ResourceSelectors)
if err != nil {
return err
}

// When updating fields other than ResourceSelector, should first find the corresponding ResourceBinding
// and add the bound object to the processor's queue for reconciliation to make sure that
// PropagationPolicy's updates can be synchronized to ResourceBinding.
resourceBindings, err := d.listPPDerivedRB(policy.Namespace, policy.Name)
if err != nil {
return err
}
for _, rb := range resourceBindings.Items {
resourceKey, err := helper.ConstructClusterWideKey(rb.Spec.Resource)
// if PolicyLazyReconcile disabled, add the PP related RT to processor, otherwise just skip this step
if !features.FeatureGate.Enabled(features.PolicyLazyReconcile) {
// When updating fields other than ResourceSelector, should first find the corresponding ResourceBinding
// and add the bound object to the processor's queue for reconciliation to make sure that
// PropagationPolicy's updates can be synchronized to ResourceBinding.
resourceBindings, err := d.listPPDerivedRB(policy.Namespace, policy.Name)
if err != nil {
return err
}
d.Processor.Add(resourceKey)
for _, rb := range resourceBindings.Items {
resourceKey, err := helper.ConstructClusterWideKey(rb.Spec.Resource)
if err != nil {
return err
}
d.Processor.Add(resourceKey)
}
}

// check whether there are matched RT in waiting list, is so, add it to processor
matchedKeys := d.GetMatching(policy.Spec.ResourceSelectors)
klog.Infof("Matched %d resources by policy(%s/%s)", len(matchedKeys), policy.Namespace, policy.Name)

Expand All @@ -1271,7 +1285,11 @@ func (d *ResourceDetector) HandlePropagationPolicyCreationOrUpdate(policy *polic
d.Processor.Add(key)
}

// if preemption is enabled, handle the preemption process.
// If preemption is enabled, handle the preemption process.
// If this policy succeeds in preempting resource managed by other policy, the label marked on relevant resource will be replaced,
// which gives the resource template a change to match to this policy.
// However, if PolicyLazyReconcile feature enabled, the marked label will also be replaced, but the label change is by Karmada self,
// thus the event will be ignored by Detector, so the resource will not adjust to this policy immediately util the resource itself updated.
if preemptionEnabled(policy.Spec.Preemption) {
return d.handlePropagationPolicyPreemption(policy)
}
Expand All @@ -1286,6 +1304,10 @@ func (d *ResourceDetector) HandlePropagationPolicyCreationOrUpdate(policy *polic
// from waiting list and throw the object to it's reconcile queue. If not, do nothing.
// Finally, handle the cluster propagation policy preemption process if preemption is enabled.
func (d *ResourceDetector) HandleClusterPropagationPolicyCreationOrUpdate(policy *policyv1alpha1.ClusterPropagationPolicy) error {
// If ResourceSelectors of Policy changed and thus some Resource no longer match to the Policy,
// the policy label marked on Resource and Binding will be removed, and thus Resource will watch the label change and execute a reconcile.
// However, if PolicyLazyReconcile feature enabled, the marked label will also be removed, but the label change is by Karmada self,
// thus the event will be ignored by Detector, so the resource will not execute reconcile and keep status unchanged after label removed.
err := d.cleanCPPUnmatchedResourceBindings(policy.Name, policy.Spec.ResourceSelectors)
if err != nil {
return err
Expand All @@ -1296,32 +1318,14 @@ func (d *ResourceDetector) HandleClusterPropagationPolicyCreationOrUpdate(policy
return err
}

// When updating fields other than ResourceSelector, should first find the corresponding ResourceBinding/ClusterResourceBinding
// and add the bound object to the processor's queue for reconciliation to make sure that
// ClusterPropagationPolicy's updates can be synchronized to ResourceBinding/ClusterResourceBinding.
resourceBindings, err := d.listCPPDerivedRB(policy.Name)
if err != nil {
return err
}
clusterResourceBindings, err := d.listCPPDerivedCRB(policy.Name)
if err != nil {
return err
}
for _, rb := range resourceBindings.Items {
resourceKey, err := helper.ConstructClusterWideKey(rb.Spec.Resource)
if err != nil {
return err
}
d.Processor.Add(resourceKey)
}
for _, crb := range clusterResourceBindings.Items {
resourceKey, err := helper.ConstructClusterWideKey(crb.Spec.Resource)
if err != nil {
// if PolicyLazyReconcile disabled, add the PP related RT to processor, otherwise just skip this step
if !features.FeatureGate.Enabled(features.PolicyLazyReconcile) {
if err = d.addRelatedTemplateToProcessorQueue(policy); err != nil {
return err
}
d.Processor.Add(resourceKey)
}

// check whether there are matched RT in waiting list, is so, add it to processor
matchedKeys := d.GetMatching(policy.Spec.ResourceSelectors)
klog.Infof("Matched %d resources by policy(%s)", len(matchedKeys), policy.Name)

Expand All @@ -1339,14 +1343,47 @@ func (d *ResourceDetector) HandleClusterPropagationPolicyCreationOrUpdate(policy
d.Processor.Add(key)
}

// if preemption is enabled, handle the preemption process.
// If preemption is enabled, handle the preemption process.
// If this policy succeeds in preempting resource managed by other policy, the label marked on relevant resource will be replaced,
// which gives the resource template a change to match to this policy.
// However, if PolicyLazyReconcile feature enabled, the marked label will also be replaced, but the label change is by Karmada self,
// thus the event will be ignored by Detector, so the resource will not adjust to this policy immediately util the resource itself updated.
if preemptionEnabled(policy.Spec.Preemption) {
return d.handleClusterPropagationPolicyPreemption(policy)
}

return nil
}

// addRelatedTemplateToProcessorQueue When updating fields other than ResourceSelector, should first find the corresponding
// ResourceBinding/ClusterResourceBinding and add the bound object to the processor's queue for reconciliation to make sure that
// ClusterPropagationPolicy's updates can be synchronized to ResourceBinding/ClusterResourceBinding.
func (d *ResourceDetector) addRelatedTemplateToProcessorQueue(policy *policyv1alpha1.ClusterPropagationPolicy) error {
resourceBindings, err := d.listCPPDerivedRB(policy.Name)
if err != nil {
return fmt.Errorf("failed to list CPP(%s) derived RB: %+v", policy.Name, err)
}
clusterResourceBindings, err := d.listCPPDerivedCRB(policy.Name)
if err != nil {
return fmt.Errorf("failed to list CPP(%s) derived CRB: %+v", policy.Name, err)
}
for _, rb := range resourceBindings.Items {
resourceKey, err := helper.ConstructClusterWideKey(rb.Spec.Resource)
if err != nil {
return fmt.Errorf("failed to get resource ClusterWideKey from RB(%s): %+v", rb.Name, err)
}
d.Processor.Add(resourceKey)
}
for _, crb := range clusterResourceBindings.Items {
resourceKey, err := helper.ConstructClusterWideKey(crb.Spec.Resource)
if err != nil {
return fmt.Errorf("failed to get resource ClusterWideKey from CRB(%s): %+v", crb.Name, err)
}
d.Processor.Add(resourceKey)
}
return nil
}

// CleanupLabels removes labels from object referencing by objRef.
func (d *ResourceDetector) CleanupLabels(objRef workv1alpha2.ObjectReference, labelKeys ...string) error {
workload, err := helper.FetchResourceTemplate(d.DynamicClient, d.InformerManager, d.RESTMapper, objRef)
Expand Down
5 changes: 5 additions & 0 deletions pkg/features/features.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ const (

// MultiClusterService indicates if enable multi-cluster service function.
MultiClusterService featuregate.Feature = "MultiClusterService"

// PolicyLazyReconcile indicates if enable Lazy Reconcile feature for Propagation Policy.
// If enabled, the effective time of policy modification will be delayed until the resource template is modified.
PolicyLazyReconcile featuregate.Feature = "PolicyLazyReconcile"
)

var (
Expand All @@ -54,6 +58,7 @@ var (
CustomizedClusterResourceModeling: {Default: true, PreRelease: featuregate.Beta},
PolicyPreemption: {Default: false, PreRelease: featuregate.Alpha},
MultiClusterService: {Default: false, PreRelease: featuregate.Alpha},
PolicyLazyReconcile: {Default: false, PreRelease: featuregate.Alpha},
}
)

Expand Down
39 changes: 39 additions & 0 deletions pkg/util/eventfilter/eventfilter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,22 @@ package eventfilter

import (
"reflect"
"strings"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
"github.com/karmada-io/karmada/pkg/features"
)

// labelOrAnnoKeyPrefixByKarmada defines the key prefix used for labels or annotations used by karmada's own components.
const labelOrAnnoKeyPrefixByKarmada = ".karmada.io"

// labelsForUserWithKarmadaPrefix enumerates special cases that labels use the karmada prefix but are really only for users to use.
var labelsForUserWithKarmadaPrefix = map[string]struct{}{
policyv1alpha1.NamespaceSkipAutoPropagationLabel: {},
}

// SpecificationChanged check if the specification of the given object change or not
func SpecificationChanged(oldObj, newObj *unstructured.Unstructured) bool {
oldBackup := oldObj.DeepCopy()
Expand All @@ -35,5 +47,32 @@ func SpecificationChanged(oldObj, newObj *unstructured.Unstructured) bool {
unstructured.RemoveNestedField(newBackup.Object, r...)
}

// If Policy LazyReconcile feature enabled, ignore the label/annotation modifications by karmada's own components.
if features.FeatureGate.Enabled(features.PolicyLazyReconcile) {
// Remove the labels or annotations modified by karmada's own components.
for k := range oldBackup.GetLabels() {
_, isLabelForUser := labelsForUserWithKarmadaPrefix[k]
if strings.Contains(k, labelOrAnnoKeyPrefixByKarmada) && !isLabelForUser {
unstructured.RemoveNestedField(oldBackup.Object, []string{"metadata", "labels", k}...)
}
}
for k := range oldBackup.GetAnnotations() {
if strings.Contains(k, labelOrAnnoKeyPrefixByKarmada) {
unstructured.RemoveNestedField(oldBackup.Object, []string{"metadata", "annotations", k}...)
}
}
for k := range newBackup.GetLabels() {
_, isLabelForUser := labelsForUserWithKarmadaPrefix[k]
if strings.Contains(k, labelOrAnnoKeyPrefixByKarmada) && !isLabelForUser {
unstructured.RemoveNestedField(newBackup.Object, []string{"metadata", "labels", k}...)
}
}
for k := range newBackup.GetAnnotations() {
if strings.Contains(k, labelOrAnnoKeyPrefixByKarmada) {
unstructured.RemoveNestedField(newBackup.Object, []string{"metadata", "annotations", k}...)
}
}
}

return !reflect.DeepEqual(oldBackup, newBackup)
}

0 comments on commit c245c93

Please sign in to comment.