From 6e73eeb0ac8d52629e1d5109d244cdda52e72849 Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Fri, 7 Jun 2024 13:59:14 +0200 Subject: [PATCH] feat(meshservice): sync mesh service to other zones (#10380) Signed-off-by: Jakub Dyszkiewicz --- docs/generated/raw/kuma-cp.yaml | 6 ++-- .../base_endpoints/_resources.golden.json | 2 +- pkg/config/app/kuma-cp/kuma-cp.defaults.yaml | 6 ++-- .../meshservice/api/v1alpha1/meshservice.go | 1 + .../api/v1alpha1/zz_generated.resource.go | 2 +- pkg/kds/context/context.go | 14 ++++++-- pkg/kds/util/resource.go | 33 ++++++++++++++++--- test/e2e_env/multizone/meshservice/status.go | 10 +++++- test/framework/envs/multizone/env.go | 1 + .../policy-gen/generator/cmd/core_resource.go | 2 +- .../generator/pkg/parse/policyconfig.go | 6 ++++ 11 files changed, 67 insertions(+), 16 deletions(-) diff --git a/docs/generated/raw/kuma-cp.yaml b/docs/generated/raw/kuma-cp.yaml index b606e0741696..10fc5d80db6f 100644 --- a/docs/generated/raw/kuma-cp.yaml +++ b/docs/generated/raw/kuma-cp.yaml @@ -800,9 +800,9 @@ ipam: # MeshService address management meshService: # CIDR for MeshService IPs - cidr: 241.0.0.0/8 # ENV: IPAM_MESH_SERVICE_CIDR + cidr: 241.0.0.0/8 # ENV: KUMA_IPAM_MESH_SERVICE_CIDR meshExternalService: # CIDR for MeshExternalService IPs - cidr: 242.0.0.0/8 # ENV: IPAM_MESH_EXTERNAL_SERVICE_CIDR + cidr: 242.0.0.0/8 # ENV: KUMA_IPAM_MESH_EXTERNAL_SERVICE_CIDR # Interval on which Kuma will allocate new IPs for MeshServices and MeshExternalServices - allocationInterval: 5s # ENV: IPAM_ALLOCATION_INTERVAL + allocationInterval: 5s # ENV: KUMA_IPAM_ALLOCATION_INTERVAL diff --git a/pkg/api-server/testdata/base_endpoints/_resources.golden.json b/pkg/api-server/testdata/base_endpoints/_resources.golden.json index 96f2e9ef9eb4..8ea98e8b0265 100644 --- a/pkg/api-server/testdata/base_endpoints/_resources.golden.json +++ b/pkg/api-server/testdata/base_endpoints/_resources.golden.json @@ -292,7 +292,7 @@ "singularDisplayName": "Mesh Retry" }, { - "includeInFederation": true, + "includeInFederation": false, "name": "MeshService", "path": "meshservices", "pluralDisplayName": "Mesh Services", diff --git a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml index b606e0741696..10fc5d80db6f 100644 --- a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml +++ b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml @@ -800,9 +800,9 @@ ipam: # MeshService address management meshService: # CIDR for MeshService IPs - cidr: 241.0.0.0/8 # ENV: IPAM_MESH_SERVICE_CIDR + cidr: 241.0.0.0/8 # ENV: KUMA_IPAM_MESH_SERVICE_CIDR meshExternalService: # CIDR for MeshExternalService IPs - cidr: 242.0.0.0/8 # ENV: IPAM_MESH_EXTERNAL_SERVICE_CIDR + cidr: 242.0.0.0/8 # ENV: KUMA_IPAM_MESH_EXTERNAL_SERVICE_CIDR # Interval on which Kuma will allocate new IPs for MeshServices and MeshExternalServices - allocationInterval: 5s # ENV: IPAM_ALLOCATION_INTERVAL + allocationInterval: 5s # ENV: KUMA_IPAM_ALLOCATION_INTERVAL diff --git a/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go b/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go index 2a7ab0e8f579..d3f8611e557e 100644 --- a/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go +++ b/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go @@ -29,6 +29,7 @@ type Port struct { // MeshService // +kuma:policy:is_policy=false // +kuma:policy:has_status=true +// +kuma:policy:kds_flags=model.ZoneToGlobalFlag | model.GlobalToAllButOriginalZoneFlag type MeshService struct { Selector Selector `json:"selector,omitempty"` // +patchMergeKey=port diff --git a/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.resource.go b/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.resource.go index 9e42b09d3bfa..cb24324ded0e 100644 --- a/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.resource.go +++ b/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.resource.go @@ -146,7 +146,7 @@ var MeshServiceResourceTypeDescriptor = model.ResourceTypeDescriptor{ Resource: NewMeshServiceResource(), ResourceList: &MeshServiceResourceList{}, Scope: model.ScopeMesh, - KDSFlags: model.GlobalToAllZonesFlag | model.ZoneToGlobalFlag, + KDSFlags: model.ZoneToGlobalFlag | model.GlobalToAllButOriginalZoneFlag, WsPath: "meshservices", KumactlArg: "meshservice", KumactlListArg: "meshservices", diff --git a/pkg/kds/context/context.go b/pkg/kds/context/context.go index a6908d8b06b4..4aa28a682945 100644 --- a/pkg/kds/context/context.go +++ b/pkg/kds/context/context.go @@ -20,6 +20,7 @@ import ( "github.com/kumahq/kuma/pkg/core" config_manager "github.com/kumahq/kuma/pkg/core/config/manager" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + meshservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1" "github.com/kumahq/kuma/pkg/core/resources/apis/system" "github.com/kumahq/kuma/pkg/core/resources/manager" core_model "github.com/kumahq/kuma/pkg/core/resources/model" @@ -77,6 +78,9 @@ func DefaultContext( reconcile.If( reconcile.IsKubernetes(cfg.Store.Type), RemoveK8sSystemNamespaceSuffixMapper(cfg.Store.Kubernetes.SystemNamespace)), + reconcile.If( + reconcile.TypeIs(meshservice_api.MeshServiceType), + RemoveStatus()), reconcile.If( reconcile.And( reconcile.ScopeIs(core_model.ScopeMesh), @@ -192,7 +196,13 @@ func RemoveK8sSystemNamespaceSuffixMapper(k8sSystemNamespace string) reconcile.R return func(_ kds.Features, r core_model.Resource) (core_model.Resource, error) { dotSuffix := fmt.Sprintf(".%s", k8sSystemNamespace) newName := strings.TrimSuffix(r.GetMeta().GetName(), dotSuffix) - return util.CloneResource(r, util.WithName(newName)), nil + return util.CloneResource(r, util.WithResourceName(newName)), nil + } +} + +func RemoveStatus() reconcile.ResourceMapper { + return func(_ kds.Features, r core_model.Resource) (core_model.Resource, error) { + return util.CloneResource(r, util.WithoutStatus()), nil } } @@ -209,7 +219,7 @@ func HashSuffixMapper(checkKDSFeature bool, labelsToUse ...string) reconcile.Res values = append(values, r.GetMeta().GetLabels()[lbl]) } - return util.CloneResource(r, util.WithName(hash.HashedName(r.GetMeta().GetMesh(), name, values...))), nil + return util.CloneResource(r, util.WithResourceName(hash.HashedName(r.GetMeta().GetMesh(), name, values...))), nil } } diff --git a/pkg/kds/util/resource.go b/pkg/kds/util/resource.go index f80671fdb2c6..cc6df81ae2ca 100644 --- a/pkg/kds/util/resource.go +++ b/pkg/kds/util/resource.go @@ -126,7 +126,8 @@ func ZoneTag(r model.Resource) string { case *mesh_proto.ZoneEgress: return res.GetZone() default: - return "" + // todo(jakubdyszkiewicz): consider replacing this whole function with just model.ZoneOfResource(r) + return model.ZoneOfResource(r) } } @@ -168,12 +169,36 @@ func StatsOf(status *system_proto.KDSSubscriptionStatus, resourceType model.Reso return stat } -func CloneResource(res core_model.Resource, fs ...CloneResourceMetaOpt) core_model.Resource { +type cloneResource struct { + name string + withoutStatus bool +} + +func WithResourceName(name string) CloneResourceOpt { + return func(m *cloneResource) { + m.name = name + } +} + +func WithoutStatus() CloneResourceOpt { + return func(m *cloneResource) { + m.withoutStatus = true + } +} + +type CloneResourceOpt func(*cloneResource) + +func CloneResource(res core_model.Resource, fs ...CloneResourceOpt) core_model.Resource { + opts := &cloneResource{} + for _, f := range fs { + f(opts) + } + newObj := res.Descriptor().NewObject() - newMeta := CloneResourceMeta(res.GetMeta(), fs...) + newMeta := CloneResourceMeta(res.GetMeta(), WithName(opts.name)) newObj.SetMeta(newMeta) _ = newObj.SetSpec(res.GetSpec()) - if newObj.Descriptor().HasStatus { + if newObj.Descriptor().HasStatus && !opts.withoutStatus { _ = newObj.SetStatus(res.GetStatus()) } return newObj diff --git a/test/e2e_env/multizone/meshservice/status.go b/test/e2e_env/multizone/meshservice/status.go index 4991b3b92f48..1c5bd2d5c2dd 100644 --- a/test/e2e_env/multizone/meshservice/status.go +++ b/test/e2e_env/multizone/meshservice/status.go @@ -56,12 +56,20 @@ spec: g.Expect(out).To(ContainSubstring("ip: 241.0.0.")) }, "30s", "1s").Should(Succeed()) - // and MeshService is synced to global with status + // and MeshService is synced to global with the original status Eventually(func(g Gomega) { out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "meshservices", "-m", meshName, "-oyaml") g.Expect(err).ToNot(HaveOccurred()) g.Expect(out).To(ContainSubstring("kuma.io/display-name: backend")) g.Expect(out).To(ContainSubstring("ip: 241.0.0.")) }, "30s", "1s").Should(Succeed()) + + // and MeshService is synced to other zone but status is generated by other zone + Eventually(func(g Gomega) { + out, err := multizone.UniZone2.GetKumactlOptions().RunKumactlAndGetOutput("get", "meshservices", "-m", meshName, "-oyaml") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out).To(ContainSubstring("kuma.io/display-name: backend")) + g.Expect(out).To(ContainSubstring("ip: 251.0.0.")) + }, "30s", "1s").Should(Succeed()) }) } diff --git a/test/framework/envs/multizone/env.go b/test/framework/envs/multizone/env.go index 14107b917347..7aad8e87ccae 100644 --- a/test/framework/envs/multizone/env.go +++ b/test/framework/envs/multizone/env.go @@ -142,6 +142,7 @@ func SetupAndGetState() []byte { WithIngressEnvoyAdminTunnel(), WithEnv("KUMA_XDS_DATAPLANE_DEREGISTRATION_DELAY", "0s"), // we have only 1 Kuma CP instance so there is no risk setting this to 0 WithEnv("KUMA_MULTIZONE_ZONE_KDS_NACK_BACKOFF", "1s"), + WithEnv("KUMA_IPAM_MESH_SERVICE_CIDR", "251.0.0.0/8"), // just to see that the status is not synced around }, framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.UniZone2)..., ) diff --git a/tools/policy-gen/generator/cmd/core_resource.go b/tools/policy-gen/generator/cmd/core_resource.go index 0e46d075ab4f..5f4eadf15fb4 100644 --- a/tools/policy-gen/generator/cmd/core_resource.go +++ b/tools/policy-gen/generator/cmd/core_resource.go @@ -203,7 +203,7 @@ var {{.Name}}ResourceTypeDescriptor = model.ResourceTypeDescriptor{ Resource: New{{.Name}}Resource(), ResourceList: &{{.Name}}ResourceList{}, Scope: model.ScopeMesh, - KDSFlags: model.GlobalToAllZonesFlag | model.ZoneToGlobalFlag, + KDSFlags: {{.KDSFlags}}, WsPath: "{{.Path}}", KumactlArg: "{{index .AlternativeNames 0}}", KumactlListArg: "{{.Path}}", diff --git a/tools/policy-gen/generator/pkg/parse/policyconfig.go b/tools/policy-gen/generator/pkg/parse/policyconfig.go index 33a9ed2b9322..e3aa0f8401fb 100644 --- a/tools/policy-gen/generator/pkg/parse/policyconfig.go +++ b/tools/policy-gen/generator/pkg/parse/policyconfig.go @@ -30,6 +30,7 @@ type PolicyConfig struct { GoModule string ResourceDir string IsPolicy bool + KDSFlags string } func Policy(path string) (PolicyConfig, error) { @@ -135,6 +136,11 @@ func newPolicyConfig(pkg, name string, markers map[string]string, fields map[str if v, ok := parseBool(markers, "kuma:policy:has_status"); ok { res.HasStatus = v } + if v, ok := markers["kuma:policy:kds_flags"]; ok { + res.KDSFlags = v + } else { + res.KDSFlags = "model.GlobalToAllZonesFlag | model.ZoneToGlobalFlag" + } if v, ok := markers["kuma:policy:singular_display_name"]; ok { res.SingularDisplayName = v