feat(meshservice): sync mesh service to other zones (#10380)

Signed-off-by: Jakub Dyszkiewicz <jakub.dyszkiewicz@gmail.com>

docs/generated/raw/kuma-cp.yaml

@@ -800,9 +800,9 @@ ipam:
   # MeshService address management
   meshService:
     # CIDR for MeshService IPs
-    cidr: 241.0.0.0/8 # ENV: IPAM_MESH_SERVICE_CIDR
+    cidr: 241.0.0.0/8 # ENV: KUMA_IPAM_MESH_SERVICE_CIDR
   meshExternalService:
     # CIDR for MeshExternalService IPs
-    cidr: 242.0.0.0/8 # ENV: IPAM_MESH_EXTERNAL_SERVICE_CIDR
+    cidr: 242.0.0.0/8 # ENV: KUMA_IPAM_MESH_EXTERNAL_SERVICE_CIDR
   # Interval on which Kuma will allocate new IPs for MeshServices and MeshExternalServices
-  allocationInterval: 5s # ENV: IPAM_ALLOCATION_INTERVAL
+  allocationInterval: 5s # ENV: KUMA_IPAM_ALLOCATION_INTERVAL

pkg/api-server/testdata/base_endpoints/_resources.golden.json

@@ -292,7 +292,7 @@
    "singularDisplayName": "Mesh Retry"
   },
   {
-   "includeInFederation": true,
+   "includeInFederation": false,
    "name": "MeshService",
    "path": "meshservices",
    "pluralDisplayName": "Mesh Services",

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -800,9 +800,9 @@ ipam:
   # MeshService address management
   meshService:
     # CIDR for MeshService IPs
-    cidr: 241.0.0.0/8 # ENV: IPAM_MESH_SERVICE_CIDR
+    cidr: 241.0.0.0/8 # ENV: KUMA_IPAM_MESH_SERVICE_CIDR
   meshExternalService:
     # CIDR for MeshExternalService IPs
-    cidr: 242.0.0.0/8 # ENV: IPAM_MESH_EXTERNAL_SERVICE_CIDR
+    cidr: 242.0.0.0/8 # ENV: KUMA_IPAM_MESH_EXTERNAL_SERVICE_CIDR
   # Interval on which Kuma will allocate new IPs for MeshServices and MeshExternalServices
-  allocationInterval: 5s # ENV: IPAM_ALLOCATION_INTERVAL
+  allocationInterval: 5s # ENV: KUMA_IPAM_ALLOCATION_INTERVAL

pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go

@@ -29,6 +29,7 @@ type Port struct {
 // MeshService
 // +kuma:policy:is_policy=false
 // +kuma:policy:has_status=true
+// +kuma:policy:kds_flags=model.ZoneToGlobalFlag | model.GlobalToAllButOriginalZoneFlag
 type MeshService struct {
 	Selector Selector `json:"selector,omitempty"`
 	// +patchMergeKey=port

pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.resource.go

@@ -146,7 +146,7 @@ var MeshServiceResourceTypeDescriptor = model.ResourceTypeDescriptor{
 	Resource:            NewMeshServiceResource(),
 	ResourceList:        &MeshServiceResourceList{},
 	Scope:               model.ScopeMesh,
-	KDSFlags:            model.GlobalToAllZonesFlag | model.ZoneToGlobalFlag,
+	KDSFlags:            model.ZoneToGlobalFlag | model.GlobalToAllButOriginalZoneFlag,
 	WsPath:              "meshservices",
 	KumactlArg:          "meshservice",
 	KumactlListArg:      "meshservices",

pkg/kds/context/context.go

@@ -20,6 +20,7 @@ import (
 	"github.com/kumahq/kuma/pkg/core"
 	config_manager "github.com/kumahq/kuma/pkg/core/config/manager"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
+	meshservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1"
 	"github.com/kumahq/kuma/pkg/core/resources/apis/system"
 	"github.com/kumahq/kuma/pkg/core/resources/manager"
 	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
@@ -77,6 +78,9 @@ func DefaultContext(
 		reconcile.If(
 			reconcile.IsKubernetes(cfg.Store.Type),
 			RemoveK8sSystemNamespaceSuffixMapper(cfg.Store.Kubernetes.SystemNamespace)),
+		reconcile.If(
+			reconcile.TypeIs(meshservice_api.MeshServiceType),
+			RemoveStatus()),
 		reconcile.If(
 			reconcile.And(
 				reconcile.ScopeIs(core_model.ScopeMesh),
@@ -192,7 +196,13 @@ func RemoveK8sSystemNamespaceSuffixMapper(k8sSystemNamespace string) reconcile.R
 	return func(_ kds.Features, r core_model.Resource) (core_model.Resource, error) {
 		dotSuffix := fmt.Sprintf(".%s", k8sSystemNamespace)
 		newName := strings.TrimSuffix(r.GetMeta().GetName(), dotSuffix)
-		return util.CloneResource(r, util.WithName(newName)), nil
+		return util.CloneResource(r, util.WithResourceName(newName)), nil
+	}
+}
+
+func RemoveStatus() reconcile.ResourceMapper {
+	return func(_ kds.Features, r core_model.Resource) (core_model.Resource, error) {
+		return util.CloneResource(r, util.WithoutStatus()), nil
 	}
 }
 
@@ -209,7 +219,7 @@ func HashSuffixMapper(checkKDSFeature bool, labelsToUse ...string) reconcile.Res
 			values = append(values, r.GetMeta().GetLabels()[lbl])
 		}
 
-		return util.CloneResource(r, util.WithName(hash.HashedName(r.GetMeta().GetMesh(), name, values...))), nil
+		return util.CloneResource(r, util.WithResourceName(hash.HashedName(r.GetMeta().GetMesh(), name, values...))), nil
 	}
 }
 

pkg/kds/util/resource.go

@@ -126,7 +126,8 @@ func ZoneTag(r model.Resource) string {
 	case *mesh_proto.ZoneEgress:
 		return res.GetZone()
 	default:
-		return ""
+		// todo(jakubdyszkiewicz): consider replacing this whole function with just model.ZoneOfResource(r)
+		return model.ZoneOfResource(r)
 	}
 }
 
@@ -168,12 +169,36 @@ func StatsOf(status *system_proto.KDSSubscriptionStatus, resourceType model.Reso
 	return stat
 }
 
-func CloneResource(res core_model.Resource, fs ...CloneResourceMetaOpt) core_model.Resource {
+type cloneResource struct {
+	name          string
+	withoutStatus bool
+}
+
+func WithResourceName(name string) CloneResourceOpt {
+	return func(m *cloneResource) {
+		m.name = name
+	}
+}
+
+func WithoutStatus() CloneResourceOpt {
+	return func(m *cloneResource) {
+		m.withoutStatus = true
+	}
+}
+
+type CloneResourceOpt func(*cloneResource)
+
+func CloneResource(res core_model.Resource, fs ...CloneResourceOpt) core_model.Resource {
+	opts := &cloneResource{}
+	for _, f := range fs {
+		f(opts)
+	}
+
 	newObj := res.Descriptor().NewObject()
-	newMeta := CloneResourceMeta(res.GetMeta(), fs...)
+	newMeta := CloneResourceMeta(res.GetMeta(), WithName(opts.name))
 	newObj.SetMeta(newMeta)
 	_ = newObj.SetSpec(res.GetSpec())
-	if newObj.Descriptor().HasStatus {
+	if newObj.Descriptor().HasStatus && !opts.withoutStatus {
 		_ = newObj.SetStatus(res.GetStatus())
 	}
 	return newObj

test/e2e_env/multizone/meshservice/status.go

@@ -56,12 +56,20 @@ spec:
 			g.Expect(out).To(ContainSubstring("ip: 241.0.0."))
 		}, "30s", "1s").Should(Succeed())
 
-		// and MeshService is synced to global with status
+		// and MeshService is synced to global with the original status
 		Eventually(func(g Gomega) {
 			out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "meshservices", "-m", meshName, "-oyaml")
 			g.Expect(err).ToNot(HaveOccurred())
 			g.Expect(out).To(ContainSubstring("kuma.io/display-name: backend"))
 			g.Expect(out).To(ContainSubstring("ip: 241.0.0."))
 		}, "30s", "1s").Should(Succeed())
+
+		// and MeshService is synced to other zone but status is generated by other zone
+		Eventually(func(g Gomega) {
+			out, err := multizone.UniZone2.GetKumactlOptions().RunKumactlAndGetOutput("get", "meshservices", "-m", meshName, "-oyaml")
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(out).To(ContainSubstring("kuma.io/display-name: backend"))
+			g.Expect(out).To(ContainSubstring("ip: 251.0.0."))
+		}, "30s", "1s").Should(Succeed())
 	})
 }

test/framework/envs/multizone/env.go

@@ -142,6 +142,7 @@ func SetupAndGetState() []byte {
 			WithIngressEnvoyAdminTunnel(),
 			WithEnv("KUMA_XDS_DATAPLANE_DEREGISTRATION_DELAY", "0s"), // we have only 1 Kuma CP instance so there is no risk setting this to 0
 			WithEnv("KUMA_MULTIZONE_ZONE_KDS_NACK_BACKOFF", "1s"),
+			WithEnv("KUMA_IPAM_MESH_SERVICE_CIDR", "251.0.0.0/8"), // just to see that the status is not synced around
 		},
 		framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.UniZone2)...,
 	)

tools/policy-gen/generator/cmd/core_resource.go

@@ -203,7 +203,7 @@ var {{.Name}}ResourceTypeDescriptor = model.ResourceTypeDescriptor{
 		Resource: New{{.Name}}Resource(),
 		ResourceList: &{{.Name}}ResourceList{},
 		Scope: model.ScopeMesh,
-		KDSFlags: model.GlobalToAllZonesFlag | model.ZoneToGlobalFlag,
+		KDSFlags: {{.KDSFlags}},
 		WsPath: "{{.Path}}",
 		KumactlArg: "{{index .AlternativeNames 0}}",
 		KumactlListArg: "{{.Path}}",

tools/policy-gen/generator/pkg/parse/policyconfig.go

@@ -30,6 +30,7 @@ type PolicyConfig struct {
 	GoModule            string
 	ResourceDir         string
 	IsPolicy            bool
+	KDSFlags            string
 }
 
 func Policy(path string) (PolicyConfig, error) {
@@ -135,6 +136,11 @@ func newPolicyConfig(pkg, name string, markers map[string]string, fields map[str
 	if v, ok := parseBool(markers, "kuma:policy:has_status"); ok {
 		res.HasStatus = v
 	}
+	if v, ok := markers["kuma:policy:kds_flags"]; ok {
+		res.KDSFlags = v
+	} else {
+		res.KDSFlags = "model.GlobalToAllZonesFlag | model.ZoneToGlobalFlag"
+	}
 
 	if v, ok := markers["kuma:policy:singular_display_name"]; ok {
 		res.SingularDisplayName = v