diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml index af06c8d98c80..fcddd2e3ee3a 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml @@ -5714,6 +5714,14 @@ spec: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -5734,6 +5742,21 @@ spec: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone + control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml index c5f19fd66a05..ee83a6f1c14e 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml @@ -5714,6 +5714,14 @@ spec: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -5734,6 +5742,21 @@ spec: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone + control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml index bdbe4212a1cd..20105570a598 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml @@ -5734,6 +5734,14 @@ spec: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -5754,6 +5762,21 @@ spec: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone + control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml b/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml index d757d1c25e0c..4bb7ccdab043 100644 --- a/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml @@ -7268,6 +7268,14 @@ spec: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -7288,6 +7296,21 @@ spec: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone + control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/deployments/charts/kuma/crds/kuma.io_meshservices.yaml b/deployments/charts/kuma/crds/kuma.io_meshservices.yaml index c122d7c120c5..9957845d5718 100644 --- a/deployments/charts/kuma/crds/kuma.io_meshservices.yaml +++ b/deployments/charts/kuma/crds/kuma.io_meshservices.yaml @@ -91,6 +91,14 @@ spec: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -111,6 +119,21 @@ spec: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone + control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/docs/generated/openapi.yaml b/docs/generated/openapi.yaml index 1d33e9d0483e..f857d1fc2550 100644 --- a/docs/generated/openapi.yaml +++ b/docs/generated/openapi.yaml @@ -11370,6 +11370,18 @@ components: type: string type: object type: object + state: + description: >- + State of MeshService. Available if there is at least one healthy + endpoint. Otherwise, Unavailable. + + It's used for cross zone communication to check if we should + send traffic to it, when MeshService is aggregated into + MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -11390,6 +11402,23 @@ components: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: >- + Number of data plane proxies connected to the zone control + plane + type: integer + healthy: + description: >- + Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/docs/generated/raw/crds/kuma.io_meshservices.yaml b/docs/generated/raw/crds/kuma.io_meshservices.yaml index c122d7c120c5..9957845d5718 100644 --- a/docs/generated/raw/crds/kuma.io_meshservices.yaml +++ b/docs/generated/raw/crds/kuma.io_meshservices.yaml @@ -91,6 +91,14 @@ spec: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -111,6 +119,21 @@ spec: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone + control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/pkg/api-server/testdata/resources/crud/list_meshservices.golden.json b/pkg/api-server/testdata/resources/crud/list_meshservices.golden.json index c3212bb7e6a1..b791cc48e96f 100644 --- a/pkg/api-server/testdata/resources/crud/list_meshservices.golden.json +++ b/pkg/api-server/testdata/resources/crud/list_meshservices.golden.json @@ -27,7 +27,8 @@ "ip": "10.0.0.1" } ], - "tls": {} + "tls": {}, + "dataplaneProxies": {} } }, { @@ -56,7 +57,8 @@ "ip": "10.0.0.2" } ], - "tls": {} + "tls": {}, + "dataplaneProxies": {} } } ], diff --git a/pkg/api-server/testdata/resources/crud/put_meshservice_01.golden.json b/pkg/api-server/testdata/resources/crud/put_meshservice_01.golden.json index c6c512860f98..f6dcfe5cda73 100644 --- a/pkg/api-server/testdata/resources/crud/put_meshservice_01.golden.json +++ b/pkg/api-server/testdata/resources/crud/put_meshservice_01.golden.json @@ -9,6 +9,7 @@ "kuma.io/zone": "default" }, "spec": { + "state": "Unavailable", "selector": { "dataplaneTags": { "app": "redis" @@ -27,6 +28,7 @@ "ip": "10.0.0.1" } ], - "tls": {} + "tls": {}, + "dataplaneProxies": {} } } diff --git a/pkg/core/resources/apis/meshservice/api/v1alpha1/helpers.go b/pkg/core/resources/apis/meshservice/api/v1alpha1/helpers.go index 3a2f5a171a79..49269969b0d6 100644 --- a/pkg/core/resources/apis/meshservice/api/v1alpha1/helpers.go +++ b/pkg/core/resources/apis/meshservice/api/v1alpha1/helpers.go @@ -70,3 +70,10 @@ func (t *MeshServiceResource) SNIName(systemNamespace string) string { } return t.GetMeta().GetName() } + +func (t *MeshServiceResource) Default() error { + if t.Spec.State == "" { + t.Spec.State = StateUnavailable + } + return nil +} diff --git a/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go b/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go index ce5fc1c0aa0e..0176236e2af2 100644 --- a/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go +++ b/pkg/core/resources/apis/meshservice/api/v1alpha1/meshservice.go @@ -34,6 +34,9 @@ const maxNameLength = 63 // +kuma:policy:has_status=true // +kuma:policy:kds_flags=model.ZoneToGlobalFlag | model.GlobalToAllButOriginalZoneFlag type MeshService struct { + // State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + // It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + State State `json:"state,omitempty"` Selector Selector `json:"selector,omitempty"` // +patchMergeKey=port // +patchStrategy=merge @@ -48,6 +51,14 @@ type VIP struct { IP string `json:"ip,omitempty"` } +// +kubebuilder:validation:Enum=Available;Unavailable +type State string + +const ( + StateAvailable State = "Available" + StateUnavailable State = "Unavailable" +) + // +kubebuilder:validation:Enum=Ready;NotReady type TLSStatus string @@ -65,6 +76,8 @@ type MeshServiceStatus struct { VIPs []VIP `json:"vips,omitempty"` TLS TLS `json:"tls,omitempty"` HostnameGenerators []hostnamegenerator_api.HostnameGeneratorStatus `json:"hostnameGenerators,omitempty"` + // Data plane proxies statistics selected by this MeshService. + DataplaneProxies DataplaneProxies `json:"dataplaneProxies,omitempty"` } // +kubebuilder:validation:Enum=ServiceTag @@ -78,3 +91,12 @@ type MeshServiceIdentity struct { Type MeshServiceIdentityType `json:"type"` Value string `json:"value"` } + +type DataplaneProxies struct { + // Number of data plane proxies connected to the zone control plane + Connected int `json:"connected,omitempty"` + // Number of data plane proxies with all healthy inbounds selected by this MeshService. + Healthy int `json:"healthy,omitempty"` + // Total number of data plane proxies. + Total int `json:"total,omitempty"` +} diff --git a/pkg/core/resources/apis/meshservice/api/v1alpha1/schema.yaml b/pkg/core/resources/apis/meshservice/api/v1alpha1/schema.yaml index 0fdf75431f59..646aeba0c2b7 100644 --- a/pkg/core/resources/apis/meshservice/api/v1alpha1/schema.yaml +++ b/pkg/core/resources/apis/meshservice/api/v1alpha1/schema.yaml @@ -72,6 +72,14 @@ properties: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -92,6 +100,19 @@ properties: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.deepcopy.go b/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.deepcopy.go index a4ff6d165032..b7d54144e8ff 100644 --- a/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/core/resources/apis/meshservice/api/v1alpha1/zz_generated.deepcopy.go @@ -8,6 +8,21 @@ import ( apiv1alpha1 "github.com/kumahq/kuma/pkg/core/resources/apis/hostnamegenerator/api/v1alpha1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataplaneProxies) DeepCopyInto(out *DataplaneProxies) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataplaneProxies. +func (in *DataplaneProxies) DeepCopy() *DataplaneProxies { + if in == nil { + return nil + } + out := new(DataplaneProxies) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataplaneRef) DeepCopyInto(out *DataplaneRef) { *out = *in @@ -106,6 +121,7 @@ func (in *MeshServiceStatus) DeepCopyInto(out *MeshServiceStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.DataplaneProxies = in.DataplaneProxies } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MeshServiceStatus. diff --git a/pkg/core/resources/apis/meshservice/k8s/crd/kuma.io_meshservices.yaml b/pkg/core/resources/apis/meshservice/k8s/crd/kuma.io_meshservices.yaml index c122d7c120c5..9957845d5718 100644 --- a/pkg/core/resources/apis/meshservice/k8s/crd/kuma.io_meshservices.yaml +++ b/pkg/core/resources/apis/meshservice/k8s/crd/kuma.io_meshservices.yaml @@ -91,6 +91,14 @@ spec: type: string type: object type: object + state: + description: |- + State of MeshService. Available if there is at least one healthy endpoint. Otherwise, Unavailable. + It's used for cross zone communication to check if we should send traffic to it, when MeshService is aggregated into MeshMultiZoneService. + enum: + - Available + - Unavailable + type: string type: object status: description: Status is the current status of the Kuma MeshService resource. @@ -111,6 +119,21 @@ spec: type: string type: object type: array + dataplaneProxies: + description: Data plane proxies statistics selected by this MeshService. + properties: + connected: + description: Number of data plane proxies connected to the zone + control plane + type: integer + healthy: + description: Number of data plane proxies with all healthy inbounds + selected by this MeshService. + type: integer + total: + description: Total number of data plane proxies. + type: integer + type: object hostnameGenerators: items: properties: diff --git a/pkg/core/resources/apis/meshservice/status/updater.go b/pkg/core/resources/apis/meshservice/status/updater.go index ffa4d7876113..4770f3bf8d09 100644 --- a/pkg/core/resources/apis/meshservice/status/updater.go +++ b/pkg/core/resources/apis/meshservice/status/updater.go @@ -129,6 +129,21 @@ func (s *StatusUpdater) updateStatus(ctx context.Context) error { ms.Status.TLS = tls } + dataplaneProxies := buildDataplaneProxies(dpps, insightsByKey, ms.Spec.Ports) + if !reflect.DeepEqual(ms.Status.DataplaneProxies, dataplaneProxies) { + changeReasons = append(changeReasons, "data plane proxies") + ms.Status.DataplaneProxies = dataplaneProxies + } + + state := meshservice_api.StateUnavailable + if dataplaneProxies.Healthy > 0 { + state = meshservice_api.StateAvailable + } + if ms.Spec.State != state { + changeReasons = append(changeReasons, "availability state") + ms.Spec.State = state + } + if len(changeReasons) > 0 { log.Info("updating mesh service", "reason", changeReasons) if err := s.resManager.Update(ctx, ms); err != nil { @@ -139,6 +154,46 @@ func (s *StatusUpdater) updateStatus(ctx context.Context) error { return nil } +func buildDataplaneProxies( + dpps []*core_mesh.DataplaneResource, + insightsByKey map[core_model.ResourceKey]*core_mesh.DataplaneInsightResource, + ports []meshservice_api.Port, +) meshservice_api.DataplaneProxies { + result := meshservice_api.DataplaneProxies{} + for _, dpp := range dpps { + result.Total++ + if insight := insightsByKey[core_model.MetaToResourceKey(dpp.Meta)]; insight != nil { + if insight.Spec.IsOnline() { + result.Connected++ + } + } + healthyInbounds := 0 + for _, port := range ports { + if inbound := dpInboundForMeshServicePort(dpp.Spec.GetNetworking().Inbound, port); inbound != nil { + if inbound.State == mesh_proto.Dataplane_Networking_Inbound_Ready { + healthyInbounds++ + } + } + } + if healthyInbounds == len(ports) { + result.Healthy++ + } + } + return result +} + +func dpInboundForMeshServicePort(inbounds []*mesh_proto.Dataplane_Networking_Inbound, port meshservice_api.Port) *mesh_proto.Dataplane_Networking_Inbound { + for _, inbound := range inbounds { + if port.Name != "" && inbound.Name == port.Name { + return inbound + } + if port.Port == inbound.Port { + return inbound + } + } + return nil +} + func buildTLS( existing meshservice_api.TLS, dpps []*core_mesh.DataplaneResource, diff --git a/pkg/core/resources/apis/meshservice/status/updater_test.go b/pkg/core/resources/apis/meshservice/status/updater_test.go index 00d40a77ccae..2d53802a52dd 100644 --- a/pkg/core/resources/apis/meshservice/status/updater_test.go +++ b/pkg/core/resources/apis/meshservice/status/updater_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/gomega" "github.com/kumahq/kuma/api/mesh/v1alpha1" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" meshservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1" "github.com/kumahq/kuma/pkg/core/resources/manager" "github.com/kumahq/kuma/pkg/core/resources/model" @@ -18,6 +19,7 @@ import ( test_metrics "github.com/kumahq/kuma/pkg/test/metrics" "github.com/kumahq/kuma/pkg/test/resources/builders" "github.com/kumahq/kuma/pkg/test/resources/samples" + "github.com/kumahq/kuma/pkg/util/proto" ) var _ = Describe("Updater", func() { @@ -34,10 +36,10 @@ var _ = Describe("Updater", func() { updater, err := NewStatusUpdater(logr.Discard(), resManager, resManager, 50*time.Millisecond, m, "east") Expect(err).ToNot(HaveOccurred()) stopCh = make(chan struct{}) - go func() { + go func(stopCh chan struct{}) { defer GinkgoRecover() Expect(updater.Start(stopCh)).To(Succeed()) - }() + }(stopCh) Expect(samples.MeshDefaultBuilder().Create(resManager)).To(Succeed()) }) @@ -158,6 +160,109 @@ var _ = Describe("Updater", func() { }), ) + type dpProxiesTestCase struct { + meshService *builders.MeshServiceBuilder + dpps []*builders.DataplaneBuilder + insights []*builders.DataplaneInsightBuilder + expectedState meshservice_api.State + expectedDpProxies meshservice_api.DataplaneProxies + } + + DescribeTable("data plane proxies and state update", + func(given dpProxiesTestCase) { + // given + Expect(samples.MeshDefaultBuilder().WithName("test").Create(resManager)).To(Succeed()) + for _, dpp := range given.dpps { + Expect(dpp.WithMesh("test").Create(resManager)).To(Succeed()) + } + for _, insight := range given.insights { + Expect(insight.WithMesh("test").Create(resManager)).To(Succeed()) + } + Expect(given.meshService.WithMesh("test").Create(resManager)).To(Succeed()) + + Eventually(func(g Gomega) { + // when + ms := meshservice_api.NewMeshServiceResource() + err := resManager.Get(context.Background(), ms, store.GetByKey("backend", "test")) + + // then + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(ms.Status.DataplaneProxies).To(Equal(given.expectedDpProxies)) + g.Expect(ms.Spec.State).To(Equal(given.expectedState)) + }, "10s", "50ms").Should(Succeed()) + }, + Entry("should set empty stats and state unavailable", dpProxiesTestCase{ + meshService: samples.MeshServiceBackendBuilder(), + expectedState: meshservice_api.StateUnavailable, + expectedDpProxies: meshservice_api.DataplaneProxies{ + Connected: 0, + Healthy: 0, + Total: 0, + }, + }), + Entry("should count connected DPPs", dpProxiesTestCase{ + meshService: samples.MeshServiceBackendBuilder(), + dpps: []*builders.DataplaneBuilder{ + samples.DataplaneBackendBuilder().WithName("dp-connected"), + samples.DataplaneBackendBuilder().WithName("dp-disconnected"), + samples.DataplaneBackendBuilder().WithName("dp-never-connected"), + }, + insights: []*builders.DataplaneInsightBuilder{ + samples.DataplaneInsightBackendBuilder(). + WithName("dp-connected"). + AddSubscription(&v1alpha1.DiscoverySubscription{ + ConnectTime: proto.MustTimestampProto(time.Now()), + }), + samples.DataplaneInsightBackendBuilder(). + WithName("dp-disconnected"). + AddSubscription(&v1alpha1.DiscoverySubscription{ + ConnectTime: proto.MustTimestampProto(time.Now()), + DisconnectTime: proto.MustTimestampProto(time.Now()), + }), + samples.DataplaneInsightBackendBuilder(). + WithName("dp-never-connected"), + }, + expectedState: meshservice_api.StateAvailable, + expectedDpProxies: meshservice_api.DataplaneProxies{ + Connected: 1, + Healthy: 3, + Total: 3, + }, + }), + Entry("should count healthy DPPs", dpProxiesTestCase{ + meshService: samples.MeshServiceBackendBuilder(). + WithDataplaneTagsSelectorKV("app", "backend"). + AddIntPort(builders.FirstInboundPort+1, builders.FirstInboundServicePort+1, core_mesh.ProtocolHTTP), + dpps: []*builders.DataplaneBuilder{ + builders.Dataplane(). + WithName("dp-all-inbounds-healthy"). + AddInboundOfTagsMap(map[string]string{"kuma.io/service": "backend-proxy", "app": "backend"}). + AddInboundOfTagsMap(map[string]string{"kuma.io/service": "backend-api", "app": "backend"}), + builders.Dataplane(). + WithName("dp-one-inbounds-healthy"). + AddInboundOfTagsMap(map[string]string{"kuma.io/service": "backend-proxy", "app": "backend"}). + AddInboundOfTagsMap(map[string]string{"kuma.io/service": "backend-api", "app": "backend"}). + With(func(resource *core_mesh.DataplaneResource) { + resource.Spec.Networking.Inbound[0].State = v1alpha1.Dataplane_Networking_Inbound_NotReady + }), + builders.Dataplane(). + WithName("dp-no-inbounds-healthy"). + AddInboundOfTagsMap(map[string]string{"kuma.io/service": "backend-proxy", "app": "backend"}). + AddInboundOfTagsMap(map[string]string{"kuma.io/service": "backend-api", "app": "backend"}). + With(func(resource *core_mesh.DataplaneResource) { + resource.Spec.Networking.Inbound[0].State = v1alpha1.Dataplane_Networking_Inbound_NotReady + resource.Spec.Networking.Inbound[1].State = v1alpha1.Dataplane_Networking_Inbound_NotReady + }), + }, + expectedState: meshservice_api.StateAvailable, + expectedDpProxies: meshservice_api.DataplaneProxies{ + Connected: 0, + Healthy: 1, + Total: 3, + }, + }), + ) + It("should emit metric", func() { Eventually(func(g Gomega) { g.Expect(test_metrics.FindMetric(metrics, "component_ms_status_updater")).ToNot(BeNil()) diff --git a/pkg/core/resources/manager/manager.go b/pkg/core/resources/manager/manager.go index 34c1b84d56a1..5611efe1fd5f 100644 --- a/pkg/core/resources/manager/manager.go +++ b/pkg/core/resources/manager/manager.go @@ -56,6 +56,11 @@ func (r *resourcesManager) Create(ctx context.Context, resource model.Resource, if existingMeta == nil { resource.SetMeta(metaFromCreateOpts(resource.Descriptor(), *opts)) } + if defaulter, ok := resource.(model.Defaulter); ok { + if err := defaulter.Default(); err != nil { + return err + } + } if err := model.Validate(resource); err != nil { return err } @@ -100,6 +105,11 @@ func DeleteAllResources(manager ResourceManager, ctx context.Context, list model } func (r *resourcesManager) Update(ctx context.Context, resource model.Resource, fs ...store.UpdateOptionsFunc) error { + if defaulter, ok := resource.(model.Defaulter); ok { + if err := defaulter.Default(); err != nil { + return err + } + } if err := model.Validate(resource); err != nil { return err } diff --git a/pkg/core/resources/manager/manager_test.go b/pkg/core/resources/manager/manager_test.go index 3d0dc3654b0f..812b68aaa232 100644 --- a/pkg/core/resources/manager/manager_test.go +++ b/pkg/core/resources/manager/manager_test.go @@ -8,10 +8,12 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1" "github.com/kumahq/kuma/pkg/core/resources/manager" "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/resources/store" "github.com/kumahq/kuma/pkg/plugins/resources/memory" + "github.com/kumahq/kuma/pkg/test/resources/samples" ) var _ = Describe("Resource Manager", func() { @@ -73,6 +75,20 @@ var _ = Describe("Resource Manager", func() { // then Expect(err.Error()).To(Equal("mesh of name mesh-1 is not found")) }) + + It("should default if a resource has a defaulter", func() { + // given + Expect(samples.MeshDefaultBuilder().Create(resManager)).To(Succeed()) + + // when + err := samples.MeshServiceBackendBuilder().WithState("").Create(resManager) + + // then + Expect(err).ToNot(HaveOccurred()) + getMs := v1alpha1.NewMeshServiceResource() + Expect(resManager.Get(context.Background(), getMs, store.GetByKey("backend", "default"))).To(Succeed()) + Expect(getMs.Spec.State).To(Equal(v1alpha1.StateUnavailable)) + }) }) Describe("DeleteAll()", func() { diff --git a/pkg/core/resources/model/resource.go b/pkg/core/resources/model/resource.go index 32467d84a528..658905a12738 100644 --- a/pkg/core/resources/model/resource.go +++ b/pkg/core/resources/model/resource.go @@ -636,3 +636,10 @@ func IndexByKey[T Resource](resources []T) map[ResourceKey]T { } return indexedResources } + +// Resource can implement defaulter to provide static default fields. +// Kubernetes Webhook and Resource Manager will make sure that Default() is called before Create/Update +type Defaulter interface { + Resource + Default() error +} diff --git a/pkg/mads/v1/reconcile/snapshot_generator_test.go b/pkg/mads/v1/reconcile/snapshot_generator_test.go index 346405ba6ab3..dda8c579b802 100644 --- a/pkg/mads/v1/reconcile/snapshot_generator_test.go +++ b/pkg/mads/v1/reconcile/snapshot_generator_test.go @@ -48,9 +48,10 @@ var _ = Describe("snapshotGenerator", func() { Mesh: "demo", Service: "backend", Targets: []*observability_v1.MonitoringAssignment_Target{{ - Name: "backend-02", - Address: "192.168.0.2:1234", - Scheme: "http", + Name: "backend-02", + Address: "192.168.0.2:1234", + Scheme: "http", + MetricsPath: "/metrics", Labels: map[string]string{ "env": "intg", "envs": ",intg,", diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/01.meshservice.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/01.meshservice.yaml index 8f7af7d729c5..2dbc78b95936 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/01.meshservice.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/01.meshservice.yaml @@ -28,6 +28,7 @@ items: dataplaneTags: k8s.kuma.io/namespace: demo status: + dataplaneProxies: {} tls: {} vips: - ip: 192.168.0.1 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/02.meshservice.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/02.meshservice.yaml index 885451fbbd15..72b1914b6050 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/02.meshservice.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/02.meshservice.yaml @@ -26,6 +26,7 @@ items: dataplaneTags: k8s.kuma.io/namespace: demo status: + dataplaneProxies: {} tls: {} vips: - ip: 192.168.0.1 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/headless.meshservice.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/headless.meshservice.yaml index fb0876d82155..707fbe4a1ac4 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/headless.meshservice.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/meshservice/headless.meshservice.yaml @@ -28,6 +28,7 @@ items: dataplaneRef: name: example-1.demo status: + dataplaneProxies: {} tls: {} vips: - ip: 192.168.0.5 @@ -60,6 +61,7 @@ items: dataplaneRef: name: example-2.demo status: + dataplaneProxies: {} tls: {} vips: - ip: 192.168.0.6 @@ -90,6 +92,7 @@ items: dataplaneRef: name: very-long-very-long-very-long-very-long-very-long-v.demo status: + dataplaneProxies: {} tls: {} vips: - ip: 192.168.0.7 diff --git a/pkg/plugins/runtime/k8s/webhooks/defaulter.go b/pkg/plugins/runtime/k8s/webhooks/defaulter.go index d232abc670d0..95f529e97878 100644 --- a/pkg/plugins/runtime/k8s/webhooks/defaulter.go +++ b/pkg/plugins/runtime/k8s/webhooks/defaulter.go @@ -14,11 +14,6 @@ import ( "github.com/kumahq/kuma/pkg/plugins/resources/k8s" ) -type Defaulter interface { - core_model.Resource - Default() error -} - func DefaultingWebhookFor(scheme *runtime.Scheme, converter k8s_common.Converter, checker ResourceAdmissionChecker) *admission.Webhook { return &admission.Webhook{ Handler: &defaultingHandler{ @@ -56,7 +51,7 @@ func (h *defaultingHandler) Handle(_ context.Context, req admission.Request) adm return admission.Errored(http.StatusInternalServerError, err) } - if defaulter, ok := resource.(Defaulter); ok { + if defaulter, ok := resource.(core_model.Defaulter); ok { if err := defaulter.Default(); err != nil { return admission.Errored(http.StatusInternalServerError, err) } diff --git a/pkg/test/resources/builders/dataplane_insight_builder.go b/pkg/test/resources/builders/dataplane_insight_builder.go index e613d113ab67..8cdfcdf45b7d 100644 --- a/pkg/test/resources/builders/dataplane_insight_builder.go +++ b/pkg/test/resources/builders/dataplane_insight_builder.go @@ -58,6 +58,11 @@ func (d *DataplaneInsightBuilder) WithMTLS(mtls *mesh_proto.DataplaneInsight_MTL return d } +func (d *DataplaneInsightBuilder) AddSubscription(sub *mesh_proto.DiscoverySubscription) *DataplaneInsightBuilder { + d.res.Spec.Subscriptions = append(d.res.Spec.Subscriptions, sub) + return d +} + func (d *DataplaneInsightBuilder) WithMTLSIssuedBackend(issuedBackend string) *DataplaneInsightBuilder { if d.res.Spec.MTLS == nil { d.res.Spec.MTLS = &mesh_proto.DataplaneInsight_MTLS{} diff --git a/pkg/test/resources/builders/meshservice_builder.go b/pkg/test/resources/builders/meshservice_builder.go index ba201367dbdb..c25ec2a774a6 100644 --- a/pkg/test/resources/builders/meshservice_builder.go +++ b/pkg/test/resources/builders/meshservice_builder.go @@ -93,6 +93,11 @@ func (m *MeshServiceBuilder) WithKumaVIP(vip string) *MeshServiceBuilder { return m } +func (m *MeshServiceBuilder) WithState(state v1alpha1.State) *MeshServiceBuilder { + m.res.Spec.State = state + return m +} + func (m *MeshServiceBuilder) WithoutVIP() *MeshServiceBuilder { m.res.Status.VIPs = []v1alpha1.VIP{} return m diff --git a/pkg/test/resources/samples/meshservice_samples.go b/pkg/test/resources/samples/meshservice_samples.go index c2ab2bafd4eb..6b32f069d13c 100644 --- a/pkg/test/resources/samples/meshservice_samples.go +++ b/pkg/test/resources/samples/meshservice_samples.go @@ -45,6 +45,7 @@ func MeshServiceSyncedBackendBuilder() *builders.MeshServiceBuilder { mesh_proto.ZoneTag: "east", mesh_proto.ResourceOriginLabel: "global", }). + WithState(v1alpha1.StateAvailable). WithKumaVIP("240.0.0.3") } diff --git a/pkg/xds/topology/outbound.go b/pkg/xds/topology/outbound.go index 83c60919510d..75da54dffd06 100644 --- a/pkg/xds/topology/outbound.go +++ b/pkg/xds/topology/outbound.go @@ -131,7 +131,7 @@ func BuildEdsEndpointMap( } // it has to be last because it reuses endpoints for other cases - fillMeshMultiZoneServices(outbound, meshServicesByName, meshMultiZoneServices) + fillMeshMultiZoneServices(outbound, meshServicesByName, meshMultiZoneServices, localZone) return outbound } @@ -140,6 +140,7 @@ func fillMeshMultiZoneServices( outbound core_xds.EndpointMap, meshServicesByName map[string]*meshservice_api.MeshServiceResource, meshMultiZoneServices []*meshmzservice_api.MeshMultiZoneServiceResource, + localZone string, ) { for _, mzSvc := range meshMultiZoneServices { for _, matchedMs := range mzSvc.Status.MeshServices { @@ -147,6 +148,12 @@ func fillMeshMultiZoneServices( if !ok { continue } + if !ms.IsLocalMeshService(localZone) && ms.Spec.State != meshservice_api.StateAvailable { + // we don't want to load balance to zones that has no available endpoints. + // we check this only for non-local services, because if service is unavailable in the local zone it has no endpoints. + // if a new local endpoint just become healthy, we can add it immediately without waiting for state to be reconciled. + continue + } for _, port := range mzSvc.Status.Ports { serviceName := mzSvc.DestinationName(port.Port) diff --git a/test/e2e_env/multizone/meshmultizoneservice/connectivity.go b/test/e2e_env/multizone/meshmultizoneservice/connectivity.go index daf75aefb914..8a6da47c21b8 100644 --- a/test/e2e_env/multizone/meshmultizoneservice/connectivity.go +++ b/test/e2e_env/multizone/meshmultizoneservice/connectivity.go @@ -171,7 +171,6 @@ spec: // then Expect(err).ToNot(HaveOccurred()) Eventually(responseFromInstance(multizone.UniZone1), "30s", "1s"). - Should(Equal("kube-test-server-2")) - // todo(jakubdyszkiewicz) add MustPassRepeatedly(5) after we solve excluding zones without any endpoints + MustPassRepeatedly(5).Should(Equal("kube-test-server-2")) }) }