From dc1daadf0a8ee930bb9d1412a56e10c5a92d4d65 Mon Sep 17 00:00:00 2001 From: Jay Jijie Chen <1180092+jijiechen@users.noreply.github.com> Date: Wed, 14 Aug 2024 23:50:53 +0800 Subject: [PATCH] feat(kuma-dp): rework on the virtual probes to support probing tcp and grpc ports (#10624) --- UPGRADE.md | 26 + api/mesh/v1alpha1/dataplane.pb.go | 2 + api/mesh/v1alpha1/dataplane.proto | 2 + app/cni/pkg/cni/annotations_linux.go | 27 + app/cni/pkg/cni/annotations_linux_test.go | 10 + app/cni/pkg/cni/main_linux.go | 15 +- app/kuma-dp/cmd/run.go | 8 + app/kuma-dp/pkg/dataplane/probes/component.go | 218 +++++++ app/kuma-dp/pkg/dataplane/probes/grpc.go | 87 +++ app/kuma-dp/pkg/dataplane/probes/http.go | 141 +++++ .../pkg/dataplane/probes/probe_suite_test.go | 11 + .../pkg/dataplane/probes/probes_test.go | 585 ++++++++++++++++++ app/kuma-dp/pkg/dataplane/probes/tcp.go | 38 ++ docs/generated/raw/kuma-cp.yaml | 8 +- docs/generated/raw/protos/Dataplane.json | 2 +- .../raw/protos/DataplaneOverview.json | 2 +- pkg/config/app/kuma-cp/kuma-cp.defaults.yaml | 8 +- pkg/config/app/kuma-dp/config.go | 22 +- .../testdata/default-config.golden.yaml | 2 + pkg/config/loader_test.go | 3 + pkg/config/plugins/runtime/k8s/config.go | 17 +- .../k8s/testdata/default-config.golden.yaml | 1 + pkg/plugins/runtime/k8s/containers/factory.go | 51 +- .../runtime/k8s/metadata/annotations.go | 16 + pkg/plugins/runtime/k8s/plugin_gateway.go | 3 +- pkg/plugins/runtime/k8s/probes/headers.go | 43 ++ .../runtime/k8s/probes/pod_probe_overrider.go | 175 ++++++ pkg/plugins/runtime/k8s/probes/probe.go | 35 ++ .../runtime/k8s/probes/probe_converter.go | 166 +++++ .../k8s/probes/probe_converter_test.go | 192 ++++++ .../runtime/k8s/webhooks/injector/injector.go | 23 +- .../k8s/webhooks/injector/injector_test.go | 27 +- .../runtime/k8s/webhooks/injector/probes.go | 34 - .../injector/testdata/inject.01.golden.yaml | 7 +- .../injector/testdata/inject.02.golden.yaml | 7 +- .../injector/testdata/inject.03.golden.yaml | 34 +- .../injector/testdata/inject.03.input.yaml | 22 +- .../injector/testdata/inject.04.golden.yaml | 7 +- .../injector/testdata/inject.05.golden.yaml | 7 +- .../injector/testdata/inject.06.golden.yaml | 1 + .../injector/testdata/inject.07.golden.yaml | 7 +- .../injector/testdata/inject.08.golden.yaml | 7 +- .../injector/testdata/inject.09.golden.yaml | 7 +- .../injector/testdata/inject.10.golden.yaml | 7 +- .../injector/testdata/inject.11.golden.yaml | 7 +- .../injector/testdata/inject.12.golden.yaml | 7 +- .../injector/testdata/inject.13.golden.yaml | 7 +- .../injector/testdata/inject.14.golden.yaml | 7 +- .../injector/testdata/inject.14.input.yaml | 2 +- .../injector/testdata/inject.15.golden.yaml | 11 +- .../injector/testdata/inject.15.input.yaml | 2 +- .../injector/testdata/inject.16.golden.yaml | 7 +- .../injector/testdata/inject.17.golden.yaml | 1 + .../injector/testdata/inject.18.golden.yaml | 1 + .../injector/testdata/inject.19.golden.yaml | 1 + .../injector/testdata/inject.20.golden.yaml | 1 + .../injector/testdata/inject.21.golden.yaml | 7 +- .../injector/testdata/inject.22.golden.yaml | 1 + .../injector/testdata/inject.23.golden.yaml | 1 + .../injector/testdata/inject.24.golden.yaml | 1 + .../injector/testdata/inject.25.golden.yaml | 1 + .../injector/testdata/inject.26.golden.yaml | 7 +- .../injector/testdata/inject.27.golden.yaml | 7 +- .../injector/testdata/inject.28.golden.yaml | 7 +- .../injector/testdata/inject.29.golden.yaml | 1 + .../injector/testdata/inject.30.golden.yaml | 1 + .../injector/testdata/inject.31.golden.yaml | 7 +- .../injector/testdata/inject.32.golden.yaml | 7 +- .../injector/testdata/inject.33.golden.yaml | 1 + .../injector/testdata/inject.34.golden.yaml | 1 + .../injector/testdata/inject.35.golden.yaml | 7 +- .../injector/testdata/inject.36.golden.yaml | 7 +- .../injector/testdata/inject.37.golden.yaml | 7 +- .../injector/testdata/inject.38.golden.yaml | 7 +- .../injector/testdata/inject.39.golden.yaml | 7 +- .../injector/testdata/inject.40.golden.yaml | 205 ++++++ .../injector/testdata/inject.40.input.yaml | 38 ++ .../testdata/inject.builtindns.config.yaml | 1 + .../injector/testdata/inject.config-cni.yaml | 1 + .../testdata/inject.config-ipv6-disabled.yaml | 1 + .../testdata/inject.config-ports.yaml | 1 + .../injector/testdata/inject.config.yaml | 1 + .../injector/testdata/inject.ebpf.config.yaml | 1 + .../testdata/inject.env-vars.config.yaml | 1 + .../inject.high-resources.config.yaml | 1 + .../inject.sidecar-feature.01.golden.yaml | 7 +- .../inject.sidecar-feature.02.golden.yaml | 7 +- .../inject.sidecar-feature.03.golden.yaml | 34 +- .../inject.sidecar-feature.04.golden.yaml | 7 +- .../inject.sidecar-feature.05.golden.yaml | 7 +- .../inject.sidecar-feature.06.golden.yaml | 1 + .../inject.sidecar-feature.07.golden.yaml | 7 +- .../inject.sidecar-feature.08.golden.yaml | 7 +- .../inject.sidecar-feature.09.golden.yaml | 7 +- .../inject.sidecar-feature.10.golden.yaml | 7 +- .../inject.sidecar-feature.11.golden.yaml | 7 +- .../inject.sidecar-feature.12.golden.yaml | 7 +- .../inject.sidecar-feature.13.golden.yaml | 7 +- .../inject.sidecar-feature.14.golden.yaml | 7 +- .../inject.sidecar-feature.15.golden.yaml | 11 +- .../inject.sidecar-feature.16.golden.yaml | 7 +- .../inject.sidecar-feature.17.golden.yaml | 1 + .../inject.sidecar-feature.18.golden.yaml | 1 + .../inject.sidecar-feature.19.golden.yaml | 1 + .../inject.sidecar-feature.20.golden.yaml | 1 + .../inject.sidecar-feature.21.golden.yaml | 7 +- .../inject.sidecar-feature.22.golden.yaml | 1 + .../inject.sidecar-feature.23.golden.yaml | 1 + .../inject.sidecar-feature.24.golden.yaml | 1 + .../inject.sidecar-feature.25.golden.yaml | 1 + .../inject.sidecar-feature.26.golden.yaml | 7 +- .../inject.sidecar-feature.27.golden.yaml | 7 +- .../inject.sidecar-feature.28.golden.yaml | 7 +- .../inject.sidecar-feature.29.golden.yaml | 1 + .../inject.sidecar-feature.30.golden.yaml | 1 + .../inject.sidecar-feature.31.golden.yaml | 7 +- .../inject.sidecar-feature.32.golden.yaml | 7 +- .../inject.sidecar-feature.33.golden.yaml | 1 + .../inject.sidecar-feature.34.golden.yaml | 1 + .../inject.sidecar-feature.35.golden.yaml | 7 +- .../inject.sidecar-feature.36.golden.yaml | 7 +- .../inject.sidecar-feature.37.golden.yaml | 7 +- .../inject.sidecar-feature.38.golden.yaml | 7 +- .../inject.sidecar-feature.39.golden.yaml | 7 +- .../inject.sidecar-feature.40.golden.yaml | 218 +++++++ .../testdata/inject.vp-disabled.config.yaml | 1 + pkg/transparentproxy/kubernetes/kubernetes.go | 18 +- test/server/cmd/grpc_server.go | 13 +- test/server/grpc/api/healthcheck.pb.go | 289 --------- test/server/grpc/api/healthcheck.proto | 23 - test/server/grpc/api/healthcheck_grpc.pb.go | 165 ----- 131 files changed, 2716 insertions(+), 740 deletions(-) create mode 100644 app/kuma-dp/pkg/dataplane/probes/component.go create mode 100644 app/kuma-dp/pkg/dataplane/probes/grpc.go create mode 100644 app/kuma-dp/pkg/dataplane/probes/http.go create mode 100644 app/kuma-dp/pkg/dataplane/probes/probe_suite_test.go create mode 100644 app/kuma-dp/pkg/dataplane/probes/probes_test.go create mode 100644 app/kuma-dp/pkg/dataplane/probes/tcp.go create mode 100644 pkg/plugins/runtime/k8s/probes/headers.go create mode 100644 pkg/plugins/runtime/k8s/probes/pod_probe_overrider.go create mode 100644 pkg/plugins/runtime/k8s/probes/probe_converter.go create mode 100644 pkg/plugins/runtime/k8s/probes/probe_converter_test.go create mode 100644 pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.golden.yaml create mode 100644 pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.input.yaml create mode 100644 pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.40.golden.yaml delete mode 100644 test/server/grpc/api/healthcheck.pb.go delete mode 100644 test/server/grpc/api/healthcheck.proto delete mode 100644 test/server/grpc/api/healthcheck_grpc.pb.go diff --git a/UPGRADE.md b/UPGRADE.md index 6d14af999f2c..cb034ece874b 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -174,6 +174,32 @@ Migration step: 3. Deploy the gateway and verify if traffic works correctly. 4. Remove the old resources. +### Introduction to Application Probe Proxy and deprecation of Virtual Probes + +To support more types of application probes on Kubernetes, in version 2.9, we introduced a new feature named "Application Probe Proxy" which supports HTTP Get, TCP Socket and gRPC application probes. Starting from `2.9.x`, Virtual Probes is deprecated, and Application Probe Proxy is enabled by default. + +Application workloads using Virtual Probes will be migrated to Application Probe Proxy automatically on next restart/redeploy on Kubernetes, without other operations. + +Application Probe Proxy will by default listen on port `9000`, the same port that Virtual Probes Listener uses. If you'd customized the Virtual Probes port, you might also want to customize the port of Application Probe Proxy. You may do so using one of these methods: + +1. Configuring on the control plane to apply on all dataplanes: set the port onto configuration key `runtime.kubernetes.injector.sidecarContainer.applicationProbeProxyPort` +1. Configuring on the control plane to apply on all dataplanes: set the port using environment variable `KUMA_RUNTIME_KUBERNETES_APPLICATION_PROBE_PROXY_PORT` +1. Configuring for certain dataplanes: set the port using pod annotation `kuma.io/application-probe-proxy-port` + +By setting the port to `0`, Application Probe Proxy feature will be disabled. + +When the Application Probe Proxy is disabled, Virtual Probes still works as usual before Virtual Probes is removed. + +Because of deprecation of Virtual Probes, the following items are considered deprecated: + +- Pod annotation `kuma.io/virtual-probes` +- Pod annotation `kuma.io/virtual-probes-port` +- Control plane configuration key `runtime.kubernetes.injector.sidecarContainer.virtualProbesEnabled` +- Control plane configuration key `runtime.kubernetes.injector.sidecarContainer.virtualProbesPort` +- Control plane environment variable `KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_ENABLED` +- Control plane environment variable `KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_PORT` +- Data field `probes` on `Dataplane` objects + ### kumactl #### Default prometheus scrape config removes `service` diff --git a/api/mesh/v1alpha1/dataplane.pb.go b/api/mesh/v1alpha1/dataplane.pb.go index ad8815e96f76..595acd70a029 100644 --- a/api/mesh/v1alpha1/dataplane.pb.go +++ b/api/mesh/v1alpha1/dataplane.pb.go @@ -211,6 +211,8 @@ type Dataplane struct { // See // https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes // for more information. + // Deprecated: this feature will be removed for Universal; on Kubernetes, it's + // not needed anymore. Probes *Dataplane_Probes `protobuf:"bytes,3,opt,name=probes,proto3" json:"probes,omitempty"` } diff --git a/api/mesh/v1alpha1/dataplane.proto b/api/mesh/v1alpha1/dataplane.proto index fe7b22ad520b..513d20eead6c 100644 --- a/api/mesh/v1alpha1/dataplane.proto +++ b/api/mesh/v1alpha1/dataplane.proto @@ -362,5 +362,7 @@ message Dataplane { // See // https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes // for more information. + // Deprecated: this feature will be removed for Universal; on Kubernetes, it's + // not needed anymore. Probes probes = 3; } diff --git a/app/cni/pkg/cni/annotations_linux.go b/app/cni/pkg/cni/annotations_linux.go index e7e07de7bbcb..f65e25e33e50 100644 --- a/app/cni/pkg/cni/annotations_linux.go +++ b/app/cni/pkg/cni/annotations_linux.go @@ -16,6 +16,7 @@ const ( defaultIPFamilyMode = "dualstack" defaultBuiltinDNSPort = "15053" defaultNoRedirectUID = "5678" + defaultAppProbeProxyPort = "9000" defaultRedirectExcludePort = defaultProxyStatusPort ) @@ -36,6 +37,7 @@ var annotationRegistry = map[string]*annotationParam{ "iptablesLogs": {"traffic.kuma.io/iptables-logs", "false", alwaysValidFunc}, "excludeInboundIPs": {"traffic.kuma.io/exclude-inbound-ips", "", validateIPs}, "excludeOutboundIPs": {"traffic.kuma.io/exclude-outbound-ips", "", validateIPs}, + "applicationProbeProxyPort": {"kuma.io/application-probe-proxy-port", defaultAppProbeProxyPort, validateSinglePort}, } type IntermediateConfig struct { @@ -148,6 +150,13 @@ func validatePortList(ports string) error { return nil } +func validateSinglePort(portString string) error { + if _, err := parsePort(portString); err != nil { + return err + } + return nil +} + func validateIpFamilyMode(val string) error { if val == "" { return errors.New("value is empty") @@ -181,6 +190,7 @@ func getAnnotationOrDefault(name string, annotations map[string]string) (string, // NewIntermediateConfig returns a new IntermediateConfig Object constructed from a list of ports and annotations func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig, error) { intermediateConfig := &IntermediateConfig{} + valDefaultProbeProxyPort := defaultAppProbeProxyPort allFields := map[string]*string{ "outboundPort": &intermediateConfig.targetPort, @@ -193,6 +203,7 @@ func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig, "builtinDNSPort": &intermediateConfig.builtinDNSPort, "excludeOutboundPortsForUIDs": &intermediateConfig.excludeOutboundPortsForUIDs, "noRedirectUID": &intermediateConfig.noRedirectUID, + "applicationProbeProxyPort": &valDefaultProbeProxyPort, "dropInvalidPackets": &intermediateConfig.dropInvalidPackets, "iptablesLogs": &intermediateConfig.iptablesLogs, "excludeInboundIPs": &intermediateConfig.excludeInboundIPs, @@ -205,6 +216,7 @@ func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig, } } + excludeAppProbeProxyPort(allFields) return intermediateConfig, nil } @@ -216,3 +228,18 @@ func mapAnnotation(annotations map[string]string, field *string, fieldName strin *field = val return nil } + +func excludeAppProbeProxyPort(allFields map[string]*string) { + inboundPortsToExclude := allFields["excludeInboundPorts"] + applicationProbeProxyPort := *allFields["applicationProbeProxyPort"] + if applicationProbeProxyPort == "0" { + return + } + + existingExcludes := *inboundPortsToExclude + if existingExcludes == "" { + *inboundPortsToExclude = applicationProbeProxyPort + } else { + *inboundPortsToExclude = fmt.Sprintf("%s,%s", existingExcludes, applicationProbeProxyPort) + } +} diff --git a/app/cni/pkg/cni/annotations_linux_test.go b/app/cni/pkg/cni/annotations_linux_test.go index 68f87f69414e..876622a39c3d 100644 --- a/app/cni/pkg/cni/annotations_linux_test.go +++ b/app/cni/pkg/cni/annotations_linux_test.go @@ -37,4 +37,14 @@ var _ = Describe("NewIntermediateConfig", func() { Expect(err).ToNot(HaveOccurred()) Expect(cfg.inboundPort).To(Equal("1234")) }) + + It("should exclude application probe proxy ports", func() { + a := map[string]string{ + "kuma.io/application-probe-proxy-port": "19988", + "traffic.kuma.io/exclude-inbound-ports": "3355", + } + cfg, err := NewIntermediateConfig(a) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.excludeInboundPorts).To(Equal("3355,19988")) + }) }) diff --git a/app/cni/pkg/cni/main_linux.go b/app/cni/pkg/cni/main_linux.go index 7bb926c981a0..4d116a7f53b4 100644 --- a/app/cni/pkg/cni/main_linux.go +++ b/app/cni/pkg/cni/main_linux.go @@ -124,7 +124,7 @@ func cmdAdd(args *skel.CmdArgs) error { ctx := context.Background() conf, err := parseConfig(args.StdinData) if err != nil { - return errors.Wrap(err, "error parsing kuma-cni cmdAdd config") + return errorLogged(log, err, "error parsing kuma-cni cmdAdd config") } mainProcessStderr, err := hijackMainProcessStderr(conf.LogLevel) @@ -139,7 +139,7 @@ func cmdAdd(args *skel.CmdArgs) error { // Determine if running under k8s by checking the CNI args k8sArgs := K8sArgs{} if err := types.LoadArgs(args.Args, &k8sArgs); err != nil { - return errors.Wrap(err, "error loading kuma-cni cmdAdd args") + return errorLogged(log, err, "error loading kuma-cni cmdAdd args") } logger := log.WithValues( "pod", string(k8sArgs.K8S_POD_NAME), @@ -162,7 +162,7 @@ func cmdAdd(args *skel.CmdArgs) error { containerCount, initContainersMap, annotations, err := getPodInfoWithRetries(ctx, conf, k8sArgs) if err != nil { - return errors.Wrap(err, "pod excluded - error getting pod info") + return errorLogged(logger, err, "pod excluded - error getting pod info") } if isInitContainerPresent(initContainersMap) { @@ -184,10 +184,10 @@ func cmdAdd(args *skel.CmdArgs) error { } if intermediateConfig, configErr := NewIntermediateConfig(annotations); configErr != nil { - return errors.Wrap(configErr, "pod excluded - pod intermediateConfig failed due to bad params") + return errorLogged(logger, configErr, "pod excluded - pod intermediateConfig failed due to bad params") } else { if err := Inject(ctx, args.Netns, intermediateConfig, logger); err != nil { - return errors.Wrap(err, "pod excluded - could not inject rules into namespace") + return errorLogged(logger, err, "pod excluded - could not inject rules into namespace") } } logger.Info("successfully injected iptables rules") @@ -208,6 +208,11 @@ func prepareResult(conf *PluginConf, logger logr.Logger) error { return types.PrintResult(result, conf.CNIVersion) } +func errorLogged(logger logr.Logger, err error, message string) error { + logger.Info(fmt.Sprintf("[WARNING] %s", message), "err", err) + return errors.Wrap(err, message) +} + func excludeByMissingSidecarInjectedAnnotation(annotations map[string]string) bool { excludePod := false val, ok := annotations[metadata.KumaSidecarInjectedAnnotation] diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index 51901baff0b4..d5b94c24d481 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -17,6 +17,7 @@ import ( "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/meshmetrics" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics" + "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/probes" kuma_cmd "github.com/kumahq/kuma/pkg/cmd" "github.com/kumahq/kuma/pkg/config" kumadp "github.com/kumahq/kuma/pkg/config/app/kuma-dp" @@ -240,6 +241,13 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { return err } + if opts.Config.ApplicationProbeProxyServer.Port > 0 { + prober := probes.NewProber(kumaSidecarConfiguration.Networking.Address, opts.Config.ApplicationProbeProxyServer.Port) + if err := rootCtx.ComponentManager.Add(prober); err != nil { + return err + } + } + stopComponents := make(chan struct{}) go func() { var draining bool diff --git a/app/kuma-dp/pkg/dataplane/probes/component.go b/app/kuma-dp/pkg/dataplane/probes/component.go new file mode 100644 index 000000000000..2b7952421c1f --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/probes/component.go @@ -0,0 +1,218 @@ +package probes + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "regexp" + "strconv" + "syscall" + "time" + + "github.com/bakito/go-log-logr-adapter/adapter" + err_pkg "github.com/pkg/errors" + kube_core "k8s.io/api/core/v1" + + "github.com/kumahq/kuma/pkg/core" + "github.com/kumahq/kuma/pkg/core/runtime/component" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/probes" +) + +var ( + logger = core.Log.WithName("virtual-probes") + tcpGRPCPathPattern = regexp.MustCompile(`^/(tcp|grpc)/(?P[0-9]+)(/.*)?$`) + httpPathPattern = regexp.MustCompile(`^/(?P[0-9]+)(?P/.*)?$`) + errLimitReached = errors.New("the read limit is reached") + + // LocalAddrIPv4 and LocalAddrIPv6 are the special IP addresses to prevent the probe traffic from being captured by the transparent proxy + LocalAddrIPv4 = &net.TCPAddr{IP: net.ParseIP("127.0.0.6")} + LocalAddrIPv6 = &net.TCPAddr{IP: net.ParseIP("::6")} +) + +const ( + pathPrefixTCP = "/tcp/" + pathPrefixGRPC = "/grpc/" + maxRespBodyLength = 10 * 1 << 10 // 10KB + + Healthy = "HEALTHY" + Unhealthy = "UNHEALTHY" + Unknown = "UNKNOWN" +) + +type Prober struct { + listenPort uint32 + podAddress string + isPodAddrIPV6 bool +} + +func NewProber(podIPAddr string, listenPort uint32) *Prober { + ipAddr := net.ParseIP(podIPAddr) + useIPv6 := false + if ipAddr != nil { + useIPv6 = len(ipAddr) == net.IPv6len && ipAddr.To4() == nil + } + + return &Prober{ + listenPort: listenPort, + podAddress: podIPAddr, + isPodAddrIPV6: useIPv6, + } +} + +func (p *Prober) Start(stop <-chan struct{}) error { + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", p.listenPort)) + if err != nil { + return err_pkg.Wrap(err, "unable to listen for the virtual probes server") + } + + logger.Info("starting Virtual Probes Server", "port", p.listenPort) + + // routes: + // /tcp/ + // /grpc/ + // // + mux := http.NewServeMux() + mux.HandleFunc(pathPrefixTCP, p.probeTCP) + mux.HandleFunc(pathPrefixGRPC, p.probeGRPC) + mux.HandleFunc("/", p.probeHTTP) + server := &http.Server{ + ReadHeaderTimeout: time.Second, + Handler: mux, + ErrorLog: adapter.ToStd(logger), + } + + errCh := make(chan error) + go func() { + if err := server.Serve(lis); err != nil { + errCh <- err + } + }() + + select { + case err := <-errCh: + return err + case <-stop: + logger.Info("stopping Virtual Probes Server") + return server.Shutdown(context.Background()) + } +} + +func (p *Prober) NeedLeaderElection() bool { + return false +} + +var _ component.Component = &Prober{} + +func matchGroups(re *regexp.Regexp, s string) map[string]string { + results := make(map[string]string) + match := re.FindStringSubmatch(s) + if match == nil { + return results + } + + for i, name := range re.SubexpNames() { + if i > 0 && name != "" { + results[name] = match[i] + } + } + + return results +} + +func getValidatedPort(portParam string) (int, string) { + port, err := strconv.Atoi(portParam) + if err != nil { + return 0, "invalid port" + err.Error() + } + if port < 0 || port > 65535 { + return 0, "invalid port value " + portParam + } + + return port, "" +} + +func getPort(req *http.Request, re *regexp.Regexp) (int, error) { + pathParams := matchGroups(re, req.URL.Path) + if portParam, ok := pathParams["port"]; ok { + port, errMsg := getValidatedPort(portParam) + if errMsg != "" { + return 0, errors.New(errMsg) + } + return port, nil + } + + return 0, errors.New("port is required") +} + +func getTimeout(req *http.Request) time.Duration { + timeoutParam := req.Header.Get(probes.HeaderNameTimeout) + if timeoutParam == "" { + timeoutParam = "1" + } + + timeout, err := strconv.Atoi(timeoutParam) + if err != nil { + timeout = 1 + } + + return time.Duration(timeout) * time.Second +} + +func getUpstreamHTTPPath(downstreamPath string) string { + pathParams := matchGroups(httpPathPattern, downstreamPath) + if pathParam, ok := pathParams["path"]; ok { + return pathParam + } + return "/" +} + +func getScheme(req *http.Request) kube_core.URIScheme { + schemeParam := req.Header.Get(probes.HeaderNameScheme) + if schemeParam == "" { + return kube_core.URISchemeHTTP + } + return kube_core.URIScheme(schemeParam) +} + +func getGRPCService(req *http.Request) string { + return req.Header.Get(probes.HeaderNameGRPCService) +} + +// copied from https://github.com/kubernetes/kubernetes/blob/v1.27.0-alpha.1/pkg/probe/dialer_others.go#L27 +// createProbeDialer returns a dialer optimized for probes to avoid lingering sockets on TIME-WAIT state. +// The dialer reduces the TIME-WAIT period to 1 seconds instead of the OS default of 60 seconds. +// Using 1 second instead of 0 because SO_LINGER socket option to 0 causes pending data to be +// discarded and the connection to be aborted with an RST rather than for the pending data to be +// transmitted and the connection closed cleanly with a FIN. +// Ref: https://issues.k8s.io/89898 +func createProbeDialer(isIPv6 bool) *net.Dialer { + dialer := &net.Dialer{ + Control: func(network, address string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + _ = syscall.SetsockoptLinger(int(fd), syscall.SOL_SOCKET, syscall.SO_LINGER, &syscall.Linger{Onoff: 1, Linger: 1}) + }) + }, + } + dialer.LocalAddr = LocalAddrIPv4 + if isIPv6 { + dialer.LocalAddr = LocalAddrIPv6 + } + return dialer +} + +func writeProbeResult(writer http.ResponseWriter, result string) { + statusCode := http.StatusServiceUnavailable + if result == Healthy { + statusCode = http.StatusOK + } + writeHTTPProbeResult(writer, result, statusCode) +} + +func writeHTTPProbeResult(writer http.ResponseWriter, result string, statusCode int) { + writer.Header().Set("Content-Type", "text/plain; charset=utf-8") + writer.Header().Set("X-Content-Type-Options", "nosniff") + writer.WriteHeader(statusCode) + _, _ = fmt.Fprintln(writer, result) +} diff --git a/app/kuma-dp/pkg/dataplane/probes/grpc.go b/app/kuma-dp/pkg/dataplane/probes/grpc.go new file mode 100644 index 000000000000..1a7038ba8c70 --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/probes/grpc.go @@ -0,0 +1,87 @@ +package probes + +import ( + "context" + "fmt" + "net" + "net/http" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + grpchealth "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "github.com/kumahq/kuma/pkg/version" +) + +func (p *Prober) probeGRPC(writer http.ResponseWriter, req *http.Request) { + // /grpc/ + + opts := []grpc.DialOption{ + grpc.WithUserAgent(fmt.Sprintf("kube-probe/%s", version.Build.Version)), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return createProbeDialer(p.isPodAddrIPV6).DialContext(ctx, "tcp", addr) + }), + } + + port, err := getPort(req, tcpGRPCPathPattern) + if err != nil { + logger.V(1).Info("invalid port number", "error", err) + writeProbeResult(writer, Unknown) + return + } + + addr := net.JoinHostPort(p.podAddress, fmt.Sprintf("%d", port)) + conn, err := grpc.NewClient(addr, opts...) + if err != nil { + logger.V(1).Info("unable to connect to upstream server", "error", err) + writeProbeResult(writer, Unhealthy) + return + } + + defer func() { + _ = conn.Close() + }() + + client := grpchealth.NewHealthClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), getTimeout(req)) + defer cancel() + resp, err := client.Check(metadata.NewOutgoingContext(ctx, make(metadata.MD)), // nolint: contextcheck + &grpchealth.HealthCheckRequest{ + Service: getGRPCService(req), + }) + if err != nil { + stat, ok := status.FromError(err) + if ok { + switch stat.Code() { + case codes.Unimplemented: + logger.V(1).Info("the upstream server does not implement the grpc health protocol (grpc.health.v1.Health)") + writeProbeResult(writer, Unhealthy) + return + case codes.DeadlineExceeded: + logger.V(1).Info("the upstream check request did not complete within the timeout") + writeProbeResult(writer, Unhealthy) + return + default: + logger.V(1).Info(fmt.Sprintf("the upstream check request failed with code %s", stat.Code().String())) + writeProbeResult(writer, Unhealthy) + return + } + } else { + logger.V(1).Info("the upstream check request failed") + writeProbeResult(writer, Unhealthy) + return + } + } + + if resp.GetStatus() != grpchealth.HealthCheckResponse_SERVING { + logger.V(1).Info("the upstream server returned as not serving", "status", resp.GetStatus()) + writeProbeResult(writer, Unhealthy) + return + } + + writeProbeResult(writer, Healthy) +} diff --git a/app/kuma-dp/pkg/dataplane/probes/http.go b/app/kuma-dp/pkg/dataplane/probes/http.go new file mode 100644 index 000000000000..95082a7719bb --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/probes/http.go @@ -0,0 +1,141 @@ +package probes + +import ( + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + + kube_core "k8s.io/api/core/v1" + + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/probes" + "github.com/kumahq/kuma/pkg/version" +) + +func (p *Prober) probeHTTP(writer http.ResponseWriter, req *http.Request) { + // // + + upstreamScheme := getScheme(req) + timeout := getTimeout(req) + + transport := createHttpTransport(upstreamScheme, p.isPodAddrIPV6) + client := &http.Client{ + Timeout: timeout, + Transport: transport, + } + + upstreamReq, err := buildUpstreamReq(req, upstreamScheme, p.podAddress) + if err != nil { + logger.V(1).Info("unable to create upstream request", "error", err) + writeProbeResult(writer, Unknown) + return + } + + res, err := client.Do(upstreamReq) + if err != nil { + logger.V(1).Info("unable to request upstream server", "error", err) + writeProbeResult(writer, Unhealthy) + return + } + defer func() { + err = res.Body.Close() + if err != nil { + logger.V(1).Info("error closing upstream request", "error", err) + } + }() + + b, err := readAtMost(res.Body, maxRespBodyLength) + if err != nil { + if errors.Is(err, errLimitReached) { + logger.V(1).Info("non fatal body truncation for %s, Response: %v", req.URL.String(), *res) + } else { + logger.V(1).Info("failed to read response body", "err", err) + writeProbeResult(writer, Unhealthy) + return + } + } + + // from [200,400) + body := string(b) + if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest { + logger.V(1).Info(fmt.Sprintf("probe succeeded for %s", upstreamReq.URL.Path), + "headers", upstreamReq.Header, "body", body) + writeProbeResult(writer, Healthy) + return + } + + logger.V(1).Info("probe failed", "path", upstreamReq.URL.Path, + "headers", upstreamReq.Header, "statusCode", res.StatusCode, "body", body) + + // for HTTP failures, we try to re-use original status code + // so that it's easier to debug from an application developer's perspective + writeHTTPProbeResult(writer, Unhealthy, res.StatusCode) +} + +func buildUpstreamReq(downstreamReq *http.Request, upstreamScheme kube_core.URIScheme, podAddr string) (*http.Request, error) { + port, err := getPort(downstreamReq, httpPathPattern) + if err != nil { + return nil, err + } + + upstreamPath := getUpstreamHTTPPath(downstreamReq.URL.Path) + upstreamURL := &url.URL{ + Scheme: strings.ToLower(string(upstreamScheme)), + Path: upstreamPath, + RawQuery: downstreamReq.URL.RawQuery, + Host: net.JoinHostPort(podAddr, strconv.Itoa(port)), + } + + upstreamReq, err := http.NewRequest(http.MethodGet, upstreamURL.String(), nil) + if err != nil { + return nil, err + } + + for key, values := range downstreamReq.Header { + if !strings.HasPrefix(key, probes.KumaProbeHeaderPrefix) { + upstreamReq.Header[key] = values + } + } + + if _, ok := downstreamReq.Header[probes.HeaderNameHost]; ok { + // User may specify a different Host header, copy it to upstream + upstreamReq.Header.Set("Host", downstreamReq.Header.Get(probes.HeaderNameHost)) + } + if _, ok := downstreamReq.Header["User-Agent"]; !ok { + // explicitly set User-Agent so it's not set to default Go value. K8s use kube-probe. + upstreamReq.Header.Set("User-Agent", fmt.Sprintf("kuma-probe/%s", version.Build.Version)) + } + return upstreamReq, nil +} + +func createHttpTransport(scheme kube_core.URIScheme, useIPv6 bool) *http.Transport { + httpTransport := &http.Transport{ + DisableKeepAlives: true, + } + if scheme == kube_core.URISchemeHTTPS { + httpTransport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, // #nosec G402 -- Intentionally ignoring tls verification in probes + } + } + + d := createProbeDialer(useIPv6) + httpTransport.DialContext = d.DialContext + return httpTransport +} + +func readAtMost(r io.Reader, limit int64) ([]byte, error) { + limitedReader := &io.LimitedReader{R: r, N: limit} + data, err := io.ReadAll(limitedReader) + if err != nil { + return data, err + } + if limitedReader.N <= 0 { + return data, errLimitReached + } + return data, nil +} diff --git a/app/kuma-dp/pkg/dataplane/probes/probe_suite_test.go b/app/kuma-dp/pkg/dataplane/probes/probe_suite_test.go new file mode 100644 index 000000000000..e3b18279589b --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/probes/probe_suite_test.go @@ -0,0 +1,11 @@ +package probes_test + +import ( + "testing" + + "github.com/kumahq/kuma/pkg/test" +) + +func TestVirtualProbes(t *testing.T) { + test.RunSpecs(t, "Virtual Probes Suite") +} diff --git a/app/kuma-dp/pkg/dataplane/probes/probes_test.go b/app/kuma-dp/pkg/dataplane/probes/probes_test.go new file mode 100644 index 000000000000..80a2d8d5ec5f --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/probes/probes_test.go @@ -0,0 +1,585 @@ +package probes_test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "strconv" + "sync/atomic" + "time" + + "github.com/bakito/go-log-logr-adapter/adapter" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + err_pkg "github.com/pkg/errors" + "google.golang.org/grpc" + grpchealth "google.golang.org/grpc/health/grpc_health_v1" + + "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/probes" + kuma_probes "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/probes" + kuma_tls "github.com/kumahq/kuma/pkg/tls" + kuma_srv "github.com/kumahq/kuma/pkg/util/http/server" +) + +const podIP = "127.0.0.1" + +var _ = Describe("Virtual Probes", func() { + probes.LocalAddrIPv4 = &net.TCPAddr{IP: net.ParseIP("127.0.0.1")} + probes.LocalAddrIPv6 = &net.TCPAddr{IP: net.ParseIP("::1")} + + Describe("Virtual Probe Listener", func() { + It("should start and stop the listener", func() { + stopCh := make(chan struct{}) + errCh := make(chan error) + prober := probes.NewProber(podIP, 9001) + + go func() { + errCh <- prober.Start(stopCh) + }() + + time.Sleep(2 * time.Second) + close(stopCh) + + var err error + select { + case err = <-errCh: + default: + } + + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Describe("HTTP Probes", func() { + var stopCh chan struct{} + var errCh chan error + vProbePort := uint32(9002) + + BeforeAll(func() { + stopCh = make(chan struct{}) + errCh = make(chan error) + prober := probes.NewProber(podIP, vProbePort) + mockApp := &mockApplication{ + HTTP: &mockHTTPServerConfig{ + ListenPort: 8080, + Path: "/healthz", + ReturnStatusCode: 200, + FailWhenHeader: "x-custom-header-triggers-failure", + ExecutionDuration: time.Duration(3) * time.Second, + }, + } + + go func() { + errCh <- prober.Start(stopCh) + }() + go func() { + errCh <- mockApp.Start(stopCh) + }() + // wait a short period of time for the servers to be ready + <-time.After(500 * time.Millisecond) + }) + AfterAll(func() { + close(stopCh) + + var err error + select { + case err = <-errCh: + default: + } + + Expect(err).ToNot(HaveOccurred()) + }) + + It("should probe HTTP upstream when it's healthy", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/8080/healthz", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameTimeout, "5") + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(200)) + }) + + It("should probe HTTP upstream when the port is not listening", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/8081/healthz", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(503)) + }) + + It("should probe HTTP upstream when the application reports a failure and return application status code", func() { + // given a header set to trigger a failure + probeReq, err := http.NewRequest("GET", virtualProbesURL("/8080/healthz", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set("x-custom-header-triggers-failure", "present") + probeReq.Header.Set(kuma_probes.HeaderNameTimeout, "5") // 5s is longer than the execution duration (3s) + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(400)) + }) + + It("should probe HTTP upstream when path does not match", func() { + // given a header set to trigger a failure + probeReq, err := http.NewRequest("GET", virtualProbesURL("/8080/bad-path", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameTimeout, "5") // 5s is longer than the execution duration (3s) + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(404)) + }) + + It("should fail with short timeout when probing", func() { + // given a timeout shorter than the execution duration + probeReq, err := http.NewRequest("GET", virtualProbesURL("/8080/healthz", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameTimeout, "2") // 2s is shorter than the execution duration (3s) + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(503)) + }) + }, Ordered) + + Describe("HTTPS Probes", func() { + vProbePort := uint32(9003) + var stopCh chan struct{} + var errCh chan error + + BeforeAll(func() { + stopCh = make(chan struct{}) + errCh = make(chan error) + prober := probes.NewProber(podIP, vProbePort) + mockApp := &mockApplication{ + HTTP: &mockHTTPServerConfig{ + Path: "/healthz?scheme=https", + HTTPS: true, + ListenPort: 18443, + ReturnStatusCode: 200, + }, + } + + go func() { + errCh <- prober.Start(stopCh) + }() + go func() { + errCh <- mockApp.Start(stopCh) + }() + // wait a short period of time for the servers to be ready + <-time.After(500 * time.Millisecond) + }) + AfterAll(func() { + close(stopCh) + + var err error + select { + case err = <-errCh: + default: + } + + Expect(err).ToNot(HaveOccurred()) + }) + + It("should probe HTTPS upstream without verifying server certificates and keep query", func() { + // time.Sleep(100 * time.Second) + probeReq, err := http.NewRequest("GET", virtualProbesURL("/18443/healthz?scheme=https", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameScheme, "HTTPS") + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(200)) + }) + }, Ordered) + + Describe("TCP Probes", func() { + var stopCh chan struct{} + var errCh chan error + vProbePort := uint32(9004) + + BeforeAll(func() { + stopCh = make(chan struct{}) + errCh = make(chan error) + prober := probes.NewProber(podIP, vProbePort) + go func() { + errCh <- prober.Start(stopCh) + }() + + mockApp := &mockApplication{ + TCP: &mockTCPServerConfig{ + ListenPort: 6379, + }, + } + + go func() { + errCh <- mockApp.Start(stopCh) + }() + // wait a short period of time for the server to be ready + <-time.After(500 * time.Millisecond) + }) + AfterAll(func() { + close(stopCh) + + var err error + select { + case err = <-errCh: + default: + } + + Expect(err).ToNot(HaveOccurred()) + }) + + It("should probe TCP server when it's healthy", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/tcp/6379", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(200)) + }) + + It("should probe TCP server when the port is not listening", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/tcp/6000", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameTimeout, "3") + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(503)) + }) + }, Ordered) + + Describe("GRPC Probes", func() { + var stopCh chan struct{} + var errCh chan error + vProbePort := uint32(9005) + + BeforeAll(func() { + stopCh = make(chan struct{}) + errCh = make(chan error) + prober := probes.NewProber(podIP, vProbePort) + go func() { + errCh <- prober.Start(stopCh) + }() + + mockApp := &mockApplication{ + GRPC: &mockGRPCServerConfig{ + ListenPort: 5678, + ServiceName: "liveness", + IsHealthy: true, + ExecutionDuration: time.Duration(3) * time.Second, + }, + } + + go func() { + errCh <- mockApp.Start(stopCh) + }() + // wait a short period of time for the server to be ready + <-time.After(500 * time.Millisecond) + }) + AfterAll(func() { + close(stopCh) + + var err error + select { + case err = <-errCh: + default: + } + + Expect(err).ToNot(HaveOccurred()) + }) + + It("should probe gRPC server when it's healthy", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/grpc/5678", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameGRPCService, "liveness") + probeReq.Header.Set(kuma_probes.HeaderNameTimeout, "5") + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(200)) + }) + + It("should fail with a short timeout when probing", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/grpc/5678", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameGRPCService, "liveness") + probeReq.Header.Set(kuma_probes.HeaderNameTimeout, "2") + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(503)) + }) + + It("should probe gRPC server when the port is not listening", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/grpc/5656", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(503)) + }) + + It("should probe gRPC server when the application reports a failure", func() { + probeReq, err := http.NewRequest("GET", virtualProbesURL("/grpc/5678", vProbePort), nil) + Expect(err).ToNot(HaveOccurred()) + probeReq.Header.Set(kuma_probes.HeaderNameGRPCService, "readiness") + + response, err := http.DefaultClient.Do(probeReq) + + Expect(err).ToNot(HaveOccurred()) + Expect(response.StatusCode).To(Equal(503)) + }) + }, Ordered) +}) + +type mockHTTPServerConfig struct { + ListenPort uint32 + HTTPS bool + ReturnStatusCode int + Path string + ExecutionDuration time.Duration + FailWhenHeader string +} + +type mockTCPServerConfig struct { + ListenPort uint32 +} + +type mockGRPCServerConfig struct { + ListenPort uint32 + ServiceName string + IsHealthy bool + ExecutionDuration time.Duration +} + +type mockApplication struct { + HTTP *mockHTTPServerConfig + TCP *mockTCPServerConfig + GRPC *mockGRPCServerConfig + + grpchealth.UnimplementedHealthServer +} + +func (m *mockApplication) Start(stop <-chan struct{}) error { + errCh := make(chan error) + + switch { + case m.HTTP != nil: + go func() { + errCh <- m.startHTTPServer(stop) + }() + case m.TCP != nil: + go func() { + errCh <- m.startTCPServer(stop) + }() + case m.GRPC != nil: + go func() { + errCh <- m.startGRPCServer(stop) + }() + } + + return <-errCh +} + +func (m *mockApplication) startHTTPServer(stop <-chan struct{}) error { + if m.HTTP == nil { + return nil + } + + var httpReady atomic.Bool + server := &http.Server{ + ReadHeaderTimeout: time.Second, + Addr: fmt.Sprintf(":%d", m.HTTP.ListenPort), + Handler: m, + ErrorLog: adapter.ToStd(GinkgoLogr), + } + if m.HTTP.HTTPS { + tlsConfig, err := configureSelfSignedServerTLS("mock-application") + if err != nil { + return err_pkg.Wrap(err, "could not configure self-signed server TLS for the mock HTTPS server") + } + + server.TLSConfig = tlsConfig + } + + return startServer(func(stopper chan func()) error { + stopper <- func() { + GinkgoLogr.Info("stopping the mock HTTP Server") + httpReady.Store(false) + _ = server.Shutdown(context.Background()) + } + + errCh := make(chan error) + GinkgoLogr.Info("starting mock HTTP Server", "address", server.Addr) + if err := kuma_srv.StartServer(GinkgoLogr, server, &httpReady, errCh); err != nil { + return err + } + return <-errCh + }, stop) +} + +func (m *mockApplication) ServeHTTP(writer http.ResponseWriter, req *http.Request) { + if m.HTTP.ExecutionDuration > 0 { + <-time.After(m.HTTP.ExecutionDuration) + } + + if m.HTTP.Path != "" && m.HTTP.Path != req.RequestURI { + writer.WriteHeader(http.StatusNotFound) + return + } + + if m.HTTP.FailWhenHeader != "" && req.Header.Get(m.HTTP.FailWhenHeader) != "" { + writer.WriteHeader(http.StatusBadRequest) + return + } + + writer.WriteHeader(m.HTTP.ReturnStatusCode) + _, _ = writer.Write([]byte("mock HTTP server response")) +} + +func configureSelfSignedServerTLS(commonName string) (*tls.Config, error) { + kp, err := kuma_tls.NewSelfSignedCert("server", kuma_tls.DefaultKeyType, commonName) + if err != nil { + return nil, err + } + + cert, err := tls.X509KeyPair(kp.CertPEM, kp.KeyPEM) + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + }, nil +} + +func (m *mockApplication) startTCPServer(stop <-chan struct{}) error { + if m.TCP == nil { + return nil + } + return startServer(func(stopper chan func()) error { + config := &net.ListenConfig{} + addr := fmt.Sprintf(":%d", m.TCP.ListenPort) + GinkgoLogr.Info("starting the mock TCP server", "address", addr) + l, err := config.Listen(context.Background(), "tcp", addr) + if err != nil { + return err + } + stopper <- func() { + GinkgoLogr.Info("stopping the mock TCP Server") + _ = l.Close() + } + + errCh := make(chan error) + go m.handleTcpConnections(l, stop, errCh) + return <-errCh + }, stop) +} + +func (m *mockApplication) handleTcpConnections(l net.Listener, cExit <-chan struct{}, cErr chan<- error) { + for { + conn, err := l.Accept() + if err != nil { + cErr <- err + return + } + + _, _ = conn.Write([]byte("connection to mock TCP server has successfully established")) + _ = conn.Close() + + select { + case <-cExit: + return + default: + } + } +} + +func (m *mockApplication) startGRPCServer(stop <-chan struct{}) error { + if m.GRPC == nil { + return nil + } + + grpcS := grpc.NewServer() + grpchealth.RegisterHealthServer(grpcS, m) + + return startServer(func(stopper chan func()) error { + stopper <- func() { + GinkgoLogr.Info("stopping the mock gRPC Server") + grpcS.Stop() + } + + addr := fmt.Sprintf(":%d", m.GRPC.ListenPort) + GinkgoLogr.Info("starting the mock gRPC server", "address", addr) + lis, err := net.Listen("tcp", addr) + if err != nil { + return err_pkg.Wrap(err, "unable to listen the mock gRPC server") + } + if err := grpcS.Serve(lis); err != nil { + return err + } + return nil + }, stop) +} + +func (m *mockApplication) Check(ctx context.Context, req *grpchealth.HealthCheckRequest) (*grpchealth.HealthCheckResponse, error) { + status := grpchealth.HealthCheckResponse_NOT_SERVING + if m.GRPC.IsHealthy { + status = grpchealth.HealthCheckResponse_SERVING + } + + if req.GetService() == m.GRPC.ServiceName { + if m.GRPC.ExecutionDuration > 0 { + <-time.After(m.GRPC.ExecutionDuration) + } + return &grpchealth.HealthCheckResponse{Status: status}, nil + } + + return &grpchealth.HealthCheckResponse{Status: grpchealth.HealthCheckResponse_NOT_SERVING}, nil +} + +func startServer(starter func(chan func()) error, stop <-chan struct{}) error { + sReady := make(chan struct{}, 1) + stopGetter := make(chan func(), 1) + sError := make(chan error, 1) + go func() { + err := starter(stopGetter) + if err != nil { + sError <- err + close(sReady) + } + }() + + <-sReady + select { + case serverErr := <-sError: + return serverErr + case <-stop: + stopper := <-stopGetter + if stopper != nil { + stopper() + } + return nil + } +} + +func virtualProbesURL(path string, virtProbePort uint32) string { + return fmt.Sprintf("http://%s%s", net.JoinHostPort(podIP, strconv.Itoa(int(virtProbePort))), path) +} diff --git a/app/kuma-dp/pkg/dataplane/probes/tcp.go b/app/kuma-dp/pkg/dataplane/probes/tcp.go new file mode 100644 index 000000000000..a973379d8423 --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/probes/tcp.go @@ -0,0 +1,38 @@ +package probes + +import ( + "fmt" + "net" + "net/http" + "strconv" +) + +func (p *Prober) probeTCP(writer http.ResponseWriter, req *http.Request) { + // /grpc/ + port, err := getPort(req, tcpGRPCPathPattern) + if err != nil { + logger.V(1).Info("invalid port number", "error", err) + writeProbeResult(writer, Unknown) + return + } + + d := createProbeDialer(p.isPodAddrIPV6) + d.Timeout = getTimeout(req) + hostPort := net.JoinHostPort(p.podAddress, strconv.Itoa(port)) + protocol := "tcp" + if p.isPodAddrIPV6 { + protocol = "tcp6" + } + conn, err := d.Dial(protocol, hostPort) + if err != nil { + logger.V(1).Info(fmt.Sprintf("unable to establish TCP connection to %s", hostPort), "error", err) + writeProbeResult(writer, Unhealthy) + return + } + + err = conn.Close() + if err != nil { + logger.V(1).Info("unable to close TCP socket", "error", err) + } + writeProbeResult(writer, Healthy) +} diff --git a/docs/generated/raw/kuma-cp.yaml b/docs/generated/raw/kuma-cp.yaml index bbdbb6c8cdd0..b098626d7cdf 100644 --- a/docs/generated/raw/kuma-cp.yaml +++ b/docs/generated/raw/kuma-cp.yaml @@ -244,12 +244,14 @@ runtime: labels: openshift.io/build.name: "*" openshift.io/deployer-pod-for.name: "*" - # VirtualProbesEnabled enables automatic converting HttpGet probes to virtual. Virtual probe - # serves on sub-path of insecure port 'virtualProbesPort', + # (Deprecated, set ApplicationProbeProxyPort to 0 to disable probe proxying) VirtualProbesEnabled enables automatic converting HttpGet probes to virtual. + # Virtual probe serves on sub-path of insecure port 'virtualProbesPort', # i.e :8080/health/readiness -> :9000/8080/health/readiness where 9000 is virtualProbesPort virtualProbesEnabled: true # ENV: KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_ENABLED - # VirtualProbesPort is a port for exposing virtual probes which are not secured by mTLS + # (Deprecated, use ApplicationProbeProxyPort instead) VirtualProbesPort is a port for exposing virtual probes which are not secured by mTLS virtualProbesPort: 9000 # ENV: KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_PORT + # ApplicationProbeProxyPort is a port for proxying application probes, it is not secured by mTLS. By setting to 0, probe proxying will be disabled. + applicationProbeProxyPort: 9000 # ENV: KUMA_RUNTIME_KUBERNETES_APPLICATION_PROBE_PROXY_PORT # CaCertFile is CA certificate which will be used to verify a connection to the control plane. caCertFile: # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_CA_CERT_FILE # SidecarContainer defines configuration of the Kuma sidecar container. diff --git a/docs/generated/raw/protos/Dataplane.json b/docs/generated/raw/protos/Dataplane.json index bb3d8f3994cb..8fffd1f21751 100644 --- a/docs/generated/raw/protos/Dataplane.json +++ b/docs/generated/raw/protos/Dataplane.json @@ -17,7 +17,7 @@ "probes": { "$ref": "#/definitions/kuma.mesh.v1alpha1.Dataplane.Probes", "additionalProperties": true, - "description": "Probes describe a list of endpoints that will be exposed without mTLS. This is useful to expose the health endpoints of the application so the orchestration system (e.g. Kubernetes) can still health check the application. See https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes for more information." + "description": "Probes describe a list of endpoints that will be exposed without mTLS. This is useful to expose the health endpoints of the application so the orchestration system (e.g. Kubernetes) can still health check the application. See https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes for more information. Deprecated: this feature will be removed for Universal; on Kubernetes, it's not needed anymore." } }, "additionalProperties": true, diff --git a/docs/generated/raw/protos/DataplaneOverview.json b/docs/generated/raw/protos/DataplaneOverview.json index 7f75f6b01323..989c0ff5f1cd 100644 --- a/docs/generated/raw/protos/DataplaneOverview.json +++ b/docs/generated/raw/protos/DataplaneOverview.json @@ -33,7 +33,7 @@ "probes": { "$ref": "#/definitions/kuma.mesh.v1alpha1.Dataplane.Probes", "additionalProperties": true, - "description": "Probes describe a list of endpoints that will be exposed without mTLS. This is useful to expose the health endpoints of the application so the orchestration system (e.g. Kubernetes) can still health check the application. See https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes for more information." + "description": "Probes describe a list of endpoints that will be exposed without mTLS. This is useful to expose the health endpoints of the application so the orchestration system (e.g. Kubernetes) can still health check the application. See https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes for more information. Deprecated: this feature will be removed for Universal; on Kubernetes, it's not needed anymore." } }, "additionalProperties": true, diff --git a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml index bbdbb6c8cdd0..b098626d7cdf 100644 --- a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml +++ b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml @@ -244,12 +244,14 @@ runtime: labels: openshift.io/build.name: "*" openshift.io/deployer-pod-for.name: "*" - # VirtualProbesEnabled enables automatic converting HttpGet probes to virtual. Virtual probe - # serves on sub-path of insecure port 'virtualProbesPort', + # (Deprecated, set ApplicationProbeProxyPort to 0 to disable probe proxying) VirtualProbesEnabled enables automatic converting HttpGet probes to virtual. + # Virtual probe serves on sub-path of insecure port 'virtualProbesPort', # i.e :8080/health/readiness -> :9000/8080/health/readiness where 9000 is virtualProbesPort virtualProbesEnabled: true # ENV: KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_ENABLED - # VirtualProbesPort is a port for exposing virtual probes which are not secured by mTLS + # (Deprecated, use ApplicationProbeProxyPort instead) VirtualProbesPort is a port for exposing virtual probes which are not secured by mTLS virtualProbesPort: 9000 # ENV: KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_PORT + # ApplicationProbeProxyPort is a port for proxying application probes, it is not secured by mTLS. By setting to 0, probe proxying will be disabled. + applicationProbeProxyPort: 9000 # ENV: KUMA_RUNTIME_KUBERNETES_APPLICATION_PROBE_PROXY_PORT # CaCertFile is CA certificate which will be used to verify a connection to the control plane. caCertFile: # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_CA_CERT_FILE # SidecarContainer defines configuration of the Kuma sidecar container. diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index acae6eb98c3b..1b12d4915ef8 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -47,6 +47,9 @@ var DefaultConfig = func() Config { PrometheusPort: 19153, CoreDNSLogging: false, }, + ApplicationProbeProxyServer: ApplicationProbeProxyServer{ + Port: 9000, + }, } } @@ -59,7 +62,8 @@ type Config struct { // DataplaneRuntime defines the context in which dataplane (Envoy) runs. DataplaneRuntime DataplaneRuntime `json:"dataplaneRuntime,omitempty"` // DNS defines a configuration for builtin DNS in Kuma DP - DNS DNS `json:"dns,omitempty"` + DNS DNS `json:"dns,omitempty"` + ApplicationProbeProxyServer ApplicationProbeProxyServer `json:"applicationProbeProxyServer,omitempty"` } func (c *Config) Sanitize() { @@ -384,3 +388,19 @@ func (d *DNS) Validate() error { } return nil } + +type ApplicationProbeProxyServer struct { + config.BaseConfig + + Port uint32 `json:"port,omitempty" envconfig:"kuma_application_probe_proxy_port"` +} + +func (p *ApplicationProbeProxyServer) Validate() error { + if p.Port == 0 { + return nil + } + if p.Port > 65353 { + return errors.New(".Port has to be in [0, 65353] range") + } + return nil +} diff --git a/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml b/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml index 7b5d28d3b918..a2fd6747933e 100644 --- a/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml +++ b/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml @@ -20,3 +20,5 @@ dns: enabled: true envoyDnsPort: 15054 prometheusPort: 19153 +applicationProbeProxyServer: + port: 9000 diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 622a569cec60..9416bc566280 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -184,6 +184,7 @@ var _ = Describe("Config loader", func() { Expect(cfg.Runtime.Kubernetes.Injector.SidecarTraffic.ExcludeOutboundIPs).To(Equal([]string{"10.0.0.1", "172.16.0.0/16", "fe80::1", "fe80::/10"})) Expect(cfg.Runtime.Kubernetes.Injector.VirtualProbesEnabled).To(BeFalse()) Expect(cfg.Runtime.Kubernetes.Injector.VirtualProbesPort).To(Equal(uint32(1111))) + Expect(cfg.Runtime.Kubernetes.Injector.ApplicationProbeProxyPort).To(Equal(uint32(1112))) Expect(cfg.Runtime.Kubernetes.Injector.CNIEnabled).To(BeTrue()) Expect(cfg.Runtime.Kubernetes.Injector.ContainerPatches).To(Equal([]string{"patch1", "patch2"})) Expect(cfg.Runtime.Kubernetes.Injector.InitContainer.Image).To(Equal("test-image:test")) @@ -509,6 +510,7 @@ runtime: caCertFile: /tmp/ca.crt virtualProbesEnabled: false virtualProbesPort: 1111 + applicationProbeProxyPort: 1112 containerPatches: ["patch1", "patch2"] initContainer: image: test-image:test @@ -919,6 +921,7 @@ meshService: "KUMA_RUNTIME_KUBERNETES_INJECTOR_NODE_LABELS_TO_COPY": "label-1,label-2", "KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_ENABLED": "false", "KUMA_RUNTIME_KUBERNETES_VIRTUAL_PROBES_PORT": "1111", + "KUMA_RUNTIME_KUBERNETES_APPLICATION_PROBE_PROXY_PORT": "1112", "KUMA_RUNTIME_KUBERNETES_EXCEPTIONS_LABELS": "openshift.io/build.name:value1,openshift.io/deployer-pod-for.name:value2", "KUMA_RUNTIME_KUBERNETES_CONTROLLERS_CONCURRENCY_POD_CONTROLLER": "10", "KUMA_RUNTIME_KUBERNETES_CLIENT_CONFIG_QPS": "100", diff --git a/pkg/config/plugins/runtime/k8s/config.go b/pkg/config/plugins/runtime/k8s/config.go index dd60afa2c0c3..40b2c84d551e 100644 --- a/pkg/config/plugins/runtime/k8s/config.go +++ b/pkg/config/plugins/runtime/k8s/config.go @@ -24,9 +24,10 @@ func DefaultKubernetesRuntimeConfig() *KubernetesRuntimeConfig { ControlPlaneServiceName: "kuma-control-plane", ServiceAccountName: defaultServiceAccountName, Injector: Injector{ - CNIEnabled: false, - VirtualProbesEnabled: true, - VirtualProbesPort: 9000, + CNIEnabled: false, + VirtualProbesEnabled: true, + VirtualProbesPort: 9000, + ApplicationProbeProxyPort: 9000, SidecarContainer: SidecarContainer{ IpFamilyMode: "dualstack", RedirectPortInbound: 15006, @@ -210,13 +211,13 @@ type Injector struct { ContainerPatches []string `json:"containerPatches" envconfig:"kuma_runtime_kubernetes_injector_container_patches"` // CNIEnabled if true runs kuma-cp in CNI compatible mode CNIEnabled bool `json:"cniEnabled" envconfig:"kuma_runtime_kubernetes_injector_cni_enabled"` - // VirtualProbesEnabled enables automatic converting HttpGet probes to virtual. Virtual probe - // serves on sub-path of insecure port 'virtualProbesPort', - // i.e :8080/health/readiness -> :9000/8080/health/readiness where 9000 is virtualProbesPort + // VirtualProbesEnabled enables automatic converting pod probes to virtual probes that is proxied by the sidecar. VirtualProbesEnabled bool `json:"virtualProbesEnabled" envconfig:"kuma_runtime_kubernetes_virtual_probes_enabled"` - // VirtualProbesPort is a port for exposing virtual probes which are not secured by mTLS + // VirtualProbesPort is a port for exposing virtual probes which are not secured by mTLS. VirtualProbesPort uint32 `json:"virtualProbesPort" envconfig:"kuma_runtime_kubernetes_virtual_probes_port"` - // SidecarTraffic is a configuration for a traffic that is intercepted by sidecar + // ApplicationProbeProxyPort is a port for proxying application probes, it is not secured by mTLS. + ApplicationProbeProxyPort uint32 `json:"applicationProbeProxyPort" envconfig:"kuma_runtime_kubernetes_application_probe_proxy_port"` + // SidecarTraffic is a configuration for traffic that is intercepted by sidecar SidecarTraffic SidecarTraffic `json:"sidecarTraffic"` // Exceptions defines list of exceptions for Kuma injection Exceptions Exceptions `json:"exceptions"` diff --git a/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml b/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml index ff8ff3c42ee1..609ab48ebb73 100644 --- a/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml +++ b/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml @@ -9,6 +9,7 @@ controlPlaneServiceName: kuma-control-plane controllersConcurrency: podController: 10 injector: + applicationProbeProxyPort: 9000 builtinDNS: enabled: true port: 15053 diff --git a/pkg/plugins/runtime/k8s/containers/factory.go b/pkg/plugins/runtime/k8s/containers/factory.go index a7b2af46bfdf..924454133df8 100644 --- a/pkg/plugins/runtime/k8s/containers/factory.go +++ b/pkg/plugins/runtime/k8s/containers/factory.go @@ -6,6 +6,7 @@ import ( "strconv" "time" + "github.com/pkg/errors" kube_core "k8s.io/api/core/v1" kube_api "k8s.io/apimachinery/pkg/api/resource" kube_intstr "k8s.io/apimachinery/pkg/util/intstr" @@ -13,6 +14,7 @@ import ( runtime_k8s "github.com/kumahq/kuma/pkg/config/plugins/runtime/k8s" "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/probes" "github.com/kumahq/kuma/pkg/util/pointer" ) @@ -25,13 +27,15 @@ func (a EnvVarsByName) Less(i, j int) bool { } type DataplaneProxyFactory struct { - ControlPlaneURL string - ControlPlaneCACert string - DefaultAdminPort uint32 - ContainerConfig runtime_k8s.DataplaneContainer - BuiltinDNS runtime_k8s.BuiltinDNS - WaitForDataplane bool - sidecarContainersEnabled bool + ControlPlaneURL string + ControlPlaneCACert string + DefaultAdminPort uint32 + ContainerConfig runtime_k8s.DataplaneContainer + BuiltinDNS runtime_k8s.BuiltinDNS + WaitForDataplane bool + sidecarContainersEnabled bool + virtualProbesEnabled bool + applicationProbeProxyPort uint32 } func NewDataplaneProxyFactory( @@ -42,15 +46,19 @@ func NewDataplaneProxyFactory( builtinDNS runtime_k8s.BuiltinDNS, waitForDataplane bool, sidecarContainersEnabled bool, + virtualProbesEnabled bool, + applicationProbeProxyPort uint32, ) *DataplaneProxyFactory { return &DataplaneProxyFactory{ - ControlPlaneURL: controlPlaneURL, - ControlPlaneCACert: controlPlaneCACert, - DefaultAdminPort: defaultAdminPort, - ContainerConfig: containerConfig, - BuiltinDNS: builtinDNS, - WaitForDataplane: waitForDataplane, - sidecarContainersEnabled: sidecarContainersEnabled, + ControlPlaneURL: controlPlaneURL, + ControlPlaneCACert: controlPlaneCACert, + DefaultAdminPort: defaultAdminPort, + ContainerConfig: containerConfig, + BuiltinDNS: builtinDNS, + WaitForDataplane: waitForDataplane, + sidecarContainersEnabled: sidecarContainersEnabled, + virtualProbesEnabled: virtualProbesEnabled, + applicationProbeProxyPort: applicationProbeProxyPort, } } @@ -293,6 +301,7 @@ func (i *DataplaneProxyFactory) sidecarEnvVars(mesh string, podAnnotations map[s Value: "false", } } + if logLevel, exist := metadata.Annotations(podAnnotations).GetString(metadata.KumaEnvoyLogLevel); exist { envVars["KUMA_DATAPLANE_RUNTIME_ENVOY_LOG_LEVEL"] = kube_core.EnvVar{ Name: "KUMA_DATAPLANE_RUNTIME_ENVOY_LOG_LEVEL", @@ -306,6 +315,20 @@ func (i *DataplaneProxyFactory) sidecarEnvVars(mesh string, podAnnotations map[s } } + annotations := make(map[string]string) + if err := probes.SetVirtualProbesEnabledAnnotation(annotations, podAnnotations, i.virtualProbesEnabled); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("unable to set %s", metadata.KumaVirtualProbesAnnotation)) + } + if err := probes.SetApplicationProbeProxyPortAnnotation(annotations, podAnnotations, i.applicationProbeProxyPort); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("unable to set %s", metadata.KumaApplicationProbeProxyPortAnnotation)) + } + if appProbeProxyPort, _, _ := metadata.Annotations(annotations).GetUint32(metadata.KumaApplicationProbeProxyPortAnnotation); appProbeProxyPort > 0 { + envVars["KUMA_APPLICATION_PROBE_PROXY_PORT"] = kube_core.EnvVar{ + Name: "KUMA_APPLICATION_PROBE_PROXY_PORT", + Value: strconv.Itoa(int(appProbeProxyPort)), + } + } + // override defaults with cfg env vars for envName, envVal := range i.ContainerConfig.EnvVars { envVars[envName] = kube_core.EnvVar{ diff --git a/pkg/plugins/runtime/k8s/metadata/annotations.go b/pkg/plugins/runtime/k8s/metadata/annotations.go index 6eaa9b2d138c..d75cbcab597a 100644 --- a/pkg/plugins/runtime/k8s/metadata/annotations.go +++ b/pkg/plugins/runtime/k8s/metadata/annotations.go @@ -55,6 +55,9 @@ const ( // KumaVirtualProbesPortAnnotation is an insecure port for listening virtual probes KumaVirtualProbesPortAnnotation = "kuma.io/virtual-probes-port" + // KumaApplicationProbeProxyPortAnnotation is a port for proxying application probes + KumaApplicationProbeProxyPortAnnotation = "kuma.io/application-probe-proxy-port" + // KumaSidecarEnvVarsAnnotation is a ; separated list of env vars that will be applied on Kuma Sidecar // Example value: TEST1=1;TEST2=2 KumaSidecarEnvVarsAnnotation = "kuma.io/sidecar-env-vars" @@ -125,6 +128,8 @@ const ( var PodAnnotationDeprecations = []Deprecation{ NewReplaceByDeprecation("kuma.io/builtindns", KumaBuiltinDNS, true), NewReplaceByDeprecation("kuma.io/builtindnsport", KumaBuiltinDNSPort, true), + NewDeprecation(KumaVirtualProbesAnnotation, false), + NewReplaceByDeprecation(KumaVirtualProbesPortAnnotation, KumaApplicationProbeProxyPortAnnotation, false), { Key: KumaSidecarInjectionAnnotation, Message: "WARNING: you are using kuma.io/sidecar-injection as annotation. This is not supported you should use it as a label instead", @@ -147,6 +152,17 @@ func NewReplaceByDeprecation(old, new string, removed bool) Deprecation { } } +func NewDeprecation(old string, removed bool) Deprecation { + msg := fmt.Sprintf("'%s' will be removed in a future release", old) + if removed { + msg = fmt.Sprintf("'%s' is no longer supported and it will be ignored, please see documentation on how to migrate", old) + } + return Deprecation{ + Key: old, + Message: msg, + } +} + // Annotations that are being automatically set by the Kuma Sidecar Injector. const ( KumaSidecarInjectedAnnotation = "kuma.io/sidecar-injected" diff --git a/pkg/plugins/runtime/k8s/plugin_gateway.go b/pkg/plugins/runtime/k8s/plugin_gateway.go index b5d63e31b280..3f9cf97c844f 100644 --- a/pkg/plugins/runtime/k8s/plugin_gateway.go +++ b/pkg/plugins/runtime/k8s/plugin_gateway.go @@ -89,7 +89,8 @@ func addGatewayReconcilers(mgr kube_ctrl.Manager, rt core_runtime.Runtime, conve } proxyFactory := containers.NewDataplaneProxyFactory( - cpURL, caCert, rt.Config().GetEnvoyAdminPort(), cfg.SidecarContainer.DataplaneContainer, cfg.BuiltinDNS, false, false, + cpURL, caCert, rt.Config().GetEnvoyAdminPort(), cfg.SidecarContainer.DataplaneContainer, cfg.BuiltinDNS, + false, false, false, 0, ) kubeConfig := mgr.GetConfig() diff --git a/pkg/plugins/runtime/k8s/probes/headers.go b/pkg/plugins/runtime/k8s/probes/headers.go new file mode 100644 index 000000000000..9ffb3dde52a2 --- /dev/null +++ b/pkg/plugins/runtime/k8s/probes/headers.go @@ -0,0 +1,43 @@ +package probes + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +const ( + KumaProbeHeaderPrefix = "X-Kuma-Probes-" + HeaderNameTimeout = KumaProbeHeaderPrefix + "Timeout" + HeaderNameHost = KumaProbeHeaderPrefix + "Host" + HeaderNameScheme = KumaProbeHeaderPrefix + "Scheme" + HeaderNameGRPCService = KumaProbeHeaderPrefix + "GRPC-Service" +) + +func TimeoutHeader(timeoutSeconds int32) corev1.HTTPHeader { + return corev1.HTTPHeader{ + Name: HeaderNameTimeout, + Value: fmt.Sprintf("%d", timeoutSeconds), + } +} + +func HostHeader(host string) corev1.HTTPHeader { + return corev1.HTTPHeader{ + Name: HeaderNameHost, + Value: host, + } +} + +func SchemeHeader(scheme corev1.URIScheme) corev1.HTTPHeader { + return corev1.HTTPHeader{ + Name: HeaderNameScheme, + Value: string(scheme), + } +} + +func GRPCServiceHeader(service string) corev1.HTTPHeader { + return corev1.HTTPHeader{ + Name: HeaderNameGRPCService, + Value: service, + } +} diff --git a/pkg/plugins/runtime/k8s/probes/pod_probe_overrider.go b/pkg/plugins/runtime/k8s/probes/pod_probe_overrider.go new file mode 100644 index 000000000000..373e0185aba8 --- /dev/null +++ b/pkg/plugins/runtime/k8s/probes/pod_probe_overrider.go @@ -0,0 +1,175 @@ +package probes + +import ( + "fmt" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + kube_core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/util" +) + +func ApplicationProbeProxyDisabled(pod *kube_core.Pod) (bool, error) { + appProbeProxyPort, _, err := metadata.Annotations(pod.Annotations).GetUint32(metadata.KumaApplicationProbeProxyPortAnnotation) + if err != nil { + return false, err + } + return appProbeProxyPort == 0, nil +} + +func SetupPodProbeProxies(pod *kube_core.Pod, log logr.Logger) error { + log.WithValues("name", pod.Name, "namespace", pod.Namespace) + appProbeProxyPort, _, err := metadata.Annotations(pod.Annotations).GetUint32(metadata.KumaApplicationProbeProxyPortAnnotation) + if err != nil { + return err + } + if appProbeProxyPort == 0 { + log.V(1).Info("skipping adding application probe proxies, because it's disabled") + return err + } + + var containersNeedingProbes []kube_core.Container + + var initContainerComesAfterKumaSidecar bool + for _, c := range pod.Spec.InitContainers { + if c.Name == util.KumaSidecarContainerName { + initContainerComesAfterKumaSidecar = true + continue + } + + if initContainerComesAfterKumaSidecar && c.RestartPolicy != nil && *c.RestartPolicy == kube_core.ContainerRestartPolicyAlways { + containersNeedingProbes = append(containersNeedingProbes, c) + } + } + for _, c := range pod.Spec.Containers { + if c.Name != util.KumaSidecarContainerName { + // we don't want to create virtual probes for Envoy container, because we generate real listener which is not protected by mTLS + containersNeedingProbes = append(containersNeedingProbes, c) + } + } + for _, c := range containersNeedingProbes { + portResolver := namedPortResolver(c.Ports) + if err := overrideProbe(c.LivenessProbe, appProbeProxyPort, + portResolver, c.Name, "liveness", log); err != nil { + return err + } + if err := overrideProbe(c.ReadinessProbe, appProbeProxyPort, + portResolver, c.Name, "readiness", log); err != nil { + return err + } + if err := overrideProbe(c.StartupProbe, appProbeProxyPort, + portResolver, c.Name, "startup", log); err != nil { + return err + } + } + return nil +} + +func namedPortResolver(containerPorts []kube_core.ContainerPort) func(kube_core.ProbeHandler) { + return func(probe kube_core.ProbeHandler) { + var portStr intstr.IntOrString + switch { + case probe.HTTPGet != nil: + portStr = probe.HTTPGet.Port + case probe.TCPSocket != nil: + portStr = probe.TCPSocket.Port + default: + return + } + + if portStr.IntValue() != 0 { + return + } + + for _, containerPort := range containerPorts { + if containerPort.Name != "" && containerPort.Name == portStr.String() { + if probe.HTTPGet != nil { + probe.HTTPGet.Port = intstr.FromInt32(containerPort.ContainerPort) + } else if probe.TCPSocket != nil { + probe.TCPSocket.Port = intstr.FromInt32(containerPort.ContainerPort) + } + + break + } + } + } +} + +func overrideProbe(probe *kube_core.Probe, virtualPort uint32, + namedPortResolver func(kube_core.ProbeHandler), containerName, probeName string, log logr.Logger, +) error { + if probe == nil { + return nil + } + + proxiedProbe := ProxiedApplicationProbe(*probe) + if !proxiedProbe.OverridingSupported() { + return nil + } + + log.V(1).Info(fmt.Sprintf("overriding %s probe", probeName), "container", containerName) + + namedPortResolver(probe.ProbeHandler) + + virtual, err := proxiedProbe.ToVirtual(virtualPort) + if err != nil { + return err + } + + probe.GRPC = nil + probe.TCPSocket = nil + probe.HTTPGet = &kube_core.HTTPGetAction{ + Port: intstr.FromInt32(int32(virtual.Port())), + Path: virtual.Path(), + HTTPHeaders: virtual.Headers(), + } + return nil +} + +func SetApplicationProbeProxyPortAnnotation(annotations metadata.Annotations, podAnnotations map[string]string, defaultAppProbeProxyPort uint32) error { + str := func(port uint32) string { + return fmt.Sprintf("%d", port) + } + + // scenarios of switching between virtual probes (vp) and application probe proxy (proxy): + // vp proxy result + // Y Y proxy + // Y N vp + // N N N + // N Y - config N + // N Y - pod proxy + + // if disabled by "kuma.io/virtual-probes", we honor it when there is no "kuma.io/application-probe-proxy-port" annotation + // this is treated as deprecated though + proxyPortAnno, proxyPortAnnoExists, err := metadata.Annotations(podAnnotations).GetUint32(metadata.KumaApplicationProbeProxyPortAnnotation) + if err != nil { + return err + } + if vpEnabled, _, _ := annotations.GetEnabled(metadata.KumaVirtualProbesAnnotation); !vpEnabled && !proxyPortAnnoExists { + annotations[metadata.KumaApplicationProbeProxyPortAnnotation] = "0" + return nil + } + appProbeProxyPort := defaultAppProbeProxyPort + if proxyPortAnnoExists { + appProbeProxyPort = proxyPortAnno + } + _, gwExists := metadata.Annotations(podAnnotations).GetString(metadata.KumaGatewayAnnotation) + if gwExists { + if proxyPortAnnoExists && proxyPortAnno > 0 { + return errors.New("application probe proxies probes can't be enabled in gateway mode") + } + annotations[metadata.KumaApplicationProbeProxyPortAnnotation] = "0" + return nil + } + + annotations[metadata.KumaApplicationProbeProxyPortAnnotation] = str(appProbeProxyPort) + // with application probe proxy enabled, we don't need virtual probes + if annotations[metadata.KumaApplicationProbeProxyPortAnnotation] != "0" { + delete(annotations, metadata.KumaVirtualProbesAnnotation) + delete(annotations, metadata.KumaVirtualProbesPortAnnotation) + } + return nil +} diff --git a/pkg/plugins/runtime/k8s/probes/probe.go b/pkg/plugins/runtime/k8s/probes/probe.go index 87a13fd8e776..cb465b0221a2 100644 --- a/pkg/plugins/runtime/k8s/probes/probe.go +++ b/pkg/plugins/runtime/k8s/probes/probe.go @@ -5,6 +5,8 @@ import ( "strconv" "strings" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata" + "github.com/pkg/errors" kube_core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -71,3 +73,36 @@ func (p KumaProbe) Port() uint32 { func (p KumaProbe) Path() string { return p.HTTPGet.Path } + +func SetVirtualProbesEnabledAnnotation(annotations metadata.Annotations, podAnnotations map[string]string, cfgVirtualProbesEnabled bool) error { + str := func(b bool) string { + if b { + return metadata.AnnotationEnabled + } + return metadata.AnnotationDisabled + } + + vpEnabled, vpExist, err := metadata.Annotations(podAnnotations).GetEnabled(metadata.KumaVirtualProbesAnnotation) + if err != nil { + return err + } + gwEnabled, _, err := metadata.Annotations(podAnnotations).GetEnabled(metadata.KumaGatewayAnnotation) + if err != nil { + return err + } + + if gwEnabled { + if vpEnabled { + return errors.New("virtual probes can't be enabled in gateway mode") + } + annotations[metadata.KumaVirtualProbesAnnotation] = metadata.AnnotationDisabled + return nil + } + + if vpExist { + annotations[metadata.KumaVirtualProbesAnnotation] = str(vpEnabled) + return nil + } + annotations[metadata.KumaVirtualProbesAnnotation] = str(cfgVirtualProbesEnabled) + return nil +} diff --git a/pkg/plugins/runtime/k8s/probes/probe_converter.go b/pkg/plugins/runtime/k8s/probes/probe_converter.go new file mode 100644 index 000000000000..8bfde18604a5 --- /dev/null +++ b/pkg/plugins/runtime/k8s/probes/probe_converter.go @@ -0,0 +1,166 @@ +package probes + +import ( + "fmt" + "slices" + "strings" + + "github.com/pkg/errors" + kube_core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// ProxiedApplicationProbe is a type which allows to manipulate Kubernetes HttpGet probes. +// Probe can be either Virtual or Real. +// +// Real probe is a probe provided by user. The only constraint existing for Real +// probes is that the port must be different from 'virtualPort'. +// +// Virtual probe is an automatically generated probe on the basis of the Real probe. +// If probe's port equal to 'virtualPort' and the first segment of probe's path is an integer +// then probe is a virtual probe. +type ProxiedApplicationProbe kube_core.Probe + +func (p ProxiedApplicationProbe) ToVirtual(virtualPort uint32) (ProxiedApplicationProbe, error) { + switch { + case p.ProbeHandler.HTTPGet != nil: + return p.httpProbeToVirtual(virtualPort) + case p.ProbeHandler.TCPSocket != nil: + return p.tcpProbeToVirtual(virtualPort) + case p.ProbeHandler.GRPC != nil: + return p.grpcProbeToVirtual(virtualPort) + default: + return ProxiedApplicationProbe{}, errors.New("unsupported probe type") + } +} + +func (p ProxiedApplicationProbe) httpProbeToVirtual(virtualPort uint32) (ProxiedApplicationProbe, error) { + appPort := uint32(p.HTTPGet.Port.IntValue()) + if appPort == virtualPort { + return ProxiedApplicationProbe{}, errors.Errorf("cannot override Pod's probes. Port for probe cannot "+ + "be set to %d. It is reserved for the dataplane that will serve pods without mTLS.", virtualPort) + } + + probePath := p.Path() + if !strings.HasPrefix(p.Path(), "/") { + probePath = fmt.Sprintf("/%s", p.Path()) + } + + var headers []kube_core.HTTPHeader + headerIdx := slices.IndexFunc(p.HTTPGet.HTTPHeaders, func(header kube_core.HTTPHeader) bool { + return header.Name == "Host" + }) + + var hostHeader kube_core.HTTPHeader + if headerIdx != -1 { + hostHeader = p.HTTPGet.HTTPHeaders[headerIdx] + headers = append(p.HTTPGet.HTTPHeaders[:headerIdx], HostHeader(hostHeader.Value)) + headers = append(headers, p.HTTPGet.HTTPHeaders[headerIdx+1:]...) + } else { + headers = append(headers, p.HTTPGet.HTTPHeaders...) + } + + if p.HTTPGet.Scheme != "" && p.HTTPGet.Scheme != kube_core.URISchemeHTTP { + headers = append(headers, SchemeHeader(p.HTTPGet.Scheme)) + } + + if p.TimeoutSeconds > 1 { + headers = append(headers, TimeoutHeader(p.TimeoutSeconds)) + } + + return ProxiedApplicationProbe{ + ProbeHandler: kube_core.ProbeHandler{ + HTTPGet: &kube_core.HTTPGetAction{ + Port: intstr.FromInt32(int32(virtualPort)), + Path: fmt.Sprintf("/%d%s", appPort, probePath), + HTTPHeaders: headers, + }, + }, + }, nil +} + +func (p ProxiedApplicationProbe) tcpProbeToVirtual(virtualPort uint32) (ProxiedApplicationProbe, error) { + appPort := uint32(p.TCPSocket.Port.IntValue()) + if appPort == virtualPort { + return ProxiedApplicationProbe{}, errors.Errorf("cannot override Pod's probes. Port for probe cannot "+ + "be set to %d. It is reserved for the dataplane that will serve pods without mTLS.", virtualPort) + } + + var headers []kube_core.HTTPHeader + + if p.TimeoutSeconds > 1 { + headers = append(headers, TimeoutHeader(p.TimeoutSeconds)) + } + + return ProxiedApplicationProbe{ + ProbeHandler: kube_core.ProbeHandler{ + HTTPGet: &kube_core.HTTPGetAction{ + Port: intstr.FromInt32(int32(virtualPort)), + Path: fmt.Sprintf("/tcp/%d", appPort), + HTTPHeaders: headers, + }, + }, + }, nil +} + +func (p ProxiedApplicationProbe) grpcProbeToVirtual(virtualPort uint32) (ProxiedApplicationProbe, error) { + appPort := uint32(p.GRPC.Port) + if appPort == virtualPort { + return ProxiedApplicationProbe{}, errors.Errorf("cannot override Pod's probes. Port for probe cannot "+ + "be set to %d. It is reserved for the dataplane that will serve pods without mTLS.", virtualPort) + } + + var headers []kube_core.HTTPHeader + + if p.TimeoutSeconds > 1 { + headers = append(headers, TimeoutHeader(p.TimeoutSeconds)) + } + + if p.GRPC.Service != nil && *p.GRPC.Service != "" { + headers = append(headers, GRPCServiceHeader(*p.GRPC.Service)) + } + + return ProxiedApplicationProbe{ + ProbeHandler: kube_core.ProbeHandler{ + HTTPGet: &kube_core.HTTPGetAction{ + Port: intstr.FromInt32(int32(virtualPort)), + Path: fmt.Sprintf("/grpc/%d", appPort), + HTTPHeaders: headers, + }, + }, + }, nil +} + +func (p ProxiedApplicationProbe) Port() uint32 { + switch { + case p.ProbeHandler.HTTPGet != nil: + return uint32(p.HTTPGet.Port.IntValue()) + case p.ProbeHandler.TCPSocket != nil: + return uint32(p.TCPSocket.Port.IntValue()) + case p.ProbeHandler.GRPC != nil: + return uint32(p.GRPC.Port) + default: + return 0 + } +} + +func (p ProxiedApplicationProbe) Path() string { + return p.HTTPGet.Path +} + +func (p ProxiedApplicationProbe) Headers() []kube_core.HTTPHeader { + return p.HTTPGet.HTTPHeaders +} + +func (p ProxiedApplicationProbe) OverridingSupported() bool { + switch { + case p.ProbeHandler.HTTPGet != nil: + return true + case p.ProbeHandler.TCPSocket != nil: + return true + case p.ProbeHandler.GRPC != nil: + return true + default: + return false + } +} diff --git a/pkg/plugins/runtime/k8s/probes/probe_converter_test.go b/pkg/plugins/runtime/k8s/probes/probe_converter_test.go new file mode 100644 index 000000000000..b53bbb53d9ec --- /dev/null +++ b/pkg/plugins/runtime/k8s/probes/probe_converter_test.go @@ -0,0 +1,192 @@ +package probes_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + kube_core "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" + + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/probes" +) + +var _ = Describe("ProxiedApplicationProbe", func() { + Describe("OverridingSupported", func() { + type testCase struct { + input kube_core.Probe + expectedSupported bool + } + + DescribeTable("should check if probe is supported to be overridden", + func(given testCase) { + virtual := probes.ProxiedApplicationProbe(given.input) + + Expect(virtual.OverridingSupported()).To(Equal(given.expectedSupported)) + }, + Entry("HTTP", testCase{ + input: kube_core.Probe{ + ProbeHandler: kube_core.ProbeHandler{ + HTTPGet: &kube_core.HTTPGetAction{}, + }, + }, + expectedSupported: true, + }), + Entry("TCPSocket", testCase{ + input: kube_core.Probe{ + ProbeHandler: kube_core.ProbeHandler{ + TCPSocket: &kube_core.TCPSocketAction{}, + }, + }, + expectedSupported: true, + }), + Entry("gRPC", testCase{ + input: kube_core.Probe{ + ProbeHandler: kube_core.ProbeHandler{ + GRPC: &kube_core.GRPCAction{}, + }, + }, + expectedSupported: true, + }), + Entry("exec", testCase{ + input: kube_core.Probe{ + ProbeHandler: kube_core.ProbeHandler{ + Exec: &kube_core.ExecAction{}, + }, + }, + expectedSupported: false, + })) + }) + + Context("ToVirtual - HTTP", func() { + It("should convert pod probe to virtual probe", func() { + podProbeYaml := ` + httpGet: + path: /c1/health/liveness + port: 8080 +` + probe := kube_core.Probe{} + err := yaml.Unmarshal([]byte(podProbeYaml), &probe) + Expect(err).ToNot(HaveOccurred()) + + virtual, err := probes.ProxiedApplicationProbe(probe).ToVirtual(9000) + Expect(err).ToNot(HaveOccurred()) + Expect(virtual.Path()).To(Equal("/8080/c1/health/liveness")) + Expect(virtual.Port()).To(Equal(uint32(9000))) + }) + + It("should convert probe with custom headers and timeout", func() { + podProbeYaml := ` + timeoutSeconds: 15 + httpGet: + scheme: HTTPS + path: /c1/healthz + port: 8080 + httpHeaders: + - name: Host + value: example.com + - name: X-Custom-Header + value: custom-value +` + probe := kube_core.Probe{} + err := yaml.Unmarshal([]byte(podProbeYaml), &probe) + Expect(err).ToNot(HaveOccurred()) + + virtual, err := probes.ProxiedApplicationProbe(probe).ToVirtual(9000) + Expect(err).ToNot(HaveOccurred()) + + Expect(virtual.Path()).To(Equal("/8080/c1/healthz")) + Expect(virtual.Port()).To(Equal(uint32(9000))) + + Expect(getHeader(virtual.Headers(), probes.HeaderNameHost)).To(Equal("example.com")) + Expect(getHeader(virtual.Headers(), probes.HeaderNameScheme)).To(Equal("HTTPS")) + Expect(getHeader(virtual.Headers(), probes.HeaderNameTimeout)).To(Equal("15")) + Expect(getHeader(virtual.Headers(), "X-Custom-Header")).To(Equal("custom-value")) + }) + + It("should return an error if virtual port is equal to real", func() { + podProbeYaml := ` + httpGet: + path: /c1/health/liveness + port: 9000 +` + probe := kube_core.Probe{} + err := yaml.Unmarshal([]byte(podProbeYaml), &probe) + Expect(err).ToNot(HaveOccurred()) + + _, err = probes.ProxiedApplicationProbe(probe).ToVirtual(9000) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cannot override Pod's probes. Port for probe cannot be set " + + "to 9000. It is reserved for the dataplane that will serve pods without mTLS.")) + }) + }) + + Context("ToVirtual - TCP Socket & gRPC", func() { + It("should convert TCP socket probe", func() { + podProbeYaml := ` + timeoutSeconds: 10 + tcpSocket: + port: 6379 +` + probe := kube_core.Probe{} + err := yaml.Unmarshal([]byte(podProbeYaml), &probe) + Expect(err).ToNot(HaveOccurred()) + + virtual, err := probes.ProxiedApplicationProbe(probe).ToVirtual(9000) + Expect(err).ToNot(HaveOccurred()) + + Expect(virtual.TCPSocket).To(BeNil()) + Expect(virtual.Path()).To(Equal("/tcp/6379")) + Expect(virtual.Port()).To(Equal(uint32(9000))) + + Expect(getHeader(virtual.Headers(), probes.HeaderNameTimeout)).To(Equal("10")) + }) + + It("should convert gRPC probe", func() { + podProbeYaml := ` + timeoutSeconds: 10 + grpc: + port: 6379 + service: liveness +` + probe := kube_core.Probe{} + err := yaml.Unmarshal([]byte(podProbeYaml), &probe) + Expect(err).ToNot(HaveOccurred()) + + virtual, err := probes.ProxiedApplicationProbe(probe).ToVirtual(9000) + Expect(err).ToNot(HaveOccurred()) + + Expect(virtual.GRPC).To(BeNil()) + Expect(virtual.Path()).To(Equal("/grpc/6379")) + Expect(virtual.Port()).To(Equal(uint32(9000))) + + Expect(getHeader(virtual.Headers(), probes.HeaderNameTimeout)).To(Equal("10")) + Expect(getHeader(virtual.Headers(), probes.HeaderNameGRPCService)).To(Equal("liveness")) + }) + }) + + Context("Prepend /", func() { + It("should convert to path with prepended /", func() { + podProbeYaml := ` + httpGet: + path: c1/hc + port: 8080 +` + probe := kube_core.Probe{} + err := yaml.Unmarshal([]byte(podProbeYaml), &probe) + Expect(err).ToNot(HaveOccurred()) + + virtual, err := probes.ProxiedApplicationProbe(probe).ToVirtual(9000) + Expect(err).ToNot(HaveOccurred()) + Expect(virtual.Path()).To(Equal("/8080/c1/hc")) + Expect(virtual.Port()).To(Equal(uint32(9000))) + }) + }) +}) + +func getHeader(headers []kube_core.HTTPHeader, name string) string { + for _, header := range headers { + if header.Name == name { + return header.Value + } + } + return "" +} diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go index 0e87d6e476d1..b839bf7e50db 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go +++ b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go @@ -24,6 +24,7 @@ import ( mesh_k8s "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1" "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/containers" "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/probes" k8s_util "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/util" tp_k8s "github.com/kumahq/kuma/pkg/transparentproxy/kubernetes" "github.com/kumahq/kuma/pkg/util/pointer" @@ -60,7 +61,9 @@ func New( converter: converter, defaultAdminPort: envoyAdminPort, proxyFactory: containers.NewDataplaneProxyFactory( - controlPlaneURL, caCert, envoyAdminPort, cfg.SidecarContainer.DataplaneContainer, cfg.BuiltinDNS, cfg.SidecarContainer.WaitForDataplaneReady, sidecarContainersEnabled, + controlPlaneURL, caCert, envoyAdminPort, cfg.SidecarContainer.DataplaneContainer, + cfg.BuiltinDNS, cfg.SidecarContainer.WaitForDataplaneReady, sidecarContainersEnabled, + cfg.VirtualProbesEnabled, cfg.ApplicationProbeProxyPort, ), systemNamespace: systemNamespace, }, nil @@ -215,9 +218,19 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error pod.Spec.InitContainers = append(append(prependInitContainers, pod.Spec.InitContainers...), appendInitContainers...) - if err := i.overrideHTTPProbes(pod); err != nil { + disabledAppProbeProxy, err := probes.ApplicationProbeProxyDisabled(pod) + if err != nil { return err } + if !disabledAppProbeProxy { + if err := probes.SetupPodProbeProxies(pod, log); err != nil { + return err + } + } else { + if err := i.overrideHTTPProbes(pod); err != nil { + return err + } + } return nil } @@ -596,13 +609,17 @@ func (i *KumaInjector) NewAnnotations(pod *kube_core.Pod, mesh string, logger lo annotations[metadata.KumaBuiltinDNSLogging] = strconv.FormatBool(logging) } - if err := setVirtualProbesEnabledAnnotation(annotations, pod, i.cfg); err != nil { + if err := probes.SetVirtualProbesEnabledAnnotation(annotations, pod.Annotations, i.cfg.VirtualProbesEnabled); err != nil { return nil, errors.Wrap(err, fmt.Sprintf("unable to set %s", metadata.KumaVirtualProbesAnnotation)) } if err := setVirtualProbesPortAnnotation(annotations, pod, i.cfg); err != nil { return nil, errors.Wrap(err, fmt.Sprintf("unable to set %s", metadata.KumaVirtualProbesPortAnnotation)) } + if err := probes.SetApplicationProbeProxyPortAnnotation(annotations, pod.Annotations, i.cfg.ApplicationProbeProxyPort); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("unable to set %s", metadata.KumaApplicationProbeProxyPortAnnotation)) + } + if val, _ := metadata.Annotations(pod.Annotations).GetStringWithDefault(portsToAnnotationValue(i.cfg.SidecarTraffic.ExcludeInboundPorts), metadata.KumaTrafficExcludeInboundPorts); val != "" { annotations[metadata.KumaTrafficExcludeInboundPorts] = val } diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go b/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go index bd4921f022a8..707d2eb4ab91 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go +++ b/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go @@ -362,7 +362,7 @@ spec: kuma.io/mesh: mesh-name-from-ns`, cfgFile: "inject.config.yaml", }), - Entry("13. Adjust Pod's probes", testCase{ + Entry("13. application probe proxy", testCase{ num: "13", mesh: ` apiVersion: kuma.io/v1alpha1 @@ -379,7 +379,7 @@ spec: kuma.io/sidecar-injection: enabled`, cfgFile: "inject.config.yaml", }), - Entry("14. virtual probes: config - 9000, pod - 19000", testCase{ + Entry("14. application probe proxy: config - 9000, pod - 19000", testCase{ num: "14", mesh: ` apiVersion: kuma.io/v1alpha1 @@ -396,7 +396,7 @@ spec: kuma.io/sidecar-injection: enabled`, cfgFile: "inject.config.yaml", }), - Entry("15. virtual probes: config - enabled, pod - disabled", testCase{ + Entry("15. application probe proxy: config - enabled, pod - disabled; fallback to virtual probe", testCase{ num: "15", mesh: ` apiVersion: kuma.io/v1alpha1 @@ -464,7 +464,7 @@ spec: kuma.io/sidecar-injection: enabled`, cfgFile: "inject.config-ports.yaml", }), - Entry("19. virtual probes: config - disabled, pod - empty", testCase{ + Entry("19. application probe proxy - disabled, virtual probes - disabled, pod - empty", testCase{ num: "19", mesh: ` apiVersion: kuma.io/v1alpha1 @@ -498,7 +498,7 @@ spec: kuma.io/sidecar-injection: enabled`, cfgFile: "inject.vp-disabled.config.yaml", }), - Entry("21. Adjust Pod's probes, named port", testCase{ + Entry("21. application probe proxy: named port", testCase{ num: "21", mesh: ` apiVersion: kuma.io/v1alpha1 @@ -810,6 +810,23 @@ spec: kuma.io/sidecar-injection: enabled`, cfgFile: "inject.config.yaml", }), + Entry("40. application probe proxy: config - disabled, pod - enabled", testCase{ + num: "40", + mesh: ` + apiVersion: kuma.io/v1alpha1 + kind: Mesh + metadata: + name: default + spec: {}`, + namespace: ` + apiVersion: v1 + kind: Namespace + metadata: + name: default + labels: + kuma.io/sidecar-injection: enabled`, + cfgFile: "inject.vp-disabled.config.yaml", + }), ) DescribeTable("should not inject Kuma into a Pod", diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/probes.go b/pkg/plugins/runtime/k8s/webhooks/injector/probes.go index 90f54b9cada5..302b83f3400b 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/probes.go +++ b/pkg/plugins/runtime/k8s/webhooks/injector/probes.go @@ -3,7 +3,6 @@ package injector import ( "strconv" - "github.com/pkg/errors" kube_core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -94,39 +93,6 @@ func overrideHTTPProbe(probe *kube_core.Probe, virtualPort uint32) error { return nil } -func setVirtualProbesEnabledAnnotation(annotations metadata.Annotations, pod *kube_core.Pod, cfg runtime_k8s.Injector) error { - str := func(b bool) string { - if b { - return metadata.AnnotationEnabled - } - return metadata.AnnotationDisabled - } - - vpEnabled, vpExist, err := metadata.Annotations(pod.Annotations).GetEnabled(metadata.KumaVirtualProbesAnnotation) - if err != nil { - return err - } - gwEnabled, _, err := metadata.Annotations(pod.Annotations).GetEnabled(metadata.KumaGatewayAnnotation) - if err != nil { - return err - } - - if gwEnabled { - if vpEnabled { - return errors.New("virtual probes can't be enabled in gateway mode") - } - annotations[metadata.KumaVirtualProbesAnnotation] = metadata.AnnotationDisabled - return nil - } - - if vpExist { - annotations[metadata.KumaVirtualProbesAnnotation] = str(vpEnabled) - return nil - } - annotations[metadata.KumaVirtualProbesAnnotation] = str(cfg.VirtualProbesEnabled) - return nil -} - func setVirtualProbesPortAnnotation(annotations metadata.Annotations, pod *kube_core.Pod, cfg runtime_k8s.Injector) error { port, _, err := metadata.Annotations(pod.Annotations).GetUint32WithDefault(cfg.VirtualProbesPort, metadata.KumaVirtualProbesPortAnnotation) if err != nil { diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml index 14d665e7b7b0..39ca121d6c6a 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -134,7 +135,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml index fd30b768bdfb..c3ae315a09c5 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -142,7 +143,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml index 47999c8c4b3d..e23334eeb368 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: coredns + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null generateName: coredns-fb8b8dccf- labels: @@ -38,6 +37,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -131,9 +132,15 @@ spec: livenessProbe: failureThreshold: 5 httpGet: - path: /8080/health + httpHeaders: + - name: x-custom-header + value: custom-value + - name: X-Kuma-Probes-Scheme + value: HTTPS + - name: X-Kuma-Probes-Timeout + value: "5" + path: /9999/health port: 9000 - scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 @@ -152,9 +159,8 @@ spec: readinessProbe: failureThreshold: 3 httpGet: - path: /8080/health + path: /tcp/1433 port: 9000 - scheme: HTTP periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 @@ -173,6 +179,20 @@ spec: - all procMount: Default readOnlyRootFilesystem: true + startupProbe: + failureThreshold: 5 + httpGet: + httpHeaders: + - name: X-Kuma-Probes-Timeout + value: "5" + - name: X-Kuma-Probes-GRPC-Service + value: /startup + path: /grpc/5678 + port: 9000 + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: @@ -196,7 +216,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.input.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.input.yaml index fb5dc4fb555a..1658285f023a 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.input.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.input.yaml @@ -55,20 +55,30 @@ spec: readOnly: true mountPath: "/var/run/secrets/kubernetes.io/serviceaccount" livenessProbe: + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 5 + timeoutSeconds: 5 httpGet: path: "/health" - port: 8080 - scheme: HTTP + port: 9999 + scheme: HTTPS + httpHeaders: + - name: x-custom-header + value: "custom-value" + startupProbe: + grpc: + service: "/startup" + port: 5678 initialDelaySeconds: 60 timeoutSeconds: 5 periodSeconds: 10 successThreshold: 1 failureThreshold: 5 readinessProbe: - httpGet: - path: "/health" - port: 8080 - scheme: HTTP + tcpSocket: + port: 1433 timeoutSeconds: 1 periodSeconds: 10 successThreshold: 1 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml index 310c8ac66d47..f07358bb4257 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: demo kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -134,7 +135,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml index 0765f1c5c0eb..a7f307daabfe 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -128,7 +129,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.06.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.06.golden.yaml index 4e5dd0d80d77..fb8874fe8522 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.06.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.06.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/gateway: enabled kuma.io/mesh: default diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.07.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.07.golden.yaml index 14d665e7b7b0..39ca121d6c6a 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.07.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.07.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -134,7 +135,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.08.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.08.golden.yaml index 2a746212d9d4..86fc7270a9bd 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.08.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.08.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" prometheus.io/path: /appmetrics prometheus.io/port: "5678" prometheus.io/scrape: "true" @@ -33,6 +32,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -137,7 +138,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.09.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.09.golden.yaml index 0819cab9d746..a3bb2359156b 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.09.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.09.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" prometheus.metrics.kuma.io/path: /custom-metrics prometheus.metrics.kuma.io/port: "5678" creationTimestamp: null @@ -32,6 +31,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -136,7 +137,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.10.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.10.golden.yaml index 3dc9f726bf0e..1e5b4ea6507c 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.10.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.10.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: kuma.io/sidecar-injection: enabled @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -135,7 +136,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.11.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.11.golden.yaml index 77fd14687f1f..6a8e3f5d9056 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.11.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.11.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: mesh-name-from-ns kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -134,7 +135,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.12.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.12.golden.yaml index ce6662432621..9a548a85d78c 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.12.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.12.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: mesh-name-from-pod kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -134,7 +135,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.13.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.13.golden.yaml index 562e22fa45f6..41a8a24f4441 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.13.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.13.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -146,7 +147,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.golden.yaml index 92efabcb1add..5424df946b1c 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "19000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "19000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "19000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -146,7 +147,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "19000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.input.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.input.yaml index b33fd7623ffd..ffae836cf379 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.input.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.input.yaml @@ -5,7 +5,7 @@ metadata: labels: run: busybox annotations: - kuma.io/virtual-probes-port: "19000" + kuma.io/application-probe-proxy-port: "19000" spec: volumes: - name: default-token-w7dxf diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.golden.yaml index 0975c61e7c14..37ede0a30e1b 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,7 +13,7 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: disabled + kuma.io/virtual-probes: enabled kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: @@ -118,15 +119,15 @@ spec: - image: busybox livenessProbe: httpGet: - path: /metrics - port: 8080 + path: /8080/metrics + port: 9000 initialDelaySeconds: 3 periodSeconds: 3 name: busybox readinessProbe: httpGet: - path: /metrics - port: 3001 + path: /3001/metrics + port: 9000 initialDelaySeconds: 3 periodSeconds: 3 resources: {} diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.input.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.input.yaml index a920ece34f4a..efd62dfe91fb 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.input.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.input.yaml @@ -5,7 +5,7 @@ metadata: labels: run: busybox annotations: - kuma.io/virtual-probes: "disabled" + kuma.io/application-probe-proxy-port: "0" spec: volumes: - name: default-token-w7dxf diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.16.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.16.golden.yaml index 1bfd367eeb43..446dcf7df7c2 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.16.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.16.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/exclude-inbound-ports: 1234,1235 traffic.kuma.io/exclude-outbound-ports: "1236" creationTimestamp: null @@ -32,6 +31,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -143,7 +144,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - 1234,1235 + - 1234,1235,9000 - --exclude-outbound-ports - "1236" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.17.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.17.golden.yaml index 3e536c94239d..3c6f5fb01f5e 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.17.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.17.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.18.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.18.golden.yaml index 5f5e3e2f6db4..1531f12be065 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.18.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.18.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.19.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.19.golden.yaml index e7726db72c5a..2a14642a7634 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.19.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.19.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.20.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.20.golden.yaml index a1e36cb1b44a..46e89fc6fd59 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.20.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.20.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.21.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.21.golden.yaml index a84da41cbf07..4efba2f0b59e 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.21.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.21.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -159,7 +160,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.22.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.22.golden.yaml index 04dd68d02fc4..b04eef09bb78 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.22.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.22.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-env-vars: KUMA_DATAPLANE_DRAIN_TIME=5s;NEW_ENV_VAR=123 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.23.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.23.golden.yaml index bd63d43f0a38..8157032456e2 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.23.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.23.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.24.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.24.golden.yaml index 145b4471eb61..7dbef2eaad0f 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.24.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.24.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.25.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.25.golden.yaml index f4665a60dee2..ae13cb36209d 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.25.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.25.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.26.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.26.golden.yaml index 7834f55002fa..87c5aa756b96 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.26.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.26.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/service-account-token-volume: token @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -32,6 +31,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -132,7 +133,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.27.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.27.golden.yaml index e5eeeb4459a8..896e4ad2aad4 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.27.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.27.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-drain-time: 10s @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -128,7 +129,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.28.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.28.golden.yaml index 205229eef087..c670dc46c927 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.28.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.28.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/container-patches: container-patch-1 kuma.io/envoy-admin-port: "9901" kuma.io/envoy-log-level: trace @@ -15,8 +16,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -33,6 +32,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -136,7 +137,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.29.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.29.golden.yaml index 8015144955e0..30f8d63041b3 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.29.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.29.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.30.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.30.golden.yaml index 48a620f23dc9..9b053d463fdb 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.30.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.30.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: ignore-check.kube-linter.io/privileged-container: ebpf requires privileged-container kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.31.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.31.golden.yaml index 3500ea6f0b2c..d9f4a9c260d2 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.31.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.31.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -138,7 +139,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.32.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.32.golden.yaml index 16585eb2df05..d4abc5af9db8 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.32.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.32.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/init-first: "true" kuma.io/mesh: default @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -128,7 +129,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.33.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.33.golden.yaml index 169218e735fd..0e399ddcef75 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.33.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.33.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.golden.yaml index 3012e955d3d9..5200e5f5f219 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: k8s.v1.cni.cncf.io/networks: kuma-cni kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/init-first: "true" kuma.io/mesh: default diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.35.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.35.golden.yaml index b54470f5f94a..68d6949de388 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.35.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.35.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: init + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -30,6 +29,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -157,7 +158,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.36.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.36.golden.yaml index 0fa2778b6499..9df5560140d2 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.36.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.36.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/drop-invalid-packets: "true" creationTimestamp: null labels: @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -142,7 +143,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.37.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.37.golden.yaml index bac11184d0d8..d653955b0d37 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.37.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.37.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/iptables-logs: "true" creationTimestamp: null labels: @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -142,7 +143,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.38.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.38.golden.yaml index a6b1a5980c34..d9ff015ee5e3 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.38.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.38.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/exclude-outbound-ips: 10.0.0.1,172.16.0.0/16,fe80::1,fe80::/10 creationTimestamp: null labels: @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -142,7 +143,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.39.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.39.golden.yaml index ab9a19cf4927..dde6ac55f77f 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.39.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.39.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/exclude-inbound-ips: 192.168.0.1,172.32.16.8/16,a81b:a033:6399:73c7:72b6:aa8c:6f22:7098,fe80::/10 creationTimestamp: null labels: @@ -31,6 +30,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- @@ -142,7 +143,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.golden.yaml new file mode 100644 index 000000000000..7b9967c1a146 --- /dev/null +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.golden.yaml @@ -0,0 +1,205 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" + kuma.io/envoy-admin-port: "9901" + kuma.io/mesh: default + kuma.io/sidecar-injected: "true" + kuma.io/sidecar-uid: "5678" + kuma.io/transparent-proxying: enabled + kuma.io/transparent-proxying-ebpf: disabled + kuma.io/transparent-proxying-inbound-port: "15006" + kuma.io/transparent-proxying-ip-family-mode: dualstack + kuma.io/transparent-proxying-outbound-port: "15001" + creationTimestamp: null + labels: + run: busybox + name: busybox +spec: + containers: + - args: + - run + - --log-level=info + - --concurrency=2 + env: + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" + - name: KUMA_CONTROL_PLANE_CA_CERT + value: | + -----BEGIN CERTIFICATE----- + MIIDMzCCAhugAwIBAgIQDhlInfsXYHamKN+29qnQvzANBgkqhkiG9w0BAQsFADAP + MQ0wCwYDVQQDEwRrdW1hMB4XDTIxMDQwMjEwMjIyNloXDTMxMDMzMTEwMjIyNlow + DzENMAsGA1UEAxMEa3VtYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB + AL4GGg+e2O7eA12F0F6v2rr8j2iVSFKepnZtL15lrCds6lqK50sXWOw8PKZp2ihA + XJVTSZzKasyLDTAR9VYQjTpE526EzvtdthSagf32QWW+wY6LMpEdexKOOCx2se55 + Rd97L33yYPfgX15OYliHPD056jjhotHLdN2lpy7+STDvQyRnXAu73YkY37Ed4hI4 + t/V6soHyEGNcDhm9p5fBGqz0njBbQkp2lTY5/kj42qB7Q6rCM2tbPsEMooeAAw5m + hyY4xj0tP9ucqlUz8gc+6o8HDNst8NeJXZktWn+COytjr/NzGgS22kvSDphisJot + o0FyoIOdAtxC1qxXXR+XuUUCAwEAAaOBijCBhzAOBgNVHQ8BAf8EBAMCAqQwHQYD + VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYD + VR0OBBYEFKRLkgIzX/OjKw9idepuQ/RMtT+AMCYGA1UdEQQfMB2CCWxvY2FsaG9z + dIcQ/QChIwAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAPs5yJZhoYlGW + CpA8dSISivM8/8iBNQ3fVwP63ft0EJLMVGu2RFZ4/UAJ/rUPSGN8xhXSk5+1d56a + /kaH9rX0HaRIHHlxA7iPUKxAj44x9LKmqPHToL3XlWY1AXzvicW9d+GM2FaQee+I + leaqLbz0AZvlnu271Z1CeaACuU9GljujvyiTTE9naHUEqvHgSpPtilJalyJ5/zIl + Z9F0+UWt3TOYMs5g+SCt0MwHTNbisbmewpcFFJzjt2kvtrc9t9dkF81xhcS19w7q + h1AeP3RRlLl7bv9EAVXEmIavih/29PA3ZSy+pbYNW7jNJHjMQ4hQ0E+xcCazU/O4 + ypWGaanvPg== + -----END CERTIFICATE----- + - name: KUMA_CONTROL_PLANE_URL + value: http://kuma-control-plane.kuma-system:5681 + - name: KUMA_DATAPLANE_DRAIN_TIME + value: 31s + - name: KUMA_DATAPLANE_MESH + value: default + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: KUMA_DNS_ENABLED + value: "false" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kuma/kuma-sidecar:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 212 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 260 + periodSeconds: 25 + successThreshold: 1 + timeoutSeconds: 23 + name: kuma-sidecar + readinessProbe: + failureThreshold: 112 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 11 + periodSeconds: 15 + successThreshold: 11 + timeoutSeconds: 13 + resources: + limits: + cpu: 1100m + ephemeral-storage: 1G + memory: 1512Mi + requests: + cpu: 150m + ephemeral-storage: 50M + memory: 164Mi + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 5678 + runAsUser: 5678 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + - mountPath: /tmp + name: kuma-sidecar-tmp + - image: busybox + livenessProbe: + httpGet: + httpHeaders: + - name: X-Kuma-Probes-GRPC-Service + value: grpc-service + path: /grpc/3333 + port: 9000 + initialDelaySeconds: 3 + periodSeconds: 3 + name: busybox + readinessProbe: + httpGet: + path: /3001/metrics + port: 9000 + initialDelaySeconds: 3 + periodSeconds: 3 + resources: {} + startupProbe: + httpGet: + path: /tcp/8081 + port: 9000 + initialDelaySeconds: 3 + periodSeconds: 3 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + initContainers: + - args: + - --config-file + - /tmp/kumactl/config + - --redirect-outbound-port + - "15001" + - --redirect-inbound=true + - --redirect-inbound-port + - "15006" + - --kuma-dp-user + - "5678" + - --exclude-inbound-ports + - "9000" + - --exclude-outbound-ports + - "" + - --verbose + - --ip-family-mode + - dualstack + command: + - /usr/bin/kumactl + - install + - transparent-proxy + env: + - name: XTABLES_LOCKFILE + value: /tmp/xtables.lock + image: kuma/kuma-init:latest + imagePullPolicy: IfNotPresent + name: kuma-init + resources: + limits: + cpu: 100m + memory: 50M + requests: + cpu: 20m + memory: 20M + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /tmp + name: kuma-init-tmp + volumes: + - name: default-token-w7dxf + secret: + secretName: default-token-w7dxf + - emptyDir: + sizeLimit: 10M + name: kuma-init-tmp + - emptyDir: + sizeLimit: 10M + name: kuma-sidecar-tmp +status: {} diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.input.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.input.yaml new file mode 100644 index 000000000000..f01f23871919 --- /dev/null +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.40.input.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Pod +metadata: + name: busybox + labels: + run: busybox + annotations: + kuma.io/application-probe-proxy-port: "9000" +spec: + volumes: + - name: default-token-w7dxf + secret: + secretName: default-token-w7dxf + containers: + - name: busybox + image: busybox + resources: {} + readinessProbe: + httpGet: + path: /metrics + port: 3001 + initialDelaySeconds: 3 + periodSeconds: 3 + livenessProbe: + grpc: + service: grpc-service + port: 3333 + initialDelaySeconds: 3 + periodSeconds: 3 + startupProbe: + tcpSocket: + port: 8081 + initialDelaySeconds: 3 + periodSeconds: 3 + volumeMounts: + - name: default-token-w7dxf + readOnly: true + mountPath: "/var/run/secrets/kubernetes.io/serviceaccount" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.builtindns.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.builtindns.config.yaml index 31c3fe98b0a1..ec9136b9ec8e 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.builtindns.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.builtindns.config.yaml @@ -37,6 +37,7 @@ builtinDNS: port: 25053 virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 0 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-cni.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-cni.yaml index 54cf2808e49b..7d1cdd68fbf1 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-cni.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-cni.yaml @@ -34,6 +34,7 @@ initContainer: image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 0 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml index 4e92d804a7f8..823822a98534 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml @@ -33,6 +33,7 @@ initContainer: image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 0 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml index fc55ebafd5e1..0dfa87247c3b 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml @@ -33,6 +33,7 @@ initContainer: image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 0 sidecarTraffic: excludeInboundPorts: - 1234 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml index a1dc4f7fec5d..5befc86cf231 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml @@ -38,6 +38,7 @@ initContainer: image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 9000 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml index d62d9b1751de..38b0916e6a5b 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml @@ -33,6 +33,7 @@ initContainer: image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 0 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml index 7c091a5045ea..94cba366dbb0 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml @@ -37,6 +37,7 @@ initContainer: image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 0 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml index d46c5144e1fa..f35aae16ed50 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml @@ -35,6 +35,7 @@ builtinDNS: port: 25053 virtualProbesEnabled: true virtualProbesPort: 9000 +applicationProbeProxyPort: 0 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.01.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.01.golden.yaml index ed488c359241..7ad604ac3dae 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.01.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.01.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -39,7 +38,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -85,6 +84,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.02.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.02.golden.yaml index f27ce0718ab5..25217a4203bc 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.02.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.02.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -40,7 +39,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -86,6 +85,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.03.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.03.golden.yaml index 227df7c75ace..5e04ccf68382 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.03.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.03.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: coredns + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null generateName: coredns-fb8b8dccf- labels: @@ -36,9 +35,15 @@ spec: livenessProbe: failureThreshold: 5 httpGet: - path: /8080/health + httpHeaders: + - name: x-custom-header + value: custom-value + - name: X-Kuma-Probes-Scheme + value: HTTPS + - name: X-Kuma-Probes-Timeout + value: "5" + path: /9999/health port: 9000 - scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 @@ -57,9 +62,8 @@ spec: readinessProbe: failureThreshold: 3 httpGet: - path: /8080/health + path: /tcp/1433 port: 9000 - scheme: HTTP periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 @@ -78,6 +82,20 @@ spec: - all procMount: Default readOnlyRootFilesystem: true + startupProbe: + failureThreshold: 5 + httpGet: + httpHeaders: + - name: X-Kuma-Probes-Timeout + value: "5" + - name: X-Kuma-Probes-GRPC-Service + value: /startup + path: /grpc/5678 + port: 9000 + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: @@ -101,7 +119,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -147,6 +165,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.04.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.04.golden.yaml index d082955edace..0975070f34db 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.04.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.04.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: demo kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -39,7 +38,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -85,6 +84,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.05.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.05.golden.yaml index 6f5987c426bb..cd8a38aca4e8 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.05.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.05.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -36,7 +35,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -82,6 +81,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.06.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.06.golden.yaml index 003c47cfe3d6..1803a41b2e13 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.06.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.06.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/gateway: enabled kuma.io/mesh: default diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.07.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.07.golden.yaml index ed488c359241..7ad604ac3dae 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.07.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.07.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -39,7 +38,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -85,6 +84,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.08.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.08.golden.yaml index c22d14dee0af..f96ce1ec5a94 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.08.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.08.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" prometheus.io/path: /appmetrics prometheus.io/port: "5678" prometheus.io/scrape: "true" @@ -42,7 +41,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -88,6 +87,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.09.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.09.golden.yaml index 0bf66065585c..c2ea6b2804f0 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.09.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.09.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" prometheus.metrics.kuma.io/path: /custom-metrics prometheus.metrics.kuma.io/port: "5678" creationTimestamp: null @@ -41,7 +40,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -87,6 +86,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.10.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.10.golden.yaml index 9d80e5d0bd9d..d6b64b601580 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.10.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.10.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: kuma.io/sidecar-injection: enabled @@ -40,7 +39,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -86,6 +85,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.11.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.11.golden.yaml index 99668f8efa14..4f373da5acbe 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.11.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.11.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: mesh-name-from-ns kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -39,7 +38,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -85,6 +84,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.12.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.12.golden.yaml index f9437b933c19..9232606903dc 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.12.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.12.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: mesh-name-from-pod kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -39,7 +38,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -85,6 +84,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.13.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.13.golden.yaml index e3575221919b..e622d9aa2b39 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.13.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.13.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -51,7 +50,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -97,6 +96,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.14.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.14.golden.yaml index cdaf09ac5341..1cb4eeb92149 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.14.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.14.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "19000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "19000" creationTimestamp: null labels: run: busybox @@ -51,7 +50,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "19000" - --exclude-outbound-ports - "" - --verbose @@ -97,6 +96,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "19000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.15.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.15.golden.yaml index 0ca388845b08..aee937535aeb 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.15.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.15.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,7 +13,7 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: disabled + kuma.io/virtual-probes: enabled kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: @@ -23,15 +24,15 @@ spec: - image: busybox livenessProbe: httpGet: - path: /metrics - port: 8080 + path: /8080/metrics + port: 9000 initialDelaySeconds: 3 periodSeconds: 3 name: busybox readinessProbe: httpGet: - path: /metrics - port: 3001 + path: /3001/metrics + port: 9000 initialDelaySeconds: 3 periodSeconds: 3 resources: {} diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.16.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.16.golden.yaml index 589ac1d39423..4d3d89ce4c96 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.16.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.16.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/exclude-inbound-ports: 1234,1235 traffic.kuma.io/exclude-outbound-ports: "1236" creationTimestamp: null @@ -41,7 +40,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - 1234,1235 + - 1234,1235,9000 - --exclude-outbound-ports - "1236" - --verbose @@ -87,6 +86,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.17.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.17.golden.yaml index ae77b8ca39fa..ca0fd9630873 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.17.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.17.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.18.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.18.golden.yaml index f2fc52ab644a..f4d26613de7f 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.18.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.18.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.19.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.19.golden.yaml index 29bcdc6c0a79..8a211e659f69 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.19.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.19.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.20.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.20.golden.yaml index 7808e354b14f..7d68d05c3efc 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.20.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.20.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.21.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.21.golden.yaml index 56e047b50c4e..0b8315cc9b71 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.21.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.21.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -64,7 +63,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -110,6 +109,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.22.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.22.golden.yaml index a9583444eb4f..4523a2e7c831 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.22.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.22.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-env-vars: KUMA_DATAPLANE_DRAIN_TIME=5s;NEW_ENV_VAR=123 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.23.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.23.golden.yaml index 8641d658fa62..f7402c4d023d 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.23.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.23.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.24.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.24.golden.yaml index ed998153d009..873207b844ba 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.24.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.24.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.25.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.25.golden.yaml index 66ae4a8994b2..a6aef85e5dd9 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.25.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.25.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: docs: Documentation kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.26.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.26.golden.yaml index 2a29b0ce172e..c49f902742e1 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.26.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.26.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/service-account-token-volume: token @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -37,7 +36,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -83,6 +82,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.27.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.27.golden.yaml index e2f9d1cb0bef..93966193ab94 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.27.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.27.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-drain-time: 10s @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -36,7 +35,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -82,6 +81,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.28.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.28.golden.yaml index 3d34fbe2ef8a..68494c712f6a 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.28.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.28.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/container-patches: container-patch-1 kuma.io/envoy-admin-port: "9901" kuma.io/envoy-log-level: trace @@ -15,8 +16,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -38,7 +37,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -83,6 +82,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.29.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.29.golden.yaml index a23c88da9b55..fb9c536af021 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.29.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.29.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/builtin-dns: enabled kuma.io/builtin-dns-logging: "false" kuma.io/builtin-dns-port: "25053" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.30.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.30.golden.yaml index 997a5566972d..c6aa84e45d32 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.30.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.30.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: ignore-check.kube-linter.io/privileged-container: ebpf requires privileged-container kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.31.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.31.golden.yaml index aac58fe6f0a4..02b7fc9a9560 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.31.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.31.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -37,7 +36,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -83,6 +82,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.32.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.32.golden.yaml index eb3cba7e88c4..814723b4174b 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.32.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.32.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/init-first: "true" kuma.io/mesh: default @@ -13,8 +14,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -36,7 +35,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -82,6 +81,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.33.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.33.golden.yaml index 309751a9161d..38efadd987ae 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.33.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.33.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.34.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.34.golden.yaml index 8d032c7e0afb..59966a45f862 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.34.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.34.golden.yaml @@ -4,6 +4,7 @@ metadata: annotations: k8s.v1.cni.cncf.io/networks: kuma-cni kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "0" kuma.io/envoy-admin-port: "9901" kuma.io/init-first: "true" kuma.io/mesh: default diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.35.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.35.golden.yaml index 353b1fc7d31f..ed6a323f34fd 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.35.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.35.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: init + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" creationTimestamp: null labels: run: busybox @@ -39,7 +38,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -85,6 +84,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.36.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.36.golden.yaml index d1be0e26f394..7406fec85542 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.36.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.36.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/drop-invalid-packets: "true" creationTimestamp: null labels: @@ -40,7 +39,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -87,6 +86,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.37.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.37.golden.yaml index 2b0deaaac1cb..5c1c456cdd7c 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.37.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.37.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/iptables-logs: "true" creationTimestamp: null labels: @@ -40,7 +39,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -87,6 +86,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.38.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.38.golden.yaml index 1a9c9d19c342..8082fcdd6638 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.38.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.38.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/exclude-outbound-ips: 10.0.0.1,172.16.0.0/16,fe80::1,fe80::/10 creationTimestamp: null labels: @@ -40,7 +39,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -88,6 +87,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.39.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.39.golden.yaml index 228ae1656668..8d32f93dcf76 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.39.golden.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.39.golden.yaml @@ -3,6 +3,7 @@ kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" kuma.io/envoy-admin-port: "9901" kuma.io/mesh: default kuma.io/sidecar-injected: "true" @@ -12,8 +13,6 @@ metadata: kuma.io/transparent-proxying-inbound-port: "15006" kuma.io/transparent-proxying-ip-family-mode: dualstack kuma.io/transparent-proxying-outbound-port: "15001" - kuma.io/virtual-probes: enabled - kuma.io/virtual-probes-port: "9000" traffic.kuma.io/exclude-inbound-ips: 192.168.0.1,172.32.16.8/16,a81b:a033:6399:73c7:72b6:aa8c:6f22:7098,fe80::/10 creationTimestamp: null labels: @@ -40,7 +39,7 @@ spec: - --kuma-dp-user - "5678" - --exclude-inbound-ports - - "" + - "9000" - --exclude-outbound-ports - "" - --verbose @@ -88,6 +87,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" - name: KUMA_CONTROL_PLANE_CA_CERT value: | -----BEGIN CERTIFICATE----- diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.40.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.40.golden.yaml new file mode 100644 index 000000000000..15120eb47290 --- /dev/null +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.40.golden.yaml @@ -0,0 +1,218 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + kubectl.kubernetes.io/default-container: busybox + kuma.io/application-probe-proxy-port: "9000" + kuma.io/envoy-admin-port: "9901" + kuma.io/mesh: default + kuma.io/sidecar-injected: "true" + kuma.io/sidecar-uid: "5678" + kuma.io/transparent-proxying: enabled + kuma.io/transparent-proxying-ebpf: disabled + kuma.io/transparent-proxying-inbound-port: "15006" + kuma.io/transparent-proxying-ip-family-mode: dualstack + kuma.io/transparent-proxying-outbound-port: "15001" + creationTimestamp: null + labels: + run: busybox + name: busybox +spec: + containers: + - image: busybox + livenessProbe: + httpGet: + httpHeaders: + - name: X-Kuma-Probes-GRPC-Service + value: grpc-service + path: /grpc/3333 + port: 9000 + initialDelaySeconds: 3 + periodSeconds: 3 + name: busybox + readinessProbe: + httpGet: + path: /3001/metrics + port: 9000 + initialDelaySeconds: 3 + periodSeconds: 3 + resources: {} + startupProbe: + httpGet: + path: /tcp/8081 + port: 9000 + initialDelaySeconds: 3 + periodSeconds: 3 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + initContainers: + - args: + - --config-file + - /tmp/kumactl/config + - --redirect-outbound-port + - "15001" + - --redirect-inbound=true + - --redirect-inbound-port + - "15006" + - --kuma-dp-user + - "5678" + - --exclude-inbound-ports + - "9000" + - --exclude-outbound-ports + - "" + - --verbose + - --ip-family-mode + - dualstack + command: + - /usr/bin/kumactl + - install + - transparent-proxy + env: + - name: XTABLES_LOCKFILE + value: /tmp/xtables.lock + image: kuma/kuma-init:latest + imagePullPolicy: IfNotPresent + name: kuma-init + resources: + limits: + cpu: 100m + memory: 50M + requests: + cpu: 20m + memory: 20M + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /tmp + name: kuma-init-tmp + - args: + - run + - --log-level=info + - --concurrency=2 + env: + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUMA_APPLICATION_PROBE_PROXY_PORT + value: "9000" + - name: KUMA_CONTROL_PLANE_CA_CERT + value: | + -----BEGIN CERTIFICATE----- + MIIDMzCCAhugAwIBAgIQDhlInfsXYHamKN+29qnQvzANBgkqhkiG9w0BAQsFADAP + MQ0wCwYDVQQDEwRrdW1hMB4XDTIxMDQwMjEwMjIyNloXDTMxMDMzMTEwMjIyNlow + DzENMAsGA1UEAxMEa3VtYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB + AL4GGg+e2O7eA12F0F6v2rr8j2iVSFKepnZtL15lrCds6lqK50sXWOw8PKZp2ihA + XJVTSZzKasyLDTAR9VYQjTpE526EzvtdthSagf32QWW+wY6LMpEdexKOOCx2se55 + Rd97L33yYPfgX15OYliHPD056jjhotHLdN2lpy7+STDvQyRnXAu73YkY37Ed4hI4 + t/V6soHyEGNcDhm9p5fBGqz0njBbQkp2lTY5/kj42qB7Q6rCM2tbPsEMooeAAw5m + hyY4xj0tP9ucqlUz8gc+6o8HDNst8NeJXZktWn+COytjr/NzGgS22kvSDphisJot + o0FyoIOdAtxC1qxXXR+XuUUCAwEAAaOBijCBhzAOBgNVHQ8BAf8EBAMCAqQwHQYD + VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYD + VR0OBBYEFKRLkgIzX/OjKw9idepuQ/RMtT+AMCYGA1UdEQQfMB2CCWxvY2FsaG9z + dIcQ/QChIwAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAPs5yJZhoYlGW + CpA8dSISivM8/8iBNQ3fVwP63ft0EJLMVGu2RFZ4/UAJ/rUPSGN8xhXSk5+1d56a + /kaH9rX0HaRIHHlxA7iPUKxAj44x9LKmqPHToL3XlWY1AXzvicW9d+GM2FaQee+I + leaqLbz0AZvlnu271Z1CeaACuU9GljujvyiTTE9naHUEqvHgSpPtilJalyJ5/zIl + Z9F0+UWt3TOYMs5g+SCt0MwHTNbisbmewpcFFJzjt2kvtrc9t9dkF81xhcS19w7q + h1AeP3RRlLl7bv9EAVXEmIavih/29PA3ZSy+pbYNW7jNJHjMQ4hQ0E+xcCazU/O4 + ypWGaanvPg== + -----END CERTIFICATE----- + - name: KUMA_CONTROL_PLANE_URL + value: http://kuma-control-plane.kuma-system:5681 + - name: KUMA_DATAPLANE_DRAIN_TIME + value: 31s + - name: KUMA_DATAPLANE_MESH + value: default + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: KUMA_DNS_ENABLED + value: "false" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kuma/kuma-sidecar:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - killall + - -USR2 + - kuma-dp + livenessProbe: + failureThreshold: 212 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 260 + periodSeconds: 25 + successThreshold: 1 + timeoutSeconds: 23 + name: kuma-sidecar + readinessProbe: + failureThreshold: 112 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 11 + periodSeconds: 15 + successThreshold: 11 + timeoutSeconds: 13 + resources: + limits: + cpu: 1100m + ephemeral-storage: 1G + memory: 1512Mi + requests: + cpu: 150m + ephemeral-storage: 50M + memory: 164Mi + restartPolicy: Always + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 5678 + runAsUser: 5678 + startupProbe: + httpGet: + path: /ready + port: 9901 + successThreshold: 1 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + - mountPath: /tmp + name: kuma-sidecar-tmp + volumes: + - name: default-token-w7dxf + secret: + secretName: default-token-w7dxf + - emptyDir: + sizeLimit: 10M + name: kuma-init-tmp + - emptyDir: + sizeLimit: 10M + name: kuma-sidecar-tmp +status: {} diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.vp-disabled.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.vp-disabled.config.yaml index 9f11f6920b79..1a9689eebfdf 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.vp-disabled.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.vp-disabled.config.yaml @@ -34,6 +34,7 @@ initContainer: image: kuma/kuma-init:latest virtualProbesEnabled: false virtualProbesPort: 9000 +applicationProbeProxyPort: 0 exceptions: labels: "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/transparentproxy/kubernetes/kubernetes.go b/pkg/transparentproxy/kubernetes/kubernetes.go index 94ba34cc2b27..897c81f43ba0 100644 --- a/pkg/transparentproxy/kubernetes/kubernetes.go +++ b/pkg/transparentproxy/kubernetes/kubernetes.go @@ -86,8 +86,7 @@ func NewPodRedirectForPod(pod *kube_core.Pod) (*PodRedirect, error) { podRedirect.RedirectInbound = false } - podRedirect.ExcludeInboundPorts, _ = metadata.Annotations(pod.Annotations).GetString(metadata.KumaTrafficExcludeInboundPorts) - + podRedirect.ExcludeInboundPorts = excludeApplicationProbeProxyPort(pod.Annotations) podRedirect.RedirectPortInbound, _, err = metadata.Annotations(pod.Annotations).GetUint32(metadata.KumaTransparentProxyingInboundPortAnnotation) if err != nil { return nil, err @@ -158,6 +157,21 @@ func NewPodRedirectForPod(pod *kube_core.Pod) (*PodRedirect, error) { return podRedirect, nil } +func excludeApplicationProbeProxyPort(annotations map[string]string) string { + // the annotations are validated/defaulted in a previous step in injector.NewAnnotations, so we can safely ignore the errors here + inboundPortsToExclude, _ := metadata.Annotations(annotations).GetString(metadata.KumaTrafficExcludeInboundPorts) + appProbeProxyPort, _ := metadata.Annotations(annotations).GetString(metadata.KumaApplicationProbeProxyPortAnnotation) + if appProbeProxyPort == "0" || appProbeProxyPort == "" { + return inboundPortsToExclude + } + + if inboundPortsToExclude == "" { + return appProbeProxyPort + } + + return fmt.Sprintf("%s,%s", inboundPortsToExclude, appProbeProxyPort) +} + func (pr *PodRedirect) AsKumactlCommandLine() []string { result := []string{ "--config-file", diff --git a/test/server/cmd/grpc_server.go b/test/server/cmd/grpc_server.go index 139212dc7ebd..7cc0f0d25242 100644 --- a/test/server/cmd/grpc_server.go +++ b/test/server/cmd/grpc_server.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" "google.golang.org/grpc" + grpchealth "google.golang.org/grpc/health/grpc_health_v1" "github.com/kumahq/kuma/pkg/core" "github.com/kumahq/kuma/test/server/grpc/api" @@ -16,16 +17,16 @@ var grpcServerLog = grpcLog.WithName("server") type grpcServer struct { api.UnimplementedGreeterServer - api.UnimplementedHealthServer + grpchealth.UnimplementedHealthServer id string } -func (g *grpcServer) Check(ctx context.Context, request *api.HealthCheckRequest) (*api.HealthCheckResponse, error) { - return &api.HealthCheckResponse{Status: api.HealthCheckResponse_SERVING}, nil +func (g *grpcServer) Check(ctx context.Context, request *grpchealth.HealthCheckRequest) (*grpchealth.HealthCheckResponse, error) { + return &grpchealth.HealthCheckResponse{Status: grpchealth.HealthCheckResponse_SERVING}, nil } -func (g *grpcServer) Watch(request *api.HealthCheckRequest, server api.Health_WatchServer) error { - return server.Send(&api.HealthCheckResponse{Status: api.HealthCheckResponse_SERVING}) +func (g *grpcServer) Watch(request *grpchealth.HealthCheckRequest, server grpchealth.Health_WatchServer) error { + return server.Send(&grpchealth.HealthCheckResponse{Status: grpchealth.HealthCheckResponse_SERVING}) } func (g *grpcServer) SayHello(ctx context.Context, request *api.HelloRequest) (*api.HelloReply, error) { @@ -74,7 +75,7 @@ test-server grpc server --port 8080 api.RegisterGreeterServer(s, &grpcServer{ id: core.NewUUID(), }) - api.RegisterHealthServer(s, &grpcServer{ + grpchealth.RegisterHealthServer(s, &grpcServer{ id: core.NewUUID(), }) diff --git a/test/server/grpc/api/healthcheck.pb.go b/test/server/grpc/api/healthcheck.pb.go deleted file mode 100644 index 860b8a13d841..000000000000 --- a/test/server/grpc/api/healthcheck.pb.go +++ /dev/null @@ -1,289 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.1 -// protoc v3.20.0 -// source: test/server/grpc/api/healthcheck.proto - -package api - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type HealthCheckResponse_ServingStatus int32 - -const ( - HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 - HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 - HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 - HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3 // Used only by the Watch method. -) - -// Enum value maps for HealthCheckResponse_ServingStatus. -var ( - HealthCheckResponse_ServingStatus_name = map[int32]string{ - 0: "UNKNOWN", - 1: "SERVING", - 2: "NOT_SERVING", - 3: "SERVICE_UNKNOWN", - } - HealthCheckResponse_ServingStatus_value = map[string]int32{ - "UNKNOWN": 0, - "SERVING": 1, - "NOT_SERVING": 2, - "SERVICE_UNKNOWN": 3, - } -) - -func (x HealthCheckResponse_ServingStatus) Enum() *HealthCheckResponse_ServingStatus { - p := new(HealthCheckResponse_ServingStatus) - *p = x - return p -} - -func (x HealthCheckResponse_ServingStatus) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (HealthCheckResponse_ServingStatus) Descriptor() protoreflect.EnumDescriptor { - return file_test_server_grpc_api_healthcheck_proto_enumTypes[0].Descriptor() -} - -func (HealthCheckResponse_ServingStatus) Type() protoreflect.EnumType { - return &file_test_server_grpc_api_healthcheck_proto_enumTypes[0] -} - -func (x HealthCheckResponse_ServingStatus) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use HealthCheckResponse_ServingStatus.Descriptor instead. -func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { - return file_test_server_grpc_api_healthcheck_proto_rawDescGZIP(), []int{1, 0} -} - -type HealthCheckRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` -} - -func (x *HealthCheckRequest) Reset() { - *x = HealthCheckRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_test_server_grpc_api_healthcheck_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckRequest) ProtoMessage() {} - -func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message { - mi := &file_test_server_grpc_api_healthcheck_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead. -func (*HealthCheckRequest) Descriptor() ([]byte, []int) { - return file_test_server_grpc_api_healthcheck_proto_rawDescGZIP(), []int{0} -} - -func (x *HealthCheckRequest) GetService() string { - if x != nil { - return x.Service - } - return "" -} - -type HealthCheckResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` -} - -func (x *HealthCheckResponse) Reset() { - *x = HealthCheckResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_test_server_grpc_api_healthcheck_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckResponse) ProtoMessage() {} - -func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message { - mi := &file_test_server_grpc_api_healthcheck_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckResponse.ProtoReflect.Descriptor instead. -func (*HealthCheckResponse) Descriptor() ([]byte, []int) { - return file_test_server_grpc_api_healthcheck_proto_rawDescGZIP(), []int{1} -} - -func (x *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus { - if x != nil { - return x.Status - } - return HealthCheckResponse_UNKNOWN -} - -var File_test_server_grpc_api_healthcheck_proto protoreflect.FileDescriptor - -var file_test_server_grpc_api_healthcheck_proto_rawDesc = []byte{ - 0x0a, 0x26, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x67, 0x72, - 0x70, 0x63, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, - 0x63, 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x22, 0x2e, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, - 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0xb1, 0x01, 0x0a, 0x13, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x49, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, - 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x4f, 0x0a, 0x0d, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, - 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x45, 0x52, - 0x56, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x4e, 0x4f, 0x54, 0x5f, 0x53, 0x45, - 0x52, 0x56, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x52, 0x56, 0x49, - 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x32, 0xae, 0x01, 0x0a, - 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x50, 0x0a, 0x05, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, - 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52, 0x0a, 0x05, 0x57, 0x61, 0x74, - 0x63, 0x68, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x2d, 0x5a, - 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, - 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_test_server_grpc_api_healthcheck_proto_rawDescOnce sync.Once - file_test_server_grpc_api_healthcheck_proto_rawDescData = file_test_server_grpc_api_healthcheck_proto_rawDesc -) - -func file_test_server_grpc_api_healthcheck_proto_rawDescGZIP() []byte { - file_test_server_grpc_api_healthcheck_proto_rawDescOnce.Do(func() { - file_test_server_grpc_api_healthcheck_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_server_grpc_api_healthcheck_proto_rawDescData) - }) - return file_test_server_grpc_api_healthcheck_proto_rawDescData -} - -var file_test_server_grpc_api_healthcheck_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_test_server_grpc_api_healthcheck_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_test_server_grpc_api_healthcheck_proto_goTypes = []interface{}{ - (HealthCheckResponse_ServingStatus)(0), // 0: grpc.health.v1.HealthCheckResponse.ServingStatus - (*HealthCheckRequest)(nil), // 1: grpc.health.v1.HealthCheckRequest - (*HealthCheckResponse)(nil), // 2: grpc.health.v1.HealthCheckResponse -} -var file_test_server_grpc_api_healthcheck_proto_depIdxs = []int32{ - 0, // 0: grpc.health.v1.HealthCheckResponse.status:type_name -> grpc.health.v1.HealthCheckResponse.ServingStatus - 1, // 1: grpc.health.v1.Health.Check:input_type -> grpc.health.v1.HealthCheckRequest - 1, // 2: grpc.health.v1.Health.Watch:input_type -> grpc.health.v1.HealthCheckRequest - 2, // 3: grpc.health.v1.Health.Check:output_type -> grpc.health.v1.HealthCheckResponse - 2, // 4: grpc.health.v1.Health.Watch:output_type -> grpc.health.v1.HealthCheckResponse - 3, // [3:5] is the sub-list for method output_type - 1, // [1:3] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name -} - -func init() { file_test_server_grpc_api_healthcheck_proto_init() } -func file_test_server_grpc_api_healthcheck_proto_init() { - if File_test_server_grpc_api_healthcheck_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_test_server_grpc_api_healthcheck_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_test_server_grpc_api_healthcheck_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_test_server_grpc_api_healthcheck_proto_rawDesc, - NumEnums: 1, - NumMessages: 2, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_test_server_grpc_api_healthcheck_proto_goTypes, - DependencyIndexes: file_test_server_grpc_api_healthcheck_proto_depIdxs, - EnumInfos: file_test_server_grpc_api_healthcheck_proto_enumTypes, - MessageInfos: file_test_server_grpc_api_healthcheck_proto_msgTypes, - }.Build() - File_test_server_grpc_api_healthcheck_proto = out.File - file_test_server_grpc_api_healthcheck_proto_rawDesc = nil - file_test_server_grpc_api_healthcheck_proto_goTypes = nil - file_test_server_grpc_api_healthcheck_proto_depIdxs = nil -} diff --git a/test/server/grpc/api/healthcheck.proto b/test/server/grpc/api/healthcheck.proto deleted file mode 100644 index f1811189e5aa..000000000000 --- a/test/server/grpc/api/healthcheck.proto +++ /dev/null @@ -1,23 +0,0 @@ -syntax = "proto3"; - -package grpc.health.v1; - -option go_package = "github.com/kumahq/kuma/test/server/grpc/api"; - -message HealthCheckRequest { string service = 1; } - -message HealthCheckResponse { - enum ServingStatus { - UNKNOWN = 0; - SERVING = 1; - NOT_SERVING = 2; - SERVICE_UNKNOWN = 3; // Used only by the Watch method. - } - ServingStatus status = 1; -} - -service Health { - rpc Check(HealthCheckRequest) returns (HealthCheckResponse); - - rpc Watch(HealthCheckRequest) returns (stream HealthCheckResponse); -} \ No newline at end of file diff --git a/test/server/grpc/api/healthcheck_grpc.pb.go b/test/server/grpc/api/healthcheck_grpc.pb.go deleted file mode 100644 index 81d48cd5d43d..000000000000 --- a/test/server/grpc/api/healthcheck_grpc.pb.go +++ /dev/null @@ -1,165 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package api - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// HealthClient is the client API for Health service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type HealthClient interface { - Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) - Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) -} - -type healthClient struct { - cc grpc.ClientConnInterface -} - -func NewHealthClient(cc grpc.ClientConnInterface) HealthClient { - return &healthClient{cc} -} - -func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { - out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) { - stream, err := c.cc.NewStream(ctx, &Health_ServiceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...) - if err != nil { - return nil, err - } - x := &healthWatchClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Health_WatchClient interface { - Recv() (*HealthCheckResponse, error) - grpc.ClientStream -} - -type healthWatchClient struct { - grpc.ClientStream -} - -func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) { - m := new(HealthCheckResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// HealthServer is the server API for Health service. -// All implementations must embed UnimplementedHealthServer -// for forward compatibility -type HealthServer interface { - Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) - Watch(*HealthCheckRequest, Health_WatchServer) error - mustEmbedUnimplementedHealthServer() -} - -// UnimplementedHealthServer must be embedded to have forward compatible implementations. -type UnimplementedHealthServer struct { -} - -func (UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Check not implemented") -} -func (UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error { - return status.Errorf(codes.Unimplemented, "method Watch not implemented") -} -func (UnimplementedHealthServer) mustEmbedUnimplementedHealthServer() {} - -// UnsafeHealthServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to HealthServer will -// result in compilation errors. -type UnsafeHealthServer interface { - mustEmbedUnimplementedHealthServer() -} - -func RegisterHealthServer(s grpc.ServiceRegistrar, srv HealthServer) { - s.RegisterService(&Health_ServiceDesc, srv) -} - -func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(HealthCheckRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(HealthServer).Check(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.health.v1.Health/Check", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(HealthCheckRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(HealthServer).Watch(m, &healthWatchServer{stream}) -} - -type Health_WatchServer interface { - Send(*HealthCheckResponse) error - grpc.ServerStream -} - -type healthWatchServer struct { - grpc.ServerStream -} - -func (x *healthWatchServer) Send(m *HealthCheckResponse) error { - return x.ServerStream.SendMsg(m) -} - -// Health_ServiceDesc is the grpc.ServiceDesc for Health service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var Health_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.health.v1.Health", - HandlerType: (*HealthServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Check", - Handler: _Health_Check_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "Watch", - Handler: _Health_Watch_Handler, - ServerStreams: true, - }, - }, - Metadata: "test/server/grpc/api/healthcheck.proto", -}