From 5766d6ac4c102d99f91ad574865e7aaf3cb31acd Mon Sep 17 00:00:00 2001 From: Vincent Deng Date: Tue, 2 Jan 2024 14:55:07 +0800 Subject: [PATCH] Sidecar container support Supports sidecar container by setting restartPolicy of init container to 'always' Closes #20372 Signed-off-by: Vincent Deng --- docs/source/markdown/podman-kube-play.1.md.in | 2 + libpod/container_api.go | 44 ++++++ libpod/pod_api.go | 24 ++- pkg/domain/infra/abi/play.go | 28 +++- pkg/k8s.io/api/core/v1/types.go | 20 +++ test/e2e/play_kube_test.go | 146 ++++++++++++++++++ 6 files changed, 249 insertions(+), 15 deletions(-) diff --git a/docs/source/markdown/podman-kube-play.1.md.in b/docs/source/markdown/podman-kube-play.1.md.in index d7ffb1a5aef6..9e6fd5e787ce 100644 --- a/docs/source/markdown/podman-kube-play.1.md.in +++ b/docs/source/markdown/podman-kube-play.1.md.in @@ -35,6 +35,8 @@ Only three volume types are supported by kube play, the *hostPath*, *emptyDir*, Note: The default restart policy for containers is `always`. You can change the default by setting the `restartPolicy` field in the spec. +Note: The restartPolicy of init container can be set to `always`. When it is set to `always`, the init container will function as sidecar container. The init container will keep running while Pod is running. However, for now, the started init container won't get restarted when it run into failure even if restartPolicy is set to `always`. + Note: When playing a kube YAML with init containers, the init container is created with init type value `once`. To change the default type, use the `io.podman.annotations.init.container.type` annotation to set the type to `always`. Note: *hostPath* volume types created by kube play is given an SELinux shared label (z), bind mounts are not relabeled (use `chcon -t container_file_t -R `). diff --git a/libpod/container_api.go b/libpod/container_api.go index b0e06566df1c..9c01880840e4 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -558,6 +558,50 @@ func (c *Container) Wait(ctx context.Context) (int32, error) { return c.WaitForExit(ctx, DefaultWaitInterval) } +// WaitForStartupProbeSuccess blocks until the container startupProbe success +func (c *Container) WaitForStartupProbeSuccess(ctx context.Context, pollInterval time.Duration) error { + if !c.valid { + return define.ErrCtrRemoved + } + + // Return immediately if StartupHealthCheckConfig is not defined + if c.config.StartupHealthCheckConfig == nil { + return nil + } + + var ( + startupPass bool + err error + ) + + // If startupHC already passed, return immediately + if startupPass, err = c.StartupHCPassed(); err != nil { + return err + } else if startupPass { + return nil + } + + // polling for startupHc result + for { + if _, _, err := c.runHealthCheck(ctx, true); err != nil { + return err + } + if startupPass, err = c.StartupHCPassed(); err != nil { + return err + } else if startupPass { + return nil + } + + select { + case <-ctx.Done(): + return fmt.Errorf("waiting for exit code of container %s canceled", c.ID()) + default: + time.Sleep(pollInterval) + continue + } + } +} + // WaitForExit blocks until the container exits and returns its exit code. The // argument is the interval at which checks the container's status. func (c *Container) WaitForExit(ctx context.Context, pollInterval time.Duration) (int32, error) { diff --git a/libpod/pod_api.go b/libpod/pod_api.go index c5bab8c2d7e2..dcf60c6f2fb9 100644 --- a/libpod/pod_api.go +++ b/libpod/pod_api.go @@ -28,14 +28,22 @@ func (p *Pod) startInitContainers(ctx context.Context) error { if err := initCon.Start(ctx, true); err != nil { return err } - // Check that the init container waited correctly and the exit - // code is good - rc, err := initCon.Wait(ctx) - if err != nil { - return err - } - if rc != 0 { - return fmt.Errorf("init container %s exited with code %d", initCon.ID(), rc) + + if initCon.config.InitContainerType == define.AlwaysInitContainer { + // For restartable init container, check startup probing is success + if err := initCon.WaitForStartupProbeSuccess(ctx, DefaultWaitInterval); err != nil { + return err + } + } else { + // Check that the init container waited correctly and the exit + // code is good + rc, err := initCon.Wait(ctx) + if err != nil { + return err + } + if rc != 0 { + return fmt.Errorf("init container %s exited with code %d", initCon.ID(), rc) + } } // If the container is a once init container, we need to remove it // after it runs diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index 1d1f3a6407ff..47fca161ffd0 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -759,10 +759,14 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY return nil, nil, fmt.Errorf("the pod %q is invalid; duplicate container name %q detected", podName, initCtr.Name) } ctrNames[initCtr.Name] = "" - // Init containers cannot have either of lifecycle, livenessProbe, readinessProbe, or startupProbe set - if initCtr.Lifecycle != nil || initCtr.LivenessProbe != nil || initCtr.ReadinessProbe != nil || initCtr.StartupProbe != nil { - return nil, nil, fmt.Errorf("cannot create an init container that has either of lifecycle, livenessProbe, readinessProbe, or startupProbe set") + + if initCtr.RestartPolicy == nil || *initCtr.RestartPolicy != v1.ContainerRestartPolicyAlways { + // Non-restartable init containers cannot have either of lifecycle, livenessProbe, readinessProbe, or startupProbe set + if initCtr.Lifecycle != nil || initCtr.LivenessProbe != nil || initCtr.ReadinessProbe != nil || initCtr.StartupProbe != nil { + return nil, nil, fmt.Errorf("cannot create an init container that has either of lifecycle, livenessProbe, readinessProbe, or startupProbe set") + } } + pulledImage, labels, err := ic.getImageAndLabelInfo(ctx, cwd, annotations, writer, initCtr, options) if err != nil { return nil, nil, err @@ -771,9 +775,19 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY for k, v := range podSpec.PodSpecGen.Labels { // add podYAML labels labels[k] = v } - initCtrType := annotations[define.InitContainerType] - if initCtrType == "" { - initCtrType = define.OneShotInitContainer + + // Supporting native sidecar container. Check https://kubernetes.io/blog/2023/08/25/native-sidecar-containers/ + var initCtrType string + var restartPolicy string + if initCtr.RestartPolicy != nil && *initCtr.RestartPolicy == v1.ContainerRestartPolicyAlways { + restartPolicy = define.RestartPolicyAlways + initCtrType = define.AlwaysInitContainer + } else { + restartPolicy = define.RestartPolicyNo + initCtrType = annotations[define.InitContainerType] + if initCtrType == "" { + initCtrType = define.OneShotInitContainer + } } specgenOpts := kube.CtrSpecGenOptions{ @@ -791,7 +805,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY PodName: podName, PodSecurityContext: podYAML.Spec.SecurityContext, ReadOnly: readOnly, - RestartPolicy: define.RestartPolicyNo, + RestartPolicy: restartPolicy, SeccompPaths: seccompPaths, SecretsManager: secretsManager, UserNSIsHost: p.Userns.IsHost(), diff --git a/pkg/k8s.io/api/core/v1/types.go b/pkg/k8s.io/api/core/v1/types.go index 904e50f18b28..8d5b0f0efc0f 100644 --- a/pkg/k8s.io/api/core/v1/types.go +++ b/pkg/k8s.io/api/core/v1/types.go @@ -1257,6 +1257,12 @@ type Container struct { // Default is false. // +optional TTY bool `json:"tty,omitempty"` + // Restart policy for the container to manage the restart behavior of each + // container within a pod. + // This may only be set for init containers. You cannot set this field on + // ephemeral containers. + // +optional + RestartPolicy *ContainerRestartPolicy `json:"restartPolicy,omitempty" protobuf:"bytes,24,opt,name=restartPolicy,casttype=ContainerRestartPolicy"` } // Handler defines a specific action that should be taken @@ -1485,6 +1491,14 @@ const ( RestartPolicyNever RestartPolicy = "Never" ) +// ContainerRestartPolicy is the restart policy for a single container. +// This may only be set for init containers and only allowed value is "Always". +type ContainerRestartPolicy string + +const ( + ContainerRestartPolicyAlways ContainerRestartPolicy = "Always" +) + // DNSPolicy defines how a pod's DNS will be configured. type DNSPolicy string @@ -2375,6 +2389,12 @@ type EphemeralContainerCommon struct { // Default is false. // +optional TTY bool `json:"tty,omitempty"` + // Restart policy for the container to manage the restart behavior of each + // container within a pod. + // This may only be set for init containers. You cannot set this field on + // ephemeral containers. + // +optional + RestartPolicy *ContainerRestartPolicy `json:"restartPolicy,omitempty" protobuf:"bytes,24,opt,name=restartPolicy,casttype=ContainerRestartPolicy"` } // EphemeralContainerCommon converts to Container. All fields must be kept in sync between diff --git a/test/e2e/play_kube_test.go b/test/e2e/play_kube_test.go index 77aed75e860d..050bb1835ac0 100644 --- a/test/e2e/play_kube_test.go +++ b/test/e2e/play_kube_test.go @@ -582,6 +582,27 @@ spec: {{ end }} image: {{ .Image }} name: {{ .Name }} +{{ if .RestartPolicy }} + restartPolicy: {{ .RestartPolicy }} + {{ if .StartupProbe }} + startupProbe: + exec: + command: + {{ range .StartupProbe.Cmd }} + - {{.}} + {{ end }} + {{ if .StartupProbe.TimeoutSeconds }} + timeoutSeconds: {{ .StartupProbe.TimeoutSeconds }} + {{ end }} + {{ end }} +{{ end }} + +{{ if .VolumeMount }} + volumeMounts: + - name: {{.VolumeName}} + mountPath: {{ .VolumeMountPath }} + readonly: {{.VolumeReadOnly}} +{{ end }} {{ end }} {{ end }} {{ if .SecurityContext }} @@ -1644,6 +1665,40 @@ func getPodNameInDeployment(d *Deployment) Pod { return p } +// HealthcheckProbe describes the options for a health check probe +type HealthCheckProbe struct { + Cmd []string + InitialDelaySeconds int + PeriodSeconds int + TimeoutSeconds int +} + +func getHealthCheckProbe(options ...healthProbeOption) *HealthCheckProbe { + probe := HealthCheckProbe{ + InitialDelaySeconds: 1, + PeriodSeconds: 1, + } + + for _, option := range options { + option(&probe) + } + return &probe +} + +type healthProbeOption func(*HealthCheckProbe) + +func WithHealthCheckCommand(cmd []string) healthProbeOption { + return func(h *HealthCheckProbe) { + h.Cmd = cmd + } +} + +func WithHeathCheckTimeout(timeout int) healthProbeOption { + return func(h *HealthCheckProbe) { + h.TimeoutSeconds = timeout + } +} + // Ctr describes the options a kube yaml can be configured at container level type Ctr struct { Name string @@ -1669,8 +1724,10 @@ type Ctr struct { Env []Env EnvFrom []EnvFrom InitCtrType string + RestartPolicy string RunAsUser string RunAsGroup string + StartupProbe *HealthCheckProbe } // getCtr takes a list of ctrOptions and returns a Ctr with sane defaults @@ -1717,6 +1774,18 @@ func withInitCtr() ctrOption { } } +func withRestartableInitCtr() ctrOption { + return func(c *Ctr) { + c.RestartPolicy = string(v1.RestartPolicyAlways) + } +} + +func withStartupProbe(probe *HealthCheckProbe) ctrOption { + return func(c *Ctr) { + c.StartupProbe = probe + } +} + func withCmd(cmd []string) ctrOption { return func(c *Ctr) { c.Cmd = cmd @@ -2498,6 +2567,83 @@ var _ = Describe("Podman kube play", func() { Expect(inspect.OutputToString()).To(ContainSubstring("running")) }) + // If always restart init container didn't define define startup probe, it is started immediately and the Pod is starting regular container + It("test with always restart init container without startup probe started immediately", func() { + pod := getPod(withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"}), withRestartableInitCtr(), withName("sidecar-container"))), withCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"})))) + err := generateKubeYaml("pod", pod, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"kube", "play", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(ExitCleanly()) + + // Expect the number of containers created to be 3, infra, restartable init container and regular container + numOfCtrs := podmanTest.NumberOfContainers() + Expect(numOfCtrs).To(Equal(3)) + + // Init container should keep running + inspect := podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", "testPod-" + "sidecar-container"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(ExitCleanly()) + Expect(inspect.OutputToString()).To(ContainSubstring("running")) + + // Regular container should be in running state + inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", "testPod-" + defaultCtrName}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(ExitCleanly()) + Expect(inspect.OutputToString()).To(ContainSubstring("running")) + }) + + // For a Pod with [initContainer1, restartableInitContainer, initContainer2, regularContainer], the containers shouldbe started in sequence. And regular container started after + // start probe of restartableInitContainer finished + It("init container started in sequence", func() { + startupProbe := getHealthCheckProbe(WithHealthCheckCommand([]string{"sh", "-c", "echo 'sidecar'>>/test/startUpSequence"}), WithHeathCheckTimeout(3)) + initContainer1 := withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"sh", "-c", "echo 'init-test1'>>/test/startUpSequence"}), withInitCtr(), withName("init-test1"), withVolumeMount("/test", "", false))) + restartableInitConatiner := withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"}), withRestartableInitCtr(), withName("sidecar-container"), withStartupProbe(startupProbe), withVolumeMount("/test", "", false))) + initContainer2 := withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"sh", "-c", "echo 'init-test2'>>/test/startUpSequence"}), withInitCtr(), withName("init-test2"), withVolumeMount("/test", "", false))) + + pod := getPod(initContainer1, restartableInitConatiner, initContainer2, withCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"}), withVolumeMount("/test", "", false))), withVolume(getEmptyDirVolume())) + err := generateKubeYaml("pod", pod, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"kube", "play", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(ExitCleanly()) + + // Expect the number of containers created to be 3, infra, restartable init container and regular container + numOfCtrs := podmanTest.NumberOfContainers() + Expect(numOfCtrs).To(Equal(3)) + + // Init container should keep running + inspect := podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", "testPod-" + "sidecar-container"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(ExitCleanly()) + Expect(inspect.OutputToString()).To(ContainSubstring("running")) + + // Regular container should be in running state + inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", getCtrNameInPod(pod)}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(ExitCleanly()) + Expect(inspect.OutputToString()).To(ContainSubstring("running")) + + // Init container started in correct sequence + inspect = podmanTest.Podman([]string{"exec", getCtrNameInPod(pod), "cat", "/test/startUpSequence"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(ExitCleanly()) + Expect(inspect.OutputToString()).To(Equal("init-test1 sidecar init-test2")) + + // Regular container started after restartable init container + inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.StartedAt}}", "testPod-" + "sidecar-container"}) + inspect.WaitWithDefaultTimeout() + startOfRestartableInitContainer, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", inspect.OutputToString()) + Expect(err).ToNot(HaveOccurred()) + inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.StartedAt}}", getCtrNameInPod(pod)}) + inspect.WaitWithDefaultTimeout() + startOfRegularContainer, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", inspect.OutputToString()) + Expect(err).ToNot(HaveOccurred()) + Expect(startOfRestartableInitContainer).To(BeTemporally("<", startOfRegularContainer)) + }) + // If you supply only args for a Container, the default Entrypoint defined in the Docker image is run with the args that you supplied. It("test correct command with only set args in yaml file", func() { pod := getPod(withCtr(getCtr(withImage(REGISTRY_IMAGE), withCmd(nil), withArg([]string{"echo", "hello"}))))