issue-20372 - Initial impl for restartable init-container

docs/source/markdown/podman-kube-play.1.md.in

@@ -35,6 +35,8 @@ Only three volume types are supported by kube play, the *hostPath*, *emptyDir*,
 
 Note: The default restart policy for containers is `always`.  You can change the default by setting the `restartPolicy` field in the spec.
 
+Note: The restartPolicy of init container can be set to `always`. When it is set to `always`, the init container will function as sidecar container. The init container will keep running while Pod is running. However, for now, the started init container won't get restarted when it run into failure even if restartPolicy is set to `always`.
+
 Note: When playing a kube YAML with init containers, the init container is created with init type value `once`. To change the default type, use the `io.podman.annotations.init.container.type` annotation to set the type to `always`.
 
 Note: *hostPath* volume types created by kube play is given an SELinux shared label (z), bind mounts are not relabeled (use `chcon -t container_file_t -R <directory>`).

libpod/container_api.go

@@ -557,6 +557,50 @@ func (c *Container) Wait(ctx context.Context) (int32, error) {
 	return c.WaitForExit(ctx, DefaultWaitInterval)
 }
 
+// WaitForStartupProbeSuccess blocks until the container startupProbe success
+func (c *Container) WaitForStartupProbeSuccess(ctx context.Context, pollInterval time.Duration) error {
+	if !c.valid {
+		return define.ErrCtrRemoved
+	}
+
+	// Return immediately if StartupHealthCheckConfig is not defined
+	if c.config.StartupHealthCheckConfig == nil {
+		return nil
+	}
+
+	var (
+		startupPass bool
+		err         error
+	)
+
+	// If startupHC already passed, return immediately
+	if startupPass, err = c.StartupHCPassed(); err != nil {
+		return err
+	} else if startupPass {
+		return nil
+	}
+
+	// polling for startupHc result
+	for {
+		if _, _, err := c.runHealthCheck(ctx, true); err != nil {
+			return err
+		}
+		if startupPass, err = c.StartupHCPassed(); err != nil {
+			return err
+		} else if startupPass {
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("waiting for exit code of container %s canceled", c.ID())
+		default:
+			time.Sleep(pollInterval)
+			continue
+		}
+	}
+}
+
 // WaitForExit blocks until the container exits and returns its exit code. The
 // argument is the interval at which checks the container's status.
 func (c *Container) WaitForExit(ctx context.Context, pollInterval time.Duration) (int32, error) {

libpod/pod_api.go

@@ -27,14 +27,22 @@ func (p *Pod) startInitContainers(ctx context.Context) error {
 		if err := initCon.Start(ctx, true); err != nil {
 			return err
 		}
-		// Check that the init container waited correctly and the exit
-		// code is good
-		rc, err := initCon.Wait(ctx)
-		if err != nil {
-			return err
-		}
-		if rc != 0 {
-			return fmt.Errorf("init container %s exited with code %d", initCon.ID(), rc)
+
+		if initCon.config.InitContainerType == define.AlwaysInitContainer {
+			// For restartable init container, check startup probing is success
+			if err := initCon.WaitForStartupProbeSuccess(ctx, DefaultWaitInterval); err != nil {
+				return err
+			}
+		} else {
+			// Check that the init container waited correctly and the exit
+			// code is good
+			rc, err := initCon.Wait(ctx)
+			if err != nil {
+				return err
+			}
+			if rc != 0 {
+				return fmt.Errorf("init container %s exited with code %d", initCon.ID(), rc)
+			}
 		}
 		// If the container is a once init container, we need to remove it
 		// after it runs

pkg/domain/infra/abi/play.go

@@ -768,10 +768,14 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			return nil, nil, fmt.Errorf("the pod %q is invalid; duplicate container name %q detected", podName, initCtr.Name)
 		}
 		ctrNames[initCtr.Name] = ""
-		// Init containers cannot have either of lifecycle, livenessProbe, readinessProbe, or startupProbe set
-		if initCtr.Lifecycle != nil || initCtr.LivenessProbe != nil || initCtr.ReadinessProbe != nil || initCtr.StartupProbe != nil {
-			return nil, nil, fmt.Errorf("cannot create an init container that has either of lifecycle, livenessProbe, readinessProbe, or startupProbe set")
+
+		if initCtr.RestartPolicy == nil || *initCtr.RestartPolicy != v1.ContainerRestartPolicyAlways {
+			// Non-restartable init containers cannot have either of lifecycle, livenessProbe, readinessProbe, or startupProbe set
+			if initCtr.Lifecycle != nil || initCtr.LivenessProbe != nil || initCtr.ReadinessProbe != nil || initCtr.StartupProbe != nil {
+				return nil, nil, fmt.Errorf("cannot create an init container that has either of lifecycle, livenessProbe, readinessProbe, or startupProbe set")
+			}
 		}
+
 		pulledImage, labels, err := ic.getImageAndLabelInfo(ctx, cwd, annotations, writer, initCtr, options)
 		if err != nil {
 			return nil, nil, err
@@ -780,9 +784,19 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		for k, v := range podSpec.PodSpecGen.Labels { // add podYAML labels
 			labels[k] = v
 		}
-		initCtrType := annotations[define.InitContainerType]
-		if initCtrType == "" {
-			initCtrType = define.OneShotInitContainer
+
+		// Supporting native sidecar container. Check https://kubernetes.io/blog/2023/08/25/native-sidecar-containers/
+		var initCtrType string
+		var restartPolicy string
+		if initCtr.RestartPolicy != nil && *initCtr.RestartPolicy == v1.ContainerRestartPolicyAlways {
+			restartPolicy = define.RestartPolicyAlways
+			initCtrType = define.AlwaysInitContainer
+		} else {
+			restartPolicy = define.RestartPolicyNo
+			initCtrType = annotations[define.InitContainerType]
+			if initCtrType == "" {
+				initCtrType = define.OneShotInitContainer
+			}
 		}
 
 		specgenOpts := kube.CtrSpecGenOptions{
@@ -800,7 +814,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			PodName:            podName,
 			PodSecurityContext: podYAML.Spec.SecurityContext,
 			ReadOnly:           readOnly,
-			RestartPolicy:      define.RestartPolicyNo,
+			RestartPolicy:      restartPolicy,
 			SeccompPaths:       seccompPaths,
 			SecretsManager:     secretsManager,
 			UserNSIsHost:       p.Userns.IsHost(),

pkg/k8s.io/api/core/v1/types.go

@@ -1257,6 +1257,12 @@ type Container struct {
 	// Default is false.
 	// +optional
 	TTY bool `json:"tty,omitempty"`
+	// Restart policy for the container to manage the restart behavior of each
+	// container within a pod.
+	// This may only be set for init containers. You cannot set this field on
+	// ephemeral containers.
+	// +optional
+	RestartPolicy *ContainerRestartPolicy `json:"restartPolicy,omitempty" protobuf:"bytes,24,opt,name=restartPolicy,casttype=ContainerRestartPolicy"`
 }
 
 // Handler defines a specific action that should be taken
@@ -1485,6 +1491,14 @@ const (
 	RestartPolicyNever     RestartPolicy = "Never"
 )
 
+// ContainerRestartPolicy is the restart policy for a single container.
+// This may only be set for init containers and only allowed value is "Always".
+type ContainerRestartPolicy string
+
+const (
+	ContainerRestartPolicyAlways ContainerRestartPolicy = "Always"
+)
+
 // DNSPolicy defines how a pod's DNS will be configured.
 type DNSPolicy string
 
@@ -2375,6 +2389,12 @@ type EphemeralContainerCommon struct {
 	// Default is false.
 	// +optional
 	TTY bool `json:"tty,omitempty"`
+	// Restart policy for the container to manage the restart behavior of each
+	// container within a pod.
+	// This may only be set for init containers. You cannot set this field on
+	// ephemeral containers.
+	// +optional
+	RestartPolicy *ContainerRestartPolicy `json:"restartPolicy,omitempty" protobuf:"bytes,24,opt,name=restartPolicy,casttype=ContainerRestartPolicy"`
 }
 
 // EphemeralContainerCommon converts to Container. All fields must be kept in sync between

test/e2e/play_kube_test.go

@@ -582,6 +582,27 @@ spec:
     {{ end }}
     image: {{ .Image }}
     name: {{ .Name }}
+{{ if .RestartPolicy }}
+    restartPolicy: {{ .RestartPolicy }}
+    {{ if .StartupProbe }}
+    startupProbe:
+      exec:
+        command:
+        {{ range .StartupProbe.Cmd }}
+        - {{.}}
+        {{ end }}
+    {{ if .StartupProbe.TimeoutSeconds }}
+      timeoutSeconds: {{ .StartupProbe.TimeoutSeconds }}
+    {{ end }}
+    {{ end }}
+{{ end }}
+
+{{ if .VolumeMount }}
+    volumeMounts:
+    - name: {{.VolumeName}}
+      mountPath: {{ .VolumeMountPath }}
+      readonly: {{.VolumeReadOnly}}
+{{ end }}
   {{ end }}
 {{ end }}
 {{ if .SecurityContext }}
@@ -1644,6 +1665,40 @@ func getPodNameInDeployment(d *Deployment) Pod {
 	return p
 }
 
+// HealthcheckProbe describes the options for a health check probe
+type HealthCheckProbe struct {
+	Cmd                 []string
+	InitialDelaySeconds int
+	PeriodSeconds       int
+	TimeoutSeconds      int
+}
+
+func getHealthCheckProbe(options ...healthProbeOption) *HealthCheckProbe {
+	probe := HealthCheckProbe{
+		InitialDelaySeconds: 1,
+		PeriodSeconds:       1,
+	}
+
+	for _, option := range options {
+		option(&probe)
+	}
+	return &probe
+}
+
+type healthProbeOption func(*HealthCheckProbe)
+
+func WithHealthCheckCommand(cmd []string) healthProbeOption {
+	return func(h *HealthCheckProbe) {
+		h.Cmd = cmd
+	}
+}
+
+func WithHeathCheckTimeout(timeout int) healthProbeOption {
+	return func(h *HealthCheckProbe) {
+		h.TimeoutSeconds = timeout
+	}
+}
+
 // Ctr describes the options a kube yaml can be configured at container level
 type Ctr struct {
 	Name            string
@@ -1669,8 +1724,10 @@ type Ctr struct {
 	Env             []Env
 	EnvFrom         []EnvFrom
 	InitCtrType     string
+	RestartPolicy   string
 	RunAsUser       string
 	RunAsGroup      string
+	StartupProbe    *HealthCheckProbe
 }
 
 // getCtr takes a list of ctrOptions and returns a Ctr with sane defaults
@@ -1717,6 +1774,18 @@ func withInitCtr() ctrOption {
 	}
 }
 
+func withRestartableInitCtr() ctrOption {
+	return func(c *Ctr) {
+		c.RestartPolicy = string(v1.RestartPolicyAlways)
+	}
+}
+
+func withStartupProbe(probe *HealthCheckProbe) ctrOption {
+	return func(c *Ctr) {
+		c.StartupProbe = probe
+	}
+}
+
 func withCmd(cmd []string) ctrOption {
 	return func(c *Ctr) {
 		c.Cmd = cmd
@@ -2498,6 +2567,84 @@ var _ = Describe("Podman kube play", func() {
 		Expect(inspect.OutputToString()).To(ContainSubstring("running"))
 	})
 
+	// If always restart init container didn't define define startup probe, it is started immediately and the Pod is starting regular container
+	It("test with always restart init container without startup probe started immediately", func() {
+		pod := getPod(withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"}), withRestartableInitCtr(), withName("sidecar-container"))), withCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"}))))
+		err := generateKubeYaml("pod", pod, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"kube", "play", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(ExitCleanly())
+
+		// Expect the number of containers created to be 3, infra, restartable init container and regular container
+		numOfCtrs := podmanTest.NumberOfContainers()
+		Expect(numOfCtrs).To(Equal(3))
+
+		// Regular container should be in running state
+		inspect := podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", "testPod-" + defaultCtrName})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect).Should(ExitCleanly())
+		Expect(inspect.OutputToString()).To(ContainSubstring("running"))
+
+		// Init container should keep running
+		inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", "testPod-" + "sidecar-container"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect).Should(ExitCleanly())
+		Expect(inspect.OutputToString()).To(ContainSubstring("running"))
+
+	})
+
+	// For a Pod with [initContainer1, restartableInitContainer, initContainer2, regularContainer], the containers shouldbe started in sequence. And regular container started after
+	// start probe of restartableInitContainer finished
+	It("init container started in sequence", func() {
+		startupProbe := getHealthCheckProbe(WithHealthCheckCommand([]string{"sh", "-c", "echo 'sidecar'>>/test/startUpSequence"}), WithHeathCheckTimeout(3))
+		initContainer1 := withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"sh", "-c", "echo 'init-test1'>>/test/startUpSequence"}), withInitCtr(), withName("init-test1"), withVolumeMount("/test", "", false)))
+		restartableInitConatiner := withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"}), withRestartableInitCtr(), withName("sidecar-container"), withStartupProbe(startupProbe), withVolumeMount("/test", "", false)))
+		initContainer2 := withPodInitCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"sh", "-c", "echo 'init-test2'>>/test/startUpSequence"}), withInitCtr(), withName("init-test2"), withVolumeMount("/test", "", false)))
+
+		pod := getPod(initContainer1, restartableInitConatiner, initContainer2, withCtr(getCtr(withImage(CITEST_IMAGE), withCmd([]string{"top"}), withVolumeMount("/test", "", false))), withVolume(getEmptyDirVolume()))
+		err := generateKubeYaml("pod", pod, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"kube", "play", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(ExitCleanly())
+
+		// Expect the number of containers created to be 3, infra, restartable init container and regular container
+		numOfCtrs := podmanTest.NumberOfContainers()
+		Expect(numOfCtrs).To(Equal(3))
+
+		// Init container should keep running
+		inspect := podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", "testPod-" + "sidecar-container"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect).Should(ExitCleanly())
+		Expect(inspect.OutputToString()).To(ContainSubstring("running"))
+
+		// Regular container should be in running state
+		inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.Status}}", getCtrNameInPod(pod)})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect).Should(ExitCleanly())
+		Expect(inspect.OutputToString()).To(ContainSubstring("running"))
+
+		// Init container started in correct sequence
+		inspect = podmanTest.Podman([]string{"exec", getCtrNameInPod(pod), "cat", "/test/startUpSequence"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect).Should(ExitCleanly())
+		Expect(inspect.OutputToString()).To(Equal("init-test1 sidecar init-test2"))
+
+		// Regular container started after restartable init container
+		inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.StartedAt}}", "testPod-" + "sidecar-container"})
+		inspect.WaitWithDefaultTimeout()
+		startOfRestartableInitContainer, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", inspect.OutputToString())
+		Expect(err).ToNot(HaveOccurred())
+		inspect = podmanTest.Podman([]string{"inspect", "--format", "{{.State.StartedAt}}", getCtrNameInPod(pod)})
+		inspect.WaitWithDefaultTimeout()
+		startOfRegularContainer, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", inspect.OutputToString())
+		Expect(err).ToNot(HaveOccurred())
+		Expect(startOfRestartableInitContainer).To(BeTemporally("<", startOfRegularContainer))
+	})
+
 	// If you supply only args for a Container, the default Entrypoint defined in the Docker image is run with the args that you supplied.
 	It("test correct command with only set args in yaml file", func() {
 		pod := getPod(withCtr(getCtr(withImage(REGISTRY_IMAGE), withCmd(nil), withArg([]string{"echo", "hello"}))))