Skip to content

Commit 8624f43

Browse files
deliahudeliahu
authored
Support configurable pre-stop command for realtime and async containers (#2403)
1 parent d0b29f3 commit 8624f43

File tree

7 files changed

+150
-34
lines changed

7 files changed

+150
-34
lines changed

‎docs/workloads/async/configuration.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@
4242
period_seconds: <int> # how often (in seconds) to perform the probe (default: 10)
4343
success_threshold: <int> # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)
4444
failure_threshold: <int> # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)
45+
pre_stop: # a pre-stop lifecycle hook for the container; will be executed before container termination (optional)
46+
http_get: # specifies an http endpoint to send a request to (only one of http_get, tcp_socket, and exec may be specified)
47+
port: <int|string> # the port to access on the container (required)
48+
path: <string> # the path to access on the HTTP server (default: /)
49+
exec: # specifies a command to run (only one of http_get, tcp_socket, and exec may be specified)
50+
command: <list[string]> # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)
4551
autoscaling: # autoscaling configuration (default: see below)
4652
min_replicas: <int> # minimum number of replicas (default: 1; min value: 0)
4753
max_replicas: <int> # maximum number of replicas (default: 100)

‎docs/workloads/realtime/configuration.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@
4545
period_seconds: <int> # how often (in seconds) to perform the probe (default: 10)
4646
success_threshold: <int> # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)
4747
failure_threshold: <int> # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)
48+
pre_stop: # a pre-stop lifecycle hook for the container; will be executed before container termination (optional)
49+
http_get: # specifies an http endpoint to send a request to (only one of http_get, tcp_socket, and exec may be specified)
50+
port: <int|string> # the port to access on the container (required)
51+
path: <string> # the path to access on the HTTP server (default: /)
52+
exec: # specifies a command to run (only one of http_get, tcp_socket, and exec may be specified)
53+
command: <list[string]> # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)
4854
autoscaling: # autoscaling configuration (default: see below)
4955
min_replicas: <int> # minimum number of replicas (default: 1)
5056
max_replicas: <int> # maximum number of replicas (default: 100)

‎pkg/types/spec/validations.go

Lines changed: 55 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,10 @@ func containersValidation(kind userconfig.Kind) *cr.StructFieldValidation {
259259
validations = append(validations, probeValidation("ReadinessProbe", false))
260260
}
261261

262+
if kind == userconfig.RealtimeAPIKind || kind == userconfig.AsyncAPIKind {
263+
validations = append(validations, preStopValidation())
264+
}
265+
262266
return &cr.StructFieldValidation{
263267
StructField: "Containers",
264268
StructListValidation: &cr.StructListValidation{
@@ -306,8 +310,8 @@ func networkingValidation() *cr.StructFieldValidation {
306310

307311
func probeValidation(structFieldName string, hasExecProbe bool) *cr.StructFieldValidation {
308312
validations := []*cr.StructFieldValidation{
309-
httpGetProbeValidation(),
310-
tcpSocketProbeValidation(),
313+
httpGetHandlerValidation(),
314+
tcpSocketHandlerValidation(),
311315
{
312316
StructField: "InitialDelaySeconds",
313317
Int32Validation: &cr.Int32Validation{
@@ -346,7 +350,7 @@ func probeValidation(structFieldName string, hasExecProbe bool) *cr.StructFieldV
346350
}
347351

348352
if hasExecProbe {
349-
validations = append(validations, execProbeValidation())
353+
validations = append(validations, execHandlerValidation())
350354
}
351355

352356
return &cr.StructFieldValidation{
@@ -360,7 +364,22 @@ func probeValidation(structFieldName string, hasExecProbe bool) *cr.StructFieldV
360364
}
361365
}
362366

363-
func httpGetProbeValidation() *cr.StructFieldValidation {
367+
func preStopValidation() *cr.StructFieldValidation {
368+
return &cr.StructFieldValidation{
369+
StructField: "PreStop",
370+
StructValidation: &cr.StructValidation{
371+
Required: false,
372+
AllowExplicitNull: true,
373+
DefaultNil: true,
374+
StructFieldValidations: []*cr.StructFieldValidation{
375+
httpGetHandlerValidation(),
376+
execHandlerValidation(),
377+
},
378+
},
379+
}
380+
}
381+
382+
func httpGetHandlerValidation() *cr.StructFieldValidation {
364383
return &cr.StructFieldValidation{
365384
StructField: "HTTPGet",
366385
StructValidation: &cr.StructValidation{
@@ -390,7 +409,7 @@ func httpGetProbeValidation() *cr.StructFieldValidation {
390409
}
391410
}
392411

393-
func tcpSocketProbeValidation() *cr.StructFieldValidation {
412+
func tcpSocketHandlerValidation() *cr.StructFieldValidation {
394413
return &cr.StructFieldValidation{
395414
StructField: "TCPSocket",
396415
StructValidation: &cr.StructValidation{
@@ -412,7 +431,7 @@ func tcpSocketProbeValidation() *cr.StructFieldValidation {
412431
}
413432
}
414433

415-
func execProbeValidation() *cr.StructFieldValidation {
434+
func execHandlerValidation() *cr.StructFieldValidation {
416435
return &cr.StructFieldValidation{
417436
StructField: "Exec",
418437
StructValidation: &cr.StructValidation{
@@ -783,6 +802,12 @@ func validateContainers(
783802
}
784803
}
785804

805+
if container.PreStop != nil {
806+
if err := validatePreStop(*container.PreStop); err != nil {
807+
return errors.Wrap(err, s.Index(i), userconfig.PreStopKey)
808+
}
809+
}
810+
786811
compute := container.Compute
787812
if compute.Shm != nil && compute.Mem != nil && compute.Shm.Cmp(compute.Mem.Quantity) > 0 {
788813
return errors.Wrap(ErrorShmCannotExceedMem(*compute.Shm, *compute.Mem), s.Index(i), userconfig.ComputeKey)
@@ -794,23 +819,39 @@ func validateContainers(
794819
}
795820

796821
func validateProbe(probe userconfig.Probe, supportsExecProbe bool) error {
797-
numSpecifiedProbes := 0
822+
numSpecifiedHandlers := 0
798823
if probe.HTTPGet != nil {
799-
numSpecifiedProbes++
824+
numSpecifiedHandlers++
800825
}
801826
if probe.TCPSocket != nil {
802-
numSpecifiedProbes++
827+
numSpecifiedHandlers++
803828
}
804829
if probe.Exec != nil {
805-
numSpecifiedProbes++
830+
numSpecifiedHandlers++
806831
}
807832

808-
if numSpecifiedProbes != 1 {
809-
validProbes := []string{userconfig.HTTPGetKey, userconfig.TCPSocketKey}
833+
if numSpecifiedHandlers != 1 {
834+
validHandlers := []string{userconfig.HTTPGetKey, userconfig.TCPSocketKey}
810835
if supportsExecProbe {
811-
validProbes = append(validProbes, userconfig.ExecKey)
836+
validHandlers = append(validHandlers, userconfig.ExecKey)
812837
}
813-
return ErrorSpecifyExactlyOneField(numSpecifiedProbes, validProbes...)
838+
return ErrorSpecifyExactlyOneField(numSpecifiedHandlers, validHandlers...)
839+
}
840+
841+
return nil
842+
}
843+
844+
func validatePreStop(preStop userconfig.PreStop) error {
845+
numSpecifiedHandlers := 0
846+
if preStop.HTTPGet != nil {
847+
numSpecifiedHandlers++
848+
}
849+
if preStop.Exec != nil {
850+
numSpecifiedHandlers++
851+
}
852+
853+
if numSpecifiedHandlers != 1 {
854+
return ErrorSpecifyExactlyOneField(numSpecifiedHandlers, userconfig.HTTPGetKey, userconfig.ExecKey)
814855
}
815856

816857
return nil

‎pkg/types/userconfig/api.go

Lines changed: 51 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,9 @@ type Container struct {
6161
Command []string `json:"command" yaml:"command"`
6262
Args []string `json:"args" yaml:"args"`
6363

64-
ReadinessProbe *Probe `json:"readiness_probe" yaml:"readiness_probe"`
65-
LivenessProbe *Probe `json:"liveness_probe" yaml:"liveness_probe"`
64+
ReadinessProbe *Probe `json:"readiness_probe" yaml:"readiness_probe"`
65+
LivenessProbe *Probe `json:"liveness_probe" yaml:"liveness_probe"`
66+
PreStop *PreStop `json:"pre_stop" yaml:"pre_stop"`
6667

6768
Compute *Compute `json:"compute" yaml:"compute"`
6869
}
@@ -78,26 +79,31 @@ type Networking struct {
7879
}
7980

8081
type Probe struct {
81-
HTTPGet *HTTPGetProbe `json:"http_get" yaml:"http_get"`
82-
TCPSocket *TCPSocketProbe `json:"tcp_socket" yaml:"tcp_socket"`
83-
Exec *ExecProbe `json:"exec" yaml:"exec"`
84-
InitialDelaySeconds int32 `json:"initial_delay_seconds" yaml:"initial_delay_seconds"`
85-
TimeoutSeconds int32 `json:"timeout_seconds" yaml:"timeout_seconds"`
86-
PeriodSeconds int32 `json:"period_seconds" yaml:"period_seconds"`
87-
SuccessThreshold int32 `json:"success_threshold" yaml:"success_threshold"`
88-
FailureThreshold int32 `json:"failure_threshold" yaml:"failure_threshold"`
82+
HTTPGet *HTTPGetHandler `json:"http_get" yaml:"http_get"`
83+
TCPSocket *TCPSocketHandler `json:"tcp_socket" yaml:"tcp_socket"`
84+
Exec *ExecHandler `json:"exec" yaml:"exec"`
85+
InitialDelaySeconds int32 `json:"initial_delay_seconds" yaml:"initial_delay_seconds"`
86+
TimeoutSeconds int32 `json:"timeout_seconds" yaml:"timeout_seconds"`
87+
PeriodSeconds int32 `json:"period_seconds" yaml:"period_seconds"`
88+
SuccessThreshold int32 `json:"success_threshold" yaml:"success_threshold"`
89+
FailureThreshold int32 `json:"failure_threshold" yaml:"failure_threshold"`
8990
}
9091

91-
type HTTPGetProbe struct {
92+
type PreStop struct {
93+
HTTPGet *HTTPGetHandler `json:"http_get" yaml:"http_get"`
94+
Exec *ExecHandler `json:"exec" yaml:"exec"`
95+
}
96+
97+
type HTTPGetHandler struct {
9298
Path string `json:"path" yaml:"path"`
9399
Port int32 `json:"port" yaml:"port"`
94100
}
95101

96-
type TCPSocketProbe struct {
102+
type TCPSocketHandler struct {
97103
Port int32 `json:"port" yaml:"port"`
98104
}
99105

100-
type ExecProbe struct {
106+
type ExecHandler struct {
101107
Command []string `json:"command" yaml:"command"`
102108
}
103109

@@ -387,6 +393,11 @@ func (container *Container) UserStr() string {
387393
sb.WriteString(s.Indent(container.LivenessProbe.UserStr(), " "))
388394
}
389395

396+
if container.PreStop != nil {
397+
sb.WriteString(fmt.Sprintf("%s:\n", PreStopKey))
398+
sb.WriteString(s.Indent(container.PreStop.UserStr(), " "))
399+
}
400+
390401
if container.Compute != nil {
391402
sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey))
392403
sb.WriteString(s.Indent(container.Compute.UserStr(), " "))
@@ -428,24 +439,39 @@ func (probe *Probe) UserStr() string {
428439
return sb.String()
429440
}
430441

431-
func (httpProbe *HTTPGetProbe) UserStr() string {
442+
func (preStop *PreStop) UserStr() string {
432443
var sb strings.Builder
433444

434-
sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, httpProbe.Path))
435-
sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, httpProbe.Port))
445+
if preStop.HTTPGet != nil {
446+
sb.WriteString(fmt.Sprintf("%s:\n", HTTPGetKey))
447+
sb.WriteString(s.Indent(preStop.HTTPGet.UserStr(), " "))
448+
}
449+
if preStop.Exec != nil {
450+
sb.WriteString(fmt.Sprintf("%s:\n", ExecKey))
451+
sb.WriteString(s.Indent(preStop.Exec.UserStr(), " "))
452+
}
436453

437454
return sb.String()
438455
}
439456

440-
func (tcpSocketProbe *TCPSocketProbe) UserStr() string {
457+
func (httpHandler *HTTPGetHandler) UserStr() string {
441458
var sb strings.Builder
442-
sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, tcpSocketProbe.Port))
459+
460+
sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, httpHandler.Path))
461+
sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, httpHandler.Port))
462+
443463
return sb.String()
444464
}
445465

446-
func (execProbe *ExecProbe) UserStr() string {
466+
func (tcpSocketHandler *TCPSocketHandler) UserStr() string {
447467
var sb strings.Builder
448-
sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(execProbe.Command)))
468+
sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, tcpSocketHandler.Port))
469+
return sb.String()
470+
}
471+
472+
func (execHandler *ExecHandler) UserStr() string {
473+
var sb strings.Builder
474+
sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(execHandler.Command)))
449475
return sb.String()
450476
}
451477

@@ -589,17 +615,22 @@ func (api *API) TelemetryEvent() map[string]interface{} {
589615

590616
var numReadinessProbes int
591617
var numLivenessProbes int
618+
var numPreStops int
592619
for _, container := range api.Pod.Containers {
593620
if container.ReadinessProbe != nil {
594621
numReadinessProbes++
595622
}
596623
if container.LivenessProbe != nil {
597624
numLivenessProbes++
598625
}
626+
if container.PreStop != nil {
627+
numPreStops++
628+
}
599629
}
600630

601631
event["pod.containers._num_readiness_probes"] = numReadinessProbes
602632
event["pod.containers._num_liveness_probes"] = numLivenessProbes
633+
event["pod.containers._num_pre_stops"] = numPreStops
603634

604635
totalCompute := GetPodComputeRequest(api)
605636
if totalCompute.CPU != nil {

‎pkg/types/userconfig/config_key.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ const (
4646
ArgsKey = "args"
4747
ReadinessProbeKey = "readiness_probe"
4848
LivenessProbeKey = "liveness_probe"
49+
PreStopKey = "pre_stop"
4950

5051
// Probe
5152
HTTPGetKey = "http_get"

‎pkg/workloads/helpers.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,36 @@ func GetProbeSpec(probe *userconfig.Probe) *kcore.Probe {
7575
}
7676
}
7777

78+
func GetLifecycleSpec(preStop *userconfig.PreStop) *kcore.Lifecycle {
79+
if preStop == nil {
80+
return nil
81+
}
82+
83+
var httpGetAction *kcore.HTTPGetAction
84+
var execAction *kcore.ExecAction
85+
86+
if preStop.HTTPGet != nil {
87+
httpGetAction = &kcore.HTTPGetAction{
88+
Path: strings.TrimPrefix(preStop.HTTPGet.Path, "/"), // the leading / is automatically added by k8s
89+
Port: intstr.IntOrString{
90+
IntVal: preStop.HTTPGet.Port,
91+
},
92+
}
93+
}
94+
if preStop.Exec != nil {
95+
execAction = &kcore.ExecAction{
96+
Command: preStop.Exec.Command,
97+
}
98+
}
99+
100+
return &kcore.Lifecycle{
101+
PreStop: &kcore.Handler{
102+
HTTPGet: httpGetAction,
103+
Exec: execAction,
104+
},
105+
}
106+
}
107+
78108
func GetReadinessProbesFromContainers(containers []*userconfig.Container) map[string]kcore.Probe {
79109
probes := map[string]kcore.Probe{}
80110

‎pkg/workloads/k8s.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,6 +402,7 @@ func userPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) {
402402
VolumeMounts: containerMounts,
403403
LivenessProbe: GetProbeSpec(container.LivenessProbe),
404404
ReadinessProbe: readinessProbe,
405+
Lifecycle: GetLifecycleSpec(container.PreStop),
405406
Resources: kcore.ResourceRequirements{
406407
Requests: containerResourceList,
407408
Limits: containerResourceLimitsList,

0 commit comments

Comments
 (0)