Skip to content

Commit

Permalink
Adding support for startup probe. (istio#24440)
Browse files Browse the repository at this point in the history
  • Loading branch information
luckoseabraham authored Jun 6, 2020
1 parent 5b55581 commit 6105906
Show file tree
Hide file tree
Showing 9 changed files with 881 additions and 6 deletions.
7 changes: 4 additions & 3 deletions pilot/cmd/pilot-agent/status/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,12 @@ func NewServer(config Config) (*Server, error) {
return s, nil
}

// FormatProberURL returns a pair of HTTP URLs that pilot agent will serve to take over Kubernetes
// FormatProberURL returns a set of HTTP URLs that pilot agent will serve to take over Kubernetes
// app probers.
func FormatProberURL(container string) (string, string) {
func FormatProberURL(container string) (string, string, string) {
return fmt.Sprintf("/app-health/%v/readyz", container),
fmt.Sprintf("/app-health/%v/livez", container)
fmt.Sprintf("/app-health/%v/livez", container),
fmt.Sprintf("/app-health/%v/startupz", container)
}

// Run opens a the status port and begins accepting probes.
Expand Down
21 changes: 18 additions & 3 deletions pkg/kube/inject/app_probe.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func DumpAppProbers(podspec *corev1.PodSpec) string {
if c.Name == ProxyContainerName {
continue
}
readyz, livez := status.FormatProberURL(c.Name)
readyz, livez, startupz := status.FormatProberURL(c.Name)
portMap := map[string]int32{}
for _, p := range c.Ports {
if p.Name != "" {
Expand All @@ -106,6 +106,10 @@ func DumpAppProbers(podspec *corev1.PodSpec) string {
if h := updateNamedPort(kubeProbeToInternalProber(c.LivenessProbe), portMap); h != nil {
out[livez] = h
}
if h := updateNamedPort(kubeProbeToInternalProber(c.StartupProbe), portMap); h != nil {
out[startupz] = h
}

}
b, err := json.Marshal(out)
if err != nil {
Expand All @@ -117,6 +121,7 @@ func DumpAppProbers(podspec *corev1.PodSpec) string {

// rewriteAppHTTPProbes modifies the app probers in place for kube-inject.
func rewriteAppHTTPProbe(annotations map[string]string, podSpec *corev1.PodSpec, spec *SidecarInjectionSpec, port int32) {

if !ShouldRewriteAppHTTPProbers(annotations, spec) {
return
}
Expand Down Expand Up @@ -147,13 +152,16 @@ func rewriteAppHTTPProbe(annotations map[string]string, podSpec *corev1.PodSpec,
if c.Name == ProxyContainerName {
continue
}
readyz, livez := status.FormatProberURL(c.Name)
readyz, livez, startupz := status.FormatProberURL(c.Name)
if rp := convertAppProber(c.ReadinessProbe, readyz, statusPort); rp != nil {
*c.ReadinessProbe = *rp
}
if lp := convertAppProber(c.LivenessProbe, livez, statusPort); lp != nil {
*c.LivenessProbe = *lp
}
if sp := convertAppProber(c.StartupProbe, startupz, statusPort); sp != nil {
*c.StartupProbe = *sp
}
}
}

Expand Down Expand Up @@ -184,7 +192,7 @@ func createProbeRewritePatch(annotations map[string]string, podSpec *corev1.PodS
for _, p := range c.Ports {
portMap[p.Name] = p.ContainerPort
}
readyz, livez := status.FormatProberURL(c.Name)
readyz, livez, startupz := status.FormatProberURL(c.Name)
if probePatch := convertAppProber(c.ReadinessProbe, readyz, statusPort); probePatch != nil {
podPatches = append(podPatches, rfc6902PatchOperation{
Op: "replace",
Expand All @@ -199,6 +207,13 @@ func createProbeRewritePatch(annotations map[string]string, podSpec *corev1.PodS
Value: *probePatch,
})
}
if probePatch := convertAppProber(c.StartupProbe, startupz, statusPort); probePatch != nil {
podPatches = append(podPatches, rfc6902PatchOperation{
Op: "replace",
Path: fmt.Sprintf("/spec/containers/%v/startupProbe", i),
Value: *probePatch,
})
}
}
return podPatches
}
Expand Down
15 changes: 15 additions & 0 deletions pkg/kube/inject/inject_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,21 @@ func TestRewriteAppProbe(t *testing.T) {
rewriteAppHTTPProbe: true,
want: "ready_live.yaml.injected",
},
{
in: "startup_only.yaml",
rewriteAppHTTPProbe: true,
want: "startup_only.yaml.injected",
},
{
in: "startup_live.yaml",
rewriteAppHTTPProbe: true,
want: "startup_live.yaml.injected",
},
{
in: "startup_ready_live.yaml",
rewriteAppHTTPProbe: true,
want: "startup_ready_live.yaml.injected",
},
// TODO(incfly): add more test case covering different -statusPort=123, --statusPort=123
// No statusport, --statusPort 123.
}
Expand Down
43 changes: 43 additions & 0 deletions pkg/kube/inject/testdata/inject/app_probe/startup_live.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: hello
spec:
replicas: 7
selector:
matchLabels:
app: hello
tier: backend
track: stable
template:
metadata:
labels:
app: hello
tier: backend
track: stable
spec:
containers:
- name: hello
image: "fake.docker.io/google-samples/hello-go-gke:1.0"
ports:
- name: http
containerPort: 80
livenessProbe:
httpGet:
port: http
startupProbe:
httpGet:
port: 3333
- name: world
image: "fake.docker.io/google-samples/hello-go-gke:1.0"
ports:
- name: http
containerPort: 90
livenessProbe:
httpGet:
port: http
startupProbe:
exec:
command:
- cat
- /tmp/healthy
246 changes: 246 additions & 0 deletions pkg/kube/inject/testdata/inject/app_probe/startup_live.yaml.injected
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
name: hello
spec:
replicas: 7
selector:
matchLabels:
app: hello
tier: backend
track: stable
strategy: {}
template:
metadata:
annotations:
sidecar.istio.io/interceptionMode: REDIRECT
sidecar.istio.io/status: '{"version":"","initContainers":["istio-init"],"containers":["istio-proxy"],"volumes":["istio-envoy","istio-data","istio-podinfo","istio-token","istiod-ca-cert"],"imagePullSecrets":null}'
traffic.sidecar.istio.io/excludeInboundPorts: "15020"
traffic.sidecar.istio.io/includeInboundPorts: 80,90
traffic.sidecar.istio.io/includeOutboundIPRanges: '*'
creationTimestamp: null
labels:
app: hello
istio.io/rev: ""
security.istio.io/tlsMode: istio
tier: backend
track: stable
spec:
containers:
- image: fake.docker.io/google-samples/hello-go-gke:1.0
livenessProbe:
httpGet:
path: /app-health/hello/livez
port: 15020
name: hello
ports:
- containerPort: 80
name: http
resources: {}
startupProbe:
httpGet:
path: /app-health/hello/startupz
port: 15020
- image: fake.docker.io/google-samples/hello-go-gke:1.0
livenessProbe:
httpGet:
path: /app-health/world/livez
port: 15020
name: world
ports:
- containerPort: 90
name: http
resources: {}
startupProbe:
exec:
command:
- cat
- /tmp/healthy
- args:
- proxy
- sidecar
- --domain
- $(POD_NAMESPACE).svc.cluster.local
- --serviceCluster
- hello.$(POD_NAMESPACE)
- --proxyLogLevel=warning
- --proxyComponentLogLevel=misc:error
- --trust-domain=cluster.local
- --concurrency
- "2"
env:
- name: JWT_POLICY
value: third-party-jwt
- name: PILOT_CERT_PROVIDER
value: istiod
- name: CA_ADDR
value: istiod.istio-system.svc:15012
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: INSTANCE_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: SERVICE_ACCOUNT
valueFrom:
fieldRef:
fieldPath: spec.serviceAccountName
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: CANONICAL_SERVICE
valueFrom:
fieldRef:
fieldPath: metadata.labels['service.istio.io/canonical-name']
- name: CANONICAL_REVISION
valueFrom:
fieldRef:
fieldPath: metadata.labels['service.istio.io/canonical-revision']
- name: PROXY_CONFIG
value: |
{}
- name: ISTIO_META_POD_PORTS
value: |-
[
{"name":"http","containerPort":80}
,{"name":"http","containerPort":90}
]
- name: ISTIO_META_APP_CONTAINERS
value: |-
[
hello,
world
]
- name: ISTIO_META_CLUSTER_ID
value: Kubernetes
- name: ISTIO_META_INTERCEPTION_MODE
value: REDIRECT
- name: ISTIO_META_WORKLOAD_NAME
value: hello
- name: ISTIO_META_OWNER
value: kubernetes://apis/apps/v1/namespaces/default/deployments/hello
- name: ISTIO_META_MESH_ID
value: cluster.local
- name: ISTIO_KUBE_APP_PROBERS
value: '{"/app-health/hello/livez":{"httpGet":{"port":80}},"/app-health/hello/startupz":{"httpGet":{"port":3333}},"/app-health/world/livez":{"httpGet":{"port":90}}}'
image: gcr.io/istio-testing/proxyv2:latest
imagePullPolicy: Always
name: istio-proxy
ports:
- containerPort: 15090
name: http-envoy-prom
protocol: TCP
readinessProbe:
failureThreshold: 30
httpGet:
path: /healthz/ready
port: 15021
initialDelaySeconds: 1
periodSeconds: 2
resources:
limits:
cpu: "2"
memory: 1Gi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsGroup: 1337
runAsNonRoot: true
runAsUser: 1337
volumeMounts:
- mountPath: /var/run/secrets/istio
name: istiod-ca-cert
- mountPath: /var/lib/istio/data
name: istio-data
- mountPath: /etc/istio/proxy
name: istio-envoy
- mountPath: /var/run/secrets/tokens
name: istio-token
- mountPath: /etc/istio/pod
name: istio-podinfo
initContainers:
- args:
- istio-iptables
- -p
- "15001"
- -z
- "15006"
- -u
- "1337"
- -m
- REDIRECT
- -i
- '*'
- -x
- ""
- -b
- '*'
- -d
- 15090,15021,15020
image: gcr.io/istio-testing/proxyv2:latest
imagePullPolicy: Always
name: istio-init
resources:
limits:
cpu: 100m
memory: 50Mi
requests:
cpu: 10m
memory: 10Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_ADMIN
- NET_RAW
drop:
- ALL
privileged: false
readOnlyRootFilesystem: false
runAsGroup: 0
runAsNonRoot: false
runAsUser: 0
securityContext:
fsGroup: 1337
volumes:
- emptyDir:
medium: Memory
name: istio-envoy
- emptyDir: {}
name: istio-data
- downwardAPI:
items:
- fieldRef:
fieldPath: metadata.labels
path: labels
- fieldRef:
fieldPath: metadata.annotations
path: annotations
name: istio-podinfo
- name: istio-token
projected:
sources:
- serviceAccountToken:
audience: istio-ca
expirationSeconds: 43200
path: istio-token
- configMap:
name: istio-ca-root-cert
name: istiod-ca-cert
status: {}
---
Loading

0 comments on commit 6105906

Please sign in to comment.