Merge pull request #955 from devkid/feature/pod-readiness-gate

Pod readiness gates

docs/examples/rbac-role.yaml

@@ -16,6 +16,7 @@ rules:
       - ingresses
       - ingresses/status
       - services
+      - pods/status
     verbs:
       - create
       - get

docs/guide/ingress/pod-conditions.md

@@ -0,0 +1,101 @@
+# Using pod conditions / pod readiness gates
+
+One can add so-called [»Pod readiness gates«](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate) to Kubernetes pods. A readiness gate can be used by e.g. a controller to mark a pod as ready or as unready by setting a custom condition on the pod.
+
+The AWS ALB ingress controller can set such a condition on your pods. This is needed under certain circumstances to achieve full zero downtime rolling deployments. Consider the following example:
+* low number of replicas in a deployment (e.g. one to three)
+* start a rolling update of the deployment
+* rollout of new pods takes less time than it takes the ALB ingress controller to register the new pods and for their health state turn »Healthy« in the target group
+* at some point during this rolling update, the target group might only have registered targets that are in »Initial« or »Draining« state; this results in service outage
+
+In order to avoid this situation, the AWS ALB ingress controller can set the before mentioned condition on the pods that constitute your ingress backend services. The condition status on a pod will only be set to `True` when the corresponding target in the ALB target group shows a health state of »Healthy«. This prevents the rolling update of a deployment from terminating old pods until the newly created pods are »Healthy« in the ALB target group and ready to take traffic.
+
+
+## Pod configuration
+
+Add a readiness gate with `conditionType: target-health.alb.ingress.k8s.aws/<ingress name>_<service name>_<service port>` to your pod.
+
+Example:
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  clusterIP: None
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: nginx
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: nginx-ingress
+  annotations:
+    kubernetes.io/ingress.class: alb
+    alb.ingress.kubernetes.io/target-type: ip
+    alb.ingress.kubernetes.io/scheme: internal
+spec:
+  rules:
+    - http:
+        paths:
+          - backend:
+              serviceName: nginx-service
+              servicePort: 80
+            path: /*
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      readinessGates:
+      - conditionType: target-health.alb.ingress.k8s.aws/nginx-ingress_nginx-service_80
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+```
+
+If your pod is part of multiple ingresses / target groups and you want to make sure your pod is `Healthy` in all of them before it is marked as `Ready`, add one `readinessGate` per ingress.
+
+
+## Checking the pod condition status
+
+The status of the readiness gates can be verified with `kubectl get pod -o wide`:
+```
+NAME                          READY   STATUS    RESTARTS   AGE   IP         NODE                       READINESS GATES
+nginx-test-5744b9ff84-7ftl9   1/1     Running   0          81s   10.1.2.3   ip-10-1-2-3.ec2.internal   0/1
+```
+
+When the target is registered and healthy in the ALB, the output will look like:
+```
+NAME                          READY   STATUS    RESTARTS   AGE   IP         NODE                       READINESS GATES
+nginx-test-5744b9ff84-7ftl9   1/1     Running   0          81s   10.1.2.3   ip-10-1-2-3.ec2.internal   1/1
+```
+
+If a readiness gate doesn't get ready, you can check the reason via:
+
+```console
+$ kubectl get pod nginx-test-5744b9ff84-7ftl9 -o yaml | grep -B5 'type: target-health'
+    - lastProbeTime: "2020-02-28T10:05:08Z"
+      lastTransitionTime: "2020-02-28T10:05:08Z"
+      message: Target registration is in progress.
+      reason: Elb.RegistrationInProgress
+      status: "False"
+      type: target-health.alb.ingress.k8s.aws/nginx-test_nginx-test_80
+```

internal/alb/tg/targetgroup.go

@@ -19,6 +19,7 @@ import (
 	extensions "k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // The port used when creating targetGroup serves as a default value for targets registered without port specified.
@@ -32,11 +33,13 @@ const targetGroupDefaultPort = 1
 type Controller interface {
 	// Reconcile ensures an targetGroup exists for specified backend of ingress.
 	Reconcile(ctx context.Context, ingress *extensions.Ingress, backend extensions.IngressBackend) (TargetGroup, error)
+	StopReconcilingPodConditionStatus(tgArn string)
 }
 
-func NewController(cloud aws.CloudAPI, store store.Storer, nameTagGen NameTagGenerator, tagsController tags.Controller, endpointResolver backend.EndpointResolver) Controller {
+func NewController(cloud aws.CloudAPI, store store.Storer, nameTagGen NameTagGenerator, tagsController tags.Controller, endpointResolver backend.EndpointResolver, client client.Client) Controller {
 	attrsController := NewAttributesController(cloud)
-	targetsController := NewTargetsController(cloud, endpointResolver)
+	targetHealthController := NewTargetHealthController(cloud, store, endpointResolver, client)
+	targetsController := NewTargetsController(cloud, endpointResolver, targetHealthController)
 	return &defaultController{
 		cloud:             cloud,
 		store:             store,
@@ -115,6 +118,10 @@ func (controller *defaultController) Reconcile(ctx context.Context, ingress *ext
 	}, nil
 }
 
+func (controller *defaultController) StopReconcilingPodConditionStatus(tgArn string) {
+	controller.targetsController.StopReconcilingPodConditionStatus(tgArn)
+}
+
 func (controller *defaultController) newTGInstance(ctx context.Context, name string, serviceAnnos *annotations.Service, healthCheckPort string) (*elbv2.TargetGroup, error) {
 	albctx.GetLogger(ctx).Infof("creating target group %v", name)
 	resp, err := controller.cloud.CreateTargetGroupWithContext(ctx, &elbv2.CreateTargetGroupInput{

internal/alb/tg/targetgroup_group.go

@@ -15,6 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // GroupController manages all target groups for one ingress.
@@ -35,8 +36,9 @@ func NewGroupController(
 	store store.Storer,
 	nameTagGen NameTagGenerator,
 	tagsController tags.Controller,
-	endpointResolver backend.EndpointResolver) GroupController {
-	tgController := NewController(cloud, store, nameTagGen, tagsController, endpointResolver)
+	endpointResolver backend.EndpointResolver,
+	client client.Client) GroupController {
+	tgController := NewController(cloud, store, nameTagGen, tagsController, endpointResolver, client)
 	return &defaultGroupController{
 		cloud:        cloud,
 		store:        store,
@@ -95,6 +97,7 @@ func (controller *defaultGroupController) GC(ctx context.Context, tgGroup Target
 	unusedTgArns := currentTgArns.Difference(usedTgArns)
 	for arn := range unusedTgArns {
 		albctx.GetLogger(ctx).Infof("deleting target group %v", arn)
+		controller.tgController.StopReconcilingPodConditionStatus(arn)
 		if err := controller.cloud.DeleteTargetGroupByArn(ctx, arn); err != nil {
 			return fmt.Errorf("failed to delete targetGroup due to %v", err)
 		}

internal/alb/tg/targetgroup_group_test.go

@@ -620,6 +620,9 @@ func TestDefaultGroupController_GC(t *testing.T) {
 		}
 		mockNameTagGen := &MockNameTagGenerator{}
 		mockTGController := &MockController{}
+		for _, call := range tc.DeleteTargetGroupByArnCalls {
+			mockTGController.On("StopReconcilingPodConditionStatus", call.Arn).Return()
+		}
 
 		controller := &defaultGroupController{
 			cloud:        cloud,
@@ -728,6 +731,9 @@ func TestDefaultGroupController_Delete(t *testing.T) {
 			mockNameTagGen.On("TagTGGroup", tc.TagTGGroupCall.Namespace, tc.TagTGGroupCall.IngressName).Return(tc.TagTGGroupCall.Tags)
 		}
 		mockTGController := &MockController{}
+		for _, call := range tc.DeleteTargetGroupByArnCalls {
+			mockTGController.On("StopReconcilingPodConditionStatus", call.Arn).Return()
+		}
 
 		controller := &defaultGroupController{
 			cloud:        cloud,