feature: zalando.org/forward-backend annotation support to enable migration to eks

docs/stack_crd.yaml

@@ -475,6 +475,22 @@ spec:
                     - type: integer
                     - type: string
                     x-kubernetes-int-or-string: true
+                  metadata:
+                    description: |-
+                      EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached
+                      to a resource. It's a slimmed down version of metav1.ObjectMeta only
+                      containing annotations.
+                    properties:
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Annotations is an unstructured key value map stored with a resource that may be
+                          set by external tools to store and retrieve arbitrary metadata. They are not
+                          queryable and should be preserved when modifying objects.
+                          More info: http://kubernetes.io/docs/user-guide/annotations
+                        type: object
+                    type: object
                 required:
                 - backendPort
                 type: object
@@ -1096,11 +1112,6 @@ spec:
                                       type: object
                                       x-kubernetes-map-type: atomic
                                     namespaces:
-                                      description: |-
-                                        namespaces specifies a static list of namespace names that the term applies to.
-                                        The term is applied to the union of the namespaces listed in this field
-                                        and the ones selected by namespaceSelector.
-                                        null or empty namespaces list and null namespaceSelector means "this pod's namespace".
                                       items:
                                         type: string
                                       type: array
@@ -6625,12 +6636,6 @@ spec:
                                   properties:
                                     name:
                                       default: ""
-                                      description: |-
-                                        Name of the referent.
-                                        This field is effectively required, but due to backwards compatibility is
-                                        allowed to be empty. Instances of this type with an empty value here are
-                                        almost certainly wrong.
-                                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
                                       type: string
                                   type: object
                                   x-kubernetes-map-type: atomic

docs/stackset_crd.yaml

@@ -67,6 +67,22 @@ spec:
                     - type: integer
                     - type: string
                     x-kubernetes-int-or-string: true
+                  metadata:
+                    description: |-
+                      EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached
+                      to a resource. It's a slimmed down version of metav1.ObjectMeta only
+                      containing annotations.
+                    properties:
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Annotations is an unstructured key value map stored with a resource that may be
+                          set by external tools to store and retrieve arbitrary metadata. They are not
+                          queryable and should be preserved when modifying objects.
+                          More info: http://kubernetes.io/docs/user-guide/annotations
+                        type: object
+                    type: object
                 required:
                 - backendPort
                 type: object
@@ -3582,12 +3598,6 @@ spec:
                                                 Must be set if and only if type is "Localhost".
                                               type: string
                                             type:
-                                              description: |-
-                                                type indicates which kind of AppArmor profile will be applied.
-                                                Valid options are:
-                                                  Localhost - a profile pre-loaded on the node.
-                                                  RuntimeDefault - the container runtime's default profile.
-                                                  Unconfined - no AppArmor enforcement.
                                               type: string
                                           required:
                                           - type
@@ -3694,13 +3704,6 @@ spec:
                                             localhostProfile:
                                               type: string
                                             type:
-                                              description: |-
-                                                type indicates which kind of seccomp profile will be applied.
-                                                Valid options are:
-
-                                                Localhost - a profile defined in a file on the node should be used.
-                                                RuntimeDefault - the container runtime default profile should be used.
-                                                Unconfined - no profile should be applied.
                                               type: string
                                           required:
                                           - type

pkg/apis/zalando.org/v1/types.go

@@ -124,7 +124,8 @@ func (s *StackSetIngressSpec) GetAnnotations() map[string]string {
 // backendport for ingress managed outside of stackset.
 // +k8s:deepcopy-gen=true
 type StackSetExternalIngressSpec struct {
-	BackendPort intstr.IntOrString `json:"backendPort"`
+	EmbeddedObjectMetaWithAnnotations `json:"metadata,omitempty"`
+	BackendPort                       intstr.IntOrString `json:"backendPort"`
 }
 
 // RouteGroupSpec defines the specification for defining a RouteGroup attached

pkg/core/stackset.go

@@ -17,6 +17,8 @@ const (
 	StackVersionLabelKey     = "stack-version"
 
 	ingressTrafficAuthoritativeAnnotation = "zalando.org/traffic-authoritative"
+	forwardBackendAnnotation              = "zalando.org/forward-backend"
+	forwardBackendName                    = "fwd"
 )
 
 var (
@@ -52,6 +54,7 @@ func sanitizeServicePorts(service *zv1.StackServiceSpec) *zv1.StackServiceSpec {
 
 // NewStack returns an (optional) stack that should be created
 func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
+	_, forwardMigration := ssc.StackSet.ObjectMeta.Annotations[forwardBackendAnnotation]
 	observedStackVersion := ssc.StackSet.Status.ObservedStackVersion
 	stackVersion := currentStackVersion(ssc.StackSet)
 	stackName := generateStackName(ssc.StackSet, stackVersion)
@@ -80,6 +83,11 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 			spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy()
 		}
 
+		stackAnnotations := ssc.StackSet.Spec.StackTemplate.Annotations
+		if forwardMigration {
+			stackAnnotations[forwardBackendAnnotation] = forwardBackendName
+		}
+
 		return &StackContainer{
 			Stack: &zv1.Stack{
 				ObjectMeta: metav1.ObjectMeta{
@@ -100,7 +108,7 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 						ssc.StackSet.Labels,
 						map[string]string{StackVersionLabelKey: stackVersion},
 					),
-					Annotations: ssc.StackSet.Spec.StackTemplate.Annotations,
+					Annotations: stackAnnotations,
 				},
 				Spec: *spec,
 			},

pkg/core/types.go

@@ -418,19 +418,44 @@ func (sc *StackContainer) updateStackResources() error {
 	return nil
 }
 
+// updateFromResources updates stack from all the containing
+// resources. On cluster migrations we do traffic switching to the
+// other cluster, so all the new pods from this stack will not receive
+// any traffic. Therefore we can set replicas to 1 and remove hpa
+// whatever was configured. Ingress and ExternaIngress will have an
+// annotation set that it will send traffic to the other
+// cluster. RouteGroup will get a patched backend, such that it does
+// the same as ingress.
 func (sc *StackContainer) updateFromResources() {
-	sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas)
+	fwdVal, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]
+
+	if clusterMigration {
+		sc.Stack.Spec.ExternalIngress.Annotations[forwardBackendAnnotation] = fwdVal
+		// we do not use these pods so reduce wasted resources!
+		sc.stackReplicas = 1
+	} else {
+		sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas)
+	}
 
 	var deploymentUpdated, serviceUpdated, ingressUpdated, routeGroupUpdated, hpaUpdated bool
 	var ingressSegmentUpdated, routeGroupSegmentUpdated bool
 
 	// deployment
 	if sc.Resources.Deployment != nil {
 		deployment := sc.Resources.Deployment
-		sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas)
-		sc.createdReplicas = deployment.Status.Replicas
-		sc.readyReplicas = deployment.Status.ReadyReplicas
-		sc.updatedReplicas = deployment.Status.UpdatedReplicas
+
+		if clusterMigration {
+			// we do not use these pods so reduce wasted resources!
+			sc.deploymentReplicas = 1
+			sc.createdReplicas = 1
+			sc.readyReplicas = 1
+			sc.updatedReplicas = 1
+		} else {
+			sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas)
+			sc.createdReplicas = deployment.Status.Replicas
+			sc.readyReplicas = deployment.Status.ReadyReplicas
+			sc.updatedReplicas = deployment.Status.UpdatedReplicas
+		}
 		deploymentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Deployment.ObjectMeta) && deployment.Status.ObservedGeneration == deployment.Generation
 	}
 
@@ -442,13 +467,20 @@ func (sc *StackContainer) updateFromResources() {
 		// the per-stack ingress must either be present and up-to-date, or not present and not expected.
 		// the per-stack ingress is not expected if the stack has no hostnames matching the cluster domain.
 		if sc.Resources.Ingress != nil {
+			if clusterMigration {
+				sc.Resources.Ingress.Annotations["zalando.org/skipper-backend"] = "forward"
+			}
 			ingressUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Ingress.ObjectMeta)
 		} else {
 			hostnames := sc.stackHostnames(sc.ingressSpec, false)
 			ingressUpdated = len(hostnames) == 0
 		}
-		ingressSegmentUpdated = sc.Resources.IngressSegment != nil &&
-			IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta)
+		if sc.Resources.IngressSegment != nil {
+			if clusterMigration {
+				sc.Resources.IngressSegment.Annotations["zalando.org/skipper-backend"] = "forward"
+			}
+			ingressSegmentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta)
+		}
 	} else {
 		// ignore if ingress is not set
 		ingressUpdated = sc.Resources.Ingress == nil
@@ -460,24 +492,33 @@ func (sc *StackContainer) updateFromResources() {
 		// the per-stack route group must either be present and up-to-date, or not present and not expected.
 		// the per-stack route group is not expected if the stack has no hostnames matching the cluster domain.
 		if sc.Resources.RouteGroup != nil {
+			if clusterMigration {
+				patchForwardBackend(&sc.Resources.RouteGroup.Spec)
+			}
 			routeGroupUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.RouteGroup.ObjectMeta)
 		} else {
 			hostnames := sc.stackHostnames(sc.routeGroupSpec, false)
 			routeGroupUpdated = len(hostnames) == 0
 		}
-		routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment != nil &&
-			IsResourceUpToDate(
+
+		if sc.Resources.RouteGroupSegment != nil {
+			if clusterMigration {
+				patchForwardBackend(&sc.Resources.RouteGroupSegment.Spec)
+			}
+
+			routeGroupSegmentUpdated = IsResourceUpToDate(
 				sc.Stack,
 				sc.Resources.RouteGroupSegment.ObjectMeta,
 			)
+		}
 	} else {
 		// ignore if route group is not set
 		routeGroupUpdated = sc.Resources.RouteGroup == nil
 		routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment == nil
 	}
 
-	// hpa
-	if sc.IsAutoscaled() {
+	// hpa not used if we migrate cluster to reduce wasted resources
+	if sc.IsAutoscaled() && !clusterMigration {
 		hpaUpdated = sc.Resources.HPA != nil && IsResourceUpToDate(sc.Stack, sc.Resources.HPA.ObjectMeta)
 	} else {
 		hpaUpdated = sc.Resources.HPA == nil
@@ -494,10 +535,27 @@ func (sc *StackContainer) updateFromResources() {
 
 	status := sc.Stack.Status
 	sc.noTrafficSince = unwrapTime(status.NoTrafficSince)
-	if status.Prescaling.Active {
+	// do not prescale on cluster migration to reduce wasted resources
+	if status.Prescaling.Active && !clusterMigration {
 		sc.prescalingActive = true
 		sc.prescalingReplicas = status.Prescaling.Replicas
 		sc.prescalingDesiredTrafficWeight = status.Prescaling.DesiredTrafficWeight
 		sc.prescalingLastTrafficIncrease = unwrapTime(status.Prescaling.LastTrafficIncrease)
 	}
 }
+
+func patchForwardBackend(rg *rgv1.RouteGroupSpec) {
+	rg.Backends = []rgv1.RouteGroupBackend{
+		{
+			Name: forwardBackendName,
+			Type: rgv1.ForwardRouteGroupBackend,
+		},
+	}
+	for _, route := range rg.Routes {
+		route.Backends = []rgv1.RouteGroupBackendReference{
+			{
+				BackendName: forwardBackendName,
+			},
+		}
+	}
+}