Wait for deployments to stabilize

Signed-off-by: Juraci Paixão Kröhling <juraci@kroehling.de>

pkg/controller/jaeger/jaeger_controller.go

@@ -160,7 +160,7 @@ func (r *ReconcileJaeger) handleDependencies(str strategy.S) error {
 			deadline = time.Duration(int64(*dep.Spec.ActiveDeadlineSeconds))
 		}
 
-		return wait.Poll(time.Second, deadline*time.Second, func() (done bool, err error) {
+		return wait.PollImmediate(time.Second, deadline*time.Second, func() (done bool, err error) {
 			batch := &batchv1.Job{}
 			err = r.client.Get(context.Background(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, batch)
 			if err != nil {
@@ -232,6 +232,17 @@ func (r *ReconcileJaeger) applyDeployments(jaeger v1alpha1.Jaeger, desired []app
 		}
 	}
 
+	for _, d := range depInventory.Create {
+		if err := r.waitForStability(d); err != nil {
+			return err
+		}
+	}
+	for _, d := range depInventory.Update {
+		if err := r.waitForStability(d); err != nil {
+			return err
+		}
+	}
+
 	for _, d := range depInventory.Delete {
 		log.WithFields(log.Fields{
 			"namespace":  jaeger.Namespace,
@@ -245,3 +256,29 @@ func (r *ReconcileJaeger) applyDeployments(jaeger v1alpha1.Jaeger, desired []app
 
 	return nil
 }
+
+func (r *ReconcileJaeger) waitForStability(dep appsv1.Deployment) error {
+	return wait.PollImmediate(time.Second, time.Minute, func() (done bool, err error) {
+		d := &appsv1.Deployment{}
+		err = r.client.Get(context.Background(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, d)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"namespace": dep.Namespace,
+				"name":      dep.Name,
+			}).WithError(err).Error("failed to get the status of the deployment")
+			return false, err
+		}
+
+		if d.Status.ReadyReplicas != d.Status.Replicas {
+			log.WithFields(log.Fields{
+				"namespace": dep.Namespace,
+				"name":      dep.Name,
+				"ready":     d.Status.ReadyReplicas,
+				"desired":   d.Status.Replicas,
+			}).Info("Waiting for deployment to estabilize")
+			return false, nil
+		}
+
+		return true, nil
+	})
+}

pkg/controller/jaeger/jaeger_controller_test.go

@@ -219,6 +219,67 @@ func TestDeleteDeployments(t *testing.T) {
 	assert.Error(t, err) // not found
 }
 
+func TestDeleteOnlyAfterSuccessfulUpdate(t *testing.T) {
+	// prepare
+	nsn := types.NamespacedName{
+		Name: "TestDeleteOnlyAfterSuccessfulUpdate",
+	}
+
+	// the deployment to be deleted
+	depToDelete := appsv1.Deployment{}
+	depToDelete.Name = nsn.Name + "-delete"
+	depToDelete.Annotations = map[string]string{
+		"app.kubernetes.io/instance":   nsn.Name,
+		"app.kubernetes.io/managed-by": "jaeger-operator",
+	}
+	objs := []runtime.Object{v1alpha1.NewJaeger(nsn.Name), &depToDelete}
+
+	s := scheme.Scheme
+	s.AddKnownTypes(v1alpha1.SchemeGroupVersion, &v1alpha1.Jaeger{})
+	cl := fake.NewFakeClient(objs...)
+	r := &ReconcileJaeger{client: cl, scheme: s}
+
+	// the deployment to be created
+	dep := appsv1.Deployment{}
+	dep.Name = nsn.Name
+	dep.Status.Replicas = 2
+	dep.Status.ReadyReplicas = 1
+
+	r.strategyChooser = func(jaeger *v1alpha1.Jaeger) strategy.S {
+		s := strategy.New().WithDeployments([]appsv1.Deployment{dep})
+		return s
+	}
+
+	// sanity check that the deployment to be removed is indeed there in the first place...
+	persistedDelete := &appsv1.Deployment{}
+	assert.NoError(t, cl.Get(context.Background(), types.NamespacedName{Name: depToDelete.Name, Namespace: depToDelete.Namespace}, persistedDelete))
+	assert.Equal(t, depToDelete.Name, persistedDelete.Name)
+
+	// test
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		// will block until it finishes, which should happen after the deployments
+		// have achieved stability and everything has been created/updated/deleted
+		_, err := r.Reconcile(reconcile.Request{NamespacedName: nsn})
+		assert.NoError(t, err)
+		wg.Done()
+	}()
+
+	dep.Status.ReadyReplicas = 2
+	wg.Wait() // will block until the reconcile logic finishes
+
+	// verify that the deployment to be created was created
+	persisted := &appsv1.Deployment{}
+	assert.NoError(t, cl.Get(context.Background(), nsn, persisted))
+	assert.Equal(t, nsn.Name, persisted.Name)
+
+	// check that the deployment to be deleted was indeed deleted
+	persistedDelete = &appsv1.Deployment{}
+	assert.Error(t, cl.Get(context.Background(), types.NamespacedName{Name: depToDelete.Name, Namespace: depToDelete.Namespace}, persistedDelete))
+	assert.Empty(t, persistedDelete.Name)
+}
+
 func TestHandleDependencies(t *testing.T) {
 	// prepare
 	nsn := types.NamespacedName{