test: Add some microvmreplicaset_controller tests

These are not comprehensive. We should move to use envtest and cover
everything properly.

api/v1alpha1/condition_consts.go

@@ -31,9 +31,15 @@ const (
 	// MicrovmReplicaSetIncompleteReason indicates the microvmreplicaset does not have all replicas yet.
 	MicrovmReplicaSetIncompleteReason = "MicrovmReplicaSetIncomplete"
 
+	// MicrovmReplicaSetProvisionFailedReason indicates that the microvm failed to provision.
+	MicrovmReplicaSetProvisionFailedReason = "MicrovmReplicaSetProvisionFailed"
+
 	// MicrovmReplicaSetDeletingReason indicates the microvmreplicaset is in a deleted state.
 	MicrovmReplicaSetDeletingReason = "MicrovmReplicaSetDeleting"
 
 	// MicrovmReplicaSetDeletedFailedReason indicates the microvmreplicaset failed to deleted cleanly.
 	MicrovmReplicaSetDeleteFailedReason = "MicrovmReplicaSetDeleteFailed"
+
+	// MicrovmReplicaSetUpdatingReason indicates the microvm is in a pending state.
+	MicrovmReplicaSetUpdatingReason = "MicrovmReplicaSetUpdating"
 )

config/crd/bases/infrastructure.liquid-metal.io_microvmreplicasets.yaml

@@ -346,13 +346,13 @@ spec:
                 description: Ready is true when Replicas is Equal to ReadyReplicas.
                 type: boolean
               readyReplicas:
-                description: readyReplicas is the number of pods targeted by this
+                description: ReadyReplicas is the number of pods targeted by this
                   ReplicaSet with a Ready Condition.
                 format: int32
                 type: integer
               replicas:
-                description: 'Replicas is the most recently observed number of replicas.
-                  More info: https://kubernetes.io/docs/concepts/workloads/controllers/replicationcontroller/#what-is-a-replicationcontroller'
+                description: Replicas is the most recently observed number of replicas
+                  which have been created.
                 format: int32
                 type: integer
             type: object

controllers/helpers_test.go

@@ -34,10 +34,11 @@ import (
 )
 
 const (
-	testNamespace     = "ns1"
-	testMicrovmName   = "mvm1"
-	testMicrovmUID    = "ABCDEF123456"
-	testBootstrapData = "somesamplebootstrapsdata"
+	testNamespace             = "ns1"
+	testMicrovmName           = "mvm1"
+	testMicrovmReplicaSetName = "rs1"
+	testMicrovmUID            = "ABCDEF123456"
+	testBootstrapData         = "somesamplebootstrapsdata"
 )
 
 func asRuntimeObject(microvm *infrav1.Microvm) []runtime.Object {
@@ -68,6 +69,34 @@ func reconcileMicrovm(client client.Client, mockAPIClient flclient.Client) (ctrl
 	return mvmController.Reconcile(context.TODO(), request)
 }
 
+func reconcileMicrovmReplicaSet(client client.Client) (ctrl.Result, error) {
+	mvmRSController := &controllers.MicrovmReplicaSetReconciler{
+		Client: client,
+		Scheme: client.Scheme(),
+	}
+
+	request := ctrl.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      testMicrovmReplicaSetName,
+			Namespace: testNamespace,
+		},
+	}
+
+	return mvmRSController.Reconcile(context.TODO(), request)
+}
+
+func reconcileMicrovmReplicaSetNTimes(g *WithT, client client.Client, count int32) error {
+	for count > 0 {
+		ensureMicrovmState(g, client)
+		if _, err := reconcileMicrovmReplicaSet(client); err != nil {
+			return err
+		}
+		count--
+	}
+
+	return nil
+}
+
 func getMicrovm(c client.Client, name, namespace string) (*infrav1.Microvm, error) {
 	key := client.ObjectKey{
 		Name:      name,
@@ -79,6 +108,23 @@ func getMicrovm(c client.Client, name, namespace string) (*infrav1.Microvm, erro
 	return mvm, err
 }
 
+func listMicrovm(c client.Client) (*infrav1.MicrovmList, error) {
+	mvm := &infrav1.MicrovmList{}
+	err := c.List(context.TODO(), mvm)
+	return mvm, err
+}
+
+func getMicrovmReplicaSet(c client.Client, name, namespace string) (*infrav1.MicrovmReplicaSet, error) {
+	key := client.ObjectKey{
+		Name:      name,
+		Namespace: namespace,
+	}
+
+	mvmRS := &infrav1.MicrovmReplicaSet{}
+	err := c.Get(context.TODO(), key, mvmRS)
+	return mvmRS, err
+}
+
 func createFakeClient(g *WithT, objects []runtime.Object) client.Client {
 	scheme := runtime.NewScheme()
 
@@ -127,6 +173,27 @@ func createMicrovm() *infrav1.Microvm {
 	}
 }
 
+func createMicrovmReplicaSet(reps int32) *infrav1.MicrovmReplicaSet {
+	mvm := createMicrovm()
+	mvm.Spec.Host = microvm.Host{}
+
+	return &infrav1.MicrovmReplicaSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      testMicrovmReplicaSetName,
+			Namespace: testNamespace,
+		},
+		Spec: infrav1.MicrovmReplicaSetSpec{
+			Host: microvm.Host{
+				Endpoint: "127.0.0.1:9090",
+			},
+			Replicas: pointer.Int32(reps),
+			Template: infrav1.MicrovmTemplateSpec{
+				Spec: mvm.Spec,
+			},
+		},
+	}
+}
+
 func withExistingMicrovm(fc *fakes.FakeClient, mvmState flintlocktypes.MicroVMStatus_MicroVMState) {
 	fc.GetMicroVMReturns(&flintlockv1.GetMicroVMResponse{
 		Microvm: &flintlocktypes.MicroVM{
@@ -185,18 +252,40 @@ func assertMicrovmReconciled(g *WithT, reconciled *infrav1.Microvm) {
 	g.Expect(reconciled.Status.Ready).To(BeTrue(), "The Ready property must be true when the mvm has been reconciled")
 }
 
+func microvmsCreated(g *WithT, c client.Client) int32 {
+	mvmList, err := listMicrovm(c)
+	g.Expect(err).NotTo(HaveOccurred())
+	return int32(len(mvmList.Items))
+}
+
+func ensureMicrovmState(g *WithT, c client.Client) {
+	// update the microvms so they report as ready to move the replicaset reconciliation along
+	mvmList, err := listMicrovm(c)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	for _, mvm := range mvmList.Items {
+		mvm.Status.Ready = true
+		g.Expect(c.Update(context.TODO(), &mvm)).To(Succeed())
+	}
+}
+
 func assertFinalizer(g *WithT, reconciled *infrav1.Microvm) {
 	g.Expect(reconciled.ObjectMeta.Finalizers).NotTo(BeEmpty(), "Expected at least one finalizer to be set")
-	g.Expect(hasMicrovmFinalizer(reconciled)).To(BeTrue(), "Expect the mvm finalizer")
+	g.Expect(hasMicrovmFinalizer(&reconciled.ObjectMeta, infrav1.MvmFinalizer)).To(BeTrue(), "Expect the mvm finalizer")
+}
+
+func assertMRSFinalizer(g *WithT, reconciled *infrav1.MicrovmReplicaSet) {
+	g.Expect(reconciled.ObjectMeta.Finalizers).NotTo(BeEmpty(), "Expected at least one finalizer to be set")
+	g.Expect(hasMicrovmFinalizer(&reconciled.ObjectMeta, infrav1.MvmRSFinalizer)).To(BeTrue(), "Expect the mvmrs finalizer")
 }
 
-func hasMicrovmFinalizer(mvm *infrav1.Microvm) bool {
-	if len(mvm.ObjectMeta.Finalizers) == 0 {
+func hasMicrovmFinalizer(meta *metav1.ObjectMeta, finalizer string) bool {
+	if len(meta.Finalizers) == 0 {
 		return false
 	}
 
-	for _, f := range mvm.ObjectMeta.Finalizers {
-		if f == infrav1.MvmFinalizer {
+	for _, f := range meta.Finalizers {
+		if f == finalizer {
 			return true
 		}
 	}

controllers/microvmreplicaset_controller.go

@@ -123,15 +123,17 @@ func (r *MicrovmReplicaSetReconciler) reconcileDelete(
 
 	for _, mvm := range mvmList {
 		// if the object is already being deleted, skip this
-		if mvm.DeletionTimestamp == nil {
-			// otherwise send a delete call
-			if err := r.Delete(ctx, &mvm); err != nil {
-				mvmReplicaSetScope.Error(err, "failed deleting microvm")
-				mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetDeleteFailedReason, "Error", "")
+		if !mvm.DeletionTimestamp.IsZero() {
+			continue
+		}
 
-				return ctrl.Result{}, err
+		// otherwise send a delete call
+		go func(m infrav1.Microvm) {
+			if err := r.Delete(ctx, &m); err != nil {
+				mvmReplicaSetScope.Error(err, "failed deleting microvm", "microvm", m.Name)
+				mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetDeleteFailedReason, "Error", "")
 			}
-		}
+		}(mvm)
 	}
 
 	// reset the number of created replicas.
@@ -184,41 +186,43 @@ func (r *MicrovmReplicaSetReconciler) reconcileNormal(
 		mvmReplicaSetScope.SetReady()
 
 		return reconcile.Result{}, nil
-	// if all desired microvms have been created, but are not quite ready yet,
-	// set the condition and requeue
-	case mvmReplicaSetScope.CreatedReplicas() == mvmReplicaSetScope.DesiredReplicas():
-		mvmReplicaSetScope.Info("MicrovmReplicaSet creating: waiting for microvms to become ready")
-		mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetIncompleteReason, "Info", "")
-
-		return ctrl.Result{RequeueAfter: requeuePeriod}, nil
 	// if we are in this branch then not all desired microvms have been created.
 	// create a new one and set the ownerref to this controller.
 	case mvmReplicaSetScope.CreatedReplicas() < mvmReplicaSetScope.DesiredReplicas():
 		mvmReplicaSetScope.Info("MicrovmReplicaSet creating: create new microvm")
 
 		if err := r.createMicrovm(ctx, mvmReplicaSetScope); err != nil {
 			mvmReplicaSetScope.Error(err, "failed creating owned microvm")
+			mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetProvisionFailedReason, "Error", "")
+
 			return reconcile.Result{}, fmt.Errorf("failed to create new microvm for replicaset: %w", err)
 		}
 
 		mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetIncompleteReason, "Info", "")
 	// if we are here then a scale down has been requested.
 	// we delete the first found until the numbers balance out.
-	// TODO this is not ideal, find a better way
+	// TODO the way this works is very naive and often ends up deleting everything
+	// if the timing is wrong/right, find a better way https://github.com/weaveworks-liquidmetal/microvm-operator/issues/17
 	case mvmReplicaSetScope.CreatedReplicas() > mvmReplicaSetScope.DesiredReplicas():
-		mvmReplicaSetScope.Info("MicrovmReplicaSet deleting: delete microvm")
+		mvmReplicaSetScope.Info("MicrovmReplicaSet updating: delete microvm")
+		mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetUpdatingReason, "Info", "")
 
 		mvm := mvmList[0]
-		if mvm.DeletionTimestamp == nil {
-			if err := r.Delete(ctx, &mvm); err != nil {
-				mvmReplicaSetScope.Error(err, "failed deleting microvm")
-				mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetDeleteFailedReason, "Error", "")
-
-				return ctrl.Result{}, err
-			}
+		if !mvm.DeletionTimestamp.IsZero() {
+			return ctrl.Result{}, nil
 		}
 
-		mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetUpdatingReason, "Info", "")
+		if err := r.Delete(ctx, &mvm); err != nil {
+			mvmReplicaSetScope.Error(err, "failed deleting microvm")
+			mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetDeleteFailedReason, "Error", "")
+
+			return ctrl.Result{}, err
+		}
+	// if all desired microvms have been created, but are not quite ready yet,
+	// set the condition and requeue
+	default:
+		mvmReplicaSetScope.Info("MicrovmReplicaSet creating: waiting for microvms to become ready")
+		mvmReplicaSetScope.SetNotReady(infrav1.MicrovmReplicaSetIncompleteReason, "Info", "")
 	}
 
 	controllerutil.AddFinalizer(mvmReplicaSetScope.MicrovmReplicaSet, infrav1.MvmRSFinalizer)