#139 do not reconcile blueprint if a restore is in progress

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#135] Support for the blueprint mask custom resource.
 - [#129] Reconciliation of the blueprint on changes of dogu-crs, ces-configMaps and ces-secrets
 - [#131] Ignore loglevel changes while debug-mode is active
+- [#131] Do not reconcile blueprint if a restore is in progress
 
 ### Changed
 - [#119] *breaking* sensitive dogu config can now only be referenced with secrets

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.4.0
 	github.com/cloudogu/ces-commons-lib v0.2.0
 	github.com/cloudogu/cesapp-lib v0.18.1
+	github.com/cloudogu/k8s-backup-lib v1.7.0
 	github.com/cloudogu/k8s-blueprint-lib/v3 v3.0.0
 	github.com/cloudogu/k8s-debug-mode-cr-lib v1.0.0
 	github.com/cloudogu/k8s-dogu-lib/v2 v2.10.0

go.sum

@@ -18,6 +18,8 @@ github.com/cloudogu/ces-commons-lib v0.2.0 h1:yOEZWFl4W9N3J/6fok4svE3UufK5GQQtyx
 github.com/cloudogu/ces-commons-lib v0.2.0/go.mod h1:4rvR2RTDDaz5a6OZ1fW27G0MOnl5I3ackeiHxt4gn3o=
 github.com/cloudogu/cesapp-lib v0.18.1 h1:LMdGktIefm/PuhdPqpLTPvjY1smO06EEGBbRSAaYi7U=
 github.com/cloudogu/cesapp-lib v0.18.1/go.mod h1:J05eXFxnz4enZblABlmiVTZaUtJ+LIhlJ2UF6l9jpDw=
+github.com/cloudogu/k8s-backup-lib v1.7.0 h1:piu0+IK7IejeKTmdIMKuBHt0QKSdLVYzCALNewTDeF4=
+github.com/cloudogu/k8s-backup-lib v1.7.0/go.mod h1:FKk+/VswDzTttHzlIFDfgpWBe7kXWngxSFdSh7dZTpg=
 github.com/cloudogu/k8s-blueprint-lib/v3 v3.0.0 h1:XDWrVVmQ1aJ00aisCNQ1nJqOHVK9LB3q92swJFMZEFg=
 github.com/cloudogu/k8s-blueprint-lib/v3 v3.0.0/go.mod h1:3E1iLra8//8+kCwBjuDi6b0iwtNoArYfOIYnzNXSFMQ=
 github.com/cloudogu/k8s-debug-mode-cr-lib v1.0.0 h1:geZjXwWQY8d8aEWA9l2is/DwlADdOHQveBokPK63XH0=
@@ -180,8 +182,6 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
 github.com/prometheus/common v0.67.1 h1:OTSON1P4DNxzTg4hmKCc37o4ZAZDv0cfXLkOt0oEowI=
 github.com/prometheus/common v0.67.1/go.mod h1:RpmT9v35q2Y+lsieQsdOh5sXZ6ajUGC8NjZAmr8vb0Q=
-github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
-github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
 github.com/prometheus/procfs v0.18.0 h1:2QTA9cKdznfYJz7EDaa7IiJobHuV7E1WzeBwcrhk0ao=
 github.com/prometheus/procfs v0.18.0/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
@@ -247,8 +247,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
 golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
-golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b h1:18qgiDvlvH7kk8Ioa8Ov+K6xCi0GMvmGfGW0sgd/SYA=
-golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
 golang.org/x/exp v0.0.0-20251017212417-90e834f514db h1:by6IehL4BH5k3e3SJmcoNbOobMey2SLpAF79iPOEBvw=
 golang.org/x/exp v0.0.0-20251017212417-90e834f514db/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

k8s/helm/templates/restore-reader-role.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+  {{- include "k8s-blueprint-operator.labels" . | nindent 4 }}
+  name: {{ include "k8s-blueprint-operator.name" . }}-restore-reader-role
+rules:
+  - apiGroups:
+      - k8s.cloudogu.com
+    resources:
+      - restores
+    verbs:
+      - get
+      - list

k8s/helm/templates/restore-reader-rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+  {{- include "k8s-blueprint-operator.labels" . | nindent 4 }}
+  name: {{ include "k8s-blueprint-operator.name" . }}-restore-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "k8s-blueprint-operator.name" . }}-restore-reader-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "k8s-blueprint-operator.name" . }}-controller-manager

pkg/adapter/kubernetes/restorecr/interfaces.go

@@ -0,0 +1,16 @@
+package restorecr
+
+import (
+	"context"
+
+	restorev1 "github.com/cloudogu/k8s-backup-lib/api/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// interface replication for generating mocks
+
+//nolint:unused
+type RestoreInterface interface {
+	// List takes label and field selectors, and returns the list of Restores that match those selectors.
+	List(ctx context.Context, opts metav1.ListOptions) (*restorev1.RestoreList, error)
+}

pkg/adapter/kubernetes/restorecr/restoreRepo.go

@@ -0,0 +1,34 @@
+package restorecr
+
+import (
+	"context"
+	"fmt"
+
+	restorev1 "github.com/cloudogu/k8s-backup-lib/api/v1"
+	"github.com/cloudogu/k8s-blueprint-operator/v2/pkg/domainservice"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type restoreRepo struct {
+	restoreClient RestoreInterface
+}
+
+// NewRestoreRepo returns a new restoreRepo to interact with the restore CR.
+func NewRestoreRepo(restoreClient RestoreInterface) domainservice.RestoreRepository {
+	return &restoreRepo{restoreClient: restoreClient}
+}
+
+func (repo *restoreRepo) IsRestoreInProgress(ctx context.Context) (bool, error) {
+	list, err := repo.restoreClient.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return false, fmt.Errorf("error while listing restore CRs: %w", err)
+	}
+
+	for _, restore := range list.Items {
+		if restore.Status.Status == restorev1.RestoreStatusInProgress {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

pkg/adapter/kubernetes/restorecr/restoreRepo_test.go

@@ -0,0 +1,93 @@
+package restorecr
+
+import (
+	"context"
+	"testing"
+
+	restorev1 "github.com/cloudogu/k8s-backup-lib/api/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var testCtx = context.Background()
+
+func TestNewRestoreRepo(t *testing.T) {
+	t.Run("should create new RestoreRepo", func(t *testing.T) {
+		mRestoreClient := NewMockRestoreInterface(t)
+
+		repo := NewRestoreRepo(mRestoreClient)
+
+		assert.NotNil(t, repo)
+		assert.Equal(t, mRestoreClient, repo.(*restoreRepo).restoreClient)
+	})
+}
+
+func Test_restoreRepo_IsRestoreInProgress(t *testing.T) {
+	t.Run("should return true if a restore is in progress", func(t *testing.T) {
+		mRestoreClient := NewMockRestoreInterface(t)
+		mRestoreClient.EXPECT().List(testCtx, metav1.ListOptions{}).Return(&restorev1.RestoreList{
+			Items: []restorev1.Restore{
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-1"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusNew}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-2"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusCompleted}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-3"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusDeleting}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-4"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusFailed}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-5"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusInProgress}},
+			},
+		}, nil)
+
+		repo := &restoreRepo{restoreClient: mRestoreClient}
+
+		result, err := repo.IsRestoreInProgress(testCtx)
+
+		require.NoError(t, err)
+		assert.True(t, result)
+	})
+
+	t.Run("should return false if no restore is in progress", func(t *testing.T) {
+		mRestoreClient := NewMockRestoreInterface(t)
+		mRestoreClient.EXPECT().List(testCtx, metav1.ListOptions{}).Return(&restorev1.RestoreList{
+			Items: []restorev1.Restore{
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-1"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusNew}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-2"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusCompleted}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-3"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusDeleting}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-4"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusFailed}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "restore-5"}, Status: restorev1.RestoreStatus{Status: restorev1.RestoreStatusCompleted}},
+			},
+		}, nil)
+
+		repo := &restoreRepo{restoreClient: mRestoreClient}
+
+		result, err := repo.IsRestoreInProgress(testCtx)
+
+		require.NoError(t, err)
+		assert.False(t, result)
+	})
+
+	t.Run("should return false if no restore exists", func(t *testing.T) {
+		mRestoreClient := NewMockRestoreInterface(t)
+		mRestoreClient.EXPECT().List(testCtx, metav1.ListOptions{}).Return(&restorev1.RestoreList{
+			Items: []restorev1.Restore{},
+		}, nil)
+
+		repo := &restoreRepo{restoreClient: mRestoreClient}
+
+		result, err := repo.IsRestoreInProgress(testCtx)
+
+		require.NoError(t, err)
+		assert.False(t, result)
+	})
+
+	t.Run("should fail if there is an error listing restores", func(t *testing.T) {
+		mRestoreClient := NewMockRestoreInterface(t)
+		mRestoreClient.EXPECT().List(testCtx, metav1.ListOptions{}).Return(nil, assert.AnError)
+
+		repo := &restoreRepo{restoreClient: mRestoreClient}
+
+		_, err := repo.IsRestoreInProgress(testCtx)
+
+		require.Error(t, err)
+		assert.ErrorIs(t, err, assert.AnError)
+		assert.ErrorContains(t, err, "error while listing restore CRs")
+	})
+}

pkg/adapter/reconciler/error_handler.go

@@ -31,6 +31,7 @@ func (h *ErrorHandler) handleError(logger logr.Logger, err error) (ctrl.Result,
 	var stateDiffNotEmptyError *domain.StateDiffNotEmptyError
 	var multipleBlueprintsError *domain.MultipleBlueprintsError
 	var dogusNotUpToDateError *domain.DogusNotUpToDateError
+	var restoreInProgressError *domain.RestoreInProgressError
 	switch {
 	case errors.As(err, &internalError):
 		return h.handleInternalError(errLogger, err)
@@ -48,6 +49,8 @@ func (h *ErrorHandler) handleError(logger logr.Logger, err error) (ctrl.Result,
 		return h.handleMultipleBlueprintsError(errLogger, err)
 	case errors.As(err, &dogusNotUpToDateError):
 		return h.handleDogusNotUpToDateError(errLogger, err)
+	case errors.As(err, &restoreInProgressError):
+		return h.handleRestoreInProgressError(errLogger, err)
 	default:
 		return h.handleUnknownError(errLogger, err)
 	}
@@ -102,6 +105,12 @@ func (h *ErrorHandler) handleDogusNotUpToDateError(logger logr.Logger, err error
 	return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
 }
 
+func (h *ErrorHandler) handleRestoreInProgressError(logger logr.Logger, err error) (ctrl.Result, error) {
+	// really normal case
+	logger.Info(fmt.Sprintf("A restore is currently in progress. Retry later: %s", err.Error()))
+	return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+}
+
 func (h *ErrorHandler) handleUnknownError(logger logr.Logger, err error) (ctrl.Result, error) {
 	logger.Error(err, "An unknown error type occurred. Retry with default backoff")
 	return ctrl.Result{}, err // automatic requeue because of non-nil err

pkg/adapter/reconciler/error_handler_test.go

@@ -150,6 +150,25 @@ func Test_decideRequeueForError(t *testing.T) {
 		assert.Equal(t, ctrl.Result{RequeueAfter: 1 * time.Second}, actual)
 		assert.Contains(t, logSinkMock.output, "0: requeue until state diff is empty")
 	})
+	t.Run("should catch wrapped RestoreInProgressError, issue a log line and requeue timely", func(t *testing.T) {
+		// given
+		logSinkMock := newTrivialTestLogSink()
+		testLogger := logr.New(logSinkMock)
+
+		intermediateErr := &domain.RestoreInProgressError{
+			Message: "a generic oh-noez",
+		}
+		errorChain := fmt.Errorf("could not do the thing: %w", intermediateErr)
+
+		// when
+		sut := NewErrorHandler()
+		actual, err := sut.handleError(testLogger, errorChain)
+
+		// then
+		require.NoError(t, err)
+		assert.Equal(t, ctrl.Result{RequeueAfter: 10 * time.Second}, actual)
+		assert.Contains(t, logSinkMock.output, "0: A restore is currently in progress. Retry later: could not do the thing: a generic oh-noez")
+	})
 	t.Run("should catch general errors, issue a log line and return requeue with error", func(t *testing.T) {
 		// given
 		logSinkMock := newTrivialTestLogSink()