✨ Add 'DatasourceIgnorePatterns' to ignore datasources in the controller

Author: bdumpp

api/config/v2alpha2/projectconfig_types.go

@@ -53,6 +53,10 @@ type ProjectConfig struct {
 	// kept disabled unless migrations need to be done.
 	EnableMigrations bool `json:"enableMigrations"`
 
+	// DatasourceIgnorePatterns is a list of regex patterns, that allow datasources in styra
+	// to be ignored based on their datasource id.
+	DatasourceIgnorePatterns []string `json:"datasourceIgnorePatterns,omitempty"`
+
 	// GitCredentials holds a list of git credential configurations. The
 	// RepoPrefix of the GitCredential will be matched angainst repository URL in
 	// order to determine which credential to use. The GitCredential with the

api/config/v2alpha2/zz_generated.deepcopy.go

@@ -27,6 +27,10 @@ systemUserRoles:
   - SystemOwner
   - SystemMetadataManager
 
+#datasourceIgnorePatterns:
+# - "^systems/[a-z0-9]+/dontdeleteme/.*"
+# - "^libraries/[a-z0-9_]+/deletemenot/.*"
+
 #decisionsExporter:
 
 #opa:

config/default/config.yaml

@@ -17,5 +17,6 @@ spec:
       name: api/authz/decision
       reason:
         path: result.reasons
-
+  datasources:
+    - path: "test"
   # TODO(user): Add fields here

config/samples/styra_v1beta1_system.yaml

@@ -19,6 +19,7 @@ package config
 
 import (
 	"os"
+	"regexp"
 
 	"github.com/bankdata/styra-controller/api/config/v2alpha2"
 	"github.com/pkg/errors"
@@ -114,3 +115,17 @@ func deserialize(data []byte, scheme *runtime.Scheme) (*v2alpha2.ProjectConfig,
 
 	return cfg, nil
 }
+
+// MatchesIgnorePattern matches a specified ignore pattern, and excludes matches from being deleted
+func MatchesIgnorePattern(ignorePatterns []string, id string) (bool, error) {
+	for _, patternString := range ignorePatterns {
+		matches, err := regexp.MatchString(patternString, id)
+		if err != nil {
+			return false, errors.Wrapf(err, "could not compile regex pattern: %s", patternString)
+		}
+		if matches {
+			return true, nil
+		}
+	}
+	return false, nil
+}

internal/config/config.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"path"
 
+	"github.com/bankdata/styra-controller/internal/config"
 	ctrlerr "github.com/bankdata/styra-controller/internal/errors"
 	"github.com/bankdata/styra-controller/internal/predicate"
 	"github.com/bankdata/styra-controller/internal/sentry"
@@ -254,8 +255,18 @@ func (r *LibraryReconciler) reconcileDatasources(ctx context.Context, log logr.L
 			continue
 		}
 
+		ignore, err := config.MatchesIgnorePattern(r.Config.DatasourceIgnorePatterns, ds.ID)
+		if err != nil {
+			return ctrl.Result{}, ctrlerr.Wrap(err, "Could not check if library datasource should be ignored")
+		}
+
+		if ignore {
+			log.Info("Datasource is ignored", "id", ds.ID)
+			continue
+		}
+
 		if _, expected := expectedByID[ds.ID]; !expected {
-			log.Info("Deleting undeclared datasource")
+			log.Info("Deleting undeclared datasource", "id", ds.ID)
 
 			if _, err := r.Styra.DeleteDatasource(ctx, ds.ID); err != nil {
 				return ctrl.Result{}, ctrlerr.Wrap(err, "Could not delete library datasource")

internal/controller/styra/library_controller.go

@@ -49,6 +49,7 @@ import (
 
 	configv2alpha2 "github.com/bankdata/styra-controller/api/config/v2alpha2"
 	"github.com/bankdata/styra-controller/api/styra/v1beta1"
+	"github.com/bankdata/styra-controller/internal/config"
 	ctrlerr "github.com/bankdata/styra-controller/internal/errors"
 	"github.com/bankdata/styra-controller/internal/fields"
 	"github.com/bankdata/styra-controller/internal/finalizer"
@@ -927,9 +928,20 @@ func (r *SystemReconciler) reconcileDatasources(
 		if ds.Optional {
 			continue
 		}
+
+		ignore, err := config.MatchesIgnorePattern(r.Config.DatasourceIgnorePatterns, ds.ID)
+		if err != nil {
+			return ctrl.Result{}, ctrlerr.Wrap(err, "Could not check if system datasource should be ignored")
+		}
+
+		if ignore {
+			log.Info("Datasource is ignored", "id", ds.ID)
+			continue
+		}
+
 		if _, ok := expectedByID[ds.ID]; !ok {
 			log := log.WithValues("datasourceID", ds.ID)
-			log.Info("Deleting undeclared datasource")
+			log.Info("Deleting undeclared datasource", "id", ds.ID)
 			if _, err := r.Styra.DeleteDatasource(ctx, ds.ID); err != nil {
 				return ctrl.Result{}, ctrlerr.Wrap(err, "Could not delete datasource").
 					WithEvent("ErrorDeleteDatasource").
@@ -1279,8 +1291,17 @@ func (r *SystemReconciler) specToSystemConfig(system *v1beta1.System) (*styra.Sy
 	}
 
 	if system.Spec.SourceControl != nil {
-		if !isURLValid(system.Spec.SourceControl.Origin.URL) {
-			return nil, ctrlerr.New("Invalid URL for source control")
+		valid, err := isURLValid(system.Spec.SourceControl.Origin.URL)
+		if err != nil {
+			return nil, ctrlerr.Wrap(err, "Error while validating URL").
+				WithEvent("ErrorUpdateSystem").
+				WithSystemCondition(v1beta1.ConditionTypeSystemConfigUpdated)
+		}
+
+		if !valid {
+			return nil, ctrlerr.New("Invalid URL for source control").
+				WithEvent("ErrorUpdateSystem").
+				WithSystemCondition(v1beta1.ConditionTypeSystemConfigUpdated)
 		}
 
 		cfg.SourceControl = &styra.SourceControlConfig{
@@ -1308,23 +1329,27 @@ func (r *SystemReconciler) specToSystemConfig(system *v1beta1.System) (*styra.Sy
 	return cfg, nil
 }
 
-func isURLValid(u string) bool {
+func isURLValid(u string) (bool, error) {
 	//empty url is valid
 	if strings.TrimSpace(u) == "" {
-		return true
+		return true, nil
 	}
 
 	parsedURL, err := url.Parse(u)
 	if err != nil {
-		return false
+		return false, err
 	}
 	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
-		return false
+		return false, nil
 	}
 
 	// Regular expression to match valid URL path characters
-	re := regexp.MustCompile(`^[a-zA-Z0-9\-._~!$&'()*+,;=:@/]*$`)
-	return re.MatchString(parsedURL.Path)
+	valid, err := regexp.MatchString(`^[a-zA-Z0-9\-._~!$&'()*+,;=:@/]*$`, parsedURL.Path)
+	if err != nil {
+		return false, err
+	}
+
+	return valid, nil
 }
 
 func (r *SystemReconciler) systemNeedsUpdate(

internal/controller/styra/system_controller.go

@@ -123,6 +123,7 @@ var _ = ginkgo.BeforeSuite(func() {
 				IdentityProvider: "AzureAD Bankdata",
 				JWTGroupsClaim:   "groups",
 			},
+			DatasourceIgnorePatterns:  []string{"^.*/ignore$"},
 			ReadOnly:                  true,
 			EnableDeltaBundlesDefault: ptr.Bool(false),
 		},
@@ -161,6 +162,7 @@ var _ = ginkgo.BeforeSuite(func() {
 				IdentityProvider: "AzureAD Bankdata",
 				JWTGroupsClaim:   "groups",
 			},
+			DatasourceIgnorePatterns: []string{"^.*/ignore$"},
 			GitCredentials: []*configv2alpha2.GitCredential{
 				{User: "test-user", Password: "test-secret"},
 			},

test/integration/controller/controller_suite_test.go

@@ -125,7 +125,16 @@ var _ = ginkgo.Describe("LibraryReconciler.Reconcile", ginkgo.Label("integration
 		styraClientMock.On("GetLibrary", mock.Anything, key.Name).Return(&styra.GetLibraryResponse{
 			Statuscode: http.StatusOK,
 			LibraryEntityExpanded: &styra.LibraryEntityExpanded{
-				DataSources:   []styra.LibraryDatasourceConfig{},
+				DataSources: []styra.LibraryDatasourceConfig{
+					{
+						Category: "rest",
+						ID:       path.Join("libraries", key.Name, "ignore"),
+					},
+					{
+						Category: "rest",
+						ID:       path.Join("libraries", key.Name, "delete"),
+					},
+				},
 				Description:   "description",
 				ID:            key.Name,
 				ReadOnly:      true,
@@ -141,6 +150,14 @@ var _ = ginkgo.Describe("LibraryReconciler.Reconcile", ginkgo.Label("integration
 			StatusCode: http.StatusOK,
 		}, nil).Once()
 
+		styraClientMock.On(
+			"DeleteDatasource",
+			mock.Anything,
+			path.Join("libraries", key.Name, "delete")).
+			Return(&styra.DeleteDatasourceResponse{
+				StatusCode: http.StatusOK,
+			}, nil).Once()
+
 		webhookMock.On(
 			"LibraryDatasourceChanged",
 			mock.Anything,
@@ -267,6 +284,7 @@ var _ = ginkgo.Describe("LibraryReconciler.Reconcile", ginkgo.Label("integration
 				createInvitation   int
 				createRoleBinding  int
 				listRoleBindings   int
+				deleteDatasource   int
 			)
 
 			for _, call := range styraClientMock.Calls {
@@ -287,6 +305,8 @@ var _ = ginkgo.Describe("LibraryReconciler.Reconcile", ginkgo.Label("integration
 					createRoleBinding++
 				case "ListRoleBindingsV2":
 					listRoleBindings++
+				case "DeleteDatasource":
+					deleteDatasource++
 				}
 			}
 
@@ -296,7 +316,6 @@ var _ = ginkgo.Describe("LibraryReconciler.Reconcile", ginkgo.Label("integration
 					datasourceChanged++
 				}
 			}
-
 			return createUpdateSecret == 2 &&
 				getLibrary == 2 &&
 				upsertLibrary == 1 &&
@@ -305,7 +324,8 @@ var _ = ginkgo.Describe("LibraryReconciler.Reconcile", ginkgo.Label("integration
 				getUser == 2 &&
 				createInvitation == 1 &&
 				listRoleBindings == 3 &&
-				createRoleBinding == 1
+				createRoleBinding == 1 &&
+				deleteDatasource == 1
 		}, timeout, interval).Should(gomega.BeTrue())
 
 		resetMock(&webhookMock.Mock)

test/integration/controller/library_controller_test.go

@@ -1134,6 +1134,16 @@ discovery:
 							Reference:   "",
 						},
 					},
+					Datasources: []*styra.DatasourceConfig{
+						{
+							Category: "rest",
+							ID:       "systems/system_id/ignore",
+						},
+						{
+							Category: "rest",
+							ID:       "systems/system_id/delete",
+						},
+					},
 					DecisionMappings: map[string]styra.DecisionMapping{
 						"": {},
 						"test": {
@@ -1183,6 +1193,12 @@ discovery:
 			StatusCode: http.StatusOK,
 		}, nil).Once()
 
+		styraClientMock.On(
+			"DeleteDatasource",
+			mock.Anything,
+			"systems/system_id/delete",
+		).Return(&styra.DeleteDatasourceResponse{}, nil).Once()
+
 		webhookMock.On(
 			"SystemDatasourceChanged",
 			mock.Anything,
@@ -1200,6 +1216,7 @@ discovery:
 				upsertDatasource   int
 				getOPAConfig       int
 				datasourceChanged  int
+				deleteDatasource   int
 			)
 			for _, call := range styraClientMock.Calls {
 				switch call.Method {
@@ -1213,6 +1230,8 @@ discovery:
 					getOPAConfig++
 				case "UpsertDatasource":
 					upsertDatasource++
+				case "DeleteDatasource":
+					deleteDatasource++
 				}
 			}
 
@@ -1228,7 +1247,8 @@ discovery:
 				listRolebindingsV2 == 1 &&
 				upsertDatasource == 1 &&
 				getOPAConfig == 1 &&
-				datasourceChanged == 1
+				datasourceChanged == 1 &&
+				deleteDatasource == 1
 		}, timeout, interval).Should(gomega.BeTrue())
 
 		resetMock(&styraClientMock.Mock)