resource: freeze resources after marked for deletion (4 of 5) (#19603)

agent/grpc-external/services/resource/delete.go

@@ -81,8 +81,11 @@ func (s *Server) Delete(ctx context.Context, req *pbresource.DeleteRequest) (*pb
 			return &pbresource.DeleteResponse{}, nil
 		}
 
-		// Mark for deletion and let controllers that put finalizers in place do their thing
-		return s.markForDeletion(ctx, existing)
+		// Mark for deletion and let controllers that put finalizers in place do their
+		// thing. Note we're passing in a clone of the recently read resource since
+		// we've not crossed a network/serialization boundary since the read and we
+		// don't want to mutate the in-mem reference.
+		return s.markForDeletion(ctx, clone(existing))
 	}
 
 	// Continue with an immediate delete
@@ -102,12 +105,8 @@ func (s *Server) Delete(ctx context.Context, req *pbresource.DeleteRequest) (*pb
 }
 
 func (s *Server) markForDeletion(ctx context.Context, res *pbresource.Resource) (*pbresource.DeleteResponse, error) {
-	if res.Metadata == nil {
-		res.Metadata = map[string]string{}
-	}
-	res.Metadata[resource.DeletionTimestampKey] = time.Now().Format(time.RFC3339)
-
 	// Write the deletion timestamp
+	res.Metadata[resource.DeletionTimestampKey] = time.Now().Format(time.RFC3339)
 	_, err := s.Write(ctx, &pbresource.WriteRequest{Resource: res})
 	if err != nil {
 		return nil, err

agent/grpc-external/services/resource/server.go

@@ -7,13 +7,14 @@ import (
 	"context"
 	"strings"
 
-	"github.com/hashicorp/go-hclog"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 
+	"github.com/hashicorp/go-hclog"
+
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/acl/resolver"
 	"github.com/hashicorp/consul/internal/resource"
@@ -272,28 +273,27 @@ func validateScopedTenancy(scope resource.Scope, resourceType *pbresource.Type,
 	return nil
 }
 
-// tenancyMarkedForDeletion returns a gRPC InvalidArgument when either partition or namespace is marked for deletion.
-func tenancyMarkedForDeletion(reg *resource.Registration, tenancyBridge TenancyBridge, tenancy *pbresource.Tenancy) error {
+func isTenancyMarkedForDeletion(reg *resource.Registration, tenancyBridge TenancyBridge, tenancy *pbresource.Tenancy) (bool, error) {
 	if reg.Scope == resource.ScopePartition || reg.Scope == resource.ScopeNamespace {
 		marked, err := tenancyBridge.IsPartitionMarkedForDeletion(tenancy.Partition)
-		switch {
-		case err != nil:
-			return err
-		case marked:
-			return status.Errorf(codes.InvalidArgument, "partition marked for deletion: %v", tenancy.Partition)
+		if err != nil {
+			return false, err
+		}
+		if marked {
+			return marked, nil
 		}
 	}
 
 	if reg.Scope == resource.ScopeNamespace {
 		marked, err := tenancyBridge.IsNamespaceMarkedForDeletion(tenancy.Partition, tenancy.Namespace)
-		switch {
-		case err != nil:
-			return err
-		case marked:
-			return status.Errorf(codes.InvalidArgument, "namespace marked for deletion: %v", tenancy.Namespace)
+		if err != nil {
+			return false, err
 		}
+		return marked, nil
 	}
-	return nil
+
+	// Cluster scope has no tenancy so always return false
+	return false, nil
 }
 
 func clone[T proto.Message](v T) T { return proto.Clone(v).(T) }

agent/grpc-external/services/resource/write.go

@@ -10,8 +10,10 @@ import (
 	"time"
 
 	"github.com/oklog/ulid/v2"
+	"golang.org/x/exp/maps"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/internal/resource"
@@ -83,9 +85,11 @@ func (s *Server) Write(ctx context.Context, req *pbresource.WriteRequest) (*pbre
 		return nil, err
 	}
 
-	// Check tenancy not marked for deletion.
-	if err = tenancyMarkedForDeletion(reg, s.TenancyBridge, req.Resource.Id.Tenancy); err != nil {
-		return nil, err
+	// This is used later in the "create" and "update" paths to block non-delete related writes
+	// when a tenancy unit has been marked for deletion.
+	tenancyMarkedForDeletion, err := isTenancyMarkedForDeletion(reg, s.TenancyBridge, req.Resource.Id.Tenancy)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed tenancy marked for deletion check: %v", err)
 	}
 
 	// At the storage backend layer, all writes are CAS operations.
@@ -127,6 +131,16 @@ func (s *Server) Write(ctx context.Context, req *pbresource.WriteRequest) (*pbre
 				return errUseWriteStatus
 			}
 
+			// Reject creation in tenancy unit marked for deletion.
+			if tenancyMarkedForDeletion {
+				return status.Errorf(codes.InvalidArgument, "tenancy marked for deletion: %v", input.Id.Tenancy.String())
+			}
+
+			// Reject attempts to create a resource with a deletionTimestamp.
+			if resource.IsMarkedForDeletion(input) {
+				return status.Errorf(codes.InvalidArgument, "resource.metadata.%s can't be set on resource creation", resource.DeletionTimestampKey)
+			}
+
 			// Generally, we expect resources with owners to be created by controllers,
 			// and they should provide the Uid. In cases where no Uid is given (e.g. the
 			// owner is specified in the resource HCL) we'll look up whatever the current
@@ -208,6 +222,13 @@ func (s *Server) Write(ctx context.Context, req *pbresource.WriteRequest) (*pbre
 				return errUseWriteStatus
 			}
 
+			// If the write is related to a deferred deletion (marking for deletion or removal
+			// of finalizers), make sure nothing else is changed.
+			if err := vetIfDeleteRelated(input, existing, tenancyMarkedForDeletion); err != nil {
+				return err
+			}
+
+			// Otherwise, let the write continue
 		default:
 			return err
 		}
@@ -310,3 +331,109 @@ func (s *Server) ensureWriteRequestValid(req *pbresource.WriteRequest) (*resourc
 
 	return reg, nil
 }
+
+func ensureMetadataSameExceptFor(input *pbresource.Resource, existing *pbresource.Resource, ignoreKey string) error {
+	// Work on copies since we're mutating them
+	inputCopy := maps.Clone(input.Metadata)
+	existingCopy := maps.Clone(existing.Metadata)
+
+	delete(inputCopy, ignoreKey)
+	delete(existingCopy, ignoreKey)
+
+	if !maps.Equal(inputCopy, existingCopy) {
+		return status.Error(codes.InvalidArgument, "cannot modify metadata")
+	}
+
+	return nil
+}
+
+func ensureDataUnchanged(input *pbresource.Resource, existing *pbresource.Resource) error {
+	// Check data last since this could potentially be the most expensive comparison.
+	if !proto.Equal(input.Data, existing.Data) {
+		return status.Error(codes.InvalidArgument, "cannot modify data")
+	}
+	return nil
+}
+
+// ensureFinalizerRemoved ensures at least one finalizer was removed.
+func ensureFinalizerRemoved(input *pbresource.Resource, existing *pbresource.Resource) error {
+	inputFinalizers := resource.GetFinalizers(input)
+	existingFinalizers := resource.GetFinalizers(existing)
+	if !inputFinalizers.IsProperSubset(existingFinalizers) {
+		return status.Error(codes.InvalidArgument, "expected at least one finalizer to be removed")
+	}
+	return nil
+}
+
+func vetIfDeleteRelated(input, existing *pbresource.Resource, tenancyMarkedForDeletion bool) error {
+	// Keep track of whether this write is a normal write or a write that is related
+	// to deferred resource deletion involving setting the deletionTimestamp or the
+	// removal of finalizers.
+	deleteRelated := false
+
+	existingMarked := resource.IsMarkedForDeletion(existing)
+	inputMarked := resource.IsMarkedForDeletion(input)
+
+	// Block removal of deletion timestamp
+	if !inputMarked && existingMarked {
+		return status.Errorf(codes.InvalidArgument, "cannot remove %s", resource.DeletionTimestampKey)
+	}
+
+	// Block modification of existing deletion timestamp
+	if existing.Metadata[resource.DeletionTimestampKey] != "" && (existing.Metadata[resource.DeletionTimestampKey] != input.Metadata[resource.DeletionTimestampKey]) {
+		return status.Errorf(codes.InvalidArgument, "cannot modify %s", resource.DeletionTimestampKey)
+	}
+
+	// Block writes that do more than just adding a deletion timestamp
+	if inputMarked && !existingMarked {
+		deleteRelated = deleteRelated || true
+		// Verify rest of resource is unchanged
+		if err := ensureMetadataSameExceptFor(input, existing, resource.DeletionTimestampKey); err != nil {
+			return err
+		}
+		if err := ensureDataUnchanged(input, existing); err != nil {
+			return err
+		}
+	}
+
+	// Block no-op writes writes to resource that already has a deletion timestamp. The
+	// only valid writes should be removal of finalizers.
+	if inputMarked && existingMarked {
+		deleteRelated = deleteRelated || true
+		// Check if a no-op
+		errMetadataSame := ensureMetadataSameExceptFor(input, existing, resource.DeletionTimestampKey)
+		errDataUnchanged := ensureDataUnchanged(input, existing)
+		if errMetadataSame == nil && errDataUnchanged == nil {
+			return status.Error(codes.InvalidArgument, "no-op write of resource marked for deletion not allowed")
+		}
+	}
+
+	// Block writes that do more than removing finalizers if previously marked for deletion.
+	if inputMarked && existingMarked && resource.HasFinalizers(existing) {
+		deleteRelated = deleteRelated || true
+		if err := ensureMetadataSameExceptFor(input, existing, resource.FinalizerKey); err != nil {
+			return err
+		}
+		if err := ensureDataUnchanged(input, existing); err != nil {
+			return err
+		}
+		if err := ensureFinalizerRemoved(input, existing); err != nil {
+			return err
+		}
+	}
+
+	// Classify writes that just remove finalizer as deleteRelated regardless of deletion state.
+	if err := ensureFinalizerRemoved(input, existing); err == nil {
+		if err := ensureDataUnchanged(input, existing); err == nil {
+			deleteRelated = deleteRelated || true
+		}
+	}
+
+	// Lastly, block writes when the resource's tenancy unit has been marked for deletion and
+	// the write is not related a valid delete scenario.
+	if tenancyMarkedForDeletion && !deleteRelated {
+		return status.Errorf(codes.InvalidArgument, "cannot write resource when tenancy marked for deletion: %s", existing.Id.Tenancy)
+	}
+
+	return nil
+}