-
Notifications
You must be signed in to change notification settings - Fork 4.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
v2tenancy: namespace deletion using finalizers (#19714)
- Loading branch information
Showing
4 changed files
with
457 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,196 @@ | ||
// Copyright (c) HashiCorp, Inc. | ||
// SPDX-License-Identifier: BUSL-1.1 | ||
|
||
// Common code shared by the partition and namespace controllers. | ||
package common | ||
|
||
import ( | ||
"context" | ||
"errors" | ||
"fmt" | ||
|
||
"github.com/hashicorp/consul/internal/controller" | ||
"github.com/hashicorp/consul/internal/resource" | ||
"github.com/hashicorp/consul/proto-public/pbresource" | ||
) | ||
|
||
const ( | ||
// ConditionAccepted indicates the tenancy unit has a finalizer | ||
// and contains a default namespace if a partition. | ||
ConditionAccepted = "accepted" | ||
ReasonAcceptedOK = "Ok" | ||
ReasonEnsureHasFinalizerFailed = "EnsureHasFinalizerFailed" | ||
|
||
// ConditionDeleted indicates that the units tenants have been | ||
// deleted. It never has a state other than false because the | ||
// resource no longer exists at that point. | ||
ConditionDeleted = "deleted" | ||
ReasonDeletionInProgress = "DeletionInProgress" | ||
) | ||
|
||
var ( | ||
ErrStillHasTenants = errors.New("still has tenants") | ||
) | ||
|
||
func EnsureHasFinalizer(ctx context.Context, rt controller.Runtime, res *pbresource.Resource, statusKey string) error { | ||
// The statusKey doubles as the finalizer name for tenancy resources. | ||
if resource.HasFinalizer(res, statusKey) { | ||
rt.Logger.Trace("already has finalizer") | ||
return nil | ||
} | ||
|
||
// Finalizer hasn't been written, so add it. | ||
resource.AddFinalizer(res, statusKey) | ||
_, err := rt.Client.Write(ctx, &pbresource.WriteRequest{Resource: res}) | ||
if err != nil { | ||
return WriteStatus(ctx, rt, res, statusKey, ConditionAccepted, ReasonEnsureHasFinalizerFailed, err) | ||
} | ||
rt.Logger.Trace("added finalizer") | ||
return err | ||
} | ||
|
||
func EnsureTenantsDeleted(ctx context.Context, rt controller.Runtime, registry resource.Registry, res *pbresource.Resource, tenantScope resource.Scope, tenancy *pbresource.Tenancy) error { | ||
// Useful stats to keep track of on every sweep | ||
numExistingHasFinalizer := 0 | ||
numExistingOwned := 0 | ||
numImmediateDeletes := 0 | ||
numDeferredDeletes := 0 | ||
|
||
// List doesn't support querying across all types so iterate through each one. | ||
for _, reg := range registry.Types() { | ||
// Skip tenants that aren't scoped to the tenancy unit. | ||
if reg.Scope != tenantScope { | ||
continue | ||
} | ||
|
||
// Get all tenants of the current type. | ||
rsp, err := rt.Client.List(ctx, &pbresource.ListRequest{Type: reg.Type, Tenancy: tenancy}) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if len(rsp.Resources) > 0 { | ||
rt.Logger.Trace(fmt.Sprintf("found %d tenant %s", len(rsp.Resources), reg.Type.Kind)) | ||
} | ||
|
||
// Delete each qualified tenant. | ||
for _, tenant := range rsp.Resources { | ||
// Owned resources will be deleted when the parent resource is deleted (tombstone reaper) | ||
// so just skip over them. | ||
if tenant.Owner != nil { | ||
numExistingOwned++ | ||
continue | ||
} | ||
|
||
// Skip anything that is already marked for deletion and has finalizers | ||
// since deletion of those resource is out of our control. | ||
if resource.IsMarkedForDeletion(tenant) && resource.HasFinalizers(tenant) { | ||
numExistingHasFinalizer++ | ||
continue | ||
} | ||
|
||
// Delete tenant with a blanket non-CAS delete since we don't care about the version that | ||
// is deleted. Since we don't know whether the delete was immediate or deferred due to the | ||
// presense of a finalizer, we can't assume that the tenant is really deleted. | ||
_, err = rt.Client.Delete(ctx, &pbresource.DeleteRequest{Id: tenant.Id, Version: ""}) | ||
if err != nil { | ||
// Bail on the first sign of trouble and retry in future reconciles. | ||
return err | ||
} | ||
|
||
// Classify the just deleted tenant since we're not fully vacated if a deferred delete occurred. | ||
if resource.HasFinalizers(tenant) { | ||
rt.Logger.Trace(fmt.Sprintf("deferred delete of %s tenant %q", res.Id.Type.Kind, tenant.Id.Name)) | ||
numDeferredDeletes++ | ||
} else { | ||
rt.Logger.Trace(fmt.Sprintf("immediate delete of %s tenant %q", res.Id.Type.Kind, tenant.Id.Name)) | ||
numImmediateDeletes++ | ||
} | ||
} | ||
} | ||
|
||
// Force re-reconcile if we have any lingering tenants by returning an error. | ||
if numExistingOwned+numExistingHasFinalizer+numDeferredDeletes > 0 { | ||
if numExistingOwned > 0 { | ||
rt.Logger.Debug(fmt.Sprintf("delete blocked on %d remaining owned tenants", numExistingOwned)) | ||
} | ||
if numExistingHasFinalizer > 0 { | ||
rt.Logger.Debug(fmt.Sprintf("delete blocked on %d remaining tenants with finalizers", numExistingHasFinalizer)) | ||
} | ||
if numDeferredDeletes > 0 { | ||
rt.Logger.Debug(fmt.Sprintf("delete blocked on %d tenants which were just marked for deletion", numDeferredDeletes)) | ||
} | ||
return ErrStillHasTenants | ||
} | ||
|
||
// We should have zero tenants and be good to continue. | ||
rt.Logger.Debug("no tenants - green light the delete") | ||
return nil | ||
} | ||
|
||
// EnsureResourceDelete makes sure a tenancy unit (partition or namespace) with no tenants is finally deleted. | ||
func EnsureResourceDeleted(ctx context.Context, rt controller.Runtime, res *pbresource.Resource, statusKey string) error { | ||
// Remove finalizer if present | ||
if resource.HasFinalizer(res, statusKey) { | ||
resource.RemoveFinalizer(res, statusKey) | ||
_, err := rt.Client.Write(ctx, &pbresource.WriteRequest{Resource: res}) | ||
if err != nil { | ||
rt.Logger.Error("failed write to remove finalizer") | ||
return WriteStatus(ctx, rt, res, statusKey, ConditionDeleted, ReasonDeletionInProgress, err) | ||
} | ||
rt.Logger.Trace("removed finalizer") | ||
} | ||
|
||
// Finally, delete the tenancy unit. | ||
_, err := rt.Client.Delete(ctx, &pbresource.DeleteRequest{Id: res.Id}) | ||
if err != nil { | ||
rt.Logger.Error("failed final delete", "error", err) | ||
return WriteStatus(ctx, rt, res, statusKey, ConditionDeleted, ReasonDeletionInProgress, err) | ||
} | ||
|
||
// Success | ||
rt.Logger.Trace("finally deleted") | ||
return nil | ||
} | ||
|
||
// WriteStatus writes the tenancy resource status only if the status has changed. | ||
// The state and message are based on whether the passed in error is nil | ||
// (state=TRUE, message="") or not (state=FALSE, message=error). The passed in | ||
// error is always returned unless the delegated call to client.WriteStatus | ||
// itself fails. | ||
func WriteStatus(ctx context.Context, rt controller.Runtime, res *pbresource.Resource, statusKey string, condition string, reason string, err error) error { | ||
state := pbresource.Condition_STATE_TRUE | ||
message := "" | ||
if err != nil { | ||
state = pbresource.Condition_STATE_FALSE | ||
message = err.Error() | ||
} | ||
|
||
newStatus := &pbresource.Status{ | ||
ObservedGeneration: res.Generation, | ||
Conditions: []*pbresource.Condition{{ | ||
Type: condition, | ||
State: state, | ||
Reason: reason, | ||
Message: message, | ||
}}, | ||
} | ||
|
||
// Skip the write if the status hasn't changed to keep write amplificiation in check. | ||
if resource.EqualStatus(res.Status[statusKey], newStatus, false) { | ||
return err | ||
} | ||
|
||
_, statusErr := rt.Client.WriteStatus(ctx, &pbresource.WriteStatusRequest{ | ||
Id: res.Id, | ||
Key: statusKey, | ||
Status: newStatus, | ||
}) | ||
|
||
if statusErr != nil { | ||
rt.Logger.Error("failed writing status", "error", statusErr) | ||
return statusErr | ||
} | ||
rt.Logger.Trace("wrote status", "status", newStatus) | ||
return err | ||
} |
95 changes: 95 additions & 0 deletions
95
internal/tenancy/internal/controllers/namespace/controller.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,95 @@ | ||
// Copyright (c) HashiCorp, Inc. | ||
// SPDX-License-Identifier: BUSL-1.1 | ||
|
||
package namespace | ||
|
||
import ( | ||
"context" | ||
|
||
"google.golang.org/grpc/codes" | ||
"google.golang.org/grpc/status" | ||
|
||
"github.com/hashicorp/consul/internal/controller" | ||
"github.com/hashicorp/consul/internal/resource" | ||
"github.com/hashicorp/consul/internal/tenancy/internal/controllers/common" | ||
"github.com/hashicorp/consul/proto-public/pbresource" | ||
pbtenancy "github.com/hashicorp/consul/proto-public/pbtenancy/v2beta1" | ||
) | ||
|
||
const ( | ||
// StatusKey also serves as the finalizer name. | ||
StatusKey = "consul.io/namespace-controller" | ||
|
||
// Conditions and reasons are shared with partitions. See | ||
// common.go for the full list. | ||
) | ||
|
||
func Controller(registry resource.Registry) controller.Controller { | ||
return controller.ForType(pbtenancy.NamespaceType). | ||
WithReconciler(&Reconciler{Registry: registry}) | ||
} | ||
|
||
type Reconciler struct { | ||
Registry resource.Registry | ||
} | ||
|
||
// Reconcile is responsible for reconciling a namespace resource. | ||
// | ||
// When a namespace is created, ensures a finalizer is added for cleanup. | ||
// | ||
// When a namespace is marked for deletion, ensures tenants are deleted and | ||
// the finalizer is removed. | ||
func (r *Reconciler) Reconcile(ctx context.Context, rt controller.Runtime, req controller.Request) error { | ||
rt.Logger = rt.Logger.With("resource", req.ID.Name, "controller", "namespace") | ||
|
||
// Never reconcile the default namespace in the default partition since it is | ||
// created on system startup or snapshot restoration. Resource validation rules | ||
// protect them being deleted. | ||
if req.ID.Tenancy.Partition == resource.DefaultPartitionName && req.ID.Name == resource.DefaultNamespaceName { | ||
rt.Logger.Trace("skipping reconcile of default namespace") | ||
return nil | ||
} | ||
|
||
// Read namespace to make sure we have the latest version. | ||
rsp, err := rt.Client.Read(ctx, &pbresource.ReadRequest{Id: req.ID}) | ||
switch { | ||
case status.Code(err) == codes.NotFound: | ||
// Namespace deleted - nothing to do. | ||
rt.Logger.Trace("namespace not found, nothing to do") | ||
return nil | ||
case err != nil: | ||
rt.Logger.Error("failed read", "error", err) | ||
return err | ||
} | ||
res := rsp.Resource | ||
|
||
if resource.IsMarkedForDeletion(res) { | ||
return ensureDeleted(ctx, rt, r.Registry, res) | ||
} | ||
|
||
if err = common.EnsureHasFinalizer(ctx, rt, res, StatusKey); err != nil { | ||
return err | ||
} | ||
|
||
return common.WriteStatus(ctx, rt, res, StatusKey, common.ConditionAccepted, common.ReasonAcceptedOK, err) | ||
} | ||
|
||
func ensureDeleted(ctx context.Context, rt controller.Runtime, registry resource.Registry, res *pbresource.Resource) error { | ||
tenancy := &pbresource.Tenancy{ | ||
Partition: res.Id.Tenancy.Partition, | ||
Namespace: res.Id.Name, | ||
PeerName: resource.DefaultPeerName, | ||
} | ||
// Delete namespace scoped tenants | ||
if err := common.EnsureTenantsDeleted(ctx, rt, registry, res, resource.ScopeNamespace, tenancy); err != nil { | ||
rt.Logger.Error("failed deleting tenants", "error", err) | ||
return common.WriteStatus(ctx, rt, res, StatusKey, common.ConditionDeleted, common.ReasonDeletionInProgress, err) | ||
} | ||
|
||
// Delete namespace resource since all namespace scoped tenants are deleted | ||
if err := common.EnsureResourceDeleted(ctx, rt, res, StatusKey); err != nil { | ||
rt.Logger.Error("failed deleting namespace", "error", err) | ||
return err | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.