Make encryption persistence timeout configurable via env var

changelog/25636.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: make the best effort timeout for encryption count tracking persistence configurable via an environment variable.
+```

vault/barrier_aes_gcm.go

@@ -14,6 +14,7 @@
 	"fmt"
 	"io"
 	"math"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -38,7 +39,8 @@
 	autoRotateCheckInterval = 5 * time.Minute
 	legacyRotateReason      = "legacy rotation"
 	// The keyring is persisted before the root key.
-	keyringTimeout = 1 * time.Second
+	defaultKeyringTimeout            = 1 * time.Second
+	bestEffortKeyringTimeoutOverride = "VAULT_ENCRYPTION_COUNT_PERSIST_TIMEOUT"
 )
 
 // Versions of the AESGCM storage methodology
@@ -91,6 +93,8 @@
 	// Used only for testing
 	RemoteEncryptions     *atomic.Int64
 	totalLocalEncryptions *atomic.Int64
+
+	bestEffortKeyringTimeout time.Duration
 }
 
 func (b *AESGCMBarrier) RotationConfig() (kc KeyRotationConfig, err error) {
@@ -115,6 +119,15 @@
 // NewAESGCMBarrier is used to construct a new barrier that uses
 // the provided physical backend for storage.
 func NewAESGCMBarrier(physical physical.Backend) (*AESGCMBarrier, error) {
+	keyringTimeout := defaultKeyringTimeout
+	keyringTimeoutStr := os.Getenv(bestEffortKeyringTimeoutOverride)
+	if keyringTimeoutStr != "" {
+		t, err := time.ParseDuration(keyringTimeoutStr)
+		if err != nil {
+			return nil, err
+		}
+		keyringTimeout = t
+	}
 	b := &AESGCMBarrier{
 		backend:                  physical,
 		sealed:                   true,
@@ -123,6 +136,7 @@
 		UnaccountedEncryptions:   atomic.NewInt64(0),
 		RemoteEncryptions:        atomic.NewInt64(0),
 		totalLocalEncryptions:    atomic.NewInt64(0),
+		bestEffortKeyringTimeout: keyringTimeout,
 	}
 	return b, nil
 }
@@ -256,7 +270,7 @@
 		// We reduce the timeout on the initial 'put' but if this succeeds we will
 		// allow longer later on when we try to persist the root key .
 		var cancelKeyring func()
-		ctxKeyring, cancelKeyring = context.WithTimeout(ctx, keyringTimeout)
+		ctxKeyring, cancelKeyring = context.WithTimeout(ctx, b.bestEffortKeyringTimeout)
 		defer cancelKeyring()
 	}
 

website/content/docs/internals/rotation.mdx

@@ -73,3 +73,11 @@ Operators can estimate the number of encryptions by summing the following:
 - The `vault.token.creation` metric where the `token_type` label is `batch`.
 - The `merkle.flushDirty.num_pages` metric.
 - The WAL index.
+
+Vault periodically persists the number of encryptions to support rotation. 
+This save operation has a 1 second timeout to prevent impact to performance
+if Vault is under heavy load.  Because persisting encryptions involves the
+seal backend (if seal wrap is enabled), some seals (such as HSMs) may take
+regularly longer than 1 second to respond.  If this is the case, operators
+may override that timeout by setting the environment variable 
+`VAULT_ENCRYPTION_COUNT_PERSIST_TIMEOUT` to a larger value, such as "5s".