diff --git a/DEPS.bzl b/DEPS.bzl index 279b27957113..6ab96672742c 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1818,10 +1818,10 @@ def go_deps(): patches = [ "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch", ], - sha256 = "7136fa698f76055d99e3cf1055026d3fc00fc0cd6c65f7c27ef202ca769ea10f", - strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20241017141447-c6b097e7e6a0", + sha256 = "8c165990dc3d4d67618b19e45e2c79f5f48ab9df4e19f881ee1cfa82cdd009df", + strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20241017195839-1d2e9e829b92", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017141447-c6b097e7e6a0.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017195839-1d2e9e829b92.zip", ], ) go_repository( diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl index 657567b4b1d2..bf391b59f57f 100644 --- a/build/bazelutil/distdir_files.bzl +++ b/build/bazelutil/distdir_files.bzl @@ -345,7 +345,7 @@ DISTDIR_FILES = { "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20230118201751-21c54148d20b.zip": "ca7776f47e5fecb4c495490a679036bfc29d95bd7625290cfdb9abb0baf97476", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017141447-c6b097e7e6a0.zip": "7136fa698f76055d99e3cf1055026d3fc00fc0cd6c65f7c27ef202ca769ea10f", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017195839-1d2e9e829b92.zip": "8c165990dc3d4d67618b19e45e2c79f5f48ab9df4e19f881ee1cfa82cdd009df", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.5.zip": "11b30528eb0dafc8bc1a5ba39d81277c257cbe6946a7564402f588357c164560", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1", diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt index 06811321d3f6..bbe69ae53f43 100644 --- a/docs/generated/settings/settings-for-tenants.txt +++ b/docs/generated/settings/settings-for-tenants.txt @@ -386,6 +386,7 @@ sql.txn.read_committed_isolation.enabled boolean true set to true to allow trans sql.txn.repeatable_read_isolation.enabled (alias: sql.txn.snapshot_isolation.enabled) boolean false set to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commands application sql.txn_fingerprint_id_cache.capacity integer 100 the maximum number of txn fingerprint IDs stored application storage.columnar_blocks.enabled boolean false set to true to enable columnar-blocks to store KVs in a columnar format system-visible +storage.delete_compaction_excise.enabled boolean true set to false to direct Pebble to not partially excise sstables in delete-only compactions system-visible storage.ingestion.value_blocks.enabled boolean true set to true to enable writing of value blocks in ingestion sstables application storage.max_sync_duration duration 20s maximum duration for disk operations; any operations that take longer than this setting trigger a warning log entry or process crash system-visible storage.max_sync_duration.fatal.enabled boolean true if true, fatal the process when a disk operation exceeds storage.max_sync_duration application diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index 84e79c96fd07..525c0e5e827d 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -340,6 +340,7 @@
sql.txn.repeatable_read_isolation.enabled
(alias: sql.txn.snapshot_isolation.enabled)
booleanfalseset to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commandsServerless/Dedicated/Self-Hosted
sql.txn_fingerprint_id_cache.capacity
integer100the maximum number of txn fingerprint IDs storedServerless/Dedicated/Self-Hosted
storage.columnar_blocks.enabled
booleanfalseset to true to enable columnar-blocks to store KVs in a columnar formatDedicated/Self-hosted (read-write); Serverless (read-only) +
storage.delete_compaction_excise.enabled
booleantrueset to false to direct Pebble to not partially excise sstables in delete-only compactionsDedicated/Self-hosted (read-write); Serverless (read-only)
storage.experimental.eventually_file_only_snapshots.enabled
booleantrueset to false to disable eventually-file-only-snapshots (kv.snapshot_receiver.excise.enabled must also be false)Dedicated/Self-Hosted
storage.ingest_split.enabled
booleantrueset to false to disable ingest-time splitting that lowers write-amplificationDedicated/Self-Hosted
storage.ingestion.value_blocks.enabled
booleantrueset to true to enable writing of value blocks in ingestion sstablesServerless/Dedicated/Self-Hosted diff --git a/go.mod b/go.mod index 4d760465b430..40921e4081a8 100644 --- a/go.mod +++ b/go.mod @@ -135,7 +135,7 @@ require ( github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 github.com/cockroachdb/gostdlib v1.19.0 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b - github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0 + github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92 github.com/cockroachdb/redact v1.1.5 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b diff --git a/go.sum b/go.sum index 244e7404625e..e1a96f46f20d 100644 --- a/go.sum +++ b/go.sum @@ -536,8 +536,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0 h1:bxW1ydleLzHAb6u55RvwOgeOfrhXOF9iaGQ3cjWzvtM= -github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0/go.mod h1:XmS8uVDd9YFw/1R7J0J/CmTUANwT7iGnBRxH9AyDA90= +github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92 h1:AEWpYdO8k0gpPWZtpP8CyTr901vv7yxKVrzkXz5Vte8= +github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92/go.mod h1:XmS8uVDd9YFw/1R7J0J/CmTUANwT7iGnBRxH9AyDA90= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= diff --git a/pkg/storage/pebble.go b/pkg/storage/pebble.go index d43d536bc8ef..936c03a0862c 100644 --- a/pkg/storage/pebble.go +++ b/pkg/storage/pebble.go @@ -111,6 +111,18 @@ var columnarBlocksEnabled = settings.RegisterBoolSetting( settings.WithPublic, ) +// deleteCompactionsCanExcise controls whether delete compactions can +// apply rangedels/rangekeydels on sstables they partially apply to, through +// an excise operation, instead of just applying the rangedels/rangekeydels +// that fully delete sstables. +var deleteCompactionsCanExcise = settings.RegisterBoolSetting( + settings.SystemVisible, + "storage.delete_compaction_excise.enabled", + "set to false to direct Pebble to not partially excise sstables in delete-only compactions", + metamorphic.ConstantWithTestBool( + "storage.delete_compaction_excise.enabled", true), /* defaultValue */ + settings.WithPublic) + // IngestAsFlushable controls whether ingested sstables that overlap the // memtable may be lazily ingested: written to the WAL and enqueued in the list // of flushables (eg, memtables, large batches and now lazily-ingested @@ -1228,6 +1240,9 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) { cfg.opts.Experimental.EnableColumnarBlocks = func() bool { return columnarBlocksEnabled.Get(&cfg.settings.SV) } + cfg.opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool { + return deleteCompactionsCanExcise.Get(&cfg.settings.SV) + } auxDir := cfg.opts.FS.PathJoin(cfg.env.Dir, base.AuxiliaryDir) if !cfg.env.IsReadOnly() {