Merge #132870

132870: storage: add cluster setting delete_compaction_excise.enabled r=aadityasondhi a=itsbilal

This change adds a cluster setting to be able to dynamically
disable excises in delete-only compactions. Excises allow
for range deletions / range key deletions to be applied to
sstables even if they don't fully overlap with the sstable;
we just produce virtual sstables for the remainder of the sstable.

Epic: none

Release note: None

Co-authored-by: Bilal Akhtar <[email protected]>

DEPS.bzl

@@ -1818,10 +1818,10 @@ def go_deps():
  patches = [
  "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
  ],
- sha256 = "7136fa698f76055d99e3cf1055026d3fc00fc0cd6c65f7c27ef202ca769ea10f",
- strip_prefix = "github.com/cockroachdb/[email protected]20241017141447-c6b097e7e6a0",
+ sha256 = "8c165990dc3d4d67618b19e45e2c79f5f48ab9df4e19f881ee1cfa82cdd009df",
+ strip_prefix = "github.com/cockroachdb/[email protected]20241017195839-1d2e9e829b92",
  urls = [
- "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017141447-c6b097e7e6a0.zip",
+ "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017195839-1d2e9e829b92.zip",
  ],
  )
  go_repository(

build/bazelutil/distdir_files.bzl

@@ -345,7 +345,7 @@ DISTDIR_FILES = {
  "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
  "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20230118201751-21c54148d20b.zip": "ca7776f47e5fecb4c495490a679036bfc29d95bd7625290cfdb9abb0baf97476",
  "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
- "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017141447-c6b097e7e6a0.zip": "7136fa698f76055d99e3cf1055026d3fc00fc0cd6c65f7c27ef202ca769ea10f",
+ "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20241017195839-1d2e9e829b92.zip": "8c165990dc3d4d67618b19e45e2c79f5f48ab9df4e19f881ee1cfa82cdd009df",
  "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.5.zip": "11b30528eb0dafc8bc1a5ba39d81277c257cbe6946a7564402f588357c164560",
  "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
  "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

docs/generated/settings/settings-for-tenants.txt

@@ -386,6 +386,7 @@ sql.txn.read_committed_isolation.enabled boolean true set to true to allow trans
 sql.txn.repeatable_read_isolation.enabled (alias: sql.txn.snapshot_isolation.enabled) boolean false set to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commands application
 sql.txn_fingerprint_id_cache.capacity integer 100 the maximum number of txn fingerprint IDs stored application
 storage.columnar_blocks.enabled boolean false set to true to enable columnar-blocks to store KVs in a columnar format system-visible
+storage.delete_compaction_excise.enabled boolean true set to false to direct Pebble to not partially excise sstables in delete-only compactions system-visible
 storage.ingestion.value_blocks.enabled boolean true set to true to enable writing of value blocks in ingestion sstables application
 storage.max_sync_duration duration 20s maximum duration for disk operations; any operations that take longer than this setting trigger a warning log entry or process crash system-visible
 storage.max_sync_duration.fatal.enabled boolean true if true, fatal the process when a disk operation exceeds storage.max_sync_duration application

docs/generated/settings/settings.html

@@ -340,6 +340,7 @@
 <tr><td><div id="setting-sql-txn-snapshot-isolation-enabled" class="anchored"><code>sql.txn.repeatable_read_isolation.enabled<br />(alias: sql.txn.snapshot_isolation.enabled)</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commands</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-txn-fingerprint-id-cache-capacity" class="anchored"><code>sql.txn_fingerprint_id_cache.capacity</code></div></td><td>integer</td><td><code>100</code></td><td>the maximum number of txn fingerprint IDs stored</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-storage-columnar-blocks-enabled" class="anchored"><code>storage.columnar_blocks.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to enable columnar-blocks to store KVs in a columnar format</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
+<tr><td><div id="setting-storage-delete-compaction-excise-enabled" class="anchored"><code>storage.delete_compaction_excise.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to direct Pebble to not partially excise sstables in delete-only compactions</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
 <tr><td><div id="setting-storage-experimental-eventually-file-only-snapshots-enabled" class="anchored"><code>storage.experimental.eventually_file_only_snapshots.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to disable eventually-file-only-snapshots (kv.snapshot_receiver.excise.enabled must also be false)</td><td>Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-storage-ingest-split-enabled" class="anchored"><code>storage.ingest_split.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to disable ingest-time splitting that lowers write-amplification</td><td>Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-storage-ingestion-value-blocks-enabled" class="anchored"><code>storage.ingestion.value_blocks.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable writing of value blocks in ingestion sstables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

go.mod

@@ -135,7 +135,7 @@ require (
  github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
  github.com/cockroachdb/gostdlib v1.19.0
  github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b
- github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0
+ github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92
  github.com/cockroachdb/redact v1.1.5
  github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
  github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

@@ -536,8 +536,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
-github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0 h1:bxW1ydleLzHAb6u55RvwOgeOfrhXOF9iaGQ3cjWzvtM=
-github.com/cockroachdb/pebble v0.0.0-20241017141447-c6b097e7e6a0/go.mod h1:XmS8uVDd9YFw/1R7J0J/CmTUANwT7iGnBRxH9AyDA90=
+github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92 h1:AEWpYdO8k0gpPWZtpP8CyTr901vv7yxKVrzkXz5Vte8=
+github.com/cockroachdb/pebble v0.0.0-20241017195839-1d2e9e829b92/go.mod h1:XmS8uVDd9YFw/1R7J0J/CmTUANwT7iGnBRxH9AyDA90=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

pkg/storage/pebble.go

@@ -111,6 +111,18 @@ var columnarBlocksEnabled = settings.RegisterBoolSetting(
  settings.WithPublic,
 )
 
+// deleteCompactionsCanExcise controls whether delete compactions can
+// apply rangedels/rangekeydels on sstables they partially apply to, through
+// an excise operation, instead of just applying the rangedels/rangekeydels
+// that fully delete sstables.
+var deleteCompactionsCanExcise = settings.RegisterBoolSetting(
+ settings.SystemVisible,
+ "storage.delete_compaction_excise.enabled",
+ "set to false to direct Pebble to not partially excise sstables in delete-only compactions",
+ metamorphic.ConstantWithTestBool(
+ "storage.delete_compaction_excise.enabled", true), /* defaultValue */
+ settings.WithPublic)
+
 // IngestAsFlushable controls whether ingested sstables that overlap the
 // memtable may be lazily ingested: written to the WAL and enqueued in the list
 // of flushables (eg, memtables, large batches and now lazily-ingested
@@ -1228,6 +1240,9 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) {
  cfg.opts.Experimental.EnableColumnarBlocks = func() bool {
  return columnarBlocksEnabled.Get(&cfg.settings.SV)
  }
+ cfg.opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool {
+ return deleteCompactionsCanExcise.Get(&cfg.settings.SV)
+ }
 
  auxDir := cfg.opts.FS.PathJoin(cfg.env.Dir, base.AuxiliaryDir)
  if !cfg.env.IsReadOnly() {