mz_compute_types/
dyncfgs.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Dyncfgs used by the compute layer.
11
12use std::time::Duration;
13
14use mz_dyncfg::{Config, ConfigSet, ParameterScope};
15
16/// Whether rendering should use `half_join2` rather than DD's `half_join` for delta joins.
17///
18/// `half_join2` avoids quadratic behavior in certain join patterns. This flag exists as an escape
19/// hatch to revert to the old implementation if issues arise.
20pub const ENABLE_HALF_JOIN2: Config<bool> = Config::new(
21    "enable_compute_half_join2",
22    true,
23    "Whether compute should use `half_join2` rather than DD's `half_join` to render delta joins.",
24    ParameterScope::Environment,
25);
26
27/// Whether rendering should collapse error multiplicities to one where it arranges errors.
28///
29/// Error semantics depend only on whether an error is present, so its multiplicity carries no
30/// information a consumer reads. Left uncollapsed, a shared collection contributes its errors once
31/// per plan path that reads it, and because those factors apply again at each level of sharing they
32/// compound multiplicatively until the `Diff` overflows.
33///
34/// Governs sharing within a dataflow only. Sharing across objects is bounded unconditionally, by
35/// normalizing at every boundary another dataflow can read, so what this flag decides is never
36/// durable state and two replicas rendering under different values still write the same thing.
37/// Environment-scoped for now because nothing needs finer granularity, not because finer would be
38/// unsafe.
39pub const ENABLE_ERROR_DISTINCT: Config<bool> = Config::new(
40    "enable_compute_error_distinct",
41    true,
42    "Whether compute rendering should collapse error multiplicities to one where it arranges \
43     errors.",
44    ParameterScope::Environment,
45);
46
47/// Use the column-paged merge batcher code path at arrange sites. When
48/// `true`, arrange operators use `Col2ValPagedBatcher` (in
49/// `mz_timely_util::columnar`) and `RowRowColPagedBuilder` (in
50/// `mz_row_spine`), the columnar-native batcher that the pager can spill
51/// (gated by [`ENABLE_COLUMN_PAGED_BATCHER_SPILL`]). When `false` (the
52/// default), the same arrange sites use the legacy `Col2ValBatcher` /
53/// `RowRowBuilder` (columnation-merger) path. Read at operator construction
54/// time. Flips take effect on dataflows created after the change.
55///
56/// Disabled by default while the new path is stabilizing.
57/// `DifferentialJoinHydration*` feature-benchmark scenarios opt in
58/// explicitly so the spill path is measured.
59pub const ENABLE_COLUMN_PAGED_BATCHER: Config<bool> = Config::new(
60    "enable_column_paged_batcher",
61    false,
62    "Use the columnar-native paged merge batcher at arrange sites. When `false` (default), \
63     arranges fall back to the legacy columnation `Col2ValBatcher` / `RowRowBuilder` path.",
64    ParameterScope::Replica,
65);
66
67/// Allow the column-paged batcher's pager to evict chunks under memory
68/// pressure. Only meaningful when [`ENABLE_COLUMN_PAGED_BATCHER`] is `true`.
69/// With the spill flag off the pager keeps every chunk resident regardless of
70/// budget.
71///
72/// This flag (or the storage-side `enable_upsert_paged_spill`) also gates
73/// installation of the process buffer pool (`mz_ore::pool`): the first
74/// configuration tick with either gate on reserves the pool's virtual
75/// address space and spawns its spill threads. Turning the gates back off
76/// stops retuning but does not tear the installed pool down.
77///
78/// Off by default, even when the batcher path itself is on, so the
79/// no-pressure case stays a pure resident operation. Tune the budget via
80/// [`COLUMN_PAGED_BATCHER_BUDGET_FRACTION`].
81pub const ENABLE_COLUMN_PAGED_BATCHER_SPILL: Config<bool> = Config::new(
82    "enable_column_paged_batcher_spill",
83    false,
84    "Allow the column-paged batcher's pager to evict chunks under memory pressure. Only \
85     meaningful when `enable_column_paged_batcher = true`.",
86    ParameterScope::Replica,
87);
88
89/// Resident-bytes budget fraction for chunk spilling. Two consumers read
90/// it: the column pager's tiered policy multiplies it against the
91/// announced memory limit, and the buffer pool (`mz_ore::pool`)
92/// multiplies it against physical RAM, since a resident budget must
93/// derive from memory that can be resident and the announced limit
94/// includes swap on swap-provisioned nodes.
95///
96/// `0.05` (5%) is a reasonable starting point: large enough that the
97/// per-call ColumnBuilder ship-threshold (~2 MiB) fits multiple chunks
98/// per worker, small enough that the merge-batcher's transient state
99/// doesn't crowd out the spine. Set lower to spill more aggressively
100/// under pressure. The computed budget is floored at 128 MiB so the
101/// no-pressure case doesn't page per chunk. Ignored when
102/// `enable_column_paged_batcher_spill` is `false`.
103pub const COLUMN_PAGED_BATCHER_BUDGET_FRACTION: Config<f64> = Config::new(
104    "column_paged_batcher_budget_fraction",
105    0.05,
106    "Budget fraction for chunk spilling: the buffer pool multiplies it against physical \
107     RAM and the column pager's tiered policy against the announced memory limit. \
108     Total pool budget = max(ram * fraction, 128 MiB).",
109    ParameterScope::Replica,
110);
111
112/// Number of buffer-pool spill threads performing eviction I/O (lz4
113/// compression plus the synchronous-reclaim `MADV_PAGEOUT`) off the threads
114/// that trip the budget. Zero evicts inline on the calling thread, which
115/// measurably convoys workers behind eviction I/O at hydration eviction
116/// rates. Thread spawning is once per process: raising the value later has
117/// no effect beyond re-enabling, and lowering it to zero falls back to
118/// inline eviction while spawned threads idle.
119pub const COLUMN_PAGED_BATCHER_SPILL_WORKER_COUNT: Config<usize> = Config::new(
120    "column_paged_batcher_spill_worker_count",
121    2,
122    "Buffer-pool spill threads for off-worker eviction I/O; 0 evicts inline on the caller.",
123    ParameterScope::Replica,
124);
125
126/// Compress chunks the column-paged batcher spills, using lz4. Only
127/// meaningful when [`ENABLE_COLUMN_PAGED_BATCHER_SPILL`] is `true`; the codec
128/// is applied on the pageout path and reversed on page-in. Trades CPU for a
129/// smaller on-storage (and, for the swap backend, resident) footprint.
130///
131/// Off by default so the spill path's cost stays a pure copy until compression
132/// is shown to pay for itself on the target workload.
133pub const COLUMN_PAGED_BATCHER_LZ4: Config<bool> = Config::new(
134    "column_paged_batcher_lz4",
135    false,
136    "Compress column-paged batcher chunks with lz4 on the spill path. Only meaningful when \
137     `enable_column_paged_batcher_spill = true`.",
138    ParameterScope::Replica,
139);
140
141/// Proactively evict the column-paged batcher's lz4-compressed spill chunks
142/// from RSS via `MADV_PAGEOUT` when spilling to the swap backend. Only
143/// meaningful when [`COLUMN_PAGED_BATCHER_LZ4`] is `true` and the active
144/// backend is swap (no scratch directory): on that path the compressed bytes
145/// stay resident in the process address space and currently receive no madvise
146/// at all, so the kernel reclaims them only lazily under LRU pressure.
147/// `MADV_PAGEOUT` instead swaps them out eagerly at spill time, holding RSS at
148/// the budget rather than letting it drift up to the pressure cliff. A later
149/// page-in re-faults the pages — cheap because lz4 shrank the byte volume,
150/// which is what makes eager eviction pay off on this path.
151///
152/// Off by default: the eager-reclaim syscall is the one kernel interaction the
153/// pager design singled out as risky, so it stays gated until proven on the
154/// target workload.
155pub const COLUMN_PAGED_BATCHER_SWAP_PAGEOUT: Config<bool> = Config::new(
156    "column_paged_batcher_swap_pageout",
157    false,
158    "Eagerly evict the column-paged batcher's lz4-compressed swap-backend spill chunks from RSS \
159     via `MADV_PAGEOUT` (they otherwise receive no madvise and are reclaimed only lazily). Only \
160     meaningful when `column_paged_batcher_lz4 = true` and the swap backend is active.",
161    ParameterScope::Replica,
162);
163
164/// Eagerly compress unbacked buffer-pool chunks to `BackedResident` on idle
165/// spill threads (write-behind). The chunk stays readable in its slot while
166/// a compressed extent accumulates on the swap device, so budget-driven
167/// eviction becomes a pure page release instead of a compression. Trades
168/// background CPU (compression of chunks that may die before pressure
169/// reaches them) for near-free pressure response.
170pub const COLUMN_PAGED_BATCHER_EAGER_BACKING: Config<bool> = Config::new(
171    "column_paged_batcher_eager_backing",
172    false,
173    "Eagerly compress buffer-pool chunks to compressed-but-resident on idle spill threads, so \
174     budget-driven eviction is a pure page release. Only meaningful with spill workers.",
175    ParameterScope::Replica,
176);
177
178/// Ceiling on the buffer pool's total RSS, as a fraction of *physical RAM*
179/// (never the announced limit, which includes swap on swap-provisioned
180/// nodes). The compressed-but-resident extent tier is the headroom above the
181/// slot budget and warm cap: chunks evicted from the budget stay in RAM
182/// compressed (~5.6x denser; reads decompress without faulting) until this
183/// ceiling forces the oldest extents out to the swap device via
184/// `MADV_PAGEOUT`. Zero collapses the tier: extents page out as soon as
185/// they are written.
186///
187/// The default pairs with the 0.05 budget default to leave ~20% of RAM for
188/// the compressed tier — the same share zswap's default compressed pool
189/// takes, and roughly RAM-sized logical coverage at the measured ~5.6x
190/// ratio — while keeping three quarters of RAM for everything else in the
191/// process.
192pub const COLUMN_PAGED_BATCHER_POOL_RSS_TARGET_FRACTION: Config<f64> = Config::new(
193    "column_paged_batcher_pool_rss_target_fraction",
194    0.25,
195    "Ceiling on the buffer pool's total RSS as a fraction of physical RAM; the headroom above \
196     the slot budget holds compressed-but-resident extents. Zero pages extents out immediately.",
197    ParameterScope::Replica,
198);
199
200/// Whether rendering should use `mz_join_core` rather than DD's `JoinCore::join_core`.
201pub const ENABLE_MZ_JOIN_CORE: Config<bool> = Config::new(
202    "enable_mz_join_core",
203    true,
204    "Whether compute should use `mz_join_core` rather than DD's `JoinCore::join_core` to render \
205     linear joins.",
206    ParameterScope::Environment,
207);
208
209/// Use sync Timely operators with Tokio tasks for the MV sink.
210pub const ENABLE_SYNC_MV_SINK: Config<bool> = Config::new(
211    "enable_compute_sync_mv_sink",
212    false,
213    "Use sync Timely operators with Tokio tasks for the MV sink.",
214    ParameterScope::Environment,
215);
216
217/// Whether rendering should use the new MV sink correction buffer implementation.
218pub const ENABLE_CORRECTION_V2: Config<bool> = Config::new(
219    "enable_compute_correction_v2",
220    true,
221    "Whether compute should use the new MV sink correction buffer implementation.",
222    ParameterScope::Environment,
223);
224
225/// The size factor of subsequent chains in the correction V2 buffer.
226pub const CORRECTION_V2_CHAIN_PROPORTIONALITY: Config<f64> = Config::new(
227    "compute_correction_v2_chain_proportionality",
228    3.0,
229    "The size factor of subsequent chains in the correction V2 buffer.",
230    ParameterScope::Replica,
231);
232
233/// The byte size of chunks in the correction V2 buffer.
234pub const CORRECTION_V2_CHUNK_SIZE: Config<usize> = Config::new(
235    "compute_correction_v2_chunk_size",
236    8 * 1024,
237    "The byte size of chunks in the correction V2 buffer.",
238    ParameterScope::Replica,
239);
240
241/// Whether to enable temporal bucketing in compute.
242pub const ENABLE_COMPUTE_TEMPORAL_BUCKETING: Config<bool> = Config::new(
243    "enable_compute_temporal_bucketing",
244    false,
245    "Whether to enable temporal bucketing in compute.",
246    ParameterScope::Environment,
247);
248
249/// The summary to apply to the frontier in temporal bucketing in compute.
250pub const TEMPORAL_BUCKETING_SUMMARY: Config<Duration> = Config::new(
251    "compute_temporal_bucketing_summary",
252    Duration::from_secs(2),
253    "The summary to apply to frontiers in temporal bucketing in compute.",
254    ParameterScope::Environment,
255);
256
257/// The yielding behavior with which linear joins should be rendered.
258pub const LINEAR_JOIN_YIELDING: Config<&str> = Config::new(
259    "linear_join_yielding",
260    "work:1000000,time:100",
261    "The yielding behavior compute rendering should apply for linear join operators. Either \
262     'work:<amount>' or 'time:<milliseconds>' or 'work:<amount>,time:<milliseconds>'. Note \
263     that omitting one of 'work' or 'time' will entirely disable join yielding by time or \
264     work, respectively, rather than falling back to some default.",
265    ParameterScope::Replica,
266);
267
268/// Enable lgalloc.
269pub const ENABLE_LGALLOC: Config<bool> = Config::new(
270    "enable_lgalloc",
271    true,
272    "Enable lgalloc.",
273    ParameterScope::Replica,
274);
275
276/// Enable lgalloc's eager memory return/reclamation feature.
277pub const ENABLE_LGALLOC_EAGER_RECLAMATION: Config<bool> = Config::new(
278    "enable_lgalloc_eager_reclamation",
279    true,
280    "Enable lgalloc's eager return behavior.",
281    ParameterScope::Replica,
282);
283
284/// The interval at which the background thread wakes.
285pub const LGALLOC_BACKGROUND_INTERVAL: Config<Duration> = Config::new(
286    "lgalloc_background_interval",
287    Duration::from_secs(1),
288    "Scheduling interval for lgalloc's background worker.",
289    ParameterScope::Replica,
290);
291
292/// Enable lgalloc's eager memory return/reclamation feature.
293pub const LGALLOC_FILE_GROWTH_DAMPENER: Config<usize> = Config::new(
294    "lgalloc_file_growth_dampener",
295    2,
296    "Lgalloc's file growth dampener parameter.",
297    ParameterScope::Replica,
298);
299
300/// Enable lgalloc's eager memory return/reclamation feature.
301pub const LGALLOC_LOCAL_BUFFER_BYTES: Config<usize> = Config::new(
302    "lgalloc_local_buffer_bytes",
303    64 << 20,
304    "Lgalloc's local buffer bytes parameter.",
305    ParameterScope::Replica,
306);
307
308/// The bytes to reclaim (slow path) per size class, for each background thread activation.
309pub const LGALLOC_SLOW_CLEAR_BYTES: Config<usize> = Config::new(
310    "lgalloc_slow_clear_bytes",
311    128 << 20,
312    "Clear byte size per size class for every invocation",
313    ParameterScope::Replica,
314);
315
316/// Interval to run the memory limiter. A zero duration disables the limiter.
317pub const MEMORY_LIMITER_INTERVAL: Config<Duration> = Config::new(
318    "memory_limiter_interval",
319    Duration::from_secs(10),
320    "Interval to run the memory limiter. A zero duration disables the limiter.",
321    ParameterScope::Replica,
322);
323
324/// Bias to the memory limiter usage factor.
325pub const MEMORY_LIMITER_USAGE_BIAS: Config<f64> = Config::new(
326    "memory_limiter_usage_bias",
327    1.,
328    "Multiplicative bias to the memory limiter's limit.",
329    ParameterScope::Replica,
330);
331
332/// Burst factor to memory limit.
333pub const MEMORY_LIMITER_BURST_FACTOR: Config<f64> = Config::new(
334    "memory_limiter_burst_factor",
335    0.,
336    "Multiplicative burst factor to the memory limiter's limit.",
337    ParameterScope::Replica,
338);
339
340/// Enable lgalloc for columnation.
341pub const ENABLE_COLUMNATION_LGALLOC: Config<bool> = Config::new(
342    "enable_columnation_lgalloc",
343    true,
344    "Enable allocating regions from lgalloc.",
345    ParameterScope::Replica,
346);
347
348/// The interval at which the compute server performs maintenance tasks.
349pub const COMPUTE_SERVER_MAINTENANCE_INTERVAL: Config<Duration> = Config::new(
350    "compute_server_maintenance_interval",
351    Duration::from_millis(10),
352    "The interval at which the compute server performs maintenance tasks. Zero enables maintenance on every iteration.",
353    ParameterScope::Replica,
354);
355
356/// Maximum number of in-flight bytes emitted by persist_sources feeding dataflows.
357pub const DATAFLOW_MAX_INFLIGHT_BYTES: Config<Option<usize>> = Config::new(
358    "compute_dataflow_max_inflight_bytes",
359    None,
360    "The maximum number of in-flight bytes emitted by persist_sources feeding \
361     compute dataflows in non-cc clusters.",
362    ParameterScope::Replica,
363);
364
365/// The "physical backpressure" of `compute_dataflow_max_inflight_bytes_cc` has
366/// been replaced in cc replicas by persist lgalloc and we intend to remove it
367/// once everything has switched to cc. In the meantime, this is a CYA to turn
368/// it back on if absolutely necessary.
369pub const DATAFLOW_MAX_INFLIGHT_BYTES_CC: Config<Option<usize>> = Config::new(
370    "compute_dataflow_max_inflight_bytes_cc",
371    None,
372    "The maximum number of in-flight bytes emitted by persist_sources feeding \
373     compute dataflows in cc clusters.",
374    ParameterScope::Replica,
375);
376
377/// The term `n` in the growth rate `1 + 1/(n + 1)` for `ConsolidatingVec`.
378/// The smallest value `0` corresponds to the greatest allowed growth, of doubling.
379pub const CONSOLIDATING_VEC_GROWTH_DAMPENER: Config<usize> = Config::new(
380    "consolidating_vec_growth_dampener",
381    1,
382    "Dampener in growth rate for consolidating vector size",
383    ParameterScope::Replica,
384);
385
386/// The number of dataflows that may hydrate concurrently.
387///
388/// Enforced in `environmentd`, by the controller's per-replica hydration
389/// interceptor withholding `Schedule` commands, rather than by the replica. The
390/// interceptor resolves it from the configuration commands it observes, which
391/// are already specialized for its replica, so the limit still follows the
392/// replica's scoped override.
393pub const HYDRATION_CONCURRENCY: Config<usize> = Config::new(
394    "compute_hydration_concurrency",
395    4,
396    "Controls how many compute dataflows may hydrate concurrently.",
397    ParameterScope::Replica,
398);
399
400/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
401pub const COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO: Config<usize> = Config::new(
402    "copy_to_s3_parquet_row_group_file_ratio",
403    20,
404    "The ratio (defined as a percentage) of row-group size to max-file-size. \
405        Must be <= 100.",
406    ParameterScope::Environment,
407);
408
409/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
410pub const COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO: Config<usize> = Config::new(
411    "copy_to_s3_arrow_builder_buffer_ratio",
412    150,
413    "The ratio (defined as a percentage) of arrow-builder size to row-group size. \
414        Must be >= 100.",
415    ParameterScope::Environment,
416);
417
418/// The size of each part in the multi-part upload to use when uploading files to S3.
419pub const COPY_TO_S3_MULTIPART_PART_SIZE_BYTES: Config<usize> = Config::new(
420    "copy_to_s3_multipart_part_size_bytes",
421    1024 * 1024 * 8,
422    "The size of each part in a multipart upload to S3.",
423    ParameterScope::Environment,
424);
425
426/// Main switch to enable or disable replica expiration.
427///
428/// Changes affect existing replicas only after restart.
429///
430/// The env-wide kill switch for the feature, read in `environmentd` when
431/// specializing `CreateInstance` for a replica. [`COMPUTE_REPLICA_EXPIRATION_OFFSET`]
432/// is the replica-scoped half of the pair.
433pub const ENABLE_COMPUTE_REPLICA_EXPIRATION: Config<bool> = Config::new(
434    "enable_compute_replica_expiration",
435    true,
436    "Main switch to disable replica expiration.",
437    ParameterScope::Environment,
438);
439
440/// The maximum lifetime of a replica configured as an offset to the replica start time.
441/// Used in temporal filters to drop diffs generated at timestamps beyond the expiration time.
442///
443/// A zero duration implies no expiration. Changing this value does not affect existing replicas,
444/// even when they are restarted.
445pub const COMPUTE_REPLICA_EXPIRATION_OFFSET: Config<Duration> = Config::new(
446    "compute_replica_expiration_offset",
447    Duration::ZERO,
448    "The expiration time offset for replicas. Zero disables expiration.",
449    ParameterScope::Replica,
450);
451
452/// When enabled, applies the column demands from a MapFilterProject onto the RelationDesc used to
453/// read out of Persist. This allows Persist to prune unneeded columns as a performance
454/// optimization.
455pub const COMPUTE_APPLY_COLUMN_DEMANDS: Config<bool> = Config::new(
456    "compute_apply_column_demands",
457    true,
458    "When enabled, passes applys column demands to the RelationDesc used to read out of Persist.",
459    ParameterScope::Environment,
460);
461
462/// The amount of output the flat-map operator produces before yielding. Set to a high value to
463/// avoid yielding, or to a low value to yield frequently.
464pub const COMPUTE_FLAT_MAP_FUEL: Config<usize> = Config::new(
465    "compute_flat_map_fuel",
466    1_000_000,
467    "The amount of output the flat-map operator produces before yielding.",
468    ParameterScope::Replica,
469);
470
471/// Whether to render `as_specific_collection` using a fueled flat-map operator.
472pub const ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION: Config<bool> = Config::new(
473    "enable_compute_render_fueled_as_specific_collection",
474    true,
475    "When enabled, renders `as_specific_collection` using a fueled flat-map operator.",
476    ParameterScope::Environment,
477);
478
479/// Whether to apply logical backpressure in compute dataflows.
480pub const ENABLE_COMPUTE_LOGICAL_BACKPRESSURE: Config<bool> = Config::new(
481    "enable_compute_logical_backpressure",
482    false,
483    "When enabled, compute dataflows will apply logical backpressure.",
484    ParameterScope::Replica,
485);
486
487/// Maximal number of capabilities retained by the logical backpressure operator.
488///
489/// Selecting this value is subtle. If it's too small, it'll diminish the effectiveness of the
490/// logical backpressure operators. If it's too big, we can slow down hydration and cause state
491/// in the operator's implementation to build up.
492///
493/// The default value represents a compromise between these two extremes. We retain some metrics
494/// for 30 days, and the metrics update every minute. The default is exactly this number.
495pub const COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES: Config<Option<usize>> =
496    Config::new(
497        "compute_logical_backpressure_max_retained_capabilities",
498        Some(30 * 24 * 60),
499        "The maximum number of capabilities retained by the logical backpressure operator.",
500        ParameterScope::Replica,
501    );
502
503/// The slack to round observed timestamps up to.
504///
505/// The default corresponds to Mz's default tick interval, but does not need to do so. Ideally,
506/// it is not smaller than the tick interval, but it can be larger.
507pub const COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK: Config<Duration> = Config::new(
508    "compute_logical_backpressure_inflight_slack",
509    Duration::from_secs(1),
510    "Round observed timestamps to slack.",
511    ParameterScope::Replica,
512);
513
514/// Enable per-column dictionary compression for row containers in arrangements.
515///
516/// The `_alpha` suffix is load-bearing: this feature is not yet considered
517/// production-ready, and the name is meant to make that unmissable at the
518/// `ALTER SYSTEM SET` call site rather than relying on out-of-band warnings.
519///
520/// Disposition: added 2026-06-09; solicit feedback for one month and remove in
521/// the absence of a positive response.
522pub const ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA: Config<bool> = Config::new(
523    "enable_arrangement_dictionary_compression_alpha",
524    true,
525    "Enable arrangement dictionary compression (alpha; not yet production-ready).",
526    ParameterScope::Replica,
527);
528
529/// Whether to enable the peek response stash, for sending back large peek
530/// responses. The response stash will only be used for results that exceed
531/// `compute_peek_response_stash_threshold_bytes`.
532pub const ENABLE_PEEK_RESPONSE_STASH: Config<bool> = Config::new(
533    "enable_compute_peek_response_stash",
534    true,
535    "Whether to enable the peek response stash, for sending back large peek responses. Will only be used for results that exceed compute_peek_response_stash_threshold_bytes.",
536    ParameterScope::Environment,
537);
538
539/// The threshold for peek response size above which we should use the peek
540/// response stash. Only used if the peek response stash is enabled _and_ if the
541/// query is "streamable" (roughly: doesn't have an ORDER BY).
542pub const PEEK_RESPONSE_STASH_THRESHOLD_BYTES: Config<usize> = Config::new(
543    "compute_peek_response_stash_threshold_bytes",
544    1024 * 10, /* 10KB */
545    "The threshold above which to use the peek response stash, for sending back large peek responses.",
546    ParameterScope::Environment,
547);
548
549/// The target number of maximum runs in the batches written to the stash.
550///
551/// Setting this reasonably low will make it so batches get consolidated/sorted
552/// concurrently with data being written. Which will in turn make it so that we
553/// have to do less work when reading/consolidating those batches in
554/// `environmentd`.
555pub const PEEK_RESPONSE_STASH_BATCH_MAX_RUNS: Config<usize> = Config::new(
556    "compute_peek_response_stash_batch_max_runs",
557    // The lowest possible setting, do as much work as possible on the
558    // `clusterd` side.
559    2,
560    "The target number of maximum runs in the batches written to the stash.",
561    ParameterScope::Environment,
562);
563
564/// The target size for batches of rows we read out of the peek stash.
565pub const PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES: Config<usize> = Config::new(
566    "compute_peek_response_stash_read_batch_size_bytes",
567    1024 * 1024 * 100, /* 100mb */
568    "The target size for batches of rows we read out of the peek stash.",
569    ParameterScope::Environment,
570);
571
572/// The memory budget for consolidating stashed peek responses in
573/// `environmentd`.
574pub const PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES: Config<usize> = Config::new(
575    "compute_peek_response_stash_read_memory_budget_bytes",
576    1024 * 1024 * 64, /* 64mb */
577    "The memory budget for consolidating stashed peek responses in environmentd.",
578    ParameterScope::Environment,
579);
580
581/// The number of batches to pump from the peek result iterator when stashing peek responses.
582pub const PEEK_STASH_NUM_BATCHES: Config<usize> = Config::new(
583    "compute_peek_stash_num_batches",
584    100,
585    "The number of batches to pump from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
586    ParameterScope::Environment,
587);
588
589/// The size of each batch, as number of rows, pumped from the peek result
590/// iterator when stashing peek responses.
591pub const PEEK_STASH_BATCH_SIZE: Config<usize> = Config::new(
592    "compute_peek_stash_batch_size",
593    100000,
594    "The size, as number of rows, of each batch pumped from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
595    ParameterScope::Environment,
596);
597
598/// The collection interval for the Prometheus metrics introspection source.
599///
600/// Set to zero to disable scraping and retract any existing data.
601pub const COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL: Config<Duration> = Config::new(
602    "compute_prometheus_introspection_scrape_interval",
603    Duration::from_secs(10),
604    "The collection interval for the Prometheus metrics introspection source. Set to zero to disable.",
605    ParameterScope::Replica,
606);
607
608/// If set, skip fetching or processing the snapshot data for subscribes when possible.
609///
610/// Read twice. At plan time in `environmentd` it gates whether snapshot elision runs at all, and
611/// at render time on the replica it gates whether an elided snapshot is honored. The replica-side
612/// read only ever puts a snapshot back, never takes one away, so the two reads disagreeing costs
613/// work rather than correctness.
614///
615/// Environment-scoped because the plan-time read has no replica in scope. Making it
616/// cluster-coherent instead would need plan-time resolution of cluster overrides for
617/// `OptimizerConfig` fields that are not `OptimizerFeatures`, which is the only place cluster
618/// overrides are resolved today.
619pub const SUBSCRIBE_SNAPSHOT_OPTIMIZATION: Config<bool> = Config::new(
620    "compute_subscribe_snapshot_optimization",
621    true,
622    "If set, skip fetching or processing the snapshot data for subscribes when possible.",
623    ParameterScope::Environment,
624);
625
626/// Temporary flag to de-risk the rollout of a release-blocker fix.
627///
628/// TODO: Remove after one, or a couple, releases.
629pub const MV_SINK_ADVANCE_PERSIST_FRONTIERS: Config<bool> = Config::new(
630    "compute_mv_sink_advance_persist_frontiers",
631    true,
632    "Whether the MV sink's write operator advances its internal persist frontiers to the as_of.",
633    ParameterScope::Environment,
634);
635
636/// Adds the full set of all compute `Config`s.
637pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
638    configs
639        .add(&ENABLE_HALF_JOIN2)
640        .add(&ENABLE_ERROR_DISTINCT)
641        .add(&ENABLE_MZ_JOIN_CORE)
642        .add(&ENABLE_SYNC_MV_SINK)
643        .add(&ENABLE_CORRECTION_V2)
644        .add(&CORRECTION_V2_CHAIN_PROPORTIONALITY)
645        .add(&CORRECTION_V2_CHUNK_SIZE)
646        .add(&ENABLE_COMPUTE_TEMPORAL_BUCKETING)
647        .add(&TEMPORAL_BUCKETING_SUMMARY)
648        .add(&LINEAR_JOIN_YIELDING)
649        .add(&ENABLE_LGALLOC)
650        .add(&LGALLOC_BACKGROUND_INTERVAL)
651        .add(&LGALLOC_FILE_GROWTH_DAMPENER)
652        .add(&LGALLOC_LOCAL_BUFFER_BYTES)
653        .add(&LGALLOC_SLOW_CLEAR_BYTES)
654        .add(&MEMORY_LIMITER_INTERVAL)
655        .add(&MEMORY_LIMITER_USAGE_BIAS)
656        .add(&MEMORY_LIMITER_BURST_FACTOR)
657        .add(&ENABLE_LGALLOC_EAGER_RECLAMATION)
658        .add(&ENABLE_COLUMNATION_LGALLOC)
659        .add(&COMPUTE_SERVER_MAINTENANCE_INTERVAL)
660        .add(&DATAFLOW_MAX_INFLIGHT_BYTES)
661        .add(&DATAFLOW_MAX_INFLIGHT_BYTES_CC)
662        .add(&HYDRATION_CONCURRENCY)
663        .add(&COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO)
664        .add(&COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO)
665        .add(&COPY_TO_S3_MULTIPART_PART_SIZE_BYTES)
666        .add(&ENABLE_COMPUTE_REPLICA_EXPIRATION)
667        .add(&COMPUTE_REPLICA_EXPIRATION_OFFSET)
668        .add(&COMPUTE_APPLY_COLUMN_DEMANDS)
669        .add(&COMPUTE_FLAT_MAP_FUEL)
670        .add(&CONSOLIDATING_VEC_GROWTH_DAMPENER)
671        .add(&ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION)
672        .add(&ENABLE_COMPUTE_LOGICAL_BACKPRESSURE)
673        .add(&COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES)
674        .add(&COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK)
675        .add(&ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA)
676        .add(&ENABLE_PEEK_RESPONSE_STASH)
677        .add(&PEEK_RESPONSE_STASH_THRESHOLD_BYTES)
678        .add(&PEEK_RESPONSE_STASH_BATCH_MAX_RUNS)
679        .add(&PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES)
680        .add(&PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES)
681        .add(&PEEK_STASH_NUM_BATCHES)
682        .add(&PEEK_STASH_BATCH_SIZE)
683        .add(&COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL)
684        .add(&SUBSCRIBE_SNAPSHOT_OPTIMIZATION)
685        .add(&MV_SINK_ADVANCE_PERSIST_FRONTIERS)
686        .add(&ENABLE_COLUMN_PAGED_BATCHER)
687        .add(&ENABLE_COLUMN_PAGED_BATCHER_SPILL)
688        .add(&COLUMN_PAGED_BATCHER_BUDGET_FRACTION)
689        .add(&COLUMN_PAGED_BATCHER_LZ4)
690        .add(&COLUMN_PAGED_BATCHER_SWAP_PAGEOUT)
691        .add(&COLUMN_PAGED_BATCHER_SPILL_WORKER_COUNT)
692        .add(&COLUMN_PAGED_BATCHER_EAGER_BACKING)
693        .add(&COLUMN_PAGED_BATCHER_POOL_RSS_TARGET_FRACTION)
694}
mz_compute_types/dyncfgs.rs

mz_compute_types/
dyncfgs.rs
