mz_compute_types/
dyncfgs.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Dyncfgs used by the compute layer.
11
12use std::time::Duration;
13
14use mz_dyncfg::{Config, ConfigSet, ParameterScope};
15
16/// Whether rendering should use `half_join2` rather than DD's `half_join` for delta joins.
17///
18/// `half_join2` avoids quadratic behavior in certain join patterns. This flag exists as an escape
19/// hatch to revert to the old implementation if issues arise.
20pub const ENABLE_HALF_JOIN2: Config<bool> = Config::new(
21    "enable_compute_half_join2",
22    true,
23    "Whether compute should use `half_join2` rather than DD's `half_join` to render delta joins.",
24);
25
26/// Use the column-paged merge batcher code path at arrange sites. When
27/// `true`, arrange operators use `Col2ValPagedBatcher` (in
28/// `mz_timely_util::columnar`) and `RowRowColPagedBuilder` (in
29/// `mz_row_spine`), the columnar-native batcher that the pager can spill
30/// (gated by [`ENABLE_COLUMN_PAGED_BATCHER_SPILL`]). When `false` (the
31/// default), the same arrange sites use the legacy `Col2ValBatcher` /
32/// `RowRowBuilder` (columnation-merger) path. Read at operator construction
33/// time. Flips take effect on dataflows created after the change.
34///
35/// Disabled by default while the new path is stabilizing.
36/// `DifferentialJoinHydration*` feature-benchmark scenarios opt in
37/// explicitly so the spill path is measured.
38pub const ENABLE_COLUMN_PAGED_BATCHER: Config<bool> = Config::new(
39    "enable_column_paged_batcher",
40    false,
41    "Use the columnar-native paged merge batcher at arrange sites. When `false` (default), \
42     arranges fall back to the legacy columnation `Col2ValBatcher` / `RowRowBuilder` path.",
43)
44.scoped(ParameterScope::Replica);
45
46/// Allow the column-paged batcher's pager to evict chunks under memory
47/// pressure. Only meaningful when [`ENABLE_COLUMN_PAGED_BATCHER`] is `true`.
48/// With the spill flag off the pager keeps every chunk resident regardless of
49/// budget.
50///
51/// This flag (or the storage-side `enable_upsert_paged_spill`) also gates
52/// installation of the process buffer pool (`mz_ore::pool`): the first
53/// configuration tick with either gate on reserves the pool's virtual
54/// address space and spawns its spill threads. Turning the gates back off
55/// stops retuning but does not tear the installed pool down.
56///
57/// Off by default, even when the batcher path itself is on, so the
58/// no-pressure case stays a pure resident operation. Tune the budget via
59/// [`COLUMN_PAGED_BATCHER_BUDGET_FRACTION`].
60pub const ENABLE_COLUMN_PAGED_BATCHER_SPILL: Config<bool> = Config::new(
61    "enable_column_paged_batcher_spill",
62    false,
63    "Allow the column-paged batcher's pager to evict chunks under memory pressure. Only \
64     meaningful when `enable_column_paged_batcher = true`.",
65)
66.scoped(ParameterScope::Replica);
67
68/// Resident-bytes budget fraction for chunk spilling. Two consumers read
69/// it: the column pager's tiered policy multiplies it against the
70/// announced memory limit, and the buffer pool (`mz_ore::pool`)
71/// multiplies it against physical RAM, since a resident budget must
72/// derive from memory that can be resident and the announced limit
73/// includes swap on swap-provisioned nodes.
74///
75/// `0.05` (5%) is a reasonable starting point: large enough that the
76/// per-call ColumnBuilder ship-threshold (~2 MiB) fits multiple chunks
77/// per worker, small enough that the merge-batcher's transient state
78/// doesn't crowd out the spine. Set lower to spill more aggressively
79/// under pressure. The computed budget is floored at 128 MiB so the
80/// no-pressure case doesn't page per chunk. Ignored when
81/// `enable_column_paged_batcher_spill` is `false`.
82pub const COLUMN_PAGED_BATCHER_BUDGET_FRACTION: Config<f64> = Config::new(
83    "column_paged_batcher_budget_fraction",
84    0.05,
85    "Budget fraction for chunk spilling: the buffer pool multiplies it against physical \
86     RAM and the column pager's tiered policy against the announced memory limit. \
87     Total pool budget = max(ram * fraction, 128 MiB).",
88)
89.scoped(ParameterScope::Replica);
90
91/// Number of buffer-pool spill threads performing eviction I/O (lz4
92/// compression plus the synchronous-reclaim `MADV_PAGEOUT`) off the threads
93/// that trip the budget. Zero evicts inline on the calling thread, which
94/// measurably convoys workers behind eviction I/O at hydration eviction
95/// rates. Thread spawning is once per process: raising the value later has
96/// no effect beyond re-enabling, and lowering it to zero falls back to
97/// inline eviction while spawned threads idle.
98pub const COLUMN_PAGED_BATCHER_SPILL_WORKER_COUNT: Config<usize> = Config::new(
99    "column_paged_batcher_spill_worker_count",
100    2,
101    "Buffer-pool spill threads for off-worker eviction I/O; 0 evicts inline on the caller.",
102)
103.scoped(ParameterScope::Replica);
104
105/// Compress chunks the column-paged batcher spills, using lz4. Only
106/// meaningful when [`ENABLE_COLUMN_PAGED_BATCHER_SPILL`] is `true`; the codec
107/// is applied on the pageout path and reversed on page-in. Trades CPU for a
108/// smaller on-storage (and, for the swap backend, resident) footprint.
109///
110/// Off by default so the spill path's cost stays a pure copy until compression
111/// is shown to pay for itself on the target workload.
112pub const COLUMN_PAGED_BATCHER_LZ4: Config<bool> = Config::new(
113    "column_paged_batcher_lz4",
114    false,
115    "Compress column-paged batcher chunks with lz4 on the spill path. Only meaningful when \
116     `enable_column_paged_batcher_spill = true`.",
117)
118.scoped(ParameterScope::Replica);
119
120/// Proactively evict the column-paged batcher's lz4-compressed spill chunks
121/// from RSS via `MADV_PAGEOUT` when spilling to the swap backend. Only
122/// meaningful when [`COLUMN_PAGED_BATCHER_LZ4`] is `true` and the active
123/// backend is swap (no scratch directory): on that path the compressed bytes
124/// stay resident in the process address space and currently receive no madvise
125/// at all, so the kernel reclaims them only lazily under LRU pressure.
126/// `MADV_PAGEOUT` instead swaps them out eagerly at spill time, holding RSS at
127/// the budget rather than letting it drift up to the pressure cliff. A later
128/// page-in re-faults the pages — cheap because lz4 shrank the byte volume,
129/// which is what makes eager eviction pay off on this path.
130///
131/// Off by default: the eager-reclaim syscall is the one kernel interaction the
132/// pager design singled out as risky, so it stays gated until proven on the
133/// target workload.
134pub const COLUMN_PAGED_BATCHER_SWAP_PAGEOUT: Config<bool> = Config::new(
135    "column_paged_batcher_swap_pageout",
136    false,
137    "Eagerly evict the column-paged batcher's lz4-compressed swap-backend spill chunks from RSS \
138     via `MADV_PAGEOUT` (they otherwise receive no madvise and are reclaimed only lazily). Only \
139     meaningful when `column_paged_batcher_lz4 = true` and the swap backend is active.",
140)
141.scoped(ParameterScope::Replica);
142
143/// Eagerly compress unbacked buffer-pool chunks to `BackedResident` on idle
144/// spill threads (write-behind). The chunk stays readable in its slot while
145/// a compressed extent accumulates on the swap device, so budget-driven
146/// eviction becomes a pure page release instead of a compression. Trades
147/// background CPU (compression of chunks that may die before pressure
148/// reaches them) for near-free pressure response.
149pub const COLUMN_PAGED_BATCHER_EAGER_BACKING: Config<bool> = Config::new(
150    "column_paged_batcher_eager_backing",
151    false,
152    "Eagerly compress buffer-pool chunks to compressed-but-resident on idle spill threads, so \
153     budget-driven eviction is a pure page release. Only meaningful with spill workers.",
154)
155.scoped(ParameterScope::Replica);
156
157/// Ceiling on the buffer pool's total RSS, as a fraction of *physical RAM*
158/// (never the announced limit, which includes swap on swap-provisioned
159/// nodes). The compressed-but-resident extent tier is the headroom above the
160/// slot budget and warm cap: chunks evicted from the budget stay in RAM
161/// compressed (~5.6x denser; reads decompress without faulting) until this
162/// ceiling forces the oldest extents out to the swap device via
163/// `MADV_PAGEOUT`. Zero collapses the tier: extents page out as soon as
164/// they are written.
165///
166/// The default pairs with the 0.05 budget default to leave ~20% of RAM for
167/// the compressed tier — the same share zswap's default compressed pool
168/// takes, and roughly RAM-sized logical coverage at the measured ~5.6x
169/// ratio — while keeping three quarters of RAM for everything else in the
170/// process.
171pub const COLUMN_PAGED_BATCHER_POOL_RSS_TARGET_FRACTION: Config<f64> = Config::new(
172    "column_paged_batcher_pool_rss_target_fraction",
173    0.25,
174    "Ceiling on the buffer pool's total RSS as a fraction of physical RAM; the headroom above \
175     the slot budget holds compressed-but-resident extents. Zero pages extents out immediately.",
176)
177.scoped(ParameterScope::Replica);
178
179/// Whether rendering should use `mz_join_core` rather than DD's `JoinCore::join_core`.
180pub const ENABLE_MZ_JOIN_CORE: Config<bool> = Config::new(
181    "enable_mz_join_core",
182    true,
183    "Whether compute should use `mz_join_core` rather than DD's `JoinCore::join_core` to render \
184     linear joins.",
185);
186
187/// Use sync Timely operators with Tokio tasks for the MV sink.
188pub const ENABLE_SYNC_MV_SINK: Config<bool> = Config::new(
189    "enable_compute_sync_mv_sink",
190    true,
191    "Use sync Timely operators with Tokio tasks for the MV sink.",
192);
193
194/// Whether rendering should use the new MV sink correction buffer implementation.
195pub const ENABLE_CORRECTION_V2: Config<bool> = Config::new(
196    "enable_compute_correction_v2",
197    true,
198    "Whether compute should use the new MV sink correction buffer implementation.",
199);
200
201/// The size factor of subsequent chains in the correction V2 buffer.
202pub const CORRECTION_V2_CHAIN_PROPORTIONALITY: Config<f64> = Config::new(
203    "compute_correction_v2_chain_proportionality",
204    3.0,
205    "The size factor of subsequent chains in the correction V2 buffer.",
206);
207
208/// The byte size of chunks in the correction V2 buffer.
209pub const CORRECTION_V2_CHUNK_SIZE: Config<usize> = Config::new(
210    "compute_correction_v2_chunk_size",
211    8 * 1024,
212    "The byte size of chunks in the correction V2 buffer.",
213);
214
215/// Whether to enable temporal bucketing in compute.
216pub const ENABLE_COMPUTE_TEMPORAL_BUCKETING: Config<bool> = Config::new(
217    "enable_compute_temporal_bucketing",
218    false,
219    "Whether to enable temporal bucketing in compute.",
220);
221
222/// The summary to apply to the frontier in temporal bucketing in compute.
223pub const TEMPORAL_BUCKETING_SUMMARY: Config<Duration> = Config::new(
224    "compute_temporal_bucketing_summary",
225    Duration::from_secs(2),
226    "The summary to apply to frontiers in temporal bucketing in compute.",
227);
228
229/// The yielding behavior with which linear joins should be rendered.
230pub const LINEAR_JOIN_YIELDING: Config<&str> = Config::new(
231    "linear_join_yielding",
232    "work:1000000,time:100",
233    "The yielding behavior compute rendering should apply for linear join operators. Either \
234     'work:<amount>' or 'time:<milliseconds>' or 'work:<amount>,time:<milliseconds>'. Note \
235     that omitting one of 'work' or 'time' will entirely disable join yielding by time or \
236     work, respectively, rather than falling back to some default.",
237);
238
239/// Enable lgalloc.
240pub const ENABLE_LGALLOC: Config<bool> =
241    Config::new("enable_lgalloc", true, "Enable lgalloc.").scoped(ParameterScope::Replica);
242
243/// Enable lgalloc's eager memory return/reclamation feature.
244pub const ENABLE_LGALLOC_EAGER_RECLAMATION: Config<bool> = Config::new(
245    "enable_lgalloc_eager_reclamation",
246    true,
247    "Enable lgalloc's eager return behavior.",
248);
249
250/// The interval at which the background thread wakes.
251pub const LGALLOC_BACKGROUND_INTERVAL: Config<Duration> = Config::new(
252    "lgalloc_background_interval",
253    Duration::from_secs(1),
254    "Scheduling interval for lgalloc's background worker.",
255);
256
257/// Enable lgalloc's eager memory return/reclamation feature.
258pub const LGALLOC_FILE_GROWTH_DAMPENER: Config<usize> = Config::new(
259    "lgalloc_file_growth_dampener",
260    2,
261    "Lgalloc's file growth dampener parameter.",
262);
263
264/// Enable lgalloc's eager memory return/reclamation feature.
265pub const LGALLOC_LOCAL_BUFFER_BYTES: Config<usize> = Config::new(
266    "lgalloc_local_buffer_bytes",
267    64 << 20,
268    "Lgalloc's local buffer bytes parameter.",
269);
270
271/// The bytes to reclaim (slow path) per size class, for each background thread activation.
272pub const LGALLOC_SLOW_CLEAR_BYTES: Config<usize> = Config::new(
273    "lgalloc_slow_clear_bytes",
274    128 << 20,
275    "Clear byte size per size class for every invocation",
276);
277
278/// Interval to run the memory limiter. A zero duration disables the limiter.
279pub const MEMORY_LIMITER_INTERVAL: Config<Duration> = Config::new(
280    "memory_limiter_interval",
281    Duration::from_secs(10),
282    "Interval to run the memory limiter. A zero duration disables the limiter.",
283);
284
285/// Bias to the memory limiter usage factor.
286pub const MEMORY_LIMITER_USAGE_BIAS: Config<f64> = Config::new(
287    "memory_limiter_usage_bias",
288    1.,
289    "Multiplicative bias to the memory limiter's limit.",
290);
291
292/// Burst factor to memory limit.
293pub const MEMORY_LIMITER_BURST_FACTOR: Config<f64> = Config::new(
294    "memory_limiter_burst_factor",
295    0.,
296    "Multiplicative burst factor to the memory limiter's limit.",
297);
298
299/// Enable lgalloc for columnation.
300pub const ENABLE_COLUMNATION_LGALLOC: Config<bool> = Config::new(
301    "enable_columnation_lgalloc",
302    true,
303    "Enable allocating regions from lgalloc.",
304);
305
306/// The interval at which the compute server performs maintenance tasks.
307pub const COMPUTE_SERVER_MAINTENANCE_INTERVAL: Config<Duration> = Config::new(
308    "compute_server_maintenance_interval",
309    Duration::from_millis(10),
310    "The interval at which the compute server performs maintenance tasks. Zero enables maintenance on every iteration.",
311);
312
313/// Maximum number of in-flight bytes emitted by persist_sources feeding dataflows.
314pub const DATAFLOW_MAX_INFLIGHT_BYTES: Config<Option<usize>> = Config::new(
315    "compute_dataflow_max_inflight_bytes",
316    None,
317    "The maximum number of in-flight bytes emitted by persist_sources feeding \
318     compute dataflows in non-cc clusters.",
319);
320
321/// The "physical backpressure" of `compute_dataflow_max_inflight_bytes_cc` has
322/// been replaced in cc replicas by persist lgalloc and we intend to remove it
323/// once everything has switched to cc. In the meantime, this is a CYA to turn
324/// it back on if absolutely necessary.
325pub const DATAFLOW_MAX_INFLIGHT_BYTES_CC: Config<Option<usize>> = Config::new(
326    "compute_dataflow_max_inflight_bytes_cc",
327    None,
328    "The maximum number of in-flight bytes emitted by persist_sources feeding \
329     compute dataflows in cc clusters.",
330);
331
332/// The term `n` in the growth rate `1 + 1/(n + 1)` for `ConsolidatingVec`.
333/// The smallest value `0` corresponds to the greatest allowed growth, of doubling.
334pub const CONSOLIDATING_VEC_GROWTH_DAMPENER: Config<usize> = Config::new(
335    "consolidating_vec_growth_dampener",
336    1,
337    "Dampener in growth rate for consolidating vector size",
338);
339
340/// The number of dataflows that may hydrate concurrently.
341pub const HYDRATION_CONCURRENCY: Config<usize> = Config::new(
342    "compute_hydration_concurrency",
343    4,
344    "Controls how many compute dataflows may hydrate concurrently.",
345);
346
347/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
348pub const COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO: Config<usize> = Config::new(
349    "copy_to_s3_parquet_row_group_file_ratio",
350    20,
351    "The ratio (defined as a percentage) of row-group size to max-file-size. \
352        Must be <= 100.",
353);
354
355/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
356pub const COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO: Config<usize> = Config::new(
357    "copy_to_s3_arrow_builder_buffer_ratio",
358    150,
359    "The ratio (defined as a percentage) of arrow-builder size to row-group size. \
360        Must be >= 100.",
361);
362
363/// The size of each part in the multi-part upload to use when uploading files to S3.
364pub const COPY_TO_S3_MULTIPART_PART_SIZE_BYTES: Config<usize> = Config::new(
365    "copy_to_s3_multipart_part_size_bytes",
366    1024 * 1024 * 8,
367    "The size of each part in a multipart upload to S3.",
368);
369
370/// Main switch to enable or disable replica expiration.
371///
372/// Changes affect existing replicas only after restart.
373pub const ENABLE_COMPUTE_REPLICA_EXPIRATION: Config<bool> = Config::new(
374    "enable_compute_replica_expiration",
375    true,
376    "Main switch to disable replica expiration.",
377);
378
379/// The maximum lifetime of a replica configured as an offset to the replica start time.
380/// Used in temporal filters to drop diffs generated at timestamps beyond the expiration time.
381///
382/// A zero duration implies no expiration. Changing this value does not affect existing replicas,
383/// even when they are restarted.
384pub const COMPUTE_REPLICA_EXPIRATION_OFFSET: Config<Duration> = Config::new(
385    "compute_replica_expiration_offset",
386    Duration::ZERO,
387    "The expiration time offset for replicas. Zero disables expiration.",
388);
389
390/// When enabled, applies the column demands from a MapFilterProject onto the RelationDesc used to
391/// read out of Persist. This allows Persist to prune unneeded columns as a performance
392/// optimization.
393pub const COMPUTE_APPLY_COLUMN_DEMANDS: Config<bool> = Config::new(
394    "compute_apply_column_demands",
395    true,
396    "When enabled, passes applys column demands to the RelationDesc used to read out of Persist.",
397);
398
399/// The amount of output the flat-map operator produces before yielding. Set to a high value to
400/// avoid yielding, or to a low value to yield frequently.
401pub const COMPUTE_FLAT_MAP_FUEL: Config<usize> = Config::new(
402    "compute_flat_map_fuel",
403    1_000_000,
404    "The amount of output the flat-map operator produces before yielding.",
405);
406
407/// Whether to render `as_specific_collection` using a fueled flat-map operator.
408pub const ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION: Config<bool> = Config::new(
409    "enable_compute_render_fueled_as_specific_collection",
410    true,
411    "When enabled, renders `as_specific_collection` using a fueled flat-map operator.",
412);
413
414/// Whether to apply logical backpressure in compute dataflows.
415pub const ENABLE_COMPUTE_LOGICAL_BACKPRESSURE: Config<bool> = Config::new(
416    "enable_compute_logical_backpressure",
417    false,
418    "When enabled, compute dataflows will apply logical backpressure.",
419);
420
421/// Maximal number of capabilities retained by the logical backpressure operator.
422///
423/// Selecting this value is subtle. If it's too small, it'll diminish the effectiveness of the
424/// logical backpressure operators. If it's too big, we can slow down hydration and cause state
425/// in the operator's implementation to build up.
426///
427/// The default value represents a compromise between these two extremes. We retain some metrics
428/// for 30 days, and the metrics update every minute. The default is exactly this number.
429pub const COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES: Config<Option<usize>> =
430    Config::new(
431        "compute_logical_backpressure_max_retained_capabilities",
432        Some(30 * 24 * 60),
433        "The maximum number of capabilities retained by the logical backpressure operator.",
434    );
435
436/// The slack to round observed timestamps up to.
437///
438/// The default corresponds to Mz's default tick interval, but does not need to do so. Ideally,
439/// it is not smaller than the tick interval, but it can be larger.
440pub const COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK: Config<Duration> = Config::new(
441    "compute_logical_backpressure_inflight_slack",
442    Duration::from_secs(1),
443    "Round observed timestamps to slack.",
444);
445
446/// Enable per-column dictionary compression for row containers in arrangements.
447///
448/// The `_alpha` suffix is load-bearing: this feature is not yet considered
449/// production-ready, and the name is meant to make that unmissable at the
450/// `ALTER SYSTEM SET` call site rather than relying on out-of-band warnings.
451///
452/// Disposition: added 2026-06-09, default off; solicit feedback for one month
453/// and remove in the absence of a positive response.
454pub const ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA: Config<bool> = Config::new(
455    "enable_arrangement_dictionary_compression_alpha",
456    false,
457    "Enable arrangement dictionary compression (alpha; not yet production-ready).",
458);
459
460/// Whether to enable the peek response stash, for sending back large peek
461/// responses. The response stash will only be used for results that exceed
462/// `compute_peek_response_stash_threshold_bytes`.
463pub const ENABLE_PEEK_RESPONSE_STASH: Config<bool> = Config::new(
464    "enable_compute_peek_response_stash",
465    true,
466    "Whether to enable the peek response stash, for sending back large peek responses. Will only be used for results that exceed compute_peek_response_stash_threshold_bytes.",
467);
468
469/// The threshold for peek response size above which we should use the peek
470/// response stash. Only used if the peek response stash is enabled _and_ if the
471/// query is "streamable" (roughly: doesn't have an ORDER BY).
472pub const PEEK_RESPONSE_STASH_THRESHOLD_BYTES: Config<usize> = Config::new(
473    "compute_peek_response_stash_threshold_bytes",
474    1024 * 10, /* 10KB */
475    "The threshold above which to use the peek response stash, for sending back large peek responses.",
476);
477
478/// The target number of maximum runs in the batches written to the stash.
479///
480/// Setting this reasonably low will make it so batches get consolidated/sorted
481/// concurrently with data being written. Which will in turn make it so that we
482/// have to do less work when reading/consolidating those batches in
483/// `environmentd`.
484pub const PEEK_RESPONSE_STASH_BATCH_MAX_RUNS: Config<usize> = Config::new(
485    "compute_peek_response_stash_batch_max_runs",
486    // The lowest possible setting, do as much work as possible on the
487    // `clusterd` side.
488    2,
489    "The target number of maximum runs in the batches written to the stash.",
490);
491
492/// The target size for batches of rows we read out of the peek stash.
493pub const PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES: Config<usize> = Config::new(
494    "compute_peek_response_stash_read_batch_size_bytes",
495    1024 * 1024 * 100, /* 100mb */
496    "The target size for batches of rows we read out of the peek stash.",
497);
498
499/// The memory budget for consolidating stashed peek responses in
500/// `environmentd`.
501pub const PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES: Config<usize> = Config::new(
502    "compute_peek_response_stash_read_memory_budget_bytes",
503    1024 * 1024 * 64, /* 64mb */
504    "The memory budget for consolidating stashed peek responses in environmentd.",
505);
506
507/// The number of batches to pump from the peek result iterator when stashing peek responses.
508pub const PEEK_STASH_NUM_BATCHES: Config<usize> = Config::new(
509    "compute_peek_stash_num_batches",
510    100,
511    "The number of batches to pump from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
512);
513
514/// The size of each batch, as number of rows, pumped from the peek result
515/// iterator when stashing peek responses.
516pub const PEEK_STASH_BATCH_SIZE: Config<usize> = Config::new(
517    "compute_peek_stash_batch_size",
518    100000,
519    "The size, as number of rows, of each batch pumped from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
520);
521
522/// The collection interval for the Prometheus metrics introspection source.
523///
524/// Set to zero to disable scraping and retract any existing data.
525pub const COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL: Config<Duration> = Config::new(
526    "compute_prometheus_introspection_scrape_interval",
527    Duration::from_secs(10),
528    "The collection interval for the Prometheus metrics introspection source. Set to zero to disable.",
529);
530
531/// If set, skip fetching or processing the snapshot data for subscribes when possible.
532pub const SUBSCRIBE_SNAPSHOT_OPTIMIZATION: Config<bool> = Config::new(
533    "compute_subscribe_snapshot_optimization",
534    true,
535    "If set, skip fetching or processing the snapshot data for subscribes when possible.",
536);
537
538/// Temporary flag to de-risk the rollout of a release-blocker fix.
539///
540/// TODO: Remove after one, or a couple, releases.
541pub const MV_SINK_ADVANCE_PERSIST_FRONTIERS: Config<bool> = Config::new(
542    "compute_mv_sink_advance_persist_frontiers",
543    true,
544    "Whether the MV sink's write operator advances its internal persist frontiers to the as_of.",
545);
546
547/// Adds the full set of all compute `Config`s.
548pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
549    configs
550        .add(&ENABLE_HALF_JOIN2)
551        .add(&ENABLE_MZ_JOIN_CORE)
552        .add(&ENABLE_SYNC_MV_SINK)
553        .add(&ENABLE_CORRECTION_V2)
554        .add(&CORRECTION_V2_CHAIN_PROPORTIONALITY)
555        .add(&CORRECTION_V2_CHUNK_SIZE)
556        .add(&ENABLE_COMPUTE_TEMPORAL_BUCKETING)
557        .add(&TEMPORAL_BUCKETING_SUMMARY)
558        .add(&LINEAR_JOIN_YIELDING)
559        .add(&ENABLE_LGALLOC)
560        .add(&LGALLOC_BACKGROUND_INTERVAL)
561        .add(&LGALLOC_FILE_GROWTH_DAMPENER)
562        .add(&LGALLOC_LOCAL_BUFFER_BYTES)
563        .add(&LGALLOC_SLOW_CLEAR_BYTES)
564        .add(&MEMORY_LIMITER_INTERVAL)
565        .add(&MEMORY_LIMITER_USAGE_BIAS)
566        .add(&MEMORY_LIMITER_BURST_FACTOR)
567        .add(&ENABLE_LGALLOC_EAGER_RECLAMATION)
568        .add(&ENABLE_COLUMNATION_LGALLOC)
569        .add(&COMPUTE_SERVER_MAINTENANCE_INTERVAL)
570        .add(&DATAFLOW_MAX_INFLIGHT_BYTES)
571        .add(&DATAFLOW_MAX_INFLIGHT_BYTES_CC)
572        .add(&HYDRATION_CONCURRENCY)
573        .add(&COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO)
574        .add(&COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO)
575        .add(&COPY_TO_S3_MULTIPART_PART_SIZE_BYTES)
576        .add(&ENABLE_COMPUTE_REPLICA_EXPIRATION)
577        .add(&COMPUTE_REPLICA_EXPIRATION_OFFSET)
578        .add(&COMPUTE_APPLY_COLUMN_DEMANDS)
579        .add(&COMPUTE_FLAT_MAP_FUEL)
580        .add(&CONSOLIDATING_VEC_GROWTH_DAMPENER)
581        .add(&ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION)
582        .add(&ENABLE_COMPUTE_LOGICAL_BACKPRESSURE)
583        .add(&COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES)
584        .add(&COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK)
585        .add(&ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA)
586        .add(&ENABLE_PEEK_RESPONSE_STASH)
587        .add(&PEEK_RESPONSE_STASH_THRESHOLD_BYTES)
588        .add(&PEEK_RESPONSE_STASH_BATCH_MAX_RUNS)
589        .add(&PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES)
590        .add(&PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES)
591        .add(&PEEK_STASH_NUM_BATCHES)
592        .add(&PEEK_STASH_BATCH_SIZE)
593        .add(&COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL)
594        .add(&SUBSCRIBE_SNAPSHOT_OPTIMIZATION)
595        .add(&MV_SINK_ADVANCE_PERSIST_FRONTIERS)
596        .add(&ENABLE_COLUMN_PAGED_BATCHER)
597        .add(&ENABLE_COLUMN_PAGED_BATCHER_SPILL)
598        .add(&COLUMN_PAGED_BATCHER_BUDGET_FRACTION)
599        .add(&COLUMN_PAGED_BATCHER_LZ4)
600        .add(&COLUMN_PAGED_BATCHER_SWAP_PAGEOUT)
601        .add(&COLUMN_PAGED_BATCHER_SPILL_WORKER_COUNT)
602        .add(&COLUMN_PAGED_BATCHER_EAGER_BACKING)
603        .add(&COLUMN_PAGED_BATCHER_POOL_RSS_TARGET_FRACTION)
604}
mz_compute_types/dyncfgs.rs

mz_compute_types/
dyncfgs.rs
