mz_compute_types/
dyncfgs.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Dyncfgs used by the compute layer.
11
12use std::time::Duration;
13
14use mz_dyncfg::{Config, ConfigSet, ParameterScope};
15
16/// Whether rendering should use `half_join2` rather than DD's `half_join` for delta joins.
17///
18/// `half_join2` avoids quadratic behavior in certain join patterns. This flag exists as an escape
19/// hatch to revert to the old implementation if issues arise.
20pub const ENABLE_HALF_JOIN2: Config<bool> = Config::new(
21    "enable_compute_half_join2",
22    true,
23    "Whether compute should use `half_join2` rather than DD's `half_join` to render delta joins.",
24);
25
26/// Whether rendering should collapse error multiplicities to one where it arranges errors.
27///
28/// Error semantics depend only on whether an error is present, so its multiplicity carries no
29/// information a consumer reads. Left uncollapsed, a shared collection contributes its errors once
30/// per plan path that reads it, and because those factors apply again at each level of sharing they
31/// compound multiplicatively until the `Diff` overflows.
32///
33/// Governs sharing within a dataflow only. Sharing across objects is bounded unconditionally, by
34/// normalizing at every boundary another dataflow can read, so what this flag decides is never
35/// durable state and two replicas rendering under different values still write the same thing.
36/// Environment-scoped for now because nothing needs finer granularity, not because finer would be
37/// unsafe.
38pub const ENABLE_ERROR_DISTINCT: Config<bool> = Config::new(
39    "enable_compute_error_distinct",
40    true,
41    "Whether compute rendering should collapse error multiplicities to one where it arranges \
42     errors.",
43);
44
45/// Use the column-paged merge batcher code path at arrange sites. When
46/// `true`, arrange operators use `Col2ValPagedBatcher` (in
47/// `mz_timely_util::columnar`) and `RowRowColPagedBuilder` (in
48/// `mz_row_spine`), the columnar-native batcher that the pager can spill
49/// (gated by [`ENABLE_COLUMN_PAGED_BATCHER_SPILL`]). When `false` (the
50/// default), the same arrange sites use the legacy `Col2ValBatcher` /
51/// `RowRowBuilder` (columnation-merger) path. Read at operator construction
52/// time. Flips take effect on dataflows created after the change.
53///
54/// Disabled by default while the new path is stabilizing.
55/// `DifferentialJoinHydration*` feature-benchmark scenarios opt in
56/// explicitly so the spill path is measured.
57pub const ENABLE_COLUMN_PAGED_BATCHER: Config<bool> = Config::new(
58    "enable_column_paged_batcher",
59    false,
60    "Use the columnar-native paged merge batcher at arrange sites. When `false` (default), \
61     arranges fall back to the legacy columnation `Col2ValBatcher` / `RowRowBuilder` path.",
62)
63.scoped(ParameterScope::Replica);
64
65/// Allow the column-paged batcher's pager to evict chunks under memory
66/// pressure. Only meaningful when [`ENABLE_COLUMN_PAGED_BATCHER`] is `true`.
67/// With the spill flag off the pager keeps every chunk resident regardless of
68/// budget.
69///
70/// This flag (or the storage-side `enable_upsert_paged_spill`) also gates
71/// installation of the process buffer pool (`mz_ore::pool`): the first
72/// configuration tick with either gate on reserves the pool's virtual
73/// address space and spawns its spill threads. Turning the gates back off
74/// stops retuning but does not tear the installed pool down.
75///
76/// Off by default, even when the batcher path itself is on, so the
77/// no-pressure case stays a pure resident operation. Tune the budget via
78/// [`COLUMN_PAGED_BATCHER_BUDGET_FRACTION`].
79pub const ENABLE_COLUMN_PAGED_BATCHER_SPILL: Config<bool> = Config::new(
80    "enable_column_paged_batcher_spill",
81    false,
82    "Allow the column-paged batcher's pager to evict chunks under memory pressure. Only \
83     meaningful when `enable_column_paged_batcher = true`.",
84)
85.scoped(ParameterScope::Replica);
86
87/// Resident-bytes budget fraction for chunk spilling. Two consumers read
88/// it: the column pager's tiered policy multiplies it against the
89/// announced memory limit, and the buffer pool (`mz_ore::pool`)
90/// multiplies it against physical RAM, since a resident budget must
91/// derive from memory that can be resident and the announced limit
92/// includes swap on swap-provisioned nodes.
93///
94/// `0.05` (5%) is a reasonable starting point: large enough that the
95/// per-call ColumnBuilder ship-threshold (~2 MiB) fits multiple chunks
96/// per worker, small enough that the merge-batcher's transient state
97/// doesn't crowd out the spine. Set lower to spill more aggressively
98/// under pressure. The computed budget is floored at 128 MiB so the
99/// no-pressure case doesn't page per chunk. Ignored when
100/// `enable_column_paged_batcher_spill` is `false`.
101pub const COLUMN_PAGED_BATCHER_BUDGET_FRACTION: Config<f64> = Config::new(
102    "column_paged_batcher_budget_fraction",
103    0.05,
104    "Budget fraction for chunk spilling: the buffer pool multiplies it against physical \
105     RAM and the column pager's tiered policy against the announced memory limit. \
106     Total pool budget = max(ram * fraction, 128 MiB).",
107)
108.scoped(ParameterScope::Replica);
109
110/// Number of buffer-pool spill threads performing eviction I/O (lz4
111/// compression plus the synchronous-reclaim `MADV_PAGEOUT`) off the threads
112/// that trip the budget. Zero evicts inline on the calling thread, which
113/// measurably convoys workers behind eviction I/O at hydration eviction
114/// rates. Thread spawning is once per process: raising the value later has
115/// no effect beyond re-enabling, and lowering it to zero falls back to
116/// inline eviction while spawned threads idle.
117pub const COLUMN_PAGED_BATCHER_SPILL_WORKER_COUNT: Config<usize> = Config::new(
118    "column_paged_batcher_spill_worker_count",
119    2,
120    "Buffer-pool spill threads for off-worker eviction I/O; 0 evicts inline on the caller.",
121)
122.scoped(ParameterScope::Replica);
123
124/// Compress chunks the column-paged batcher spills, using lz4. Only
125/// meaningful when [`ENABLE_COLUMN_PAGED_BATCHER_SPILL`] is `true`; the codec
126/// is applied on the pageout path and reversed on page-in. Trades CPU for a
127/// smaller on-storage (and, for the swap backend, resident) footprint.
128///
129/// Off by default so the spill path's cost stays a pure copy until compression
130/// is shown to pay for itself on the target workload.
131pub const COLUMN_PAGED_BATCHER_LZ4: Config<bool> = Config::new(
132    "column_paged_batcher_lz4",
133    false,
134    "Compress column-paged batcher chunks with lz4 on the spill path. Only meaningful when \
135     `enable_column_paged_batcher_spill = true`.",
136)
137.scoped(ParameterScope::Replica);
138
139/// Proactively evict the column-paged batcher's lz4-compressed spill chunks
140/// from RSS via `MADV_PAGEOUT` when spilling to the swap backend. Only
141/// meaningful when [`COLUMN_PAGED_BATCHER_LZ4`] is `true` and the active
142/// backend is swap (no scratch directory): on that path the compressed bytes
143/// stay resident in the process address space and currently receive no madvise
144/// at all, so the kernel reclaims them only lazily under LRU pressure.
145/// `MADV_PAGEOUT` instead swaps them out eagerly at spill time, holding RSS at
146/// the budget rather than letting it drift up to the pressure cliff. A later
147/// page-in re-faults the pages — cheap because lz4 shrank the byte volume,
148/// which is what makes eager eviction pay off on this path.
149///
150/// Off by default: the eager-reclaim syscall is the one kernel interaction the
151/// pager design singled out as risky, so it stays gated until proven on the
152/// target workload.
153pub const COLUMN_PAGED_BATCHER_SWAP_PAGEOUT: Config<bool> = Config::new(
154    "column_paged_batcher_swap_pageout",
155    false,
156    "Eagerly evict the column-paged batcher's lz4-compressed swap-backend spill chunks from RSS \
157     via `MADV_PAGEOUT` (they otherwise receive no madvise and are reclaimed only lazily). Only \
158     meaningful when `column_paged_batcher_lz4 = true` and the swap backend is active.",
159)
160.scoped(ParameterScope::Replica);
161
162/// Eagerly compress unbacked buffer-pool chunks to `BackedResident` on idle
163/// spill threads (write-behind). The chunk stays readable in its slot while
164/// a compressed extent accumulates on the swap device, so budget-driven
165/// eviction becomes a pure page release instead of a compression. Trades
166/// background CPU (compression of chunks that may die before pressure
167/// reaches them) for near-free pressure response.
168pub const COLUMN_PAGED_BATCHER_EAGER_BACKING: Config<bool> = Config::new(
169    "column_paged_batcher_eager_backing",
170    false,
171    "Eagerly compress buffer-pool chunks to compressed-but-resident on idle spill threads, so \
172     budget-driven eviction is a pure page release. Only meaningful with spill workers.",
173)
174.scoped(ParameterScope::Replica);
175
176/// Ceiling on the buffer pool's total RSS, as a fraction of *physical RAM*
177/// (never the announced limit, which includes swap on swap-provisioned
178/// nodes). The compressed-but-resident extent tier is the headroom above the
179/// slot budget and warm cap: chunks evicted from the budget stay in RAM
180/// compressed (~5.6x denser; reads decompress without faulting) until this
181/// ceiling forces the oldest extents out to the swap device via
182/// `MADV_PAGEOUT`. Zero collapses the tier: extents page out as soon as
183/// they are written.
184///
185/// The default pairs with the 0.05 budget default to leave ~20% of RAM for
186/// the compressed tier — the same share zswap's default compressed pool
187/// takes, and roughly RAM-sized logical coverage at the measured ~5.6x
188/// ratio — while keeping three quarters of RAM for everything else in the
189/// process.
190pub const COLUMN_PAGED_BATCHER_POOL_RSS_TARGET_FRACTION: Config<f64> = Config::new(
191    "column_paged_batcher_pool_rss_target_fraction",
192    0.25,
193    "Ceiling on the buffer pool's total RSS as a fraction of physical RAM; the headroom above \
194     the slot budget holds compressed-but-resident extents. Zero pages extents out immediately.",
195)
196.scoped(ParameterScope::Replica);
197
198/// Whether rendering should use `mz_join_core` rather than DD's `JoinCore::join_core`.
199pub const ENABLE_MZ_JOIN_CORE: Config<bool> = Config::new(
200    "enable_mz_join_core",
201    true,
202    "Whether compute should use `mz_join_core` rather than DD's `JoinCore::join_core` to render \
203     linear joins.",
204);
205
206/// Use sync Timely operators with Tokio tasks for the MV sink.
207pub const ENABLE_SYNC_MV_SINK: Config<bool> = Config::new(
208    "enable_compute_sync_mv_sink",
209    false,
210    "Use sync Timely operators with Tokio tasks for the MV sink.",
211);
212
213/// Whether rendering should use the new MV sink correction buffer implementation.
214pub const ENABLE_CORRECTION_V2: Config<bool> = Config::new(
215    "enable_compute_correction_v2",
216    true,
217    "Whether compute should use the new MV sink correction buffer implementation.",
218);
219
220/// The size factor of subsequent chains in the correction V2 buffer.
221pub const CORRECTION_V2_CHAIN_PROPORTIONALITY: Config<f64> = Config::new(
222    "compute_correction_v2_chain_proportionality",
223    3.0,
224    "The size factor of subsequent chains in the correction V2 buffer.",
225);
226
227/// The byte size of chunks in the correction V2 buffer.
228pub const CORRECTION_V2_CHUNK_SIZE: Config<usize> = Config::new(
229    "compute_correction_v2_chunk_size",
230    8 * 1024,
231    "The byte size of chunks in the correction V2 buffer.",
232);
233
234/// Whether to enable temporal bucketing in compute.
235pub const ENABLE_COMPUTE_TEMPORAL_BUCKETING: Config<bool> = Config::new(
236    "enable_compute_temporal_bucketing",
237    false,
238    "Whether to enable temporal bucketing in compute.",
239);
240
241/// The summary to apply to the frontier in temporal bucketing in compute.
242pub const TEMPORAL_BUCKETING_SUMMARY: Config<Duration> = Config::new(
243    "compute_temporal_bucketing_summary",
244    Duration::from_secs(2),
245    "The summary to apply to frontiers in temporal bucketing in compute.",
246);
247
248/// The yielding behavior with which linear joins should be rendered.
249pub const LINEAR_JOIN_YIELDING: Config<&str> = Config::new(
250    "linear_join_yielding",
251    "work:1000000,time:100",
252    "The yielding behavior compute rendering should apply for linear join operators. Either \
253     'work:<amount>' or 'time:<milliseconds>' or 'work:<amount>,time:<milliseconds>'. Note \
254     that omitting one of 'work' or 'time' will entirely disable join yielding by time or \
255     work, respectively, rather than falling back to some default.",
256);
257
258/// Enable lgalloc.
259pub const ENABLE_LGALLOC: Config<bool> =
260    Config::new("enable_lgalloc", true, "Enable lgalloc.").scoped(ParameterScope::Replica);
261
262/// Enable lgalloc's eager memory return/reclamation feature.
263pub const ENABLE_LGALLOC_EAGER_RECLAMATION: Config<bool> = Config::new(
264    "enable_lgalloc_eager_reclamation",
265    true,
266    "Enable lgalloc's eager return behavior.",
267);
268
269/// The interval at which the background thread wakes.
270pub const LGALLOC_BACKGROUND_INTERVAL: Config<Duration> = Config::new(
271    "lgalloc_background_interval",
272    Duration::from_secs(1),
273    "Scheduling interval for lgalloc's background worker.",
274);
275
276/// Enable lgalloc's eager memory return/reclamation feature.
277pub const LGALLOC_FILE_GROWTH_DAMPENER: Config<usize> = Config::new(
278    "lgalloc_file_growth_dampener",
279    2,
280    "Lgalloc's file growth dampener parameter.",
281);
282
283/// Enable lgalloc's eager memory return/reclamation feature.
284pub const LGALLOC_LOCAL_BUFFER_BYTES: Config<usize> = Config::new(
285    "lgalloc_local_buffer_bytes",
286    64 << 20,
287    "Lgalloc's local buffer bytes parameter.",
288);
289
290/// The bytes to reclaim (slow path) per size class, for each background thread activation.
291pub const LGALLOC_SLOW_CLEAR_BYTES: Config<usize> = Config::new(
292    "lgalloc_slow_clear_bytes",
293    128 << 20,
294    "Clear byte size per size class for every invocation",
295);
296
297/// Interval to run the memory limiter. A zero duration disables the limiter.
298pub const MEMORY_LIMITER_INTERVAL: Config<Duration> = Config::new(
299    "memory_limiter_interval",
300    Duration::from_secs(10),
301    "Interval to run the memory limiter. A zero duration disables the limiter.",
302);
303
304/// Bias to the memory limiter usage factor.
305pub const MEMORY_LIMITER_USAGE_BIAS: Config<f64> = Config::new(
306    "memory_limiter_usage_bias",
307    1.,
308    "Multiplicative bias to the memory limiter's limit.",
309);
310
311/// Burst factor to memory limit.
312pub const MEMORY_LIMITER_BURST_FACTOR: Config<f64> = Config::new(
313    "memory_limiter_burst_factor",
314    0.,
315    "Multiplicative burst factor to the memory limiter's limit.",
316);
317
318/// Enable lgalloc for columnation.
319pub const ENABLE_COLUMNATION_LGALLOC: Config<bool> = Config::new(
320    "enable_columnation_lgalloc",
321    true,
322    "Enable allocating regions from lgalloc.",
323);
324
325/// The interval at which the compute server performs maintenance tasks.
326pub const COMPUTE_SERVER_MAINTENANCE_INTERVAL: Config<Duration> = Config::new(
327    "compute_server_maintenance_interval",
328    Duration::from_millis(10),
329    "The interval at which the compute server performs maintenance tasks. Zero enables maintenance on every iteration.",
330);
331
332/// Maximum number of in-flight bytes emitted by persist_sources feeding dataflows.
333pub const DATAFLOW_MAX_INFLIGHT_BYTES: Config<Option<usize>> = Config::new(
334    "compute_dataflow_max_inflight_bytes",
335    None,
336    "The maximum number of in-flight bytes emitted by persist_sources feeding \
337     compute dataflows in non-cc clusters.",
338);
339
340/// The "physical backpressure" of `compute_dataflow_max_inflight_bytes_cc` has
341/// been replaced in cc replicas by persist lgalloc and we intend to remove it
342/// once everything has switched to cc. In the meantime, this is a CYA to turn
343/// it back on if absolutely necessary.
344pub const DATAFLOW_MAX_INFLIGHT_BYTES_CC: Config<Option<usize>> = Config::new(
345    "compute_dataflow_max_inflight_bytes_cc",
346    None,
347    "The maximum number of in-flight bytes emitted by persist_sources feeding \
348     compute dataflows in cc clusters.",
349);
350
351/// The term `n` in the growth rate `1 + 1/(n + 1)` for `ConsolidatingVec`.
352/// The smallest value `0` corresponds to the greatest allowed growth, of doubling.
353pub const CONSOLIDATING_VEC_GROWTH_DAMPENER: Config<usize> = Config::new(
354    "consolidating_vec_growth_dampener",
355    1,
356    "Dampener in growth rate for consolidating vector size",
357);
358
359/// The number of dataflows that may hydrate concurrently.
360pub const HYDRATION_CONCURRENCY: Config<usize> = Config::new(
361    "compute_hydration_concurrency",
362    4,
363    "Controls how many compute dataflows may hydrate concurrently.",
364);
365
366/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
367pub const COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO: Config<usize> = Config::new(
368    "copy_to_s3_parquet_row_group_file_ratio",
369    20,
370    "The ratio (defined as a percentage) of row-group size to max-file-size. \
371        Must be <= 100.",
372);
373
374/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
375pub const COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO: Config<usize> = Config::new(
376    "copy_to_s3_arrow_builder_buffer_ratio",
377    150,
378    "The ratio (defined as a percentage) of arrow-builder size to row-group size. \
379        Must be >= 100.",
380);
381
382/// The size of each part in the multi-part upload to use when uploading files to S3.
383pub const COPY_TO_S3_MULTIPART_PART_SIZE_BYTES: Config<usize> = Config::new(
384    "copy_to_s3_multipart_part_size_bytes",
385    1024 * 1024 * 8,
386    "The size of each part in a multipart upload to S3.",
387);
388
389/// Main switch to enable or disable replica expiration.
390///
391/// Changes affect existing replicas only after restart.
392pub const ENABLE_COMPUTE_REPLICA_EXPIRATION: Config<bool> = Config::new(
393    "enable_compute_replica_expiration",
394    true,
395    "Main switch to disable replica expiration.",
396);
397
398/// The maximum lifetime of a replica configured as an offset to the replica start time.
399/// Used in temporal filters to drop diffs generated at timestamps beyond the expiration time.
400///
401/// A zero duration implies no expiration. Changing this value does not affect existing replicas,
402/// even when they are restarted.
403pub const COMPUTE_REPLICA_EXPIRATION_OFFSET: Config<Duration> = Config::new(
404    "compute_replica_expiration_offset",
405    Duration::ZERO,
406    "The expiration time offset for replicas. Zero disables expiration.",
407);
408
409/// When enabled, applies the column demands from a MapFilterProject onto the RelationDesc used to
410/// read out of Persist. This allows Persist to prune unneeded columns as a performance
411/// optimization.
412pub const COMPUTE_APPLY_COLUMN_DEMANDS: Config<bool> = Config::new(
413    "compute_apply_column_demands",
414    true,
415    "When enabled, passes applys column demands to the RelationDesc used to read out of Persist.",
416);
417
418/// The amount of output the flat-map operator produces before yielding. Set to a high value to
419/// avoid yielding, or to a low value to yield frequently.
420pub const COMPUTE_FLAT_MAP_FUEL: Config<usize> = Config::new(
421    "compute_flat_map_fuel",
422    1_000_000,
423    "The amount of output the flat-map operator produces before yielding.",
424);
425
426/// Whether to render `as_specific_collection` using a fueled flat-map operator.
427pub const ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION: Config<bool> = Config::new(
428    "enable_compute_render_fueled_as_specific_collection",
429    true,
430    "When enabled, renders `as_specific_collection` using a fueled flat-map operator.",
431);
432
433/// Whether to apply logical backpressure in compute dataflows.
434pub const ENABLE_COMPUTE_LOGICAL_BACKPRESSURE: Config<bool> = Config::new(
435    "enable_compute_logical_backpressure",
436    false,
437    "When enabled, compute dataflows will apply logical backpressure.",
438);
439
440/// Maximal number of capabilities retained by the logical backpressure operator.
441///
442/// Selecting this value is subtle. If it's too small, it'll diminish the effectiveness of the
443/// logical backpressure operators. If it's too big, we can slow down hydration and cause state
444/// in the operator's implementation to build up.
445///
446/// The default value represents a compromise between these two extremes. We retain some metrics
447/// for 30 days, and the metrics update every minute. The default is exactly this number.
448pub const COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES: Config<Option<usize>> =
449    Config::new(
450        "compute_logical_backpressure_max_retained_capabilities",
451        Some(30 * 24 * 60),
452        "The maximum number of capabilities retained by the logical backpressure operator.",
453    );
454
455/// The slack to round observed timestamps up to.
456///
457/// The default corresponds to Mz's default tick interval, but does not need to do so. Ideally,
458/// it is not smaller than the tick interval, but it can be larger.
459pub const COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK: Config<Duration> = Config::new(
460    "compute_logical_backpressure_inflight_slack",
461    Duration::from_secs(1),
462    "Round observed timestamps to slack.",
463);
464
465/// Enable per-column dictionary compression for row containers in arrangements.
466///
467/// The `_alpha` suffix is load-bearing: this feature is not yet considered
468/// production-ready, and the name is meant to make that unmissable at the
469/// `ALTER SYSTEM SET` call site rather than relying on out-of-band warnings.
470///
471/// Disposition: added 2026-06-09; solicit feedback for one month and remove in
472/// the absence of a positive response.
473pub const ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA: Config<bool> = Config::new(
474    "enable_arrangement_dictionary_compression_alpha",
475    true,
476    "Enable arrangement dictionary compression (alpha; not yet production-ready).",
477);
478
479/// Whether to enable the peek response stash, for sending back large peek
480/// responses. The response stash will only be used for results that exceed
481/// `compute_peek_response_stash_threshold_bytes`.
482pub const ENABLE_PEEK_RESPONSE_STASH: Config<bool> = Config::new(
483    "enable_compute_peek_response_stash",
484    true,
485    "Whether to enable the peek response stash, for sending back large peek responses. Will only be used for results that exceed compute_peek_response_stash_threshold_bytes.",
486);
487
488/// The threshold for peek response size above which we should use the peek
489/// response stash. Only used if the peek response stash is enabled _and_ if the
490/// query is "streamable" (roughly: doesn't have an ORDER BY).
491pub const PEEK_RESPONSE_STASH_THRESHOLD_BYTES: Config<usize> = Config::new(
492    "compute_peek_response_stash_threshold_bytes",
493    1024 * 10, /* 10KB */
494    "The threshold above which to use the peek response stash, for sending back large peek responses.",
495);
496
497/// The target number of maximum runs in the batches written to the stash.
498///
499/// Setting this reasonably low will make it so batches get consolidated/sorted
500/// concurrently with data being written. Which will in turn make it so that we
501/// have to do less work when reading/consolidating those batches in
502/// `environmentd`.
503pub const PEEK_RESPONSE_STASH_BATCH_MAX_RUNS: Config<usize> = Config::new(
504    "compute_peek_response_stash_batch_max_runs",
505    // The lowest possible setting, do as much work as possible on the
506    // `clusterd` side.
507    2,
508    "The target number of maximum runs in the batches written to the stash.",
509);
510
511/// The target size for batches of rows we read out of the peek stash.
512pub const PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES: Config<usize> = Config::new(
513    "compute_peek_response_stash_read_batch_size_bytes",
514    1024 * 1024 * 100, /* 100mb */
515    "The target size for batches of rows we read out of the peek stash.",
516);
517
518/// The memory budget for consolidating stashed peek responses in
519/// `environmentd`.
520pub const PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES: Config<usize> = Config::new(
521    "compute_peek_response_stash_read_memory_budget_bytes",
522    1024 * 1024 * 64, /* 64mb */
523    "The memory budget for consolidating stashed peek responses in environmentd.",
524);
525
526/// The number of batches to pump from the peek result iterator when stashing peek responses.
527pub const PEEK_STASH_NUM_BATCHES: Config<usize> = Config::new(
528    "compute_peek_stash_num_batches",
529    100,
530    "The number of batches to pump from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
531);
532
533/// The size of each batch, as number of rows, pumped from the peek result
534/// iterator when stashing peek responses.
535pub const PEEK_STASH_BATCH_SIZE: Config<usize> = Config::new(
536    "compute_peek_stash_batch_size",
537    100000,
538    "The size, as number of rows, of each batch pumped from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
539);
540
541/// The collection interval for the Prometheus metrics introspection source.
542///
543/// Set to zero to disable scraping and retract any existing data.
544pub const COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL: Config<Duration> = Config::new(
545    "compute_prometheus_introspection_scrape_interval",
546    Duration::from_secs(10),
547    "The collection interval for the Prometheus metrics introspection source. Set to zero to disable.",
548);
549
550/// If set, skip fetching or processing the snapshot data for subscribes when possible.
551pub const SUBSCRIBE_SNAPSHOT_OPTIMIZATION: Config<bool> = Config::new(
552    "compute_subscribe_snapshot_optimization",
553    true,
554    "If set, skip fetching or processing the snapshot data for subscribes when possible.",
555);
556
557/// Temporary flag to de-risk the rollout of a release-blocker fix.
558///
559/// TODO: Remove after one, or a couple, releases.
560pub const MV_SINK_ADVANCE_PERSIST_FRONTIERS: Config<bool> = Config::new(
561    "compute_mv_sink_advance_persist_frontiers",
562    true,
563    "Whether the MV sink's write operator advances its internal persist frontiers to the as_of.",
564);
565
566/// Adds the full set of all compute `Config`s.
567pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
568    configs
569        .add(&ENABLE_HALF_JOIN2)
570        .add(&ENABLE_ERROR_DISTINCT)
571        .add(&ENABLE_MZ_JOIN_CORE)
572        .add(&ENABLE_SYNC_MV_SINK)
573        .add(&ENABLE_CORRECTION_V2)
574        .add(&CORRECTION_V2_CHAIN_PROPORTIONALITY)
575        .add(&CORRECTION_V2_CHUNK_SIZE)
576        .add(&ENABLE_COMPUTE_TEMPORAL_BUCKETING)
577        .add(&TEMPORAL_BUCKETING_SUMMARY)
578        .add(&LINEAR_JOIN_YIELDING)
579        .add(&ENABLE_LGALLOC)
580        .add(&LGALLOC_BACKGROUND_INTERVAL)
581        .add(&LGALLOC_FILE_GROWTH_DAMPENER)
582        .add(&LGALLOC_LOCAL_BUFFER_BYTES)
583        .add(&LGALLOC_SLOW_CLEAR_BYTES)
584        .add(&MEMORY_LIMITER_INTERVAL)
585        .add(&MEMORY_LIMITER_USAGE_BIAS)
586        .add(&MEMORY_LIMITER_BURST_FACTOR)
587        .add(&ENABLE_LGALLOC_EAGER_RECLAMATION)
588        .add(&ENABLE_COLUMNATION_LGALLOC)
589        .add(&COMPUTE_SERVER_MAINTENANCE_INTERVAL)
590        .add(&DATAFLOW_MAX_INFLIGHT_BYTES)
591        .add(&DATAFLOW_MAX_INFLIGHT_BYTES_CC)
592        .add(&HYDRATION_CONCURRENCY)
593        .add(&COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO)
594        .add(&COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO)
595        .add(&COPY_TO_S3_MULTIPART_PART_SIZE_BYTES)
596        .add(&ENABLE_COMPUTE_REPLICA_EXPIRATION)
597        .add(&COMPUTE_REPLICA_EXPIRATION_OFFSET)
598        .add(&COMPUTE_APPLY_COLUMN_DEMANDS)
599        .add(&COMPUTE_FLAT_MAP_FUEL)
600        .add(&CONSOLIDATING_VEC_GROWTH_DAMPENER)
601        .add(&ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION)
602        .add(&ENABLE_COMPUTE_LOGICAL_BACKPRESSURE)
603        .add(&COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES)
604        .add(&COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK)
605        .add(&ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA)
606        .add(&ENABLE_PEEK_RESPONSE_STASH)
607        .add(&PEEK_RESPONSE_STASH_THRESHOLD_BYTES)
608        .add(&PEEK_RESPONSE_STASH_BATCH_MAX_RUNS)
609        .add(&PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES)
610        .add(&PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES)
611        .add(&PEEK_STASH_NUM_BATCHES)
612        .add(&PEEK_STASH_BATCH_SIZE)
613        .add(&COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL)
614        .add(&SUBSCRIBE_SNAPSHOT_OPTIMIZATION)
615        .add(&MV_SINK_ADVANCE_PERSIST_FRONTIERS)
616        .add(&ENABLE_COLUMN_PAGED_BATCHER)
617        .add(&ENABLE_COLUMN_PAGED_BATCHER_SPILL)
618        .add(&COLUMN_PAGED_BATCHER_BUDGET_FRACTION)
619        .add(&COLUMN_PAGED_BATCHER_LZ4)
620        .add(&COLUMN_PAGED_BATCHER_SWAP_PAGEOUT)
621        .add(&COLUMN_PAGED_BATCHER_SPILL_WORKER_COUNT)
622        .add(&COLUMN_PAGED_BATCHER_EAGER_BACKING)
623        .add(&COLUMN_PAGED_BATCHER_POOL_RSS_TARGET_FRACTION)
624}
mz_compute_types/dyncfgs.rs

mz_compute_types/
dyncfgs.rs
