1use std::time::Duration;
13
14use mz_dyncfg::{Config, ConfigSet, ParameterScope};
15
16pub const ENABLE_HALF_JOIN2: Config<bool> = Config::new(
21 "enable_compute_half_join2",
22 true,
23 "Whether compute should use `half_join2` rather than DD's `half_join` to render delta joins.",
24);
25
26pub const ENABLE_COLUMN_PAGED_BATCHER: Config<bool> = Config::new(
40 "enable_column_paged_batcher",
41 false,
42 "Use the columnar-native paged merge batcher at arrange sites. When `false` (default), \
43 arranges fall back to the legacy columnation `Col2ValBatcher` / `RowRowBuilder` path.",
44)
45.scoped(ParameterScope::Replica);
46
47pub const ENABLE_COLUMN_PAGED_BATCHER_SPILL: Config<bool> = Config::new(
56 "enable_column_paged_batcher_spill",
57 false,
58 "Allow the column-paged batcher's pager to evict chunks under memory pressure. Only \
59 meaningful when `enable_column_paged_batcher = true`.",
60)
61.scoped(ParameterScope::Replica);
62
63pub const COLUMN_PAGED_BATCHER_BUDGET_FRACTION: Config<f64> = Config::new(
78 "column_paged_batcher_budget_fraction",
79 0.05,
80 "Fraction of replica memory the column-paged batcher's tiered policy may hold resident \
81 before spilling to the backend. Total budget = max(mem_limit * fraction, 128 MiB).",
82);
83
84pub const COLUMN_PAGED_BATCHER_LZ4: Config<bool> = Config::new(
92 "column_paged_batcher_lz4",
93 false,
94 "Compress column-paged batcher chunks with lz4 on the spill path. Only meaningful when \
95 `enable_column_paged_batcher_spill = true`.",
96)
97.scoped(ParameterScope::Replica);
98
99pub const COLUMN_PAGED_BATCHER_SWAP_PAGEOUT: Config<bool> = Config::new(
114 "column_paged_batcher_swap_pageout",
115 false,
116 "Eagerly evict the column-paged batcher's lz4-compressed swap-backend spill chunks from RSS \
117 via `MADV_PAGEOUT` (they otherwise receive no madvise and are reclaimed only lazily). Only \
118 meaningful when `column_paged_batcher_lz4 = true` and the swap backend is active.",
119);
120
121pub const ENABLE_MZ_JOIN_CORE: Config<bool> = Config::new(
123 "enable_mz_join_core",
124 true,
125 "Whether compute should use `mz_join_core` rather than DD's `JoinCore::join_core` to render \
126 linear joins.",
127);
128
129pub const ENABLE_SYNC_MV_SINK: Config<bool> = Config::new(
131 "enable_compute_sync_mv_sink",
132 true,
133 "Use sync Timely operators with Tokio tasks for the MV sink.",
134);
135
136pub const ENABLE_CORRECTION_V2: Config<bool> = Config::new(
138 "enable_compute_correction_v2",
139 true,
140 "Whether compute should use the new MV sink correction buffer implementation.",
141);
142
143pub const CORRECTION_V2_CHAIN_PROPORTIONALITY: Config<f64> = Config::new(
145 "compute_correction_v2_chain_proportionality",
146 3.0,
147 "The size factor of subsequent chains in the correction V2 buffer.",
148);
149
150pub const CORRECTION_V2_CHUNK_SIZE: Config<usize> = Config::new(
152 "compute_correction_v2_chunk_size",
153 8 * 1024,
154 "The byte size of chunks in the correction V2 buffer.",
155);
156
157pub const ENABLE_COMPUTE_TEMPORAL_BUCKETING: Config<bool> = Config::new(
159 "enable_compute_temporal_bucketing",
160 false,
161 "Whether to enable temporal bucketing in compute.",
162);
163
164pub const TEMPORAL_BUCKETING_SUMMARY: Config<Duration> = Config::new(
166 "compute_temporal_bucketing_summary",
167 Duration::from_secs(2),
168 "The summary to apply to frontiers in temporal bucketing in compute.",
169);
170
171pub const LINEAR_JOIN_YIELDING: Config<&str> = Config::new(
173 "linear_join_yielding",
174 "work:1000000,time:100",
175 "The yielding behavior compute rendering should apply for linear join operators. Either \
176 'work:<amount>' or 'time:<milliseconds>' or 'work:<amount>,time:<milliseconds>'. Note \
177 that omitting one of 'work' or 'time' will entirely disable join yielding by time or \
178 work, respectively, rather than falling back to some default.",
179);
180
181pub const ENABLE_LGALLOC: Config<bool> =
183 Config::new("enable_lgalloc", true, "Enable lgalloc.").scoped(ParameterScope::Replica);
184
185pub const ENABLE_LGALLOC_EAGER_RECLAMATION: Config<bool> = Config::new(
187 "enable_lgalloc_eager_reclamation",
188 true,
189 "Enable lgalloc's eager return behavior.",
190);
191
192pub const LGALLOC_BACKGROUND_INTERVAL: Config<Duration> = Config::new(
194 "lgalloc_background_interval",
195 Duration::from_secs(1),
196 "Scheduling interval for lgalloc's background worker.",
197);
198
199pub const LGALLOC_FILE_GROWTH_DAMPENER: Config<usize> = Config::new(
201 "lgalloc_file_growth_dampener",
202 2,
203 "Lgalloc's file growth dampener parameter.",
204);
205
206pub const LGALLOC_LOCAL_BUFFER_BYTES: Config<usize> = Config::new(
208 "lgalloc_local_buffer_bytes",
209 64 << 20,
210 "Lgalloc's local buffer bytes parameter.",
211);
212
213pub const LGALLOC_SLOW_CLEAR_BYTES: Config<usize> = Config::new(
215 "lgalloc_slow_clear_bytes",
216 128 << 20,
217 "Clear byte size per size class for every invocation",
218);
219
220pub const MEMORY_LIMITER_INTERVAL: Config<Duration> = Config::new(
222 "memory_limiter_interval",
223 Duration::from_secs(10),
224 "Interval to run the memory limiter. A zero duration disables the limiter.",
225);
226
227pub const MEMORY_LIMITER_USAGE_BIAS: Config<f64> = Config::new(
229 "memory_limiter_usage_bias",
230 1.,
231 "Multiplicative bias to the memory limiter's limit.",
232);
233
234pub const MEMORY_LIMITER_BURST_FACTOR: Config<f64> = Config::new(
236 "memory_limiter_burst_factor",
237 0.,
238 "Multiplicative burst factor to the memory limiter's limit.",
239);
240
241pub const ENABLE_COLUMNATION_LGALLOC: Config<bool> = Config::new(
243 "enable_columnation_lgalloc",
244 true,
245 "Enable allocating regions from lgalloc.",
246);
247
248pub const COMPUTE_SERVER_MAINTENANCE_INTERVAL: Config<Duration> = Config::new(
250 "compute_server_maintenance_interval",
251 Duration::from_millis(10),
252 "The interval at which the compute server performs maintenance tasks. Zero enables maintenance on every iteration.",
253);
254
255pub const DATAFLOW_MAX_INFLIGHT_BYTES: Config<Option<usize>> = Config::new(
257 "compute_dataflow_max_inflight_bytes",
258 None,
259 "The maximum number of in-flight bytes emitted by persist_sources feeding \
260 compute dataflows in non-cc clusters.",
261);
262
263pub const DATAFLOW_MAX_INFLIGHT_BYTES_CC: Config<Option<usize>> = Config::new(
268 "compute_dataflow_max_inflight_bytes_cc",
269 None,
270 "The maximum number of in-flight bytes emitted by persist_sources feeding \
271 compute dataflows in cc clusters.",
272);
273
274pub const CONSOLIDATING_VEC_GROWTH_DAMPENER: Config<usize> = Config::new(
277 "consolidating_vec_growth_dampener",
278 1,
279 "Dampener in growth rate for consolidating vector size",
280);
281
282pub const HYDRATION_CONCURRENCY: Config<usize> = Config::new(
284 "compute_hydration_concurrency",
285 4,
286 "Controls how many compute dataflows may hydrate concurrently.",
287);
288
289pub const COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO: Config<usize> = Config::new(
291 "copy_to_s3_parquet_row_group_file_ratio",
292 20,
293 "The ratio (defined as a percentage) of row-group size to max-file-size. \
294 Must be <= 100.",
295);
296
297pub const COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO: Config<usize> = Config::new(
299 "copy_to_s3_arrow_builder_buffer_ratio",
300 150,
301 "The ratio (defined as a percentage) of arrow-builder size to row-group size. \
302 Must be >= 100.",
303);
304
305pub const COPY_TO_S3_MULTIPART_PART_SIZE_BYTES: Config<usize> = Config::new(
307 "copy_to_s3_multipart_part_size_bytes",
308 1024 * 1024 * 8,
309 "The size of each part in a multipart upload to S3.",
310);
311
312pub const ENABLE_COMPUTE_REPLICA_EXPIRATION: Config<bool> = Config::new(
316 "enable_compute_replica_expiration",
317 true,
318 "Main switch to disable replica expiration.",
319);
320
321pub const COMPUTE_REPLICA_EXPIRATION_OFFSET: Config<Duration> = Config::new(
327 "compute_replica_expiration_offset",
328 Duration::ZERO,
329 "The expiration time offset for replicas. Zero disables expiration.",
330);
331
332pub const COMPUTE_APPLY_COLUMN_DEMANDS: Config<bool> = Config::new(
336 "compute_apply_column_demands",
337 true,
338 "When enabled, passes applys column demands to the RelationDesc used to read out of Persist.",
339);
340
341pub const COMPUTE_FLAT_MAP_FUEL: Config<usize> = Config::new(
344 "compute_flat_map_fuel",
345 1_000_000,
346 "The amount of output the flat-map operator produces before yielding.",
347);
348
349pub const ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION: Config<bool> = Config::new(
351 "enable_compute_render_fueled_as_specific_collection",
352 true,
353 "When enabled, renders `as_specific_collection` using a fueled flat-map operator.",
354);
355
356pub const ENABLE_COMPUTE_LOGICAL_BACKPRESSURE: Config<bool> = Config::new(
358 "enable_compute_logical_backpressure",
359 false,
360 "When enabled, compute dataflows will apply logical backpressure.",
361);
362
363pub const COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES: Config<Option<usize>> =
372 Config::new(
373 "compute_logical_backpressure_max_retained_capabilities",
374 Some(30 * 24 * 60),
375 "The maximum number of capabilities retained by the logical backpressure operator.",
376 );
377
378pub const COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK: Config<Duration> = Config::new(
383 "compute_logical_backpressure_inflight_slack",
384 Duration::from_secs(1),
385 "Round observed timestamps to slack.",
386);
387
388pub const ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA: Config<bool> = Config::new(
397 "enable_arrangement_dictionary_compression_alpha",
398 false,
399 "Enable arrangement dictionary compression (alpha; not yet production-ready).",
400);
401
402pub const ENABLE_PEEK_RESPONSE_STASH: Config<bool> = Config::new(
406 "enable_compute_peek_response_stash",
407 true,
408 "Whether to enable the peek response stash, for sending back large peek responses. Will only be used for results that exceed compute_peek_response_stash_threshold_bytes.",
409);
410
411pub const PEEK_RESPONSE_STASH_THRESHOLD_BYTES: Config<usize> = Config::new(
415 "compute_peek_response_stash_threshold_bytes",
416 1024 * 10, "The threshold above which to use the peek response stash, for sending back large peek responses.",
418);
419
420pub const PEEK_RESPONSE_STASH_BATCH_MAX_RUNS: Config<usize> = Config::new(
427 "compute_peek_response_stash_batch_max_runs",
428 2,
431 "The target number of maximum runs in the batches written to the stash.",
432);
433
434pub const PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES: Config<usize> = Config::new(
436 "compute_peek_response_stash_read_batch_size_bytes",
437 1024 * 1024 * 100, "The target size for batches of rows we read out of the peek stash.",
439);
440
441pub const PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES: Config<usize> = Config::new(
444 "compute_peek_response_stash_read_memory_budget_bytes",
445 1024 * 1024 * 64, "The memory budget for consolidating stashed peek responses in environmentd.",
447);
448
449pub const PEEK_STASH_NUM_BATCHES: Config<usize> = Config::new(
451 "compute_peek_stash_num_batches",
452 100,
453 "The number of batches to pump from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
454);
455
456pub const PEEK_STASH_BATCH_SIZE: Config<usize> = Config::new(
459 "compute_peek_stash_batch_size",
460 100000,
461 "The size, as number of rows, of each batch pumped from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
462);
463
464pub const COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL: Config<Duration> = Config::new(
468 "compute_prometheus_introspection_scrape_interval",
469 Duration::from_secs(10),
470 "The collection interval for the Prometheus metrics introspection source. Set to zero to disable.",
471);
472
473pub const SUBSCRIBE_SNAPSHOT_OPTIMIZATION: Config<bool> = Config::new(
475 "compute_subscribe_snapshot_optimization",
476 true,
477 "If set, skip fetching or processing the snapshot data for subscribes when possible.",
478);
479
480pub const MV_SINK_ADVANCE_PERSIST_FRONTIERS: Config<bool> = Config::new(
484 "compute_mv_sink_advance_persist_frontiers",
485 true,
486 "Whether the MV sink's write operator advances its internal persist frontiers to the as_of.",
487);
488
489pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
491 configs
492 .add(&ENABLE_HALF_JOIN2)
493 .add(&ENABLE_MZ_JOIN_CORE)
494 .add(&ENABLE_SYNC_MV_SINK)
495 .add(&ENABLE_CORRECTION_V2)
496 .add(&CORRECTION_V2_CHAIN_PROPORTIONALITY)
497 .add(&CORRECTION_V2_CHUNK_SIZE)
498 .add(&ENABLE_COMPUTE_TEMPORAL_BUCKETING)
499 .add(&TEMPORAL_BUCKETING_SUMMARY)
500 .add(&LINEAR_JOIN_YIELDING)
501 .add(&ENABLE_LGALLOC)
502 .add(&LGALLOC_BACKGROUND_INTERVAL)
503 .add(&LGALLOC_FILE_GROWTH_DAMPENER)
504 .add(&LGALLOC_LOCAL_BUFFER_BYTES)
505 .add(&LGALLOC_SLOW_CLEAR_BYTES)
506 .add(&MEMORY_LIMITER_INTERVAL)
507 .add(&MEMORY_LIMITER_USAGE_BIAS)
508 .add(&MEMORY_LIMITER_BURST_FACTOR)
509 .add(&ENABLE_LGALLOC_EAGER_RECLAMATION)
510 .add(&ENABLE_COLUMNATION_LGALLOC)
511 .add(&COMPUTE_SERVER_MAINTENANCE_INTERVAL)
512 .add(&DATAFLOW_MAX_INFLIGHT_BYTES)
513 .add(&DATAFLOW_MAX_INFLIGHT_BYTES_CC)
514 .add(&HYDRATION_CONCURRENCY)
515 .add(©_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO)
516 .add(©_TO_S3_ARROW_BUILDER_BUFFER_RATIO)
517 .add(©_TO_S3_MULTIPART_PART_SIZE_BYTES)
518 .add(&ENABLE_COMPUTE_REPLICA_EXPIRATION)
519 .add(&COMPUTE_REPLICA_EXPIRATION_OFFSET)
520 .add(&COMPUTE_APPLY_COLUMN_DEMANDS)
521 .add(&COMPUTE_FLAT_MAP_FUEL)
522 .add(&CONSOLIDATING_VEC_GROWTH_DAMPENER)
523 .add(&ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION)
524 .add(&ENABLE_COMPUTE_LOGICAL_BACKPRESSURE)
525 .add(&COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES)
526 .add(&COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK)
527 .add(&ENABLE_ARRANGEMENT_DICTIONARY_COMPRESSION_ALPHA)
528 .add(&ENABLE_PEEK_RESPONSE_STASH)
529 .add(&PEEK_RESPONSE_STASH_THRESHOLD_BYTES)
530 .add(&PEEK_RESPONSE_STASH_BATCH_MAX_RUNS)
531 .add(&PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES)
532 .add(&PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES)
533 .add(&PEEK_STASH_NUM_BATCHES)
534 .add(&PEEK_STASH_BATCH_SIZE)
535 .add(&COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL)
536 .add(&SUBSCRIBE_SNAPSHOT_OPTIMIZATION)
537 .add(&MV_SINK_ADVANCE_PERSIST_FRONTIERS)
538 .add(&ENABLE_COLUMN_PAGED_BATCHER)
539 .add(&ENABLE_COLUMN_PAGED_BATCHER_SPILL)
540 .add(&COLUMN_PAGED_BATCHER_BUDGET_FRACTION)
541 .add(&COLUMN_PAGED_BATCHER_LZ4)
542 .add(&COLUMN_PAGED_BATCHER_SWAP_PAGEOUT)
543}