1use std::time::Duration;
13
14use mz_dyncfg::{Config, ConfigSet};
15
16pub const ENABLE_MZ_JOIN_CORE: Config<bool> = Config::new(
18 "enable_mz_join_core",
19 true,
20 "Whether compute should use `mz_join_core` rather than DD's `JoinCore::join_core` to render \
21 linear joins.",
22);
23
24pub const ENABLE_MZ_JOIN_CORE_V2: Config<bool> = Config::new(
26 "enable_mz_join_core_v2",
27 true,
28 "Whether compute should use `mz_join_core_v2` rather than DD's `JoinCore::join_core` to render \
29 linear joins.",
30);
31
32pub const ENABLE_CORRECTION_V2: Config<bool> = Config::new(
34 "enable_compute_correction_v2",
35 false,
36 "Whether compute should use the new MV sink correction buffer implementation.",
37);
38
39pub const ENABLE_TEMPORAL_BUCKETING: Config<bool> = Config::new(
41 "enable_compute_temporal_bucketing",
42 false,
43 "Whether to enable temporal bucketing in compute.",
44);
45
46pub const TEMPORAL_BUCKETING_SUMMARY: Config<Duration> = Config::new(
48 "compute_temporal_bucketing_summary",
49 Duration::from_secs(2),
50 "The summary to apply to frontiers in temporal bucketing in compute.",
51);
52
53pub const LINEAR_JOIN_YIELDING: Config<&str> = Config::new(
55 "linear_join_yielding",
56 "work:1000000,time:100",
57 "The yielding behavior compute rendering should apply for linear join operators. Either \
58 'work:<amount>' or 'time:<milliseconds>' or 'work:<amount>,time:<milliseconds>'. Note \
59 that omitting one of 'work' or 'time' will entirely disable join yielding by time or \
60 work, respectively, rather than falling back to some default.",
61);
62
63pub const ENABLE_LGALLOC: Config<bool> = Config::new("enable_lgalloc", true, "Enable lgalloc.");
65
66pub const ENABLE_LGALLOC_EAGER_RECLAMATION: Config<bool> = Config::new(
68 "enable_lgalloc_eager_reclamation",
69 true,
70 "Enable lgalloc's eager return behavior.",
71);
72
73pub const LGALLOC_BACKGROUND_INTERVAL: Config<Duration> = Config::new(
75 "lgalloc_background_interval",
76 Duration::from_secs(1),
77 "Scheduling interval for lgalloc's background worker.",
78);
79
80pub const LGALLOC_FILE_GROWTH_DAMPENER: Config<usize> = Config::new(
82 "lgalloc_file_growth_dampener",
83 2,
84 "Lgalloc's file growth dampener parameter.",
85);
86
87pub const LGALLOC_LOCAL_BUFFER_BYTES: Config<usize> = Config::new(
89 "lgalloc_local_buffer_bytes",
90 64 << 20,
91 "Lgalloc's local buffer bytes parameter.",
92);
93
94pub const LGALLOC_SLOW_CLEAR_BYTES: Config<usize> = Config::new(
96 "lgalloc_slow_clear_bytes",
97 128 << 20,
98 "Clear byte size per size class for every invocation",
99);
100
101pub const MEMORY_LIMITER_INTERVAL: Config<Duration> = Config::new(
103 "memory_limiter_interval",
104 Duration::from_secs(10),
105 "Interval to run the memory limiter. A zero duration disables the limiter.",
106);
107
108pub const MEMORY_LIMITER_USAGE_BIAS: Config<f64> = Config::new(
110 "memory_limiter_usage_bias",
111 1.,
112 "Multiplicative bias to the memory limiter's limit.",
113);
114
115pub const MEMORY_LIMITER_BURST_FACTOR: Config<f64> = Config::new(
117 "memory_limiter_burst_factor",
118 0.,
119 "Multiplicative burst factor to the memory limiter's limit.",
120);
121
122pub const ENABLE_COLUMNATION_LGALLOC: Config<bool> = Config::new(
124 "enable_columnation_lgalloc",
125 true,
126 "Enable allocating regions from lgalloc.",
127);
128
129pub const ENABLE_COLUMNAR_LGALLOC: Config<bool> = Config::new(
131 "enable_columnar_lgalloc",
132 true,
133 "Enable allocating aligned regions in columnar from lgalloc.",
134);
135
136pub const COMPUTE_SERVER_MAINTENANCE_INTERVAL: Config<Duration> = Config::new(
138 "compute_server_maintenance_interval",
139 Duration::from_millis(10),
140 "The interval at which the compute server performs maintenance tasks. Zero enables maintenance on every iteration.",
141);
142
143pub const DATAFLOW_MAX_INFLIGHT_BYTES: Config<Option<usize>> = Config::new(
145 "compute_dataflow_max_inflight_bytes",
146 None,
147 "The maximum number of in-flight bytes emitted by persist_sources feeding \
148 compute dataflows in non-cc clusters.",
149);
150
151pub const DATAFLOW_MAX_INFLIGHT_BYTES_CC: Config<Option<usize>> = Config::new(
156 "compute_dataflow_max_inflight_bytes_cc",
157 None,
158 "The maximum number of in-flight bytes emitted by persist_sources feeding \
159 compute dataflows in cc clusters.",
160);
161
162pub const CONSOLIDATING_VEC_GROWTH_DAMPENER: Config<usize> = Config::new(
165 "consolidating_vec_growth_dampener",
166 1,
167 "Dampener in growth rate for consolidating vector size",
168);
169
170pub const HYDRATION_CONCURRENCY: Config<usize> = Config::new(
172 "compute_hydration_concurrency",
173 4,
174 "Controls how many compute dataflows may hydrate concurrently.",
175);
176
177pub const COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO: Config<usize> = Config::new(
179 "copy_to_s3_parquet_row_group_file_ratio",
180 20,
181 "The ratio (defined as a percentage) of row-group size to max-file-size. \
182 Must be <= 100.",
183);
184
185pub const COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO: Config<usize> = Config::new(
187 "copy_to_s3_arrow_builder_buffer_ratio",
188 150,
189 "The ratio (defined as a percentage) of arrow-builder size to row-group size. \
190 Must be >= 100.",
191);
192
193pub const COPY_TO_S3_MULTIPART_PART_SIZE_BYTES: Config<usize> = Config::new(
195 "copy_to_s3_multipart_part_size_bytes",
196 1024 * 1024 * 8,
197 "The size of each part in a multipart upload to S3.",
198);
199
200pub const ENABLE_COMPUTE_REPLICA_EXPIRATION: Config<bool> = Config::new(
204 "enable_compute_replica_expiration",
205 true,
206 "Main switch to disable replica expiration.",
207);
208
209pub const COMPUTE_REPLICA_EXPIRATION_OFFSET: Config<Duration> = Config::new(
215 "compute_replica_expiration_offset",
216 Duration::ZERO,
217 "The expiration time offset for replicas. Zero disables expiration.",
218);
219
220pub const COMPUTE_APPLY_COLUMN_DEMANDS: Config<bool> = Config::new(
224 "compute_apply_column_demands",
225 true,
226 "When enabled, passes applys column demands to the RelationDesc used to read out of Persist.",
227);
228
229pub const COMPUTE_FLAT_MAP_FUEL: Config<usize> = Config::new(
232 "compute_flat_map_fuel",
233 1_000_000,
234 "The amount of output the flat-map operator produces before yielding.",
235);
236
237pub const ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION: Config<bool> = Config::new(
239 "enable_compute_render_fueled_as_specific_collection",
240 true,
241 "When enabled, renders `as_specific_collection` using a fueled flat-map operator.",
242);
243
244pub const ENABLE_COMPUTE_LOGICAL_BACKPRESSURE: Config<bool> = Config::new(
246 "enable_compute_logical_backpressure",
247 false,
248 "When enabled, compute dataflows will apply logical backpressure.",
249);
250
251pub const COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES: Config<Option<usize>> =
260 Config::new(
261 "compute_logical_backpressure_max_retained_capabilities",
262 Some(30 * 24 * 60),
263 "The maximum number of capabilities retained by the logical backpressure operator.",
264 );
265
266pub const COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK: Config<Duration> = Config::new(
271 "compute_logical_backpressure_inflight_slack",
272 Duration::from_secs(1),
273 "Round observed timestamps to slack.",
274);
275
276pub const ENABLE_ACTIVE_DATAFLOW_CANCELATION: Config<bool> = Config::new(
278 "enable_compute_active_dataflow_cancelation",
279 false,
280 "Whether to use `drop_dataflow` to actively cancel compute dataflows.",
281);
282
283pub const ENABLE_PEEK_RESPONSE_STASH: Config<bool> = Config::new(
287 "enable_compute_peek_response_stash",
288 false,
289 "Whether to enable the peek response stash, for sending back large peek responses. Will only be used for results that exceed compute_peek_response_stash_threshold_bytes.",
290);
291
292pub const PEEK_RESPONSE_STASH_THRESHOLD_BYTES: Config<usize> = Config::new(
296 "compute_peek_response_stash_threshold_bytes",
297 1024 * 1024 * 300, "The threshold above which to use the peek response stash, for sending back large peek responses.",
299);
300
301pub const PEEK_RESPONSE_STASH_BATCH_MAX_RUNS: Config<usize> = Config::new(
308 "compute_peek_response_stash_batch_max_runs",
309 2,
312 "The target number of maximum runs in the batches written to the stash.",
313);
314
315pub const PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES: Config<usize> = Config::new(
317 "compute_peek_response_stash_read_batch_size_bytes",
318 1024 * 1024 * 100, "The target size for batches of rows we read out of the peek stash.",
320);
321
322pub const PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES: Config<usize> = Config::new(
325 "compute_peek_response_stash_read_memory_budget_bytes",
326 1024 * 1024 * 64, "The memory budget for consolidating stashed peek responses in environmentd.",
328);
329
330pub const PEEK_STASH_NUM_BATCHES: Config<usize> = Config::new(
332 "compute_peek_stash_num_batches",
333 100,
334 "The number of batches to pump from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
335);
336
337pub const PEEK_STASH_BATCH_SIZE: Config<usize> = Config::new(
340 "compute_peek_stash_batch_size",
341 100000,
342 "The size, as number of rows, of each batch pumped from the peek result iterator (in one iteration through the worker loop) when stashing peek responses.",
343);
344
345pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
347 configs
348 .add(&ENABLE_MZ_JOIN_CORE)
349 .add(&ENABLE_MZ_JOIN_CORE_V2)
350 .add(&ENABLE_CORRECTION_V2)
351 .add(&ENABLE_TEMPORAL_BUCKETING)
352 .add(&TEMPORAL_BUCKETING_SUMMARY)
353 .add(&LINEAR_JOIN_YIELDING)
354 .add(&ENABLE_LGALLOC)
355 .add(&LGALLOC_BACKGROUND_INTERVAL)
356 .add(&LGALLOC_FILE_GROWTH_DAMPENER)
357 .add(&LGALLOC_LOCAL_BUFFER_BYTES)
358 .add(&LGALLOC_SLOW_CLEAR_BYTES)
359 .add(&MEMORY_LIMITER_INTERVAL)
360 .add(&MEMORY_LIMITER_USAGE_BIAS)
361 .add(&MEMORY_LIMITER_BURST_FACTOR)
362 .add(&ENABLE_LGALLOC_EAGER_RECLAMATION)
363 .add(&ENABLE_COLUMNATION_LGALLOC)
364 .add(&ENABLE_COLUMNAR_LGALLOC)
365 .add(&COMPUTE_SERVER_MAINTENANCE_INTERVAL)
366 .add(&DATAFLOW_MAX_INFLIGHT_BYTES)
367 .add(&DATAFLOW_MAX_INFLIGHT_BYTES_CC)
368 .add(&HYDRATION_CONCURRENCY)
369 .add(©_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO)
370 .add(©_TO_S3_ARROW_BUILDER_BUFFER_RATIO)
371 .add(©_TO_S3_MULTIPART_PART_SIZE_BYTES)
372 .add(&ENABLE_COMPUTE_REPLICA_EXPIRATION)
373 .add(&COMPUTE_REPLICA_EXPIRATION_OFFSET)
374 .add(&COMPUTE_APPLY_COLUMN_DEMANDS)
375 .add(&COMPUTE_FLAT_MAP_FUEL)
376 .add(&CONSOLIDATING_VEC_GROWTH_DAMPENER)
377 .add(&ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION)
378 .add(&ENABLE_COMPUTE_LOGICAL_BACKPRESSURE)
379 .add(&COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES)
380 .add(&COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK)
381 .add(&ENABLE_ACTIVE_DATAFLOW_CANCELATION)
382 .add(&ENABLE_PEEK_RESPONSE_STASH)
383 .add(&PEEK_RESPONSE_STASH_THRESHOLD_BYTES)
384 .add(&PEEK_RESPONSE_STASH_BATCH_MAX_RUNS)
385 .add(&PEEK_RESPONSE_STASH_READ_BATCH_SIZE_BYTES)
386 .add(&PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES)
387 .add(&PEEK_STASH_NUM_BATCHES)
388 .add(&PEEK_STASH_BATCH_SIZE)
389}