mz_compute_types/
dyncfgs.rs

Help
1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Dyncfgs used by the compute layer.
11
12use std::time::Duration;
13
14use mz_dyncfg::{Config, ConfigSet};
15
16/// Whether rendering should use `mz_join_core` rather than DD's `JoinCore::join_core`.
17pub const ENABLE_MZ_JOIN_CORE: Config<bool> = Config::new(
18    "enable_mz_join_core",
19    true,
20    "Whether compute should use `mz_join_core` rather than DD's `JoinCore::join_core` to render \
21     linear joins.",
22);
23
24/// Whether rendering should use the new MV sink correction buffer implementation.
25pub const ENABLE_CORRECTION_V2: Config<bool> = Config::new(
26    "enable_compute_correction_v2",
27    false,
28    "Whether compute should use the new MV sink correction buffer implementation.",
29);
30
31/// Whether the MV sink should distribute appends among workers.
32pub const ENABLE_MV_APPEND_SMEARING: Config<bool> = Config::new(
33    "enable_compute_mv_append_smearing",
34    true,
35    "Whether the MV sink should distribute appends among workers.",
36);
37
38/// The yielding behavior with which linear joins should be rendered.
39pub const LINEAR_JOIN_YIELDING: Config<&str> = Config::new(
40    "linear_join_yielding",
41    "work:1000000,time:100",
42    "The yielding behavior compute rendering should apply for linear join operators. Either \
43     'work:<amount>' or 'time:<milliseconds>' or 'work:<amount>,time:<milliseconds>'. Note \
44     that omitting one of 'work' or 'time' will entirely disable join yielding by time or \
45     work, respectively, rather than falling back to some default.",
46);
47
48/// Enable lgalloc.
49pub const ENABLE_LGALLOC: Config<bool> = Config::new("enable_lgalloc", true, "Enable lgalloc.");
50
51/// Enable lgalloc's eager memory return/reclamation feature.
52pub const ENABLE_LGALLOC_EAGER_RECLAMATION: Config<bool> = Config::new(
53    "enable_lgalloc_eager_reclamation",
54    true,
55    "Enable lgalloc's eager return behavior.",
56);
57
58/// The interval at which the background thread wakes.
59pub const LGALLOC_BACKGROUND_INTERVAL: Config<Duration> = Config::new(
60    "lgalloc_background_interval",
61    Duration::from_secs(1),
62    "Scheduling interval for lgalloc's background worker.",
63);
64
65/// Enable lgalloc's eager memory return/reclamation feature.
66pub const LGALLOC_FILE_GROWTH_DAMPENER: Config<usize> = Config::new(
67    "lgalloc_file_growth_dampener",
68    0,
69    "Lgalloc's file growth dampener parameter.",
70);
71
72/// Enable lgalloc's eager memory return/reclamation feature.
73pub const LGALLOC_LOCAL_BUFFER_BYTES: Config<usize> = Config::new(
74    "lgalloc_local_buffer_bytes",
75    32 << 20,
76    "Lgalloc's local buffer bytes parameter.",
77);
78
79/// The bytes to reclaim (slow path) per size class, for each background thread activation.
80pub const LGALLOC_SLOW_CLEAR_BYTES: Config<usize> = Config::new(
81    "lgalloc_slow_clear_bytes",
82    32 << 20,
83    "Clear byte size per size class for every invocation",
84);
85
86/// Enable lgalloc for columnation.
87pub const ENABLE_COLUMNATION_LGALLOC: Config<bool> = Config::new(
88    "enable_columnation_lgalloc",
89    true,
90    "Enable allocating regions from lgalloc.",
91);
92
93/// Enable lgalloc for columnar.
94pub const ENABLE_COLUMNAR_LGALLOC: Config<bool> = Config::new(
95    "enable_columnar_lgalloc",
96    true,
97    "Enable allocating aligned regions in columnar from lgalloc.",
98);
99
100/// The interval at which the compute server performs maintenance tasks.
101pub const COMPUTE_SERVER_MAINTENANCE_INTERVAL: Config<Duration> = Config::new(
102    "compute_server_maintenance_interval",
103    Duration::from_millis(10),
104    "The interval at which the compute server performs maintenance tasks. Zero enables maintenance on every iteration.",
105);
106
107/// Maximum number of in-flight bytes emitted by persist_sources feeding dataflows.
108pub const DATAFLOW_MAX_INFLIGHT_BYTES: Config<Option<usize>> = Config::new(
109    "compute_dataflow_max_inflight_bytes",
110    None,
111    "The maximum number of in-flight bytes emitted by persist_sources feeding \
112     compute dataflows in non-cc clusters.",
113);
114
115/// The "physical backpressure" of `compute_dataflow_max_inflight_bytes_cc` has
116/// been replaced in cc replicas by persist lgalloc and we intend to remove it
117/// once everything has switched to cc. In the meantime, this is a CYA to turn
118/// it back on if absolutely necessary.
119pub const DATAFLOW_MAX_INFLIGHT_BYTES_CC: Config<Option<usize>> = Config::new(
120    "compute_dataflow_max_inflight_bytes_cc",
121    None,
122    "The maximum number of in-flight bytes emitted by persist_sources feeding \
123     compute dataflows in cc clusters.",
124);
125
126/// The term `n` in the growth rate `1 + 1/(n + 1)` for `ConsolidatingVec`.
127/// The smallest value `0` corresponds to the greatest allowed growth, of doubling.
128pub const CONSOLIDATING_VEC_GROWTH_DAMPENER: Config<usize> = Config::new(
129    "consolidating_vec_growth_dampener",
130    0,
131    "Dampener in growth rate for consolidating vector size",
132);
133
134/// The number of dataflows that may hydrate concurrently.
135pub const HYDRATION_CONCURRENCY: Config<usize> = Config::new(
136    "compute_hydration_concurrency",
137    4,
138    "Controls how many compute dataflows may hydrate concurrently.",
139);
140
141/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
142pub const COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO: Config<usize> = Config::new(
143    "copy_to_s3_parquet_row_group_file_ratio",
144    20,
145    "The ratio (defined as a percentage) of row-group size to max-file-size. \
146        Must be <= 100.",
147);
148
149/// See `src/storage-operators/src/s3_oneshot_sink/parquet.rs` for more details.
150pub const COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO: Config<usize> = Config::new(
151    "copy_to_s3_arrow_builder_buffer_ratio",
152    150,
153    "The ratio (defined as a percentage) of arrow-builder size to row-group size. \
154        Must be >= 100.",
155);
156
157/// The size of each part in the multi-part upload to use when uploading files to S3.
158pub const COPY_TO_S3_MULTIPART_PART_SIZE_BYTES: Config<usize> = Config::new(
159    "copy_to_s3_multipart_part_size_bytes",
160    1024 * 1024 * 8,
161    "The size of each part in a multipart upload to S3.",
162);
163
164/// Main switch to enable or disable replica expiration.
165///
166/// Changes affect existing replicas only after restart.
167pub const ENABLE_COMPUTE_REPLICA_EXPIRATION: Config<bool> = Config::new(
168    "enable_compute_replica_expiration",
169    true,
170    "Main switch to disable replica expiration.",
171);
172
173/// The maximum lifetime of a replica configured as an offset to the replica start time.
174/// Used in temporal filters to drop diffs generated at timestamps beyond the expiration time.
175///
176/// A zero duration implies no expiration. Changing this value does not affect existing replicas,
177/// even when they are restarted.
178pub const COMPUTE_REPLICA_EXPIRATION_OFFSET: Config<Duration> = Config::new(
179    "compute_replica_expiration_offset",
180    Duration::ZERO,
181    "The expiration time offset for replicas. Zero disables expiration.",
182);
183
184/// When enabled, applies the column demands from a MapFilterProject onto the RelationDesc used to
185/// read out of Persist. This allows Persist to prune unneeded columns as a performance
186/// optimization.
187pub const COMPUTE_APPLY_COLUMN_DEMANDS: Config<bool> = Config::new(
188    "compute_apply_column_demands",
189    true,
190    "When enabled, passes applys column demands to the RelationDesc used to read out of Persist.",
191);
192
193/// Whether to render `as_specific_collection` using a fueled flat-map operator.
194pub const ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION: Config<bool> = Config::new(
195    "enable_compute_render_fueled_as_specific_collection",
196    true,
197    "When enabled, renders `as_specific_collection` using a fueled flat-map operator.",
198);
199
200/// Whether to apply logical backpressure in compute dataflows.
201pub const ENABLE_COMPUTE_LOGICAL_BACKPRESSURE: Config<bool> = Config::new(
202    "enable_compute_logical_backpressure",
203    false,
204    "When enabled, compute dataflows will apply logical backpressure.",
205);
206
207/// Maximal number of capabilities retained by the logical backpressure operator.
208///
209/// Selecting this value is subtle. If it's too small, it'll diminish the effectiveness of the
210/// logical backpressure operators. If it's too big, we can slow down hydration and cause state
211/// in the operator's implementation to build up.
212///
213/// The default value represents a compromise between these two extremes. We retain some metrics
214/// for 30 days, and the metrics update every minute. The default is exactly this number.
215pub const COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES: Config<Option<usize>> =
216    Config::new(
217        "compute_logical_backpressure_max_retained_capabilities",
218        Some(30 * 24 * 60),
219        "The maximum number of capabilities retained by the logical backpressure operator.",
220    );
221
222/// The slack to round observed timestamps up to.
223///
224/// The default corresponds to Mz's default tick interval, but does not need to do so. Ideally,
225/// it is not smaller than the tick interval, but it can be larger.
226pub const COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK: Config<Duration> = Config::new(
227    "compute_logical_backpressure_inflight_slack",
228    Duration::from_secs(1),
229    "Round observed timestamps to slack.",
230);
231
232/// Adds the full set of all compute `Config`s.
233pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
234    configs
235        .add(&ENABLE_MZ_JOIN_CORE)
236        .add(&ENABLE_CORRECTION_V2)
237        .add(&ENABLE_MV_APPEND_SMEARING)
238        .add(&LINEAR_JOIN_YIELDING)
239        .add(&ENABLE_LGALLOC)
240        .add(&LGALLOC_BACKGROUND_INTERVAL)
241        .add(&LGALLOC_FILE_GROWTH_DAMPENER)
242        .add(&LGALLOC_LOCAL_BUFFER_BYTES)
243        .add(&LGALLOC_SLOW_CLEAR_BYTES)
244        .add(&ENABLE_LGALLOC_EAGER_RECLAMATION)
245        .add(&ENABLE_COLUMNATION_LGALLOC)
246        .add(&ENABLE_COLUMNAR_LGALLOC)
247        .add(&COMPUTE_SERVER_MAINTENANCE_INTERVAL)
248        .add(&DATAFLOW_MAX_INFLIGHT_BYTES)
249        .add(&DATAFLOW_MAX_INFLIGHT_BYTES_CC)
250        .add(&HYDRATION_CONCURRENCY)
251        .add(&COPY_TO_S3_PARQUET_ROW_GROUP_FILE_RATIO)
252        .add(&COPY_TO_S3_ARROW_BUILDER_BUFFER_RATIO)
253        .add(&COPY_TO_S3_MULTIPART_PART_SIZE_BYTES)
254        .add(&ENABLE_COMPUTE_REPLICA_EXPIRATION)
255        .add(&COMPUTE_REPLICA_EXPIRATION_OFFSET)
256        .add(&COMPUTE_APPLY_COLUMN_DEMANDS)
257        .add(&CONSOLIDATING_VEC_GROWTH_DAMPENER)
258        .add(&ENABLE_COMPUTE_RENDER_FUELED_AS_SPECIFIC_COLLECTION)
259        .add(&ENABLE_COMPUTE_LOGICAL_BACKPRESSURE)
260        .add(&COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES)
261        .add(&COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK)
262}
mz_compute_types/dyncfgs.rs

mz_compute_types/
dyncfgs.rs
