mz_deploy/
secret_resolver.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Client-side secret resolution for mz-deploy.
11//!
12//! Secret values in SQL files may reference client-side providers like `env_var('MY_VAR')`
13//! instead of inline string literals. This module resolves those references at execution
14//! time (not compile time), so `mz-deploy compile` works without access to secrets.
15//!
16//! Unknown functions and other expressions pass through unchanged to Materialize.
17//!
18//! ## Providers
19//!
20//! Each provider is a submodule that implements [`SecretProvider`]:
21//!
22//! - `env_var::EnvVarProvider` — reads from environment variables
23//! - `aws_secret::AwsSecretProvider` — reads from AWS Secrets Manager
24//! - `aws_secret::UnconfiguredAwsProvider` — placeholder when `aws_profile` is not set
25
26mod aws_secret;
27mod env_var;
28mod json_field;
29
30use crate::cli::CliError;
31use crate::config::SecurityConfig;
32use crate::project::ast::Statement;
33use async_trait::async_trait;
34use aws_secret::{AwsSecretProvider, UnconfiguredAwsProvider};
35use env_var::EnvVarProvider;
36use mz_sql_parser::ast::{CreateSecretStatement, Expr, FunctionArgs, Raw, RawItemName, Value};
37use std::collections::BTreeMap;
38use std::ops::RangeInclusive;
39use thiserror::Error;
40
41/// Errors that can occur during secret resolution.
42#[derive(Debug, Error)]
43pub enum SecretResolveError {
44    /// A known provider was called with the wrong number of arguments.
45    #[error("function '{name}' expects {expected} argument(s), got {got}")]
46    WrongArgCount {
47        name: String,
48        expected: String,
49        got: usize,
50    },
51    /// A known provider was called with a non-literal argument.
52    #[error("function '{name}' requires string literal arguments")]
53    NonLiteralArg { name: String },
54    /// A known provider failed to resolve.
55    #[error("failed to resolve '{name}': {reason}")]
56    ResolutionFailed { name: String, reason: String },
57}
58
59/// A provider that can resolve secret values from an external source.
60#[async_trait]
61pub(crate) trait SecretProvider: Send + Sync {
62    /// The function name this provider handles (e.g. `"env_var"`).
63    fn name(&self) -> &str;
64    /// The argument counts this provider accepts.
65    fn accepted_args(&self) -> RangeInclusive<usize>;
66    /// Resolve the secret value from the given arguments.
67    async fn resolve(&self, args: &[String]) -> Result<String, SecretResolveError>;
68}
69
70/// Render an accepted-args range for error messages.
71///
72/// `1..=1` → `"1"`, `1..=2` → `"1 or 2"`, otherwise `"N..=M"`.
73fn format_arg_range(range: &RangeInclusive<usize>) -> String {
74    let (start, end) = (*range.start(), *range.end());
75    if start == end {
76        start.to_string()
77    } else if end == start + 1 {
78        format!("{} or {}", start, end)
79    } else {
80        format!("{}..={}", start, end)
81    }
82}
83
84/// Resolves client-side secret provider functions in SQL expressions.
85///
86/// Known providers (like `env_var`) are resolved to string literals.
87/// Unknown functions and other expressions pass through unchanged to Materialize.
88pub(crate) struct SecretResolver {
89    providers: BTreeMap<String, Box<dyn SecretProvider>>,
90}
91
92impl SecretResolver {
93    /// Creates a new resolver with providers configured from the given security config.
94    ///
95    /// Always registers `env_var`. For AWS, registers the real Secrets
96    /// Manager provider when `aws_profile` is set, or a placeholder that
97    /// errors clearly when it isn't. All credentials are loaded lazily.
98    pub(crate) fn new(config: &SecurityConfig) -> Self {
99        let mut resolver = Self {
100            providers: BTreeMap::new(),
101        };
102        resolver.register(Box::new(EnvVarProvider));
103
104        if let Some(profile) = config.aws_profile() {
105            resolver.register(Box::new(AwsSecretProvider::new(profile)));
106        } else {
107            resolver.register(Box::new(UnconfiguredAwsProvider));
108        }
109
110        resolver
111    }
112
113    fn register(&mut self, provider: Box<dyn SecretProvider>) {
114        self.providers.insert(provider.name().to_string(), provider);
115    }
116
117    /// Resolve client-side provider functions in an expression.
118    ///
119    /// - `Expr::Function` matching a registered provider: validate and resolve to `Expr::Value(Value::String(...))`
120    /// - Everything else: pass through unchanged
121    pub(crate) async fn resolve_expr(
122        &self,
123        expr: Expr<Raw>,
124    ) -> Result<Expr<Raw>, SecretResolveError> {
125        match &expr {
126            Expr::Function(func) => {
127                let func_name = match &func.name {
128                    RawItemName::Name(name) if name.0.len() == 1 => {
129                        Some(name.0[0].as_str().to_string())
130                    }
131                    _ => None,
132                };
133
134                let func_name = match func_name {
135                    Some(name) => name,
136                    None => return Ok(expr),
137                };
138
139                let provider = match self.providers.get(&func_name) {
140                    Some(p) => p,
141                    None => return Ok(expr),
142                };
143
144                // Validate args
145                let accepted = provider.accepted_args();
146                let arg_exprs = match &func.args {
147                    FunctionArgs::Star => {
148                        return Err(SecretResolveError::WrongArgCount {
149                            name: func_name,
150                            expected: format_arg_range(&accepted),
151                            got: 0,
152                        });
153                    }
154                    FunctionArgs::Args { args, .. } => args,
155                };
156
157                if !accepted.contains(&arg_exprs.len()) {
158                    return Err(SecretResolveError::WrongArgCount {
159                        name: func_name,
160                        expected: format_arg_range(&accepted),
161                        got: arg_exprs.len(),
162                    });
163                }
164
165                let mut string_args = Vec::with_capacity(arg_exprs.len());
166                for arg in arg_exprs {
167                    match arg {
168                        Expr::Value(Value::String(s)) => string_args.push(s.clone()),
169                        _ => {
170                            return Err(SecretResolveError::NonLiteralArg { name: func_name });
171                        }
172                    }
173                }
174
175                let resolved = provider.resolve(&string_args).await?;
176                Ok(Expr::Value(Value::String(resolved)))
177            }
178            _ => Ok(expr),
179        }
180    }
181
182    /// Resolves client-side provider functions in a `CREATE SECRET` statement.
183    ///
184    /// Returns a new statement with the value field resolved.
185    pub(crate) async fn resolve_create_secret(
186        &self,
187        stmt: &CreateSecretStatement<Raw>,
188    ) -> Result<CreateSecretStatement<Raw>, SecretResolveError> {
189        let mut resolved = stmt.clone();
190        resolved.value = self.resolve_expr(stmt.value.clone()).await?;
191        Ok(resolved)
192    }
193
194    /// Resolves a `CREATE SECRET` statement and maps errors to [`CliError`].
195    pub(crate) async fn resolve_secret_for_cli(
196        &self,
197        stmt: &CreateSecretStatement<Raw>,
198    ) -> Result<CreateSecretStatement<Raw>, CliError> {
199        self.resolve_create_secret(stmt)
200            .await
201            .map_err(|e| CliError::SecretResolution {
202                secret_name: stmt.name.to_string(),
203                source: e,
204            })
205    }
206
207    /// Resolves a [`Statement`], returning the resolved version.
208    ///
209    /// Secret statements have their provider functions resolved;
210    /// all other statements are returned unchanged.
211    pub(crate) async fn resolve_statement_for_cli(
212        &self,
213        stmt: &Statement,
214    ) -> Result<Statement, CliError> {
215        match stmt {
216            Statement::CreateSecret(create_stmt) => {
217                let resolved = self.resolve_secret_for_cli(create_stmt).await?;
218                Ok(Statement::CreateSecret(resolved))
219            }
220            _ => Ok(stmt.clone()),
221        }
222    }
223}
224
225#[cfg(test)]
226mod tests {
227    use super::*;
228    use mz_sql_parser::ast::{Function, Ident, UnresolvedItemName};
229
230    /// Helper to build `env_var('var_name')` as an `Expr<Raw>`.
231    fn make_env_var_expr(var_name: &str) -> Expr<Raw> {
232        Expr::Function(Function {
233            name: RawItemName::Name(UnresolvedItemName(vec![Ident::new("env_var").unwrap()])),
234            args: FunctionArgs::Args {
235                args: vec![Expr::Value(Value::String(var_name.to_string()))],
236                order_by: vec![],
237            },
238            filter: None,
239            over: None,
240            distinct: false,
241        })
242    }
243
244    fn make_function_expr(name: &str, args: Vec<Expr<Raw>>) -> Expr<Raw> {
245        Expr::Function(Function {
246            name: RawItemName::Name(UnresolvedItemName(vec![Ident::new(name).unwrap()])),
247            args: FunctionArgs::Args {
248                args,
249                order_by: vec![],
250            },
251            filter: None,
252            over: None,
253            distinct: false,
254        })
255    }
256
257    #[mz_ore::test(tokio::test)]
258    async fn test_resolve_string_literal_passthrough() {
259        let resolver = SecretResolver::new(&Default::default());
260        let expr = Expr::Value(Value::String("hello".to_string()));
261        let original = format!("{}", expr);
262        let resolved = resolver.resolve_expr(expr).await.unwrap();
263        assert_eq!(format!("{}", resolved), original);
264    }
265
266    #[mz_ore::test(tokio::test)]
267    async fn test_resolve_env_var_success() {
268        // SAFETY: test-only; no other thread reads this variable.
269        unsafe { std::env::set_var("MZ_TEST_SECRET_123", "my_secret_value") };
270        let resolver = SecretResolver::new(&Default::default());
271        let expr = make_env_var_expr("MZ_TEST_SECRET_123");
272        let resolved = resolver.resolve_expr(expr).await.unwrap();
273        match resolved {
274            Expr::Value(Value::String(s)) => assert_eq!(s, "my_secret_value"),
275            other => panic!("expected string literal, got: {:?}", other),
276        }
277    }
278
279    #[mz_ore::test(tokio::test)]
280    async fn test_resolve_env_var_not_set() {
281        let resolver = SecretResolver::new(&Default::default());
282        let expr = make_env_var_expr("MZ_DEFINITELY_NOT_SET_XYZ_999");
283        let err = resolver.resolve_expr(expr).await.unwrap_err();
284        assert!(matches!(err, SecretResolveError::ResolutionFailed { .. }));
285    }
286
287    #[mz_ore::test(tokio::test)]
288    async fn test_resolve_unknown_function_passthrough() {
289        let resolver = SecretResolver::new(&Default::default());
290        let expr = make_function_expr("vault", vec![Expr::Value(Value::String("foo".to_string()))]);
291        let original = format!("{}", expr);
292        let resolved = resolver.resolve_expr(expr).await.unwrap();
293        assert_eq!(format!("{}", resolved), original);
294    }
295
296    #[mz_ore::test(tokio::test)]
297    async fn test_resolve_arbitrary_expr_passthrough() {
298        let resolver = SecretResolver::new(&Default::default());
299
300        // Number literal
301        let expr = Expr::Value(Value::Number("42".to_string()));
302        let resolved = resolver.resolve_expr(expr).await.unwrap();
303        assert_eq!(format!("{}", resolved), "42");
304
305        // Identifier
306        let expr = Expr::Identifier(vec![Ident::new("some_col").unwrap()]);
307        let original = format!("{}", expr);
308        let resolved = resolver.resolve_expr(expr).await.unwrap();
309        assert_eq!(format!("{}", resolved), original);
310    }
311
312    #[mz_ore::test(tokio::test)]
313    async fn test_resolve_wrong_arg_count_zero() {
314        let resolver = SecretResolver::new(&Default::default());
315        let expr = make_function_expr("env_var", vec![]);
316        let err = resolver.resolve_expr(expr).await.unwrap_err();
317        match err {
318            SecretResolveError::WrongArgCount {
319                name,
320                expected,
321                got,
322            } => {
323                assert_eq!(name, "env_var");
324                assert_eq!(expected, "1");
325                assert_eq!(got, 0);
326            }
327            other => panic!("expected WrongArgCount, got: {:?}", other),
328        }
329    }
330
331    #[mz_ore::test(tokio::test)]
332    async fn test_resolve_wrong_arg_count_two() {
333        let resolver = SecretResolver::new(&Default::default());
334        let expr = make_function_expr(
335            "env_var",
336            vec![
337                Expr::Value(Value::String("A".to_string())),
338                Expr::Value(Value::String("B".to_string())),
339            ],
340        );
341        let err = resolver.resolve_expr(expr).await.unwrap_err();
342        match err {
343            SecretResolveError::WrongArgCount {
344                name,
345                expected,
346                got,
347            } => {
348                assert_eq!(name, "env_var");
349                assert_eq!(expected, "1");
350                assert_eq!(got, 2);
351            }
352            other => panic!("expected WrongArgCount, got: {:?}", other),
353        }
354    }
355
356    #[mz_ore::test(tokio::test)]
357    async fn test_resolve_aws_secret_wrong_arg_count() {
358        // aws_secret accepts 1 or 2 args; zero and three should both error
359        // with a message that reflects the range.
360        let resolver = SecretResolver::new(&Default::default());
361
362        let expr = make_function_expr("aws_secret", vec![]);
363        match resolver.resolve_expr(expr).await.unwrap_err() {
364            SecretResolveError::WrongArgCount {
365                name,
366                expected,
367                got,
368            } => {
369                assert_eq!(name, "aws_secret");
370                assert_eq!(expected, "1 or 2");
371                assert_eq!(got, 0);
372            }
373            other => panic!("expected WrongArgCount, got: {:?}", other),
374        }
375
376        let expr = make_function_expr(
377            "aws_secret",
378            vec![
379                Expr::Value(Value::String("a".into())),
380                Expr::Value(Value::String("b".into())),
381                Expr::Value(Value::String("c".into())),
382            ],
383        );
384        match resolver.resolve_expr(expr).await.unwrap_err() {
385            SecretResolveError::WrongArgCount {
386                name,
387                expected,
388                got,
389            } => {
390                assert_eq!(name, "aws_secret");
391                assert_eq!(expected, "1 or 2");
392                assert_eq!(got, 3);
393            }
394            other => panic!("expected WrongArgCount, got: {:?}", other),
395        }
396    }
397
398    #[mz_ore::test(tokio::test)]
399    async fn test_resolve_non_literal_arg() {
400        let resolver = SecretResolver::new(&Default::default());
401        let expr = make_function_expr(
402            "env_var",
403            vec![Expr::Identifier(vec![Ident::new("col").unwrap()])],
404        );
405        let err = resolver.resolve_expr(expr).await.unwrap_err();
406        assert!(matches!(err, SecretResolveError::NonLiteralArg { .. }));
407    }
408
409    #[mz_ore::test(tokio::test)]
410    async fn test_resolve_star_args() {
411        let resolver = SecretResolver::new(&Default::default());
412        let expr = Expr::Function(Function {
413            name: RawItemName::Name(UnresolvedItemName(vec![Ident::new("env_var").unwrap()])),
414            args: FunctionArgs::Star,
415            filter: None,
416            over: None,
417            distinct: false,
418        });
419        let err = resolver.resolve_expr(expr).await.unwrap_err();
420        match err {
421            SecretResolveError::WrongArgCount { got: 0, .. } => {}
422            other => panic!("expected WrongArgCount with got=0, got: {:?}", other),
423        }
424    }
425
426    #[mz_ore::test(tokio::test)]
427    async fn test_resolve_create_secret_with_env_var() {
428        // SAFETY: test-only; no other thread reads this variable.
429        unsafe { std::env::set_var("MZ_TEST_SECRET_456", "resolved_value") };
430        let resolver = SecretResolver::new(&Default::default());
431        let stmt = CreateSecretStatement::<Raw> {
432            name: UnresolvedItemName(vec![Ident::new("my_secret").unwrap()]),
433            if_not_exists: false,
434            value: make_env_var_expr("MZ_TEST_SECRET_456"),
435        };
436        let resolved = resolver.resolve_create_secret(&stmt).await.unwrap();
437        match &resolved.value {
438            Expr::Value(Value::String(s)) => assert_eq!(s, "resolved_value"),
439            other => panic!("expected string literal, got: {:?}", other),
440        }
441        assert_eq!(resolved.name.0[0].as_str(), stmt.name.0[0].as_str());
442    }
443
444    #[mz_ore::test(tokio::test)]
445    async fn test_resolve_create_secret_plain_string() {
446        let resolver = SecretResolver::new(&Default::default());
447        let stmt = CreateSecretStatement::<Raw> {
448            name: UnresolvedItemName(vec![Ident::new("my_secret").unwrap()]),
449            if_not_exists: false,
450            value: Expr::Value(Value::String("plain_value".to_string())),
451        };
452        let resolved = resolver.resolve_create_secret(&stmt).await.unwrap();
453        match &resolved.value {
454            Expr::Value(Value::String(s)) => assert_eq!(s, "plain_value"),
455            other => panic!("expected string literal, got: {:?}", other),
456        }
457    }
458
459    #[mz_ore::test(tokio::test)]
460    async fn test_unconfigured_aws_provider_error() {
461        let resolver = SecretResolver::new(&Default::default());
462        let expr = make_function_expr("aws_secret", vec![Expr::Value(Value::String("foo".into()))]);
463        let err = resolver.resolve_expr(expr).await.unwrap_err();
464        match err {
465            SecretResolveError::ResolutionFailed { name, reason } => {
466                assert_eq!(name, "aws_secret");
467                assert!(
468                    reason.contains("aws_profile"),
469                    "error should mention aws_profile, got: {}",
470                    reason
471                );
472            }
473            other => panic!("expected ResolutionFailed, got: {:?}", other),
474        }
475    }
476
477    #[mz_ore::test(tokio::test)]
478    async fn test_aws_secret_passthrough_when_unconfigured() {
479        // env_var still works even when AWS is unconfigured
480        unsafe { std::env::set_var("MZ_TEST_AWS_PASSTHROUGH", "works") };
481        let resolver = SecretResolver::new(&Default::default());
482        let expr = make_env_var_expr("MZ_TEST_AWS_PASSTHROUGH");
483        let resolved = resolver.resolve_expr(expr).await.unwrap();
484        match resolved {
485            Expr::Value(Value::String(s)) => assert_eq!(s, "works"),
486            other => panic!("expected string literal, got: {:?}", other),
487        }
488    }
489}
mz_deploy/secret_resolver.rs

mz_deploy/
secret_resolver.rs
