iceberg/io/storage/config/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// TODO Add specific configs
19//! Storage configuration for storage backends.
20//!
21//! This module provides configuration types for various storage backends.
22//! The configuration types are designed to be used with the `StorageFactory`
23//! trait to create storage instances.
24//!
25//! # Available Configurations
26//!
27//! - [`StorageConfig`]: Base configuration containing properties for storage backends
28//! - [`S3Config`]: Amazon S3 specific configuration
29//! - [`GcsConfig`]: Google Cloud Storage specific configuration
30//! - [`OssConfig`]: Alibaba Cloud OSS specific configuration
31//! - [`AzdlsConfig`]: Azure Data Lake Storage specific configuration
32
33mod azdls;
34mod gcs;
35mod hf;
36mod oss;
37mod s3;
38
39use std::collections::HashMap;
40
41pub use azdls::*;
42pub use gcs::*;
43pub use hf::*;
44pub use oss::*;
45pub use s3::*;
46use serde::{Deserialize, Serialize};
47
48/// Configuration properties for storage backends.
49///
50/// This struct contains only configuration properties without specifying
51/// which storage backend to use. The storage type is determined by the
52/// explicit factory selection.
53/// ```
54#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
55pub struct StorageConfig {
56    /// Configuration properties for the storage backend
57    props: HashMap<String, String>,
58}
59
60impl std::fmt::Debug for StorageConfig {
61    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62        // Property values may hold vended credentials (e.g. `s3.secret-access-key`,
63        // `s3.session-token`). Debug is reachable through the `FileIO`/`Table`
64        // derives, so print only the keys and never the secret values.
65        f.debug_struct("StorageConfig")
66            .field("keys", &self.props.keys().collect::<Vec<_>>())
67            .finish_non_exhaustive()
68    }
69}
70
71impl StorageConfig {
72    /// Create a new empty StorageConfig.
73    pub fn new() -> Self {
74        Self {
75            props: HashMap::new(),
76        }
77    }
78
79    /// Create a StorageConfig from existing properties.
80    ///
81    /// # Arguments
82    ///
83    /// * `props` - Configuration properties for the storage backend
84    pub fn from_props(props: HashMap<String, String>) -> Self {
85        Self { props }
86    }
87
88    /// Get all configuration properties.
89    pub fn props(&self) -> &HashMap<String, String> {
90        &self.props
91    }
92
93    /// Get a specific configuration property by key.
94    ///
95    /// # Arguments
96    ///
97    /// * `key` - The property key to look up
98    ///
99    /// # Returns
100    ///
101    /// An `Option` containing a reference to the property value if it exists.
102    pub fn get(&self, key: &str) -> Option<&String> {
103        self.props.get(key)
104    }
105
106    /// Add a configuration property.
107    ///
108    /// This is a builder-style method that returns `self` for chaining.
109    ///
110    /// # Arguments
111    ///
112    /// * `key` - The property key
113    /// * `value` - The property value
114    pub fn with_prop(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
115        self.props.insert(key.into(), value.into());
116        self
117    }
118
119    /// Add multiple configuration properties.
120    ///
121    /// This is a builder-style method that returns `self` for chaining.
122    ///
123    /// # Arguments
124    ///
125    /// * `props` - An iterator of key-value pairs to add
126    pub fn with_props(
127        mut self,
128        props: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
129    ) -> Self {
130        self.props
131            .extend(props.into_iter().map(|(k, v)| (k.into(), v.into())));
132        self
133    }
134}
135
136#[cfg(test)]
137mod tests {
138    use super::*;
139
140    #[test]
141    fn test_storage_config_new() {
142        let config = StorageConfig::new();
143
144        assert!(config.props().is_empty());
145    }
146
147    #[test]
148    fn test_storage_config_from_props() {
149        let props = HashMap::from([
150            ("region".to_string(), "us-east-1".to_string()),
151            ("bucket".to_string(), "my-bucket".to_string()),
152        ]);
153        let config = StorageConfig::from_props(props.clone());
154
155        assert_eq!(config.props(), &props);
156    }
157
158    #[test]
159    fn test_storage_config_default() {
160        let config = StorageConfig::default();
161
162        assert!(config.props().is_empty());
163    }
164
165    #[test]
166    fn test_debug_redacts_credential_values() {
167        let config = StorageConfig::from_props(HashMap::from([
168            ("s3.access-key-id".to_string(), "vended-key".to_string()),
169            (
170                "s3.secret-access-key".to_string(),
171                "super-secret".to_string(),
172            ),
173            ("s3.session-token".to_string(), "vended-token".to_string()),
174        ]));
175
176        let rendered = format!("{config:?}");
177        // Secret values must never appear in Debug output (reachable via FileIO/Table).
178        assert!(
179            !rendered.contains("super-secret"),
180            "leaked secret: {rendered}"
181        );
182        assert!(
183            !rendered.contains("vended-token"),
184            "leaked token: {rendered}"
185        );
186        // Keys stay visible so routing/config is still diagnosable.
187        assert!(
188            rendered.contains("s3.secret-access-key"),
189            "keys hidden: {rendered}"
190        );
191    }
192
193    #[test]
194    fn test_storage_config_get() {
195        let config = StorageConfig::new().with_prop("region", "us-east-1");
196
197        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
198        assert_eq!(config.get("nonexistent"), None);
199    }
200
201    #[test]
202    fn test_storage_config_with_prop() {
203        let config = StorageConfig::new()
204            .with_prop("region", "us-east-1")
205            .with_prop("bucket", "my-bucket");
206
207        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
208        assert_eq!(config.get("bucket"), Some(&"my-bucket".to_string()));
209    }
210
211    #[test]
212    fn test_storage_config_with_props() {
213        let additional_props = vec![("key1", "value1"), ("key2", "value2")];
214        let config = StorageConfig::new().with_props(additional_props);
215
216        assert_eq!(config.get("key1"), Some(&"value1".to_string()));
217        assert_eq!(config.get("key2"), Some(&"value2".to_string()));
218    }
219
220    #[test]
221    fn test_storage_config_clone() {
222        let config = StorageConfig::new().with_prop("region", "us-east-1");
223        let cloned = config.clone();
224
225        assert_eq!(config, cloned);
226        assert_eq!(cloned.get("region"), Some(&"us-east-1".to_string()));
227    }
228
229    #[test]
230    fn test_storage_config_serialization_roundtrip() {
231        let config = StorageConfig::new()
232            .with_prop("region", "us-east-1")
233            .with_prop("bucket", "my-bucket");
234
235        let serialized = serde_json::to_string(&config).unwrap();
236        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
237
238        assert_eq!(config, deserialized);
239    }
240
241    #[test]
242    fn test_storage_config_clone_independence() {
243        let original = StorageConfig::new().with_prop("region", "us-east-1");
244        let mut cloned = original.clone();
245
246        // Modify the clone
247        cloned = cloned.with_prop("region", "eu-west-1");
248        cloned = cloned.with_prop("new_key", "new_value");
249
250        // Original should be unchanged
251        assert_eq!(original.get("region"), Some(&"us-east-1".to_string()));
252        assert_eq!(original.get("new_key"), None);
253
254        // Clone should have the new values
255        assert_eq!(cloned.get("region"), Some(&"eu-west-1".to_string()));
256        assert_eq!(cloned.get("new_key"), Some(&"new_value".to_string()));
257    }
258
259    #[test]
260    fn test_storage_config_from_props_empty() {
261        let config = StorageConfig::from_props(HashMap::new());
262
263        assert!(config.props().is_empty());
264    }
265
266    #[test]
267    fn test_storage_config_serialization_empty() {
268        let config = StorageConfig::new();
269
270        let serialized = serde_json::to_string(&config).unwrap();
271        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
272
273        assert_eq!(config, deserialized);
274        assert!(deserialized.props().is_empty());
275    }
276}
iceberg/io/storage/config/mod.rs

iceberg/io/storage/config/
mod.rs
