iceberg/io/storage/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Storage interfaces for Iceberg.
19
20mod config;
21mod local_fs;
22mod memory;
23
24use std::fmt::Debug;
25use std::sync::Arc;
26
27use async_trait::async_trait;
28use bytes::Bytes;
29pub use config::*;
30pub use local_fs::{LocalFsStorage, LocalFsStorageFactory};
31pub use memory::{MemoryStorage, MemoryStorageFactory};
32
33use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile};
34use crate::Result;
35
36/// Trait for storage operations in Iceberg.
37///
38/// The trait supports serialization via `typetag`, allowing storage instances to be
39/// serialized and deserialized across process boundaries.
40///
41/// Third-party implementations can implement this trait to provide custom storage backends.
42///
43/// # Implementing Custom Storage
44///
45/// To implement a custom storage backend:
46///
47/// 1. Create a struct that implements this trait
48/// 2. Add `#[typetag::serde]` attribute for serialization support
49/// 3. Implement all required methods
50///
51/// # Example
52///
53/// ```rust,ignore
54/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
55/// struct MyStorage {
56///     // custom fields
57/// }
58///
59/// #[async_trait]
60/// #[typetag::serde]
61/// impl Storage for MyStorage {
62///     async fn exists(&self, path: &str) -> Result<bool> {
63///         // implementation
64///         todo!()
65///     }
66///     // ... implement other methods
67/// }
68/// ```
69#[async_trait]
70#[typetag::serde(tag = "type")]
71pub trait Storage: Debug + Send + Sync {
72    /// Check if a file exists at the given path
73    async fn exists(&self, path: &str) -> Result<bool>;
74
75    /// Get metadata from an input path
76    async fn metadata(&self, path: &str) -> Result<FileMetadata>;
77
78    /// Read bytes from a path
79    async fn read(&self, path: &str) -> Result<Bytes>;
80
81    /// Get FileRead from a path
82    async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>>;
83
84    /// Write bytes to an output path
85    async fn write(&self, path: &str, bs: Bytes) -> Result<()>;
86
87    /// Get FileWrite from a path
88    async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>>;
89
90    /// Delete a file at the given path
91    async fn delete(&self, path: &str) -> Result<()>;
92
93    /// Delete all files with the given prefix
94    async fn delete_prefix(&self, path: &str) -> Result<()>;
95
96    /// Create a new input file for reading
97    fn new_input(&self, path: &str) -> Result<InputFile>;
98
99    /// Create a new output file for writing
100    fn new_output(&self, path: &str) -> Result<OutputFile>;
101}
102
103/// Factory for creating Storage instances from configuration.
104///
105/// Implement this trait to provide custom storage backends. The factory pattern
106/// allows for lazy initialization of storage instances and enables users to
107/// inject custom storage implementations into catalogs.
108///
109/// # Example
110///
111/// ```rust,ignore
112/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
113/// struct MyCustomStorageFactory {
114///     // custom configuration
115/// }
116///
117/// #[typetag::serde]
118/// impl StorageFactory for MyCustomStorageFactory {
119///     fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>> {
120///         // Create and return custom storage implementation
121///         todo!()
122///     }
123/// }
124/// ```
125#[typetag::serde(tag = "type")]
126pub trait StorageFactory: Debug + Send + Sync {
127    /// Build a new Storage instance from the given configuration.
128    ///
129    /// # Arguments
130    ///
131    /// * `config` - The storage configuration containing scheme and properties
132    ///
133    /// # Returns
134    ///
135    /// A `Result` containing an `Arc<dyn Storage>` on success, or an error
136    /// if the storage could not be created.
137    fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>>;
138}
iceberg/io/storage/mod.rs

iceberg/io/storage/
mod.rs
