pub trait OneshotFormat: Clone {
type WorkRequest<S>: Debug + Clone + Send + Serialize + DeserializeOwned + 'static
where S: OneshotSource;
type RecordChunk: Debug + Clone + Send + Serialize + DeserializeOwned + 'static;
// Required methods
fn split_work<S: OneshotSource + Send>(
&self,
source: S,
object: S::Object,
checksum: S::Checksum,
) -> impl Future<Output = Result<Vec<Self::WorkRequest<S>>, StorageErrorX>> + Send;
fn fetch_work<'a, S: OneshotSource + Sync + 'static>(
&'a self,
source: &'a S,
request: Self::WorkRequest<S>,
) -> BoxStream<'a, Result<Self::RecordChunk, StorageErrorX>>;
fn decode_chunk(
&self,
chunk: Self::RecordChunk,
rows: &mut Vec<Row>,
) -> Result<usize, StorageErrorX>;
}
Expand description
Defines a format that we fetch for a “one time” ingestion.
Required Associated Types§
Sourcetype WorkRequest<S>: Debug + Clone + Send + Serialize + DeserializeOwned + 'static
where
S: OneshotSource
type WorkRequest<S>: Debug + Clone + Send + Serialize + DeserializeOwned + 'static where S: OneshotSource
A single unit of work for decoding this format, e.g. a single Parquet RowGroup.
Sourcetype RecordChunk: Debug + Clone + Send + Serialize + DeserializeOwned + 'static
type RecordChunk: Debug + Clone + Send + Serialize + DeserializeOwned + 'static
A chunk of records in this format that can be decoded into Rows.
Required Methods§
Sourcefn split_work<S: OneshotSource + Send>(
&self,
source: S,
object: S::Object,
checksum: S::Checksum,
) -> impl Future<Output = Result<Vec<Self::WorkRequest<S>>, StorageErrorX>> + Send
fn split_work<S: OneshotSource + Send>( &self, source: S, object: S::Object, checksum: S::Checksum, ) -> impl Future<Output = Result<Vec<Self::WorkRequest<S>>, StorageErrorX>> + Send
Given an upstream object, defines how we should parse this object in parallel.
Note: It’s totally fine to not process an object in parallel, and just return a single
Self::WorkRequest
here.
Sourcefn fetch_work<'a, S: OneshotSource + Sync + 'static>(
&'a self,
source: &'a S,
request: Self::WorkRequest<S>,
) -> BoxStream<'a, Result<Self::RecordChunk, StorageErrorX>>
fn fetch_work<'a, S: OneshotSource + Sync + 'static>( &'a self, source: &'a S, request: Self::WorkRequest<S>, ) -> BoxStream<'a, Result<Self::RecordChunk, StorageErrorX>>
Given a work request, fetch data from the OneshotSource
and return it in a format that
can later be decoded.
Sourcefn decode_chunk(
&self,
chunk: Self::RecordChunk,
rows: &mut Vec<Row>,
) -> Result<usize, StorageErrorX>
fn decode_chunk( &self, chunk: Self::RecordChunk, rows: &mut Vec<Row>, ) -> Result<usize, StorageErrorX>
Decode a chunk of records into Row
s.
Dyn Compatibility§
This trait is not dyn compatible.
In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.