Struct dataflow::source::S3SourceReader

pub struct S3SourceReader {
    source_name: String,
    id: SourceInstanceId,
    receiver_stream: Receiver<Result<InternalMessage, S3Error>>,
    dataflow_status: Sender<DataflowStatus>,
    offset: S3Offset,
}

Information required to load data from S3

Fields

source_name: String

The name of the source that the user entered

id: SourceInstanceId

Unique source ID

receiver_stream: Receiver<Result<InternalMessage, S3Error>>

Receiver channel that ingests records

dataflow_status: Sender<DataflowStatus>

offset: S3Offset

Total number of records that this source has read

Trait Implementations

impl Drop for S3SourceReader

fn drop(&mut self)

Executes the destructor for this type.

impl SourceReader for S3SourceReader

type Key = ()

type Value = MessagePayload

fn new(
    source_name: String,
    source_id: SourceInstanceId,
    worker_id: usize,
    _worker_count: usize,
    consumer_activator: SyncActivator,
    connector: ExternalSourceConnector,
    _restored_offsets: Vec<(PartitionId, Option<MzOffset>)>,
    _encoding: SourceDataEncoding,
    _: Option<Logger>,
    metrics: SourceBaseMetrics
) -> Result<(S3SourceReader, Option<PartitionId>), Error>

Create a new source reader.

fn get_next_message(
    &mut self
) -> Result<NextMessage<Self::Key, Self::Value>, Error>

Returns the next message available from the source.