Struct mz_storage::source::KinesisSourceReader
source · pub struct KinesisSourceReader {
tokio_handle: TokioHandle,
kinesis_client: KinesisClient,
stream_name: String,
shard_set: BTreeMap<String, ShardMetrics>,
shard_queue: VecDeque<(String, Option<String>)>,
last_checked_shards: Instant,
buffered_messages: VecDeque<(SourceMessage<(), Option<Vec<u8>>>, MzOffset)>,
processed_message_count: u64,
base_metrics: KinesisMetrics,
active_read_worker: bool,
data_capability: Capability<MzOffset>,
upper_capability: Capability<MzOffset>,
}
Expand description
Contains all information necessary to ingest data from Kinesis
Fields§
§tokio_handle: TokioHandle
§kinesis_client: KinesisClient
Kinesis client used to obtain records
stream_name: String
The name of the stream
shard_set: BTreeMap<String, ShardMetrics>
The set of active shards
shard_queue: VecDeque<(String, Option<String>)>
A queue representing the next shard to read from. This is necessary to ensure that all shards are read from uniformly
last_checked_shards: Instant
The time at which we last refreshed metadata TODO(natacha): this should be moved to timestamper
buffered_messages: VecDeque<(SourceMessage<(), Option<Vec<u8>>>, MzOffset)>
Storage for messages that have not yet been timestamped
processed_message_count: u64
Count of processed message
base_metrics: KinesisMetrics
Metrics from which per-shard metrics get created.
active_read_worker: bool
§data_capability: Capability<MzOffset>
Capabilities used to produce messages
upper_capability: Capability<MzOffset>
Implementations§
source§impl KinesisSourceReader
impl KinesisSourceReader
async fn update_shard_information(&mut self) -> Result<(), Error>
sourceasync fn get_records(
&self,
shard_iterator: &str
) -> Result<GetRecordsOutput, SdkError<GetRecordsError>>
async fn get_records(
&self,
shard_iterator: &str
) -> Result<GetRecordsOutput, SdkError<GetRecordsError>>
Obtains the next record for this shard given a shard iterator
Trait Implementations§
source§impl SourceReader for KinesisSourceReader
impl SourceReader for KinesisSourceReader
type Key = ()
type Value = Option<Vec<u8, Global>>
type Time = MzOffset
type Diff = ()
source§fn get_next_message(
&mut self
) -> NextMessage<Self::Key, Self::Value, Self::Time, Self::Diff>
fn get_next_message(
&mut self
) -> NextMessage<Self::Key, Self::Value, Self::Time, Self::Diff>
Returns the next message available from the source. Read more
source§fn next<'life0, 'async_trait>(
&'life0 mut self,
timestamp_granularity: Duration
) -> Pin<Box<dyn Future<Output = Option<SourceMessageType<Self::Key, Self::Value, Self::Time, Self::Diff>>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn next<'life0, 'async_trait>(
&'life0 mut self,
timestamp_granularity: Duration
) -> Pin<Box<dyn Future<Output = Option<SourceMessageType<Self::Key, Self::Value, Self::Time, Self::Diff>>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Returns the next message available from the source.
source§fn into_stream<'a>(
self,
timestamp_granularity: Duration
) -> LocalBoxStream<'a, SourceMessageType<Self::Key, Self::Value, Self::Time, Self::Diff>>where
Self: Sized + 'a,
fn into_stream<'a>(
self,
timestamp_granularity: Duration
) -> LocalBoxStream<'a, SourceMessageType<Self::Key, Self::Value, Self::Time, Self::Diff>>where
Self: Sized + 'a,
Returns an adapter that treats the source as a stream. Read more
Auto Trait Implementations§
impl !RefUnwindSafe for KinesisSourceReader
impl !Send for KinesisSourceReader
impl !Sync for KinesisSourceReader
impl Unpin for KinesisSourceReader
impl !UnwindSafe for KinesisSourceReader
Blanket Implementations§
source§impl<T> FutureExt for T
impl<T> FutureExt for T
source§fn with_context(self, otel_cx: Context) -> WithContext<Self>
fn with_context(self, otel_cx: Context) -> WithContext<Self>
source§fn with_current_context(self) -> WithContext<Self>
fn with_current_context(self) -> WithContext<Self>
source§impl<T> Instrument for T
impl<T> Instrument for T
source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
source§impl<T> IntoRequest<T> for T
impl<T> IntoRequest<T> for T
source§fn into_request(self) -> Request<T>
fn into_request(self) -> Request<T>
Wrap the input message
T
in a tonic::Request
source§impl<T> Pointable for T
impl<T> Pointable for T
source§impl<P, R> ProtoType<R> for Pwhere
R: RustType<P>,
impl<P, R> ProtoType<R> for Pwhere
R: RustType<P>,
source§fn into_rust(self) -> Result<R, TryFromProtoError>
fn into_rust(self) -> Result<R, TryFromProtoError>
See
RustType::from_proto
.source§fn from_rust(rust: &R) -> P
fn from_rust(rust: &R) -> P
See
RustType::into_proto
.