Struct mz_storage::statistics::AggregatedStatistics

source ·

pub struct AggregatedStatistics {
    worker_id: usize,
    worker_count: usize,
    local_source_statistics: BTreeMap<GlobalId, (usize, SourceStatistics)>,
    local_sink_statistics: BTreeMap<GlobalId, (usize, SinkStatistics)>,
    global_source_statistics: BTreeMap<GlobalId, (usize, Vec<Option<SourceStatisticsUpdate>>)>,
    global_sink_statistics: BTreeMap<GlobalId, (usize, Vec<Option<SinkStatisticsUpdate>>)>,
}

Expand description

A structure that keeps track of local statistics, as well as aggregating statistics into a single worker (currently worker 0).

This is because we ONLY want to emit statistics for gauge-style-statistics when ALL workers have caught up to the currently running instance of a source/sink and have reported an accurate gauge. This prevents issues like 1 worker reporting snapshot_committed=true before the others workers have caught up to report that they haven’t finished snapshotting.

The API flow is:

Initialize sources/sinks with initialize_source or initialize_sink. This advances the local epoch.
Advance the global epoch with advance_global_epoch. This should always be called strictly before reinitializing objects.
Emit a snapshot of local data with emit_local, which can be broadcasted to other workers.
Ingest data from other workers with ingest.
Emit a snapshot of global data (i.e. skipping objects that don’t have all workers caught up) with snapshot.

All functions can and should be called from ALL workers.

Note that we hand-roll a statistics epoch here, but in the future, when the storage controller has top-level support for dataflow epochs, this logic can be entirely moved there.

Fields§

§worker_id: usize§worker_count: usize§local_source_statistics: BTreeMap<GlobalId, (usize, SourceStatistics)>§local_sink_statistics: BTreeMap<GlobalId, (usize, SinkStatistics)>§global_source_statistics: BTreeMap<GlobalId, (usize, Vec<Option<SourceStatisticsUpdate>>)>§global_sink_statistics: BTreeMap<GlobalId, (usize, Vec<Option<SinkStatisticsUpdate>>)>

Struct mz_storage::statistics::AggregatedStatisticsCopy item path

Fields§

Implementations§

impl AggregatedStatistics

pub fn new(worker_id: usize, worker_count: usize) -> Self

pub fn get_source(&self, id: &GlobalId) -> Option<&SourceStatistics>

pub fn get_sink(&self, id: &GlobalId) -> Option<&SinkStatistics>

pub fn deinitialize(&mut self, id: GlobalId)

pub fn advance_global_epoch(&mut self, id: GlobalId)

pub fn initialize_source<F: FnOnce() -> SourceStatistics>( &mut self, id: GlobalId, stats: F, )

pub fn initialize_sink<F: FnOnce() -> SinkStatistics>( &mut self, id: GlobalId, stats: F, )

pub fn ingest( &mut self, source_statistics: Vec<(usize, SourceStatisticsRecord)>, sink_statistics: Vec<(usize, SinkStatisticsRecord)>, )

fn _emit_local( &mut self, ) -> (Vec<(usize, SourceStatisticsRecord)>, Vec<(usize, SinkStatisticsRecord)>)

pub fn emit_local( &mut self, ) -> (Vec<(usize, SourceStatisticsRecord)>, Vec<(usize, SinkStatisticsRecord)>)

pub fn snapshot( &mut self, ) -> (Vec<SourceStatisticsUpdate>, Vec<SinkStatisticsUpdate>)

Auto Trait Implementations§

impl Freeze for AggregatedStatistics

impl !RefUnwindSafe for AggregatedStatistics

impl !Send for AggregatedStatistics

impl !Sync for AggregatedStatistics

impl Unpin for AggregatedStatistics

impl !UnwindSafe for AggregatedStatistics

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T, U> CastInto<U> for Twhere U: CastFrom<T>,

fn cast_into(self) -> U

impl<T> Conv for T

fn conv<T>(self) -> Twhere Self: Into<T>,

impl<T> FmtForward for T

fn fmt_binary(self) -> FmtBinary<Self>where Self: Binary,

fn fmt_display(self) -> FmtDisplay<Self>where Self: Display,

fn fmt_lower_exp(self) -> FmtLowerExp<Self>where Self: LowerExp,

fn fmt_lower_hex(self) -> FmtLowerHex<Self>where Self: LowerHex,

fn fmt_octal(self) -> FmtOctal<Self>where Self: Octal,

fn fmt_pointer(self) -> FmtPointer<Self>where Self: Pointer,

fn fmt_upper_exp(self) -> FmtUpperExp<Self>where Self: UpperExp,

fn fmt_upper_hex(self) -> FmtUpperHex<Self>where Self: UpperHex,

fn fmt_list(self) -> FmtList<Self>where &'a Self: for<'a> IntoIterator,

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> FutureExt for T

fn with_context(self, otel_cx: Context) -> WithContext<Self>

fn with_current_context(self) -> WithContext<Self>

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoRequest<T> for T

fn into_request(self) -> Request<T>

impl<Unshared, Shared> IntoShared<Shared> for Unsharedwhere Shared: FromUnshared<Unshared>,

fn into_shared(self) -> Shared

impl<T, U> OverrideFrom<Option<&T>> for Uwhere U: OverrideFrom<T>,

fn override_from(self, layer: &Option<&T>) -> U

impl<T> Pipe for Twhere T: ?Sized,

fn pipe<R>(self, func: impl FnOnce(Self) -> R) -> Rwhere Self: Sized,

fn pipe_ref<'a, R>(&'a self, func: impl FnOnce(&'a Self) -> R) -> Rwhere R: 'a,

fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> Rwhere R: 'a,

fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> Rwhere Self: Borrow<B>, B: 'a + ?Sized, R: 'a,

fn pipe_borrow_mut<'a, B, R>( &'a mut self, func: impl FnOnce(&'a mut B) -> R, ) -> Rwhere Self: BorrowMut<B>, B: 'a + ?Sized, R: 'a,

fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> Rwhere Self: AsRef<U>, U: 'a + ?Sized, R: 'a,

fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> Rwhere Self: AsMut<U>, U: 'a + ?Sized, R: 'a,

fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> Rwhere Self: Deref<Target = T>, T: 'a + ?Sized, R: 'a,

fn pipe_deref_mut<'a, T, R>( &'a mut self, func: impl FnOnce(&'a mut T) -> R, ) -> Rwhere Self: DerefMut<Target = T> + Deref, T: 'a + ?Sized, R: 'a,

impl<T> Pointable for T

const ALIGN: usize = _

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<P, R> ProtoType<R> for Pwhere R: RustType<P>,

fn into_rust(self) -> Result<R, TryFromProtoError>

fn from_rust(rust: &R) -> P

impl<T> Same for T

Struct mz_storage::statistics::AggregatedStatistics

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T, U> CastInto<U> for T
where U: CastFrom<T>,

fn conv<T>(self) -> T
where Self: Into<T>,

fn fmt_binary(self) -> FmtBinary<Self>
where Self: Binary,

fn fmt_display(self) -> FmtDisplay<Self>
where Self: Display,

fn fmt_lower_exp(self) -> FmtLowerExp<Self>
where Self: LowerExp,

fn fmt_lower_hex(self) -> FmtLowerHex<Self>
where Self: LowerHex,

fn fmt_octal(self) -> FmtOctal<Self>
where Self: Octal,

fn fmt_pointer(self) -> FmtPointer<Self>
where Self: Pointer,

fn fmt_upper_exp(self) -> FmtUpperExp<Self>
where Self: UpperExp,

fn fmt_upper_hex(self) -> FmtUpperHex<Self>
where Self: UpperHex,

fn fmt_list(self) -> FmtList<Self>
where &'a Self: for<'a> IntoIterator,

impl<T, U> Into<U> for T
where U: From<T>,

impl<Unshared, Shared> IntoShared<Shared> for Unshared
where Shared: FromUnshared<Unshared>,

impl<T, U> OverrideFrom<Option<&T>> for U
where U: OverrideFrom<T>,

impl<T> Pipe for T
where T: ?Sized,

fn pipe<R>(self, func: impl FnOnce(Self) -> R) -> R
where Self: Sized,

fn pipe_ref<'a, R>(&'a self, func: impl FnOnce(&'a Self) -> R) -> R
where R: 'a,

fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R
where R: 'a,

fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> R
where Self: Borrow<B>, B: 'a + ?Sized, R: 'a,

fn pipe_borrow_mut<'a, B, R>( &'a mut self, func: impl FnOnce(&'a mut B) -> R, ) -> R
where Self: BorrowMut<B>, B: 'a + ?Sized, R: 'a,

fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> R
where Self: AsRef<U>, U: 'a + ?Sized, R: 'a,

fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> R
where Self: AsMut<U>, U: 'a + ?Sized, R: 'a,

fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R
where Self: Deref<Target = T>, T: 'a + ?Sized, R: 'a,

fn pipe_deref_mut<'a, T, R>( &'a mut self, func: impl FnOnce(&'a mut T) -> R, ) -> R
where Self: DerefMut<Target = T> + Deref, T: 'a + ?Sized, R: 'a,

impl<P, R> ProtoType<R> for P
where R: RustType<P>,

impl<'a, S, T> Semigroup<&'a S> for T
where T: Semigroup<S>,

fn tap_borrow<B>(self, func: impl FnOnce(&B)) -> Self
where Self: Borrow<B>, B: ?Sized,

fn tap_borrow_mut<B>(self, func: impl FnOnce(&mut B)) -> Self
where Self: BorrowMut<B>, B: ?Sized,

fn tap_ref<R>(self, func: impl FnOnce(&R)) -> Self
where Self: AsRef<R>, R: ?Sized,

fn tap_ref_mut<R>(self, func: impl FnOnce(&mut R)) -> Self
where Self: AsMut<R>, R: ?Sized,

fn tap_deref<T>(self, func: impl FnOnce(&T)) -> Self
where Self: Deref<Target = T>, T: ?Sized,

fn tap_deref_mut<T>(self, func: impl FnOnce(&mut T)) -> Self
where Self: DerefMut<Target = T> + Deref, T: ?Sized,

fn tap_borrow_dbg<B>(self, func: impl FnOnce(&B)) -> Self
where Self: Borrow<B>, B: ?Sized,

fn tap_borrow_mut_dbg<B>(self, func: impl FnOnce(&mut B)) -> Self
where Self: BorrowMut<B>, B: ?Sized,

fn tap_ref_dbg<R>(self, func: impl FnOnce(&R)) -> Self
where Self: AsRef<R>, R: ?Sized,

fn tap_ref_mut_dbg<R>(self, func: impl FnOnce(&mut R)) -> Self
where Self: AsMut<R>, R: ?Sized,

fn tap_deref_dbg<T>(self, func: impl FnOnce(&T)) -> Self
where Self: Deref<Target = T>, T: ?Sized,

fn tap_deref_mut_dbg<T>(self, func: impl FnOnce(&mut T)) -> Self
where Self: DerefMut<Target = T> + Deref, T: ?Sized,

fn try_conv<T>(self) -> Result<T, Self::Error>
where Self: TryInto<T>,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,