mz_storage_types/
stats.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Types and traits that connect up our mz-repr types with the stats that persist maintains.
11
12use mz_expr::{ColumnSpecs, Interpreter, MapFilterProject, ResultSpec, UnmaterializableFunc};
13use mz_persist_types::stats::{
14    BytesStats, ColumnStatKinds, JsonStats, PartStats, PartStatsMetrics,
15};
16use mz_repr::{
17    ColumnIndex, Datum, RelationDesc, ReprRelationType, RowArena, SqlColumnType, SqlScalarType,
18};
19
20/// Bundles together a relation desc with the stats for a specific part, and translates between
21/// Persist's stats representation and the `ResultSpec`s that are used for eg. filter pushdown.
22#[derive(Debug)]
23pub struct RelationPartStats<'a> {
24    pub(crate) name: &'a str,
25    pub(crate) metrics: &'a PartStatsMetrics,
26    pub(crate) desc: &'a RelationDesc,
27    pub(crate) stats: &'a PartStats,
28}
29
30impl<'a> RelationPartStats<'a> {
31    pub fn new(
32        name: &'a str,
33        metrics: &'a PartStatsMetrics,
34        desc: &'a RelationDesc,
35        stats: &'a PartStats,
36    ) -> Self {
37        Self {
38            name,
39            metrics,
40            desc,
41            stats,
42        }
43    }
44}
45
46impl RelationPartStats<'_> {
47    pub fn may_match_mfp<'a>(&'a self, time_range: ResultSpec<'a>, mfp: &MapFilterProject) -> bool {
48        let arena = RowArena::new();
49        let relation = ReprRelationType::from(self.desc.typ());
50        let mut ranges = ColumnSpecs::new(&relation, &arena);
51        ranges.push_unmaterializable(UnmaterializableFunc::MzNow, time_range);
52
53        // If the error collection is nonempty, we always keep the part.
54        // Missing err stats mean errors cannot be ruled out, so they count as
55        // "may error" too, matching the storage read path's `filter_result`.
56        if self.err_count().is_none_or(|count| count > 0) {
57            return true;
58        }
59
60        for (pos, (idx, _name, _typ)) in self.desc.iter_all().enumerate() {
61            let result_spec = self.col_stats(idx, &arena);
62            ranges.push_column(pos, result_spec);
63        }
64        let result = ranges.mfp_filter(mfp).range;
65        result.may_contain(Datum::True) || result.may_fail()
66    }
67
68    fn json_spec<'a>(len: usize, stats: &'a JsonStats, arena: &'a RowArena) -> ResultSpec<'a> {
69        match stats {
70            JsonStats::JsonNulls => ResultSpec::value(Datum::JsonNull),
71            JsonStats::Bools(bools) => {
72                ResultSpec::value_between(bools.lower.into(), bools.upper.into())
73            }
74            JsonStats::Strings(strings) => ResultSpec::value_between(
75                Datum::String(strings.lower.as_str()),
76                Datum::String(strings.upper.as_str()),
77            ),
78            JsonStats::Numerics(numerics) => {
79                match mz_repr::stats::decode_numeric(numerics, arena) {
80                    Ok((lower, upper)) => ResultSpec::value_between(lower, upper),
81                    Err(err) => {
82                        tracing::error!(%err, "failed to decode Json Numeric stats!");
83                        ResultSpec::anything()
84                    }
85                }
86            }
87            JsonStats::Maps(maps) => {
88                ResultSpec::map_spec(
89                    maps.into_iter()
90                        .map(|(k, v)| {
91                            let mut v_spec = Self::json_spec(v.len, &v.stats, arena);
92                            if v.len != len {
93                                // This field is not always present, so assume
94                                // that accessing it might be null.
95                                v_spec = v_spec.union(ResultSpec::null());
96                            }
97                            let key = arena.make_datum(|r| r.push(Datum::String(k.as_str())));
98                            (key, v_spec)
99                        })
100                        .collect(),
101                )
102            }
103            JsonStats::None => ResultSpec::nothing(),
104            JsonStats::Lists | JsonStats::Mixed => ResultSpec::anything(),
105        }
106    }
107
108    pub fn col_stats<'a>(&'a self, idx: &ColumnIndex, arena: &'a RowArena) -> ResultSpec<'a> {
109        let value_range = match self.col_values(idx, arena) {
110            Some(spec) => spec,
111            None => ResultSpec::anything(),
112        };
113        let json_range = self
114            .col_json(idx, arena)
115            .unwrap_or_else(ResultSpec::anything);
116
117        // If this is not a JSON column or we don't have JSON stats, json_range is
118        // [ResultSpec::anything] and this is a noop.
119        value_range.intersect(json_range)
120    }
121
122    fn col_json<'a>(&'a self, idx: &ColumnIndex, arena: &'a RowArena) -> Option<ResultSpec<'a>> {
123        let name = self.desc.get_name_idx(idx);
124        let typ = &self.desc.get_type(idx);
125
126        let ok_stats = self.stats.key.col("ok")?;
127        // These stats come straight off durable state, so a corrupt or
128        // version-skewed encoding can carry any shape here. Report the column
129        // range as unknown rather than panicking the process reading it.
130        let ok_stats = match ok_stats.try_as_optional_struct() {
131            Ok(ok_stats) => ok_stats,
132            Err(err) => {
133                self.metrics.mismatched_count.inc();
134                tracing::error!(
135                    "expected nullable struct stats for the 'ok' column of {}: {err}",
136                    self.name
137                );
138                return None;
139            }
140        };
141        let col_stats = ok_stats.some.cols.get(name.as_str())?;
142
143        if let SqlColumnType {
144            scalar_type: SqlScalarType::Jsonb,
145            nullable,
146        } = typ
147        {
148            let value_range = match &col_stats.values {
149                ColumnStatKinds::Bytes(BytesStats::Json(json_stats)) => {
150                    Self::json_spec(ok_stats.some.len, json_stats, arena)
151                }
152                ColumnStatKinds::Bytes(
153                    BytesStats::Primitive(_) | BytesStats::Atomic(_) | BytesStats::FixedSize(_),
154                ) => ResultSpec::anything(),
155                other => {
156                    self.metrics.mismatched_count.inc();
157                    tracing::error!(
158                        "expected BytesStats for JSON column {}, found {other:?}",
159                        self.name
160                    );
161                    return None;
162                }
163            };
164            let null_range = match (nullable, col_stats.nulls) {
165                (false, None) => ResultSpec::nothing(),
166                (true, Some(nulls)) if nulls.count == 0 => ResultSpec::nothing(),
167                (true, Some(_)) => ResultSpec::null(),
168                (col_null, stats_null) => {
169                    self.metrics.mismatched_count.inc();
170                    tracing::error!(
171                        "JSON column nullability mismatch, col {} null: {col_null}, stats: {stats_null:?}",
172                        self.name
173                    );
174                    return None;
175                }
176            };
177
178            Some(null_range.union(value_range))
179        } else {
180            None
181        }
182    }
183
184    pub fn len(&self) -> Option<usize> {
185        Some(self.stats.key.len)
186    }
187
188    pub fn ok_count(&self) -> Option<usize> {
189        // The number of OKs is the number of rows whose error is None.
190        // Malformed or wrong-shaped err stats (corrupt or version-skewed
191        // durable state) count as unknown, which callers treat as
192        // "may contain errors", rather than panicking the replica.
193        let stats = self.stats.key.col("err")?.try_as_optional_bytes().ok()?;
194        Some(stats.none)
195    }
196
197    pub fn err_count(&self) -> Option<usize> {
198        // Counter-intuitive: We can easily calculate the number of errors that
199        // were None from the column stats, but not how many were Some. So, what
200        // we do is count the number of Nones, which is the number of Oks, and
201        // then subtract that from the total.
202        let num_results = self.stats.key.len;
203        let num_oks = self.ok_count();
204        // An ok count exceeding the part length is corrupt stats; report the
205        // err count as unknown (callers keep the part) instead of underflowing.
206        num_oks.and_then(|num_oks| num_results.checked_sub(num_oks))
207    }
208
209    fn col_values<'a>(&'a self, idx: &ColumnIndex, arena: &'a RowArena) -> Option<ResultSpec<'a>> {
210        let name = self.desc.get_name_idx(idx);
211        let typ = self.desc.get_type(idx);
212
213        let ok_stats = self.stats.key.cols.get("ok")?;
214        // See the note in `col_json`: durable stats can be any shape, so a
215        // wrong-shaped 'ok' column makes the range unknown, not a panic.
216        let ColumnStatKinds::Struct(ok_stats) = &ok_stats.values else {
217            self.metrics.mismatched_count.inc();
218            tracing::error!(
219                "expected struct stats for the 'ok' column of {}, found {:?}",
220                self.name,
221                ok_stats.values
222            );
223            return None;
224        };
225        let col_stats = ok_stats.cols.get(name.as_str())?;
226
227        let min_max = mz_repr::stats::col_values(&typ.scalar_type, &col_stats.values, arena);
228        let null_count = col_stats.nulls.as_ref().map_or(0, |nulls| nulls.count);
229        let total_count = self.len();
230
231        let values = match (total_count, min_max) {
232            (Some(total_count), _) if total_count == null_count => ResultSpec::nothing(),
233            (_, Some((min, max))) => ResultSpec::value_between(min, max),
234            _ => ResultSpec::value_all(),
235        };
236        let nulls = if null_count > 0 {
237            ResultSpec::null()
238        } else {
239            ResultSpec::nothing()
240        };
241
242        Some(values.union(nulls))
243    }
244}
245
246#[cfg(test)]
247mod tests {
248    use arrow::array::AsArray;
249    use mz_ore::metrics::MetricsRegistry;
250    use mz_persist_types::codec_impls::UnitSchema;
251    use mz_persist_types::columnar::{ColumnDecoder, Schema};
252    use mz_persist_types::part::PartBuilder;
253    use mz_persist_types::stats::{PartStats, ProtoStructStats, TrimStats, trim_to_budget};
254    use mz_proto::RustType;
255    use mz_repr::{Datum, RelationDesc, Row, RowArena, SqlColumnType, SqlScalarType};
256    use mz_repr::{SqlRelationType, arb_datum_for_column};
257    use proptest::prelude::*;
258    use proptest::strategy::ValueTree;
259
260    use super::*;
261    use crate::sources::SourceData;
262
263    fn validate_stats(column_type: &SqlColumnType, datums: &[Datum<'_>]) -> Result<(), String> {
264        let schema = RelationDesc::builder()
265            .with_column("col", column_type.clone())
266            .finish();
267
268        let mut builder = PartBuilder::new(&schema, &UnitSchema);
269        let mut row = SourceData(Ok(Row::default()));
270        for datum in datums {
271            row.as_mut().unwrap().packer().push(datum);
272            builder.push(&row, &(), 1u64, 1i64);
273        }
274        let part = builder.finish();
275
276        let key_col = part.key.as_struct();
277        let decoder = <RelationDesc as Schema<SourceData>>::decoder(&schema, key_col.clone())
278            .expect("success");
279        let key_stats = decoder.stats();
280
281        // Trimming may widen bounds or drop them entirely, but must never
282        // narrow them, so the containment check below has to hold after every
283        // trimming pass: the lossy-but-column-preserving `trim`, and
284        // `trim_to_budget` at budgets all the way down to one that drops
285        // every column. The force-keep column matches the production default
286        // of never trimming the err column's stats.
287        let proto: ProtoStructStats = RustType::into_proto(&key_stats);
288        let mut variants = vec![("collected".to_string(), key_stats)];
289        {
290            let mut trimmed = proto.clone();
291            trimmed.trim();
292            variants.push((
293                "trimmed".to_string(),
294                RustType::from_proto(trimmed).expect("valid proto"),
295            ));
296        }
297        let full = prost::Message::encoded_len(&proto);
298        for budget in [full / 2, full / 4, 16, 0] {
299            let mut trimmed = proto.clone();
300            trim_to_budget(&mut trimmed, budget, |col| col == "err");
301            variants.push((
302                format!("trim_to_budget({budget})"),
303                RustType::from_proto(trimmed).expect("valid proto"),
304            ));
305        }
306
307        let metrics = PartStatsMetrics::new(&MetricsRegistry::new());
308        for (label, key_stats) in variants {
309            let stats = RelationPartStats {
310                name: "test",
311                metrics: &metrics,
312                stats: &PartStats { key: key_stats },
313                desc: &schema,
314            };
315            let arena = RowArena::default();
316
317            // Validate that the stats would include all of the provided datums.
318            for datum in datums {
319                let spec = stats.col_stats(&ColumnIndex::from_raw(0), &arena);
320                if !spec.may_contain(*datum) {
321                    return Err(format!(
322                        "{label} stats-derived spec claims {datum:?} is absent from a part that \
323                         contains it (type: {:?}, part: {datums:?}, spec: {spec:?})",
324                        column_type.scalar_type,
325                    ));
326                }
327            }
328        }
329
330        Ok(())
331    }
332
333    fn scalar_type_stats_roundtrip(scalar_type: SqlScalarType) {
334        // Non-nullable version of the column.
335        let column_type = scalar_type.clone().nullable(false);
336        for datum in scalar_type.interesting_datums() {
337            assert_eq!(validate_stats(&column_type, &[datum]), Ok(()));
338        }
339
340        // Nullable version of the column.
341        let column_type = scalar_type.clone().nullable(true);
342        for datum in scalar_type.interesting_datums() {
343            assert_eq!(validate_stats(&column_type, &[datum]), Ok(()));
344        }
345        assert_eq!(validate_stats(&column_type, &[Datum::Null]), Ok(()));
346    }
347
348    #[mz_ore::test]
349    #[cfg_attr(miri, ignore)] // too slow
350    fn all_scalar_types_stats_roundtrip() {
351        proptest!(|(scalar_type in any::<SqlScalarType>())| {
352            // The proptest! macro interferes with rustfmt.
353            scalar_type_stats_roundtrip(scalar_type)
354        });
355    }
356
357    /// Deterministic sweep over multi-datum parts: every pair of interesting
358    /// datums and the full set, packed into a single part per type.
359    ///
360    /// Part bounds are computed over the whole part, so a value whose ordering
361    /// the stats collection disagrees on (e.g. -NaN, which arrow's total order
362    /// puts below -Infinity but `OrderedFloat` ranks above every finite value)
363    /// can invalidate the bounds for *other* values in the part. Single-datum
364    /// parts, as covered by `all_scalar_types_stats_roundtrip`, can never
365    /// catch that class of bug.
366    #[mz_ore::test]
367    #[cfg_attr(miri, ignore)] // too slow
368    fn interesting_datum_combinations_stats_roundtrip() {
369        for scalar_type in SqlScalarType::enumerate() {
370            let datums: Vec<_> = scalar_type.interesting_datums().collect();
371            if datums.is_empty() {
372                continue;
373            }
374            for nullable in [false, true] {
375                let column_type = scalar_type.clone().nullable(nullable);
376                for (i, a) in datums.iter().enumerate() {
377                    for b in &datums[i + 1..] {
378                        assert_eq!(validate_stats(&column_type, &[*a, *b]), Ok(()));
379                    }
380                    if nullable {
381                        assert_eq!(validate_stats(&column_type, &[*a, Datum::Null]), Ok(()));
382                    }
383                }
384                let mut all = datums.clone();
385                if nullable {
386                    all.push(Datum::Null);
387                }
388                assert_eq!(validate_stats(&column_type, &all[..]), Ok(()));
389            }
390        }
391    }
392
393    #[mz_ore::test]
394    #[cfg_attr(miri, ignore)] // too slow
395    fn all_datums_produce_valid_stats() {
396        // A strategy that will return a Vec of Datums for an arbitrary SqlColumnType.
397        let datums = any::<SqlColumnType>().prop_flat_map(|ty| {
398            prop::collection::vec(arb_datum_for_column(ty.clone()), 0..128)
399                .prop_map(move |datums| (ty.clone(), datums))
400        });
401
402        proptest!(
403            ProptestConfig::with_cases(80),
404            |((ty, datums) in datums)| {
405                // The proptest! macro interferes with rustfmt.
406                let datums: Vec<_> = datums.iter().map(Datum::from).collect();
407                prop_assert_eq!(validate_stats(&ty, &datums[..]), Ok(()));
408            }
409        )
410    }
411
412    /// The err column's stats are force-kept from trimming by default, but
413    /// that list is configurable, so they can be absent. When they are,
414    /// `may_match_mfp` must treat the part as possibly containing errors,
415    /// exactly like `filter_result` does: error rows must surface regardless
416    /// of any filter, so a part that may hold one can never be skipped.
417    #[mz_ore::test]
418    #[cfg_attr(miri, ignore)] // too slow
419    fn may_match_mfp_missing_err_stats_keeps_part() {
420        use mz_expr::{BinaryFunc, EvalError, MirScalarExpr, func};
421        use mz_repr::ReprScalarType;
422
423        use crate::errors::DataflowError;
424
425        let schema = RelationDesc::builder()
426            .with_column("col", SqlScalarType::Int32.nullable(false))
427            .finish();
428        let mut builder = PartBuilder::new(&schema, &UnitSchema);
429        builder.push(
430            &SourceData(Ok(Row::pack_slice(&[Datum::Int32(1)]))),
431            &(),
432            1u64,
433            1i64,
434        );
435        builder.push(
436            &SourceData(Err(DataflowError::from(EvalError::DivisionByZero))),
437            &(),
438            1u64,
439            1i64,
440        );
441        let part = builder.finish();
442        let key_col = part.key.as_struct();
443        let decoder = <RelationDesc as Schema<SourceData>>::decoder(&schema, key_col.clone())
444            .expect("success");
445        let mut key_stats = decoder.stats();
446        // Simulate the err column's stats having been trimmed away.
447        key_stats.cols.remove("err").expect("err stats present");
448
449        let metrics = PartStatsMetrics::new(&MetricsRegistry::new());
450        let stats = RelationPartStats {
451            name: "test",
452            metrics: &metrics,
453            stats: &PartStats { key: key_stats },
454            desc: &schema,
455        };
456        // No Ok row matches this filter: only the error row makes the part
457        // relevant, and with the err stats missing it cannot be ruled out.
458        let mfp = MapFilterProject::new(1).filter(std::iter::once(MirScalarExpr::CallBinary {
459            func: BinaryFunc::Eq(func::Eq),
460            expr1: Box::new(MirScalarExpr::column(0)),
461            expr2: Box::new(MirScalarExpr::literal_ok(
462                Datum::Int32(999),
463                ReprScalarType::Int32,
464            )),
465        }));
466        assert!(stats.may_match_mfp(ResultSpec::anything(), &mfp));
467    }
468
469    /// Wrong-shaped err-column stats (corrupt or version-skewed durable
470    /// state) must read as "err count unknown", which fails open to keeping
471    /// the part, not panic the replica.
472    #[mz_ore::test]
473    #[cfg_attr(miri, ignore)] // too slow
474    fn malformed_err_stats_fail_open() {
475        use mz_persist_types::stats::{ColumnNullStats, ColumnarStats, PrimitiveStats};
476
477        let schema = RelationDesc::builder()
478            .with_column("col", SqlScalarType::Int32.nullable(false))
479            .finish();
480        let mut builder = PartBuilder::new(&schema, &UnitSchema);
481        builder.push(
482            &SourceData(Ok(Row::pack_slice(&[Datum::Int32(1)]))),
483            &(),
484            1u64,
485            1i64,
486        );
487        let part = builder.finish();
488        let key_col = part.key.as_struct();
489        let decoder = <RelationDesc as Schema<SourceData>>::decoder(&schema, key_col.clone())
490            .expect("success");
491        let mut key_stats = decoder.stats();
492        // Overwrite the err column's stats with a wrong-shaped entry.
493        key_stats.cols.insert(
494            "err".to_string(),
495            ColumnarStats {
496                nulls: Some(ColumnNullStats { count: 0 }),
497                values: PrimitiveStats {
498                    lower: 0i32,
499                    upper: 0i32,
500                }
501                .into(),
502            },
503        );
504
505        let metrics = PartStatsMetrics::new(&MetricsRegistry::new());
506        let stats = RelationPartStats {
507            name: "test",
508            metrics: &metrics,
509            stats: &PartStats { key: key_stats },
510            desc: &schema,
511        };
512        assert_eq!(stats.ok_count(), None);
513        assert_eq!(stats.err_count(), None);
514
515        // Well-shaped err stats whose none count exceeds the part length
516        // (corrupt or version-skewed) must read as unknown, not underflow.
517        let mut key_stats = decoder.stats();
518        match key_stats.cols.get_mut("err") {
519            Some(err_stats) => match &mut err_stats.values {
520                ColumnStatKinds::Bytes(BytesStats::Primitive(_)) => {
521                    err_stats.nulls = Some(mz_persist_types::stats::ColumnNullStats {
522                        count: key_stats.len + 1,
523                    });
524                }
525                other => panic!("unexpected err stats {other:?}"),
526            },
527            None => panic!("err stats missing"),
528        }
529        let stats = RelationPartStats {
530            name: "test",
531            metrics: &metrics,
532            stats: &PartStats { key: key_stats },
533            desc: &schema,
534        };
535        assert_eq!(stats.err_count(), None);
536    }
537
538    /// Wrong-shaped ok-column stats must read as "column range unknown",
539    /// which fails open to keeping the part. Both column paths run here: a
540    /// plain column goes through `col_values`, a JSON one adds `col_json`,
541    /// and neither may panic the process reading durable state.
542    #[mz_ore::test]
543    #[cfg_attr(miri, ignore)] // too slow
544    fn malformed_ok_stats_fail_open() {
545        use mz_expr::{BinaryFunc, MirScalarExpr, func};
546        use mz_persist_types::stats::{ColumnNullStats, ColumnarStats, PrimitiveStats};
547        use mz_repr::ReprScalarType;
548
549        let schema = RelationDesc::builder()
550            .with_column("col", SqlScalarType::Int32.nullable(false))
551            .with_column("json", SqlScalarType::Jsonb.nullable(true))
552            .finish();
553        let mut builder = PartBuilder::new(&schema, &UnitSchema);
554        builder.push(
555            &SourceData(Ok(Row::pack_slice(&[Datum::Int32(1), Datum::JsonNull]))),
556            &(),
557            1u64,
558            1i64,
559        );
560        let part = builder.finish();
561        let key_col = part.key.as_struct();
562        let decoder = <RelationDesc as Schema<SourceData>>::decoder(&schema, key_col.clone())
563            .expect("success");
564
565        let json_idx = schema
566            .iter_all()
567            .map(|(idx, _name, _typ)| idx)
568            .nth(1)
569            .expect("two columns");
570        // A filter no Ok row in this part satisfies. With well-shaped stats
571        // the part is skipped, so keeping it proves the fallback ran.
572        let mfp = MapFilterProject::new(2).filter(std::iter::once(MirScalarExpr::CallBinary {
573            func: BinaryFunc::Eq(func::Eq),
574            expr1: Box::new(MirScalarExpr::column(0)),
575            expr2: Box::new(MirScalarExpr::literal_ok(
576                Datum::Int32(999),
577                ReprScalarType::Int32,
578            )),
579        }));
580
581        let metrics = PartStatsMetrics::new(&MetricsRegistry::new());
582        let arena = RowArena::new();
583        let well_shaped = PartStats {
584            key: decoder.stats(),
585        };
586        let well_shaped = RelationPartStats::new("test", &metrics, &schema, &well_shaped);
587        assert!(!well_shaped.may_match_mfp(ResultSpec::anything(), &mfp));
588
589        // `col_values` matched the ok column's kind infallibly. A wrong kind
590        // makes every column's range unknown.
591        let mut not_a_struct = decoder.stats();
592        not_a_struct.cols.insert(
593            "ok".to_string(),
594            ColumnarStats {
595                nulls: Some(ColumnNullStats { count: 0 }),
596                values: PrimitiveStats {
597                    lower: 0i32,
598                    upper: 0i32,
599                }
600                .into(),
601            },
602        );
603        let not_a_struct = PartStats { key: not_a_struct };
604        let not_a_struct = RelationPartStats::new("test", &metrics, &schema, &not_a_struct);
605        for (idx, _name, _typ) in schema.iter_all() {
606            assert_eq!(not_a_struct.col_stats(idx, &arena), ResultSpec::anything());
607        }
608        assert!(not_a_struct.may_match_mfp(ResultSpec::anything(), &mfp));
609
610        // `col_json` additionally required the ok column to be nullable. A
611        // struct that is not widens the JSON range alone, so assert on that
612        // rather than on the part-level decision.
613        let mut not_nullable = decoder.stats();
614        match not_nullable.cols.get_mut("ok") {
615            Some(ok_stats) => ok_stats.nulls = None,
616            None => panic!("ok stats missing"),
617        }
618        let not_nullable = PartStats { key: not_nullable };
619        let not_nullable = RelationPartStats::new("test", &metrics, &schema, &not_nullable);
620        let other_json = Datum::String("a");
621        assert!(
622            !well_shaped
623                .col_stats(json_idx, &arena)
624                .may_contain(other_json)
625        );
626        assert!(
627            not_nullable
628                .col_stats(json_idx, &arena)
629                .may_contain(other_json)
630        );
631    }
632
633    #[mz_ore::test]
634    #[ignore] // TODO(parkmycar): Re-enable this test with a smaller sample size.
635    fn statistics_stability() {
636        /// This is the seed [`proptest`] uses for their deterministic RNG. We
637        /// copy it here to prevent breaking this test if [`proptest`] changes.
638        const RNG_SEED: [u8; 32] = [
639            0xf4, 0x16, 0x16, 0x48, 0xc3, 0xac, 0x77, 0xac, 0x72, 0x20, 0x0b, 0xea, 0x99, 0x67,
640            0x2d, 0x6d, 0xca, 0x9f, 0x76, 0xaf, 0x1b, 0x09, 0x73, 0xa0, 0x59, 0x22, 0x6d, 0xc5,
641            0x46, 0x39, 0x1c, 0x4a,
642        ];
643
644        let rng = proptest::test_runner::TestRng::from_seed(
645            proptest::test_runner::RngAlgorithm::ChaCha,
646            &RNG_SEED,
647        );
648        // Generate a collection of Rows.
649        let config = proptest::test_runner::Config {
650            // We let the loop below drive how much data we generate.
651            cases: u32::MAX,
652            rng_algorithm: proptest::test_runner::RngAlgorithm::ChaCha,
653            ..Default::default()
654        };
655        let mut runner = proptest::test_runner::TestRunner::new_with_rng(config, rng);
656
657        let max_cols = 4;
658        let max_rows = 8;
659        let test_cases = 1000;
660
661        // Note: We don't use the `Arbitrary` impl for `RelationDesc` because
662        // it generates large column names which is not interesting to us.
663        let strat = proptest::collection::vec(any::<SqlColumnType>(), 1..max_cols)
664            .prop_map(|cols| {
665                let col_names = (0..cols.len()).map(|i| i.to_string());
666                RelationDesc::new(SqlRelationType::new(cols), col_names)
667            })
668            .prop_flat_map(|desc| {
669                let rows = desc
670                    .typ()
671                    .columns()
672                    .iter()
673                    .cloned()
674                    .map(arb_datum_for_column)
675                    .collect::<Vec<_>>()
676                    .prop_map(|datums| Row::pack(datums.iter().map(Datum::from)));
677                proptest::collection::vec(rows, 1..max_rows)
678                    .prop_map(move |rows| (desc.clone(), rows))
679            });
680
681        let mut all_stats = Vec::new();
682        for _ in 0..test_cases {
683            let value_tree = strat.new_tree(&mut runner).unwrap();
684            let (desc, rows) = value_tree.current();
685
686            let mut builder = PartBuilder::new(&desc, &UnitSchema);
687            for row in &rows {
688                builder.push(&SourceData(Ok(row.clone())), &(), 1u64, 1i64);
689            }
690            let part = builder.finish();
691
692            let key_col = part.key.as_struct();
693            let decoder = <RelationDesc as Schema<SourceData>>::decoder(&desc, key_col.clone())
694                .expect("success");
695            let key_stats = decoder.stats();
696
697            all_stats.push(key_stats);
698        }
699
700        insta::assert_json_snapshot!(all_stats);
701    }
702}
mz_storage_types/stats.rs

mz_storage_types/
stats.rs
