use std::fmt;
use crate::format::Statistics as TStatistics;
use crate::basic::Type;
use crate::data_type::private::ParquetValueType;
use crate::data_type::*;
use crate::errors::{ParquetError, Result};
use crate::util::bit_util::from_le_slice;
pub(crate) mod private {
use super::*;
pub trait MakeStatistics {
fn make_statistics(statistics: ValueStatistics<Self>) -> Statistics
where
Self: Sized;
}
macro_rules! gen_make_statistics {
($value_ty:ty, $stat:ident) => {
impl MakeStatistics for $value_ty {
fn make_statistics(statistics: ValueStatistics<Self>) -> Statistics
where
Self: Sized,
{
Statistics::$stat(statistics)
}
}
};
}
gen_make_statistics!(bool, Boolean);
gen_make_statistics!(i32, Int32);
gen_make_statistics!(i64, Int64);
gen_make_statistics!(Int96, Int96);
gen_make_statistics!(f32, Float);
gen_make_statistics!(f64, Double);
gen_make_statistics!(ByteArray, ByteArray);
gen_make_statistics!(FixedLenByteArray, FixedLenByteArray);
}
macro_rules! statistics_new_func {
($func:ident, $vtype:ty, $stat:ident) => {
pub fn $func(
min: $vtype,
max: $vtype,
distinct: Option<u64>,
nulls: u64,
is_deprecated: bool,
) -> Self {
Statistics::$stat(ValueStatistics::new(
min,
max,
distinct,
nulls,
is_deprecated,
))
}
};
}
macro_rules! statistics_enum_func {
($self:ident, $func:ident) => {{
match *$self {
Statistics::Boolean(ref typed) => typed.$func(),
Statistics::Int32(ref typed) => typed.$func(),
Statistics::Int64(ref typed) => typed.$func(),
Statistics::Int96(ref typed) => typed.$func(),
Statistics::Float(ref typed) => typed.$func(),
Statistics::Double(ref typed) => typed.$func(),
Statistics::ByteArray(ref typed) => typed.$func(),
Statistics::FixedLenByteArray(ref typed) => typed.$func(),
}
}};
}
pub fn from_thrift(
physical_type: Type,
thrift_stats: Option<TStatistics>,
) -> Result<Option<Statistics>> {
Ok(match thrift_stats {
Some(stats) => {
let null_count = stats.null_count.unwrap_or(0);
if null_count < 0 {
return Err(ParquetError::General(format!(
"Statistics null count is negative {}",
null_count
)));
}
let null_count = null_count as u64;
let distinct_count = stats.distinct_count.map(|value| value as u64);
let old_format = stats.min_value.is_none() && stats.max_value.is_none();
let min = if old_format {
stats.min
} else {
stats.min_value
};
let max = if old_format {
stats.max
} else {
stats.max_value
};
let res = match physical_type {
Type::BOOLEAN => Statistics::boolean(
min.map(|data| data[0] != 0),
max.map(|data| data[0] != 0),
distinct_count,
null_count,
old_format,
),
Type::INT32 => Statistics::int32(
min.map(|data| i32::from_le_bytes(data[..4].try_into().unwrap())),
max.map(|data| i32::from_le_bytes(data[..4].try_into().unwrap())),
distinct_count,
null_count,
old_format,
),
Type::INT64 => Statistics::int64(
min.map(|data| i64::from_le_bytes(data[..8].try_into().unwrap())),
max.map(|data| i64::from_le_bytes(data[..8].try_into().unwrap())),
distinct_count,
null_count,
old_format,
),
Type::INT96 => {
let min = min.map(|data| {
assert_eq!(data.len(), 12);
from_le_slice::<Int96>(&data)
});
let max = max.map(|data| {
assert_eq!(data.len(), 12);
from_le_slice::<Int96>(&data)
});
Statistics::int96(min, max, distinct_count, null_count, old_format)
}
Type::FLOAT => Statistics::float(
min.map(|data| f32::from_le_bytes(data[..4].try_into().unwrap())),
max.map(|data| f32::from_le_bytes(data[..4].try_into().unwrap())),
distinct_count,
null_count,
old_format,
),
Type::DOUBLE => Statistics::double(
min.map(|data| f64::from_le_bytes(data[..8].try_into().unwrap())),
max.map(|data| f64::from_le_bytes(data[..8].try_into().unwrap())),
distinct_count,
null_count,
old_format,
),
Type::BYTE_ARRAY => Statistics::ByteArray(
ValueStatistics::new(
min.map(ByteArray::from),
max.map(ByteArray::from),
distinct_count,
null_count,
old_format,
)
.with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
.with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
),
Type::FIXED_LEN_BYTE_ARRAY => Statistics::FixedLenByteArray(
ValueStatistics::new(
min.map(ByteArray::from).map(FixedLenByteArray::from),
max.map(ByteArray::from).map(FixedLenByteArray::from),
distinct_count,
null_count,
old_format,
)
.with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
.with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
),
};
Some(res)
}
None => None,
})
}
pub fn to_thrift(stats: Option<&Statistics>) -> Option<TStatistics> {
let stats = stats?;
let mut thrift_stats = TStatistics {
max: None,
min: None,
null_count: if stats.has_nulls() {
Some(stats.null_count() as i64)
} else {
None
},
distinct_count: stats.distinct_count().map(|value| value as i64),
max_value: None,
min_value: None,
is_max_value_exact: None,
is_min_value_exact: None,
};
let (min, max, min_exact, max_exact) = if stats.has_min_max_set() {
(
Some(stats.min_bytes().to_vec()),
Some(stats.max_bytes().to_vec()),
Some(stats.min_is_exact()),
Some(stats.max_is_exact()),
)
} else {
(None, None, None, None)
};
if stats.is_min_max_backwards_compatible() {
thrift_stats.min = min.clone();
thrift_stats.max = max.clone();
}
if !stats.is_min_max_deprecated() {
thrift_stats.min_value = min;
thrift_stats.max_value = max;
}
thrift_stats.is_min_value_exact = min_exact;
thrift_stats.is_max_value_exact = max_exact;
Some(thrift_stats)
}
#[derive(Debug, Clone, PartialEq)]
pub enum Statistics {
Boolean(ValueStatistics<bool>),
Int32(ValueStatistics<i32>),
Int64(ValueStatistics<i64>),
Int96(ValueStatistics<Int96>),
Float(ValueStatistics<f32>),
Double(ValueStatistics<f64>),
ByteArray(ValueStatistics<ByteArray>),
FixedLenByteArray(ValueStatistics<FixedLenByteArray>),
}
impl<T: ParquetValueType> From<ValueStatistics<T>> for Statistics {
fn from(t: ValueStatistics<T>) -> Self {
T::make_statistics(t)
}
}
impl Statistics {
pub fn new<T: ParquetValueType>(
min: Option<T>,
max: Option<T>,
distinct_count: Option<u64>,
null_count: u64,
is_deprecated: bool,
) -> Self {
Self::from(ValueStatistics::new(
min,
max,
distinct_count,
null_count,
is_deprecated,
))
}
statistics_new_func![boolean, Option<bool>, Boolean];
statistics_new_func![int32, Option<i32>, Int32];
statistics_new_func![int64, Option<i64>, Int64];
statistics_new_func![int96, Option<Int96>, Int96];
statistics_new_func![float, Option<f32>, Float];
statistics_new_func![double, Option<f64>, Double];
statistics_new_func![byte_array, Option<ByteArray>, ByteArray];
statistics_new_func![
fixed_len_byte_array,
Option<FixedLenByteArray>,
FixedLenByteArray
];
pub fn is_min_max_deprecated(&self) -> bool {
statistics_enum_func![self, is_min_max_deprecated]
}
pub fn is_min_max_backwards_compatible(&self) -> bool {
statistics_enum_func![self, is_min_max_backwards_compatible]
}
pub fn distinct_count(&self) -> Option<u64> {
statistics_enum_func![self, distinct_count]
}
pub fn null_count(&self) -> u64 {
statistics_enum_func![self, null_count]
}
pub fn has_nulls(&self) -> bool {
self.null_count() > 0
}
pub fn has_min_max_set(&self) -> bool {
statistics_enum_func![self, has_min_max_set]
}
pub fn min_is_exact(&self) -> bool {
statistics_enum_func![self, min_is_exact]
}
pub fn max_is_exact(&self) -> bool {
statistics_enum_func![self, max_is_exact]
}
pub fn min_bytes(&self) -> &[u8] {
statistics_enum_func![self, min_bytes]
}
pub fn max_bytes(&self) -> &[u8] {
statistics_enum_func![self, max_bytes]
}
pub fn physical_type(&self) -> Type {
match self {
Statistics::Boolean(_) => Type::BOOLEAN,
Statistics::Int32(_) => Type::INT32,
Statistics::Int64(_) => Type::INT64,
Statistics::Int96(_) => Type::INT96,
Statistics::Float(_) => Type::FLOAT,
Statistics::Double(_) => Type::DOUBLE,
Statistics::ByteArray(_) => Type::BYTE_ARRAY,
Statistics::FixedLenByteArray(_) => Type::FIXED_LEN_BYTE_ARRAY,
}
}
}
impl fmt::Display for Statistics {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Statistics::Boolean(typed) => write!(f, "{typed}"),
Statistics::Int32(typed) => write!(f, "{typed}"),
Statistics::Int64(typed) => write!(f, "{typed}"),
Statistics::Int96(typed) => write!(f, "{typed}"),
Statistics::Float(typed) => write!(f, "{typed}"),
Statistics::Double(typed) => write!(f, "{typed}"),
Statistics::ByteArray(typed) => write!(f, "{typed}"),
Statistics::FixedLenByteArray(typed) => write!(f, "{typed}"),
}
}
}
pub type TypedStatistics<T> = ValueStatistics<<T as DataType>::T>;
#[derive(Clone, Eq, PartialEq)]
pub struct ValueStatistics<T> {
min: Option<T>,
max: Option<T>,
distinct_count: Option<u64>,
null_count: u64,
is_max_value_exact: bool,
is_min_value_exact: bool,
is_min_max_deprecated: bool,
is_min_max_backwards_compatible: bool,
}
impl<T: ParquetValueType> ValueStatistics<T> {
pub fn new(
min: Option<T>,
max: Option<T>,
distinct_count: Option<u64>,
null_count: u64,
is_min_max_deprecated: bool,
) -> Self {
Self {
is_max_value_exact: max.is_some(),
is_min_value_exact: min.is_some(),
min,
max,
distinct_count,
null_count,
is_min_max_deprecated,
is_min_max_backwards_compatible: is_min_max_deprecated,
}
}
pub fn with_min_is_exact(self, is_min_value_exact: bool) -> Self {
Self {
is_min_value_exact,
..self
}
}
pub fn with_max_is_exact(self, is_max_value_exact: bool) -> Self {
Self {
is_max_value_exact,
..self
}
}
pub fn with_backwards_compatible_min_max(self, backwards_compatible: bool) -> Self {
Self {
is_min_max_backwards_compatible: backwards_compatible,
..self
}
}
pub fn min(&self) -> &T {
self.min.as_ref().unwrap()
}
pub fn max(&self) -> &T {
self.max.as_ref().unwrap()
}
pub fn min_bytes(&self) -> &[u8] {
self.min().as_bytes()
}
pub fn max_bytes(&self) -> &[u8] {
self.max().as_bytes()
}
pub fn has_min_max_set(&self) -> bool {
self.min.is_some() && self.max.is_some()
}
pub fn max_is_exact(&self) -> bool {
self.max.is_some() && self.is_max_value_exact
}
pub fn min_is_exact(&self) -> bool {
self.min.is_some() && self.is_min_value_exact
}
pub fn distinct_count(&self) -> Option<u64> {
self.distinct_count
}
pub fn null_count(&self) -> u64 {
self.null_count
}
fn is_min_max_deprecated(&self) -> bool {
self.is_min_max_deprecated
}
pub fn is_min_max_backwards_compatible(&self) -> bool {
self.is_min_max_backwards_compatible
}
}
impl<T: ParquetValueType> fmt::Display for ValueStatistics<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{{")?;
write!(f, "min: ")?;
match self.min {
Some(ref value) => write!(f, "{value}")?,
None => write!(f, "N/A")?,
}
write!(f, ", max: ")?;
match self.max {
Some(ref value) => write!(f, "{value}")?,
None => write!(f, "N/A")?,
}
write!(f, ", distinct_count: ")?;
match self.distinct_count {
Some(value) => write!(f, "{value}")?,
None => write!(f, "N/A")?,
}
write!(f, ", null_count: {}", self.null_count)?;
write!(f, ", min_max_deprecated: {}", self.is_min_max_deprecated)?;
write!(f, ", max_value_exact: {}", self.is_max_value_exact)?;
write!(f, ", min_value_exact: {}", self.is_min_value_exact)?;
write!(f, "}}")
}
}
impl<T: ParquetValueType> fmt::Debug for ValueStatistics<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"{{min: {:?}, max: {:?}, distinct_count: {:?}, null_count: {}, \
min_max_deprecated: {}, min_max_backwards_compatible: {}, max_value_exact: {}, min_value_exact: {}}}",
self.min,
self.max,
self.distinct_count,
self.null_count,
self.is_min_max_deprecated,
self.is_min_max_backwards_compatible,
self.is_max_value_exact,
self.is_min_value_exact
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_statistics_min_max_bytes() {
let stats = Statistics::int32(Some(-123), Some(234), None, 1, false);
assert!(stats.has_min_max_set());
assert_eq!(stats.min_bytes(), (-123).as_bytes());
assert_eq!(stats.max_bytes(), 234.as_bytes());
let stats = Statistics::byte_array(
Some(ByteArray::from(vec![1, 2, 3])),
Some(ByteArray::from(vec![3, 4, 5])),
None,
1,
true,
);
assert!(stats.has_min_max_set());
assert_eq!(stats.min_bytes(), &[1, 2, 3]);
assert_eq!(stats.max_bytes(), &[3, 4, 5]);
}
#[test]
#[should_panic(expected = "General(\"Statistics null count is negative -10\")")]
fn test_statistics_negative_null_count() {
let thrift_stats = TStatistics {
max: None,
min: None,
null_count: Some(-10),
distinct_count: None,
max_value: None,
min_value: None,
is_max_value_exact: None,
is_min_value_exact: None,
};
from_thrift(Type::INT32, Some(thrift_stats)).unwrap();
}
#[test]
fn test_statistics_thrift_none() {
assert_eq!(from_thrift(Type::INT32, None).unwrap(), None);
assert_eq!(from_thrift(Type::BYTE_ARRAY, None).unwrap(), None);
}
#[test]
fn test_statistics_debug() {
let stats = Statistics::int32(Some(1), Some(12), None, 12, true);
assert_eq!(
format!("{stats:?}"),
"Int32({min: Some(1), max: Some(12), distinct_count: None, null_count: 12, \
min_max_deprecated: true, min_max_backwards_compatible: true, max_value_exact: true, min_value_exact: true})"
);
let stats = Statistics::int32(None, None, None, 7, false);
assert_eq!(
format!("{stats:?}"),
"Int32({min: None, max: None, distinct_count: None, null_count: 7, \
min_max_deprecated: false, min_max_backwards_compatible: false, max_value_exact: false, min_value_exact: false})"
)
}
#[test]
fn test_statistics_display() {
let stats = Statistics::int32(Some(1), Some(12), None, 12, true);
assert_eq!(
format!("{stats}"),
"{min: 1, max: 12, distinct_count: N/A, null_count: 12, min_max_deprecated: true, max_value_exact: true, min_value_exact: true}"
);
let stats = Statistics::int64(None, None, None, 7, false);
assert_eq!(
format!("{stats}"),
"{min: N/A, max: N/A, distinct_count: N/A, null_count: 7, min_max_deprecated: \
false, max_value_exact: false, min_value_exact: false}"
);
let stats = Statistics::int96(
Some(Int96::from(vec![1, 0, 0])),
Some(Int96::from(vec![2, 3, 4])),
None,
3,
true,
);
assert_eq!(
format!("{stats}"),
"{min: [1, 0, 0], max: [2, 3, 4], distinct_count: N/A, null_count: 3, \
min_max_deprecated: true, max_value_exact: true, min_value_exact: true}"
);
let stats = Statistics::ByteArray(
ValueStatistics::new(
Some(ByteArray::from(vec![1u8])),
Some(ByteArray::from(vec![2u8])),
Some(5),
7,
false,
)
.with_max_is_exact(false)
.with_min_is_exact(false),
);
assert_eq!(
format!("{stats}"),
"{min: [1], max: [2], distinct_count: 5, null_count: 7, min_max_deprecated: false, max_value_exact: false, min_value_exact: false}"
);
}
#[test]
fn test_statistics_partial_eq() {
let expected = Statistics::int32(Some(12), Some(45), None, 11, true);
assert!(Statistics::int32(Some(12), Some(45), None, 11, true) == expected);
assert!(Statistics::int32(Some(11), Some(45), None, 11, true) != expected);
assert!(Statistics::int32(Some(12), Some(44), None, 11, true) != expected);
assert!(Statistics::int32(Some(12), Some(45), None, 23, true) != expected);
assert!(Statistics::int32(Some(12), Some(45), None, 11, false) != expected);
assert!(
Statistics::int32(Some(12), Some(45), None, 11, false)
!= Statistics::int64(Some(12), Some(45), None, 11, false)
);
assert!(
Statistics::boolean(Some(false), Some(true), None, 0, true)
!= Statistics::double(Some(1.2), Some(4.5), None, 0, true)
);
assert!(
Statistics::byte_array(
Some(ByteArray::from(vec![1, 2, 3])),
Some(ByteArray::from(vec![1, 2, 3])),
None,
0,
true
) != Statistics::fixed_len_byte_array(
Some(ByteArray::from(vec![1, 2, 3]).into()),
Some(ByteArray::from(vec![1, 2, 3]).into()),
None,
0,
true,
)
);
assert!(
Statistics::byte_array(
Some(ByteArray::from(vec![1, 2, 3])),
Some(ByteArray::from(vec![1, 2, 3])),
None,
0,
true,
) != Statistics::ByteArray(
ValueStatistics::new(
Some(ByteArray::from(vec![1, 2, 3])),
Some(ByteArray::from(vec![1, 2, 3])),
None,
0,
true,
)
.with_max_is_exact(false)
)
);
assert!(
Statistics::fixed_len_byte_array(
Some(FixedLenByteArray::from(vec![1, 2, 3])),
Some(FixedLenByteArray::from(vec![1, 2, 3])),
None,
0,
true,
) != Statistics::FixedLenByteArray(
ValueStatistics::new(
Some(FixedLenByteArray::from(vec![1, 2, 3])),
Some(FixedLenByteArray::from(vec![1, 2, 3])),
None,
0,
true,
)
.with_min_is_exact(false)
)
);
}
#[test]
fn test_statistics_from_thrift() {
fn check_stats(stats: Statistics) {
let tpe = stats.physical_type();
let thrift_stats = to_thrift(Some(&stats));
assert_eq!(from_thrift(tpe, thrift_stats).unwrap(), Some(stats));
}
check_stats(Statistics::boolean(Some(false), Some(true), None, 7, true));
check_stats(Statistics::boolean(Some(false), Some(true), None, 7, true));
check_stats(Statistics::boolean(Some(false), Some(true), None, 0, false));
check_stats(Statistics::boolean(Some(true), Some(true), None, 7, true));
check_stats(Statistics::boolean(Some(false), Some(false), None, 7, true));
check_stats(Statistics::boolean(None, None, None, 7, true));
check_stats(Statistics::int32(Some(-100), Some(500), None, 7, true));
check_stats(Statistics::int32(Some(-100), Some(500), None, 0, false));
check_stats(Statistics::int32(None, None, None, 7, true));
check_stats(Statistics::int64(Some(-100), Some(200), None, 7, true));
check_stats(Statistics::int64(Some(-100), Some(200), None, 0, false));
check_stats(Statistics::int64(None, None, None, 7, true));
check_stats(Statistics::float(Some(1.2), Some(3.4), None, 7, true));
check_stats(Statistics::float(Some(1.2), Some(3.4), None, 0, false));
check_stats(Statistics::float(None, None, None, 7, true));
check_stats(Statistics::double(Some(1.2), Some(3.4), None, 7, true));
check_stats(Statistics::double(Some(1.2), Some(3.4), None, 0, false));
check_stats(Statistics::double(None, None, None, 7, true));
check_stats(Statistics::byte_array(
Some(ByteArray::from(vec![1, 2, 3])),
Some(ByteArray::from(vec![3, 4, 5])),
None,
7,
true,
));
check_stats(Statistics::byte_array(None, None, None, 7, true));
check_stats(Statistics::fixed_len_byte_array(
Some(ByteArray::from(vec![1, 2, 3]).into()),
Some(ByteArray::from(vec![3, 4, 5]).into()),
None,
7,
true,
));
check_stats(Statistics::fixed_len_byte_array(None, None, None, 7, true));
}
}