mz_repr/adt/
numeric.rs

Help
1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Functions related to Materialize's numeric type, which is largely a wrapper
11//! around [`rust-dec`].
12//!
13//! [`rust-dec`]: https://github.com/MaterializeInc/rust-dec/
14
15use std::error::Error;
16use std::fmt;
17use std::sync::LazyLock;
18
19use anyhow::bail;
20use dec::{Context, Decimal};
21use mz_lowertest::MzReflect;
22use mz_ore::cast;
23use mz_persist_types::columnar::FixedSizeCodec;
24use mz_proto::{ProtoType, RustType, TryFromProtoError};
25use proptest_derive::Arbitrary;
26use serde::{Deserialize, Serialize};
27
28include!(concat!(env!("OUT_DIR"), "/mz_repr.adt.numeric.rs"));
29
30/// The number of internal decimal units in a [`Numeric`] value.
31pub const NUMERIC_DATUM_WIDTH: u8 = 13;
32
33/// The value of [`NUMERIC_DATUM_WIDTH`] as a [`u8`].
34pub const NUMERIC_DATUM_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_DATUM_WIDTH);
35
36/// The maximum number of digits expressable in a [`Numeric`] value.
37pub const NUMERIC_DATUM_MAX_PRECISION: u8 = NUMERIC_DATUM_WIDTH * 3;
38
39/// A numeric value.
40pub type Numeric = Decimal<NUMERIC_DATUM_WIDTH_USIZE>;
41
42/// The number of internal decimal units in a [`NumericAgg`] value.
43pub const NUMERIC_AGG_WIDTH: u8 = 27;
44
45/// The value of [`NUMERIC_AGG_WIDTH`] as a [`u8`].
46pub const NUMERIC_AGG_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_AGG_WIDTH);
47
48/// The maximum number of digits expressable in a [`NumericAgg`] value.
49pub const NUMERIC_AGG_MAX_PRECISION: u8 = NUMERIC_AGG_WIDTH * 3;
50
51/// A double-width version of [`Numeric`] for use in aggregations.
52pub type NumericAgg = Decimal<NUMERIC_AGG_WIDTH_USIZE>;
53
54static CX_DATUM: LazyLock<Context<Numeric>> = LazyLock::new(|| {
55    let mut cx = Context::<Numeric>::default();
56    cx.set_max_exponent(isize::from(NUMERIC_DATUM_MAX_PRECISION - 1))
57        .unwrap();
58    cx.set_min_exponent(-isize::from(NUMERIC_DATUM_MAX_PRECISION))
59        .unwrap();
60    cx
61});
62static CX_AGG: LazyLock<Context<NumericAgg>> = LazyLock::new(|| {
63    let mut cx = Context::<NumericAgg>::default();
64    cx.set_max_exponent(isize::from(NUMERIC_AGG_MAX_PRECISION - 1))
65        .unwrap();
66    cx.set_min_exponent(-isize::from(NUMERIC_AGG_MAX_PRECISION))
67        .unwrap();
68    cx
69});
70static U128_SPLITTER_DATUM: LazyLock<Numeric> = LazyLock::new(|| {
71    let mut cx = Numeric::context();
72    // 1 << 128
73    cx.parse("340282366920938463463374607431768211456").unwrap()
74});
75static U128_SPLITTER_AGG: LazyLock<NumericAgg> = LazyLock::new(|| {
76    let mut cx = NumericAgg::context();
77    // 1 << 128
78    cx.parse("340282366920938463463374607431768211456").unwrap()
79});
80
81/// Module to simplify serde'ing a `Numeric` through its string representation.
82pub mod str_serde {
83    use std::str::FromStr;
84
85    use serde::Deserialize;
86
87    use super::Numeric;
88
89    /// Deserializing a [`Numeric`] value from its `String` representation.
90    pub fn deserialize<'de, D>(deserializer: D) -> Result<Numeric, D::Error>
91    where
92        D: serde::Deserializer<'de>,
93    {
94        let buf = String::deserialize(deserializer)?;
95        Numeric::from_str(&buf).map_err(serde::de::Error::custom)
96    }
97}
98
99/// The `max_scale` of a [`ScalarType::Numeric`].
100///
101/// This newtype wrapper ensures that the scale is within the valid range.
102///
103/// [`ScalarType::Numeric`]: crate::ScalarType::Numeric
104#[derive(
105    Arbitrary,
106    Debug,
107    Clone,
108    Copy,
109    Eq,
110    PartialEq,
111    Ord,
112    PartialOrd,
113    Hash,
114    Serialize,
115    Deserialize,
116    MzReflect,
117)]
118pub struct NumericMaxScale(pub(crate) u8);
119
120impl NumericMaxScale {
121    /// A max scale of zero.
122    pub const ZERO: NumericMaxScale = NumericMaxScale(0);
123
124    /// Consumes the newtype wrapper, returning the inner `u8`.
125    pub fn into_u8(self) -> u8 {
126        self.0
127    }
128}
129
130impl TryFrom<i64> for NumericMaxScale {
131    type Error = InvalidNumericMaxScaleError;
132
133    fn try_from(max_scale: i64) -> Result<Self, Self::Error> {
134        match u8::try_from(max_scale) {
135            Ok(max_scale) if max_scale <= NUMERIC_DATUM_MAX_PRECISION => {
136                Ok(NumericMaxScale(max_scale))
137            }
138            _ => Err(InvalidNumericMaxScaleError),
139        }
140    }
141}
142
143impl TryFrom<usize> for NumericMaxScale {
144    type Error = InvalidNumericMaxScaleError;
145
146    fn try_from(max_scale: usize) -> Result<Self, Self::Error> {
147        Self::try_from(i64::try_from(max_scale).map_err(|_| InvalidNumericMaxScaleError)?)
148    }
149}
150
151impl RustType<ProtoNumericMaxScale> for NumericMaxScale {
152    fn into_proto(&self) -> ProtoNumericMaxScale {
153        ProtoNumericMaxScale {
154            value: self.0.into_proto(),
155        }
156    }
157
158    fn from_proto(max_scale: ProtoNumericMaxScale) -> Result<Self, TryFromProtoError> {
159        Ok(NumericMaxScale(max_scale.value.into_rust()?))
160    }
161}
162
163impl RustType<ProtoOptionalNumericMaxScale> for Option<NumericMaxScale> {
164    fn into_proto(&self) -> ProtoOptionalNumericMaxScale {
165        ProtoOptionalNumericMaxScale {
166            value: self.into_proto(),
167        }
168    }
169
170    fn from_proto(max_scale: ProtoOptionalNumericMaxScale) -> Result<Self, TryFromProtoError> {
171        max_scale.value.into_rust()
172    }
173}
174
175/// The error returned when constructing a [`NumericMaxScale`] from an invalid
176/// value.
177#[derive(Debug, Clone)]
178pub struct InvalidNumericMaxScaleError;
179
180impl fmt::Display for InvalidNumericMaxScaleError {
181    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
182        write!(
183            f,
184            "scale for type numeric must be between 0 and {}",
185            NUMERIC_DATUM_MAX_PRECISION
186        )
187    }
188}
189
190impl Error for InvalidNumericMaxScaleError {}
191
192/// Traits to generalize converting [`Decimal`] values to and from their
193/// coefficients' two's complements.
194pub trait Dec<const N: usize> {
195    // The number of bytes required to represent the min/max value of a decimal
196    // using two's complement.
197    const TWOS_COMPLEMENT_BYTE_WIDTH: usize;
198    // Convenience method for generating appropriate default contexts.
199    fn context() -> Context<Decimal<N>>;
200    // Provides value to break decimal into units of `u128`s for binary
201    // encoding/decoding.
202    fn u128_splitter() -> &'static Decimal<N>;
203}
204
205impl Dec<NUMERIC_DATUM_WIDTH_USIZE> for Numeric {
206    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 17;
207    fn context() -> Context<Numeric> {
208        CX_DATUM.clone()
209    }
210    fn u128_splitter() -> &'static Numeric {
211        &U128_SPLITTER_DATUM
212    }
213}
214
215impl Dec<NUMERIC_AGG_WIDTH_USIZE> for NumericAgg {
216    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 33;
217    fn context() -> Context<NumericAgg> {
218        CX_AGG.clone()
219    }
220    fn u128_splitter() -> &'static NumericAgg {
221        &U128_SPLITTER_AGG
222    }
223}
224
225/// Returns a new context appropriate for operating on numeric datums.
226pub fn cx_datum() -> Context<Numeric> {
227    CX_DATUM.clone()
228}
229
230/// Returns a new context appropriate for operating on numeric aggregates.
231pub fn cx_agg() -> Context<NumericAgg> {
232    CX_AGG.clone()
233}
234
235fn twos_complement_be_to_u128(input: &[u8]) -> u128 {
236    assert!(input.len() <= 16);
237    let mut buf = [0; 16];
238    buf[16 - input.len()..16].copy_from_slice(input);
239    u128::from_be_bytes(buf)
240}
241
242/// Using negative binary numbers can require more digits of precision than
243/// [`Numeric`] offers, so we need to have the option to swap bytes' signs at the
244/// byte- rather than the library-level.
245fn negate_twos_complement_le<'a, I>(b: I)
246where
247    I: Iterator<Item = &'a mut u8>,
248{
249    let mut seen_first_one = false;
250    for i in b {
251        if seen_first_one {
252            *i = *i ^ 0xFF;
253        } else if *i > 0 {
254            seen_first_one = true;
255            if i == &0x80 {
256                continue;
257            }
258            let tz = i.trailing_zeros();
259            *i = *i ^ (0xFF << tz + 1);
260        }
261    }
262}
263
264/// Converts an [`Numeric`] into its big endian two's complement representation.
265pub fn numeric_to_twos_complement_be(
266    mut numeric: Numeric,
267) -> [u8; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH] {
268    let mut buf = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
269    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
270    // as zeroes so as to avoid erroring (encoding values is meant to be
271    // infallible) and retain downstream associativity/commutativity.
272    if numeric.is_special() {
273        return buf;
274    }
275
276    let mut cx = Numeric::context();
277
278    // Ensure `numeric` is a canonical coefficient.
279    if numeric.exponent() < 0 {
280        let s = Numeric::from(-numeric.exponent());
281        cx.scaleb(&mut numeric, &s);
282    }
283
284    numeric_to_twos_complement_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(
285        numeric, &mut cx, &mut buf,
286    );
287    buf
288}
289
290/// Converts an [`Numeric`] into a big endian two's complement representation where
291/// the encoded value has [`NUMERIC_AGG_MAX_PRECISION`] digits and a scale of
292/// [`NUMERIC_DATUM_MAX_PRECISION`].
293///
294/// This representation is appropriate to use in
295/// contexts requiring two's complement representation but `Numeric` values' scale
296/// isn't known, e.g. when working with columns with an explicitly defined
297/// scale.
298pub fn numeric_to_twos_complement_wide(
299    numeric: Numeric,
300) -> [u8; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH] {
301    let mut buf = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
302    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
303    // as zeroes so as to avoid erroring (encoding values is meant to be
304    // infallible) and retain downstream associativity/commutativity.
305    if numeric.is_special() {
306        return buf;
307    }
308    let mut cx = NumericAgg::context();
309    let mut d = cx.to_width(numeric);
310    let mut scaler = NumericAgg::from(NUMERIC_DATUM_MAX_PRECISION);
311    cx.neg(&mut scaler);
312    // Shape `d` so that its exponent is -NUMERIC_DATUM_MAX_PRECISION
313    cx.rescale(&mut d, &scaler);
314    // Adjust `d` so it is a canonical coefficient, i.e. its exact value can be
315    // recovered by setting its exponent to -39.
316    cx.abs(&mut scaler);
317    cx.scaleb(&mut d, &scaler);
318
319    numeric_to_twos_complement_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(d, &mut cx, &mut buf);
320    buf
321}
322
323fn numeric_to_twos_complement_inner<D: Dec<N>, const N: usize>(
324    mut d: Decimal<N>,
325    cx: &mut Context<Decimal<N>>,
326    buf: &mut [u8],
327) {
328    // Adjust negative values to be writable as series of `u128`.
329    let is_neg = if d.is_negative() {
330        cx.neg(&mut d);
331        true
332    } else {
333        false
334    };
335
336    // Values have all been made into canonical coefficients.
337    assert!(d.exponent() >= 0);
338
339    let mut buf_cursor = 0;
340    while !d.is_zero() {
341        let mut w = d.clone();
342        // Take the remainder; this represents one of our "units" to take the coefficient of, i.e. d & u128::MAX
343        cx.rem(&mut w, D::u128_splitter());
344
345        // Take the `u128` version of the coefficient, which will always be what
346        // we want given that we adjusted negative values to have an unsigned
347        // integer representation.
348        let c = w.coefficient::<u128>().unwrap();
349
350        // Determine the width of the coefficient we want to take, i.e. the full
351        // coefficient or a part of it to fill the buffer.
352        let e = std::cmp::min(buf_cursor + 16, D::TWOS_COMPLEMENT_BYTE_WIDTH);
353
354        // We're putting less significant bytes at index 0, which is little endian.
355        buf[buf_cursor..e].copy_from_slice(&c.to_le_bytes()[0..e - buf_cursor]);
356        // Advance cursor; ok that it will go past buffer on final + 1th iteration.
357        buf_cursor += 16;
358
359        // Take the quotient to represent the next unit, i.e. d >> 128
360        cx.div_integer(&mut d, D::u128_splitter());
361    }
362
363    if is_neg {
364        negate_twos_complement_le(buf.iter_mut());
365    }
366
367    // Convert from little endian to big endian.
368    buf.reverse();
369}
370
371pub fn twos_complement_be_to_numeric(
372    input: &mut [u8],
373    scale: u8,
374) -> Result<Numeric, anyhow::Error> {
375    let mut cx = cx_datum();
376    if input.len() <= 17 {
377        if let Ok(mut n) =
378            twos_complement_be_to_numeric_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(input)
379        {
380            n.set_exponent(-i32::from(scale));
381            return Ok(n);
382        }
383    }
384    // If bytes were invalid for narrower representation, try to use wider
385    // representation in case e.g. simply has more trailing zeroes.
386    let mut n = twos_complement_be_to_numeric_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(input)?;
387    // Exponent must be set before converting to `Numeric` width, otherwise values can overflow 39 dop.
388    n.set_exponent(-i32::from(scale));
389    let d = cx.to_width(n);
390    if cx.status().inexact() {
391        bail!("Value exceeds maximum numeric value")
392    }
393    Ok(d)
394}
395
396/// Parses a buffer of two's complement digits in big-endian order and converts
397/// them to [`Decimal<N>`].
398pub fn twos_complement_be_to_numeric_inner<D: Dec<N>, const N: usize>(
399    input: &mut [u8],
400) -> Result<Decimal<N>, anyhow::Error> {
401    let is_neg = if (input[0] & 0x80) != 0 {
402        // byte-level negate all negative values, guaranteeing all bytes are
403        // readable as unsigned.
404        negate_twos_complement_le(input.iter_mut().rev());
405        true
406    } else {
407        false
408    };
409
410    let head = input.len() % 16;
411    let i = twos_complement_be_to_u128(&input[0..head]);
412    let mut cx = D::context();
413    let mut d = cx.from_u128(i);
414
415    for c in input[head..].chunks(16) {
416        assert_eq!(c.len(), 16);
417        // essentially d << 128
418        cx.mul(&mut d, D::u128_splitter());
419        let i = twos_complement_be_to_u128(c);
420        let i = cx.from_u128(i);
421        cx.add(&mut d, &i);
422    }
423
424    if cx.status().inexact() {
425        bail!("Value exceeds maximum numeric value")
426    } else if cx.status().any() {
427        bail!("unexpected status {:?}", cx.status());
428    }
429    if is_neg {
430        cx.neg(&mut d);
431    }
432    Ok(d)
433}
434
435#[mz_ore::test]
436#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
437fn test_twos_complement_roundtrip() {
438    fn inner(s: &str) {
439        let mut cx = cx_datum();
440        let d = cx.parse(s).unwrap();
441        let scale = std::cmp::min(d.exponent(), 0).abs();
442        let mut b = numeric_to_twos_complement_be(d.clone());
443        let x = twos_complement_be_to_numeric(&mut b, u8::try_from(scale).unwrap()).unwrap();
444        assert_eq!(d, x);
445    }
446    inner("0");
447    inner("0.000000000000000000000000000000000012345");
448    inner("0.123456789012345678901234567890123456789");
449    inner("1.00000000000000000000000000000000000000");
450    inner("1");
451    inner("2");
452    inner("170141183460469231731687303715884105727");
453    inner("170141183460469231731687303715884105728");
454    inner("12345678901234567890.1234567890123456789");
455    inner("999999999999999999999999999999999999999");
456    inner("7e35");
457    inner("7e-35");
458    inner("-0.000000000000000000000000000000000012345");
459    inner("-0.12345678901234567890123456789012345678");
460    inner("-1.00000000000000000000000000000000000000");
461    inner("-1");
462    inner("-2");
463    inner("-170141183460469231731687303715884105727");
464    inner("-170141183460469231731687303715884105728");
465    inner("-12345678901234567890.1234567890123456789");
466    inner("-999999999999999999999999999999999999999");
467    inner("-7.2e35");
468    inner("-7.2e-35");
469}
470
471#[mz_ore::test]
472#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
473fn test_twos_comp_numeric_primitive() {
474    fn inner_inner<P>(i: P, i_be_bytes: &mut [u8])
475    where
476        P: Into<Numeric> + TryFrom<Numeric> + Eq + PartialEq + std::fmt::Debug + Copy,
477    {
478        let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
479        e[Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
480        let mut w = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
481        w[NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
482
483        let d: Numeric = i.into();
484
485        // Extend negative sign into most-significant bits
486        if d.is_negative() {
487            for i in e[..Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
488                *i = 0xFF;
489            }
490            for i in w[..NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
491                *i = 0xFF;
492            }
493        }
494
495        // Ensure decimal value's two's complement representation matches an
496        // extended version of `to_be_bytes`.
497        let d_be_bytes = numeric_to_twos_complement_be(d);
498        assert_eq!(
499            e, d_be_bytes,
500            "expected repr of {:?}, got {:?}",
501            e, d_be_bytes
502        );
503
504        // Ensure extended version of `to_be_bytes` generates same `i128`.
505        let e_numeric = twos_complement_be_to_numeric(&mut e, 0).unwrap();
506        let e_p: P = match e_numeric.try_into() {
507            Ok(e_p) => e_p,
508            Err(_) => panic!(),
509        };
510        assert_eq!(i, e_p, "expected val of {:?}, got {:?}", i, e_p);
511
512        // Wide representation produces same result.
513        let w_numeric = twos_complement_be_to_numeric(&mut w, 0).unwrap();
514        let w_p: P = match w_numeric.try_into() {
515            Ok(w_p) => w_p,
516            Err(_) => panic!(),
517        };
518        assert_eq!(i, w_p, "expected val of {:?}, got {:?}", i, e_p);
519
520        // Bytes do not need to be in `Numeric`-specific format
521        let p_numeric = twos_complement_be_to_numeric(i_be_bytes, 0).unwrap();
522        let p_p: P = match p_numeric.try_into() {
523            Ok(p_p) => p_p,
524            Err(_) => panic!(),
525        };
526        assert_eq!(i, p_p, "expected val of {:?}, got {:?}", i, p_p);
527    }
528
529    fn inner_i32(i: i32) {
530        inner_inner(i, &mut i.to_be_bytes());
531    }
532
533    fn inner_i64(i: i64) {
534        inner_inner(i, &mut i.to_be_bytes());
535    }
536
537    // We need a wrapper around i128 to implement the same traits as the other
538    // primitive types. This is less code than a second implementation of the
539    // same test that takes unwrapped i128s.
540    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
541    struct FromableI128 {
542        i: i128,
543    }
544    impl From<i128> for FromableI128 {
545        fn from(i: i128) -> FromableI128 {
546            FromableI128 { i }
547        }
548    }
549    impl From<FromableI128> for Numeric {
550        fn from(n: FromableI128) -> Numeric {
551            Numeric::try_from(n.i).unwrap()
552        }
553    }
554    impl TryFrom<Numeric> for FromableI128 {
555        type Error = ();
556        fn try_from(n: Numeric) -> Result<FromableI128, Self::Error> {
557            match i128::try_from(n) {
558                Ok(i) => Ok(FromableI128 { i }),
559                Err(_) => Err(()),
560            }
561        }
562    }
563
564    fn inner_i128(i: i128) {
565        inner_inner(FromableI128::from(i), &mut i.to_be_bytes());
566    }
567
568    inner_i32(0);
569    inner_i32(1);
570    inner_i32(2);
571    inner_i32(-1);
572    inner_i32(-2);
573    inner_i32(i32::MAX);
574    inner_i32(i32::MIN);
575    inner_i32(i32::MAX / 7 + 7);
576    inner_i32(i32::MIN / 7 + 7);
577    inner_i64(0);
578    inner_i64(1);
579    inner_i64(2);
580    inner_i64(-1);
581    inner_i64(-2);
582    inner_i64(i64::MAX);
583    inner_i64(i64::MIN);
584    inner_i64(i64::MAX / 7 + 7);
585    inner_i64(i64::MIN / 7 + 7);
586    inner_i128(0);
587    inner_i128(1);
588    inner_i128(2);
589    inner_i128(-1);
590    inner_i128(-2);
591    inner_i128(i128::from(i64::MAX));
592    inner_i128(i128::from(i64::MIN));
593    inner_i128(i128::MAX);
594    inner_i128(i128::MIN);
595    inner_i128(i128::MAX / 7 + 7);
596    inner_i128(i128::MIN / 7 + 7);
597}
598
599#[mz_ore::test]
600#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
601fn test_twos_complement_to_numeric_fail() {
602    fn inner(b: &mut [u8]) {
603        let r = twos_complement_be_to_numeric(b, 0);
604        mz_ore::assert_err!(r);
605    }
606    // 17-byte signed digit's max value exceeds 39 digits of precision
607    let mut e = [0xFF; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
608    e[0] -= 0x80;
609    inner(&mut e);
610
611    // 1 << 17 * 8 exceeds exceeds 39 digits of precision
612    let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH + 1];
613    e[0] = 1;
614    inner(&mut e);
615}
616
617#[mz_ore::test]
618#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
619fn test_wide_twos_complement_roundtrip() {
620    fn inner(s: &str) {
621        let mut cx = cx_datum();
622        let d = cx.parse(s).unwrap();
623        let mut b = numeric_to_twos_complement_wide(d.clone());
624        let x = twos_complement_be_to_numeric(&mut b, NUMERIC_DATUM_MAX_PRECISION).unwrap();
625        assert_eq!(d, x);
626    }
627    inner("0");
628    inner("0.000000000000000000000000000000000012345");
629    inner("0.123456789012345678901234567890123456789");
630    inner("1.00000000000000000000000000000000000000");
631    inner("1");
632    inner("2");
633    inner("170141183460469231731687303715884105727");
634    inner("170141183460469231731687303715884105728");
635    inner("12345678901234567890.1234567890123456789");
636    inner("999999999999999999999999999999999999999");
637    inner("-0.000000000000000000000000000000000012345");
638    inner("-0.123456789012345678901234567890123456789");
639    inner("-1.00000000000000000000000000000000000000");
640    inner("-1");
641    inner("-2");
642    inner("-170141183460469231731687303715884105727");
643    inner("-170141183460469231731687303715884105728");
644    inner("-12345678901234567890.1234567890123456789");
645    inner("-999999999999999999999999999999999999999");
646}
647
648/// Returns `n`'s precision, i.e. the total number of digits represented by `n`
649/// in standard notation not including a zero in the "one's place" in (-1,1).
650pub fn get_precision<const N: usize>(n: &Decimal<N>) -> u32 {
651    let e = n.exponent();
652    if e >= 0 {
653        // Positive exponent
654        n.digits() + u32::try_from(e).unwrap()
655    } else {
656        // Negative exponent
657        let d = n.digits();
658        let e = u32::try_from(e.abs()).unwrap();
659        // Precision is...
660        // - d if decimal point splits numbers
661        // - e if e dominates number of digits
662        std::cmp::max(d, e)
663    }
664}
665
666/// Returns `n`'s scale, i.e. the number of digits used after the decimal point.
667pub fn get_scale(n: &Numeric) -> u32 {
668    let exp = n.exponent();
669    if exp >= 0 { 0 } else { exp.unsigned_abs() }
670}
671
672/// Ensures [`Numeric`] values are:
673/// - Within `Numeric`'s max precision ([`NUMERIC_DATUM_MAX_PRECISION`]), or errors if not.
674/// - Never possible but invalid representations (i.e. never -Nan or -0).
675///
676/// Should be called after any operation that can change an [`Numeric`]'s scale or
677/// generate negative values (except addition and subtraction).
678pub fn munge_numeric(n: &mut Numeric) -> Result<(), anyhow::Error> {
679    rescale_within_max_precision(n)?;
680    if (n.is_zero() || n.is_nan()) && n.is_negative() {
681        cx_datum().neg(n);
682    }
683    Ok(())
684}
685
686/// Rescale's `n` to fit within [`Numeric`]'s max precision or error if not
687/// possible.
688fn rescale_within_max_precision(n: &mut Numeric) -> Result<(), anyhow::Error> {
689    let current_precision = get_precision(n);
690    if current_precision > u32::from(NUMERIC_DATUM_MAX_PRECISION) {
691        if n.exponent() < 0 {
692            let precision_diff = current_precision - u32::from(NUMERIC_DATUM_MAX_PRECISION);
693            let current_scale = u8::try_from(get_scale(n))?;
694            let scale_diff = current_scale - u8::try_from(precision_diff).unwrap();
695            rescale(n, scale_diff)?;
696        } else {
697            bail!(
698                "numeric value {} exceed maximum precision {}",
699                n,
700                NUMERIC_DATUM_MAX_PRECISION
701            )
702        }
703    }
704    Ok(())
705}
706
707/// Rescale `n` as an `OrderedDecimal` with the described scale, or error if:
708/// - Rescaling exceeds max precision
709/// - `n` requires > [`NUMERIC_DATUM_MAX_PRECISION`] - `scale` digits of precision
710///   left of the decimal point
711pub fn rescale(n: &mut Numeric, scale: u8) -> Result<(), anyhow::Error> {
712    let mut cx = cx_datum();
713    cx.rescale(n, &Numeric::from(-i32::from(scale)));
714    if cx.status().invalid_operation() || get_precision(n) > u32::from(NUMERIC_DATUM_MAX_PRECISION)
715    {
716        bail!(
717            "numeric value {} exceed maximum precision {}",
718            n,
719            NUMERIC_DATUM_MAX_PRECISION
720        )
721    }
722    munge_numeric(n)?;
723
724    Ok(())
725}
726
727/// A type that can represent Real Numbers. Useful for interoperability between Numeric and
728/// floating point.
729pub trait DecimalLike:
730    From<u8>
731    + From<u16>
732    + From<u32>
733    + From<i8>
734    + From<i16>
735    + From<i32>
736    + From<f32>
737    + From<f64>
738    + std::ops::Add<Output = Self>
739    + std::ops::Sub<Output = Self>
740    + std::ops::Mul<Output = Self>
741    + std::ops::Div<Output = Self>
742{
743    /// Used to do value-to-value conversions while consuming the input value. Depending on the
744    /// implementation it may be potentially lossy.
745    fn lossy_from(i: i64) -> Self;
746}
747
748impl DecimalLike for f64 {
749    // No other known way to convert `i64` to `f64`.
750    #[allow(clippy::as_conversions)]
751    fn lossy_from(i: i64) -> Self {
752        i as f64
753    }
754}
755
756impl DecimalLike for Numeric {
757    fn lossy_from(i: i64) -> Self {
758        Numeric::from(i)
759    }
760}
761
762/// An encoded packed variant of [`Numeric`].
763///
764/// Unlike other "Packed" types we _DO NOT_ uphold the invariant that
765/// [`PackedNumeric`] sorts the same as [`Numeric`].
766#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
767pub struct PackedNumeric(pub [u8; 40]);
768
769impl FixedSizeCodec<Numeric> for PackedNumeric {
770    const SIZE: usize = 40;
771
772    fn as_bytes(&self) -> &[u8] {
773        &self.0
774    }
775
776    fn from_bytes(slice: &[u8]) -> Result<Self, String> {
777        let buf: [u8; Self::SIZE] = slice.try_into().map_err(|_| {
778            format!(
779                "size for PackedNumeric is {} bytes, got {}",
780                Self::SIZE,
781                slice.len()
782            )
783        })?;
784        Ok(PackedNumeric(buf))
785    }
786
787    fn from_value(val: Numeric) -> PackedNumeric {
788        let (digits, exponent, bits, lsu) = val.to_raw_parts();
789
790        let mut buf = [0u8; 40];
791
792        buf[0..4].copy_from_slice(&digits.to_le_bytes());
793        buf[4..8].copy_from_slice(&exponent.to_le_bytes());
794
795        for i in 0..13 {
796            buf[(i * 2) + 8..(i * 2) + 10].copy_from_slice(&lsu[i].to_le_bytes());
797        }
798
799        buf[34..35].copy_from_slice(&bits.to_le_bytes());
800
801        PackedNumeric(buf)
802    }
803
804    fn into_value(self) -> Numeric {
805        let digits: [u8; 4] = self.0[0..4].try_into().unwrap();
806        let digits = u32::from_le_bytes(digits);
807
808        let exponent: [u8; 4] = self.0[4..8].try_into().unwrap();
809        let exponent = i32::from_le_bytes(exponent);
810
811        let mut lsu = [0u16; 13];
812        for i in 0..13 {
813            let x: [u8; 2] = self.0[(i * 2) + 8..(i * 2) + 10].try_into().unwrap();
814            let x = u16::from_le_bytes(x);
815            lsu[i] = x;
816        }
817
818        let bits: [u8; 1] = self.0[34..35].try_into().unwrap();
819        let bits = u8::from_le_bytes(bits);
820
821        Numeric::from_raw_parts(digits, exponent, bits, lsu)
822    }
823}
824
825#[cfg(test)]
826mod tests {
827    use mz_ore::assert_ok;
828    use mz_proto::protobuf_roundtrip;
829    use proptest::prelude::*;
830
831    use crate::scalar::arb_numeric;
832
833    use super::*;
834
835    proptest! {
836        #[mz_ore::test]
837        fn numeric_max_scale_protobuf_roundtrip(expect in any::<NumericMaxScale>()) {
838            let actual = protobuf_roundtrip::<_, ProtoNumericMaxScale>(&expect);
839            assert_ok!(actual);
840            assert_eq!(actual.unwrap(), expect);
841        }
842
843        #[mz_ore::test]
844        fn optional_numeric_max_scale_protobuf_roundtrip(expect in any::<Option<NumericMaxScale>>()) {
845            let actual = protobuf_roundtrip::<_, ProtoOptionalNumericMaxScale>(&expect);
846            assert_ok!(actual);
847            assert_eq!(actual.unwrap(), expect);
848        }
849    }
850
851    #[mz_ore::test]
852    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
853    fn smoketest_packed_numeric_roundtrips() {
854        let og = PackedNumeric::from_value(Numeric::from(-42));
855        let bytes = og.as_bytes();
856        let rnd = PackedNumeric::from_bytes(bytes).expect("valid");
857        assert_eq!(og, rnd);
858
859        // Returns an error if the size of the slice is invalid.
860        mz_ore::assert_err!(PackedNumeric::from_bytes(&[0, 0, 0, 0]));
861    }
862
863    #[mz_ore::test]
864    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
865    fn proptest_packed_numeric_roundtrip() {
866        fn test(og: Numeric) {
867            let packed = PackedNumeric::from_value(og);
868            let rnd = packed.into_value();
869
870            if og.is_nan() && rnd.is_nan() {
871                return;
872            }
873            assert_eq!(og, rnd);
874        }
875
876        proptest!(|(num in arb_numeric())| {
877            test(num);
878        });
879    }
880
881    // Note: It's expected that this test will fail if you update the strategy
882    // for generating an arbitrary Numeric. In that case feel free to
883    // regenerate the snapshot.
884    #[mz_ore::test]
885    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
886    fn packed_numeric_stability() {
887        /// This is the seed [`proptest`] uses for their deterministic RNG. We
888        /// copy it here to prevent breaking this test if [`proptest`] changes.
889        const RNG_SEED: [u8; 32] = [
890            0xf4, 0x16, 0x16, 0x48, 0xc3, 0xac, 0x77, 0xac, 0x72, 0x20, 0x0b, 0xea, 0x99, 0x67,
891            0x2d, 0x6d, 0xca, 0x9f, 0x76, 0xaf, 0x1b, 0x09, 0x73, 0xa0, 0x59, 0x22, 0x6d, 0xc5,
892            0x46, 0x39, 0x1c, 0x4a,
893        ];
894
895        let rng = proptest::test_runner::TestRng::from_seed(
896            proptest::test_runner::RngAlgorithm::ChaCha,
897            &RNG_SEED,
898        );
899        // Generate a collection of Rows.
900        let config = proptest::test_runner::Config {
901            // We let the loop below drive how much data we generate.
902            cases: u32::MAX,
903            rng_algorithm: proptest::test_runner::RngAlgorithm::ChaCha,
904            ..Default::default()
905        };
906        let mut runner = proptest::test_runner::TestRunner::new_with_rng(config, rng);
907
908        let test_cases = 2_000;
909        let strat = arb_numeric();
910
911        let mut all_numerics = Vec::new();
912        for _ in 0..test_cases {
913            let value_tree = strat.new_tree(&mut runner).unwrap();
914            let numeric = value_tree.current();
915            let packed = PackedNumeric::from_value(numeric);
916            let hex_bytes = format!("{:x?}", packed.as_bytes());
917
918            all_numerics.push((numeric, hex_bytes));
919        }
920
921        insta::assert_debug_snapshot!(all_numerics);
922    }
923}
mz_repr/adt/numeric.rs

mz_repr/adt/
numeric.rs
