mz_repr/adt/
numeric.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Functions related to Materialize's numeric type, which is largely a wrapper
11//! around [`rust-dec`].
12//!
13//! [`rust-dec`]: https://github.com/MaterializeInc/rust-dec/
14
15use std::error::Error;
16use std::fmt;
17use std::sync::LazyLock;
18
19use anyhow::bail;
20use dec::{Context, Decimal};
21use mz_lowertest::MzReflect;
22use mz_ore::cast;
23use mz_persist_types::columnar::FixedSizeCodec;
24use mz_proto::{ProtoType, RustType, TryFromProtoError};
25use proptest_derive::Arbitrary;
26use serde::{Deserialize, Serialize};
27
28include!(concat!(env!("OUT_DIR"), "/mz_repr.adt.numeric.rs"));
29
30/// The number of internal decimal units in a [`Numeric`] value.
31pub const NUMERIC_DATUM_WIDTH: u8 = 13;
32
33/// The value of [`NUMERIC_DATUM_WIDTH`] as a [`u8`].
34pub const NUMERIC_DATUM_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_DATUM_WIDTH);
35
36/// The maximum number of digits expressable in a [`Numeric`] value.
37pub const NUMERIC_DATUM_MAX_PRECISION: u8 = NUMERIC_DATUM_WIDTH * 3;
38
39/// A numeric value.
40pub type Numeric = Decimal<NUMERIC_DATUM_WIDTH_USIZE>;
41
42/// The number of internal decimal units in a [`NumericAgg`] value.
43pub const NUMERIC_AGG_WIDTH: u8 = 27;
44
45/// The value of [`NUMERIC_AGG_WIDTH`] as a [`u8`].
46pub const NUMERIC_AGG_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_AGG_WIDTH);
47
48/// The maximum number of digits expressable in a [`NumericAgg`] value.
49pub const NUMERIC_AGG_MAX_PRECISION: u8 = NUMERIC_AGG_WIDTH * 3;
50
51/// A double-width version of [`Numeric`] for use in aggregations.
52pub type NumericAgg = Decimal<NUMERIC_AGG_WIDTH_USIZE>;
53
54static CX_DATUM: LazyLock<Context<Numeric>> = LazyLock::new(|| {
55    let mut cx = Context::<Numeric>::default();
56    cx.set_max_exponent(isize::from(NUMERIC_DATUM_MAX_PRECISION - 1))
57        .unwrap();
58    cx.set_min_exponent(-isize::from(NUMERIC_DATUM_MAX_PRECISION))
59        .unwrap();
60    cx
61});
62static CX_AGG: LazyLock<Context<NumericAgg>> = LazyLock::new(|| {
63    let mut cx = Context::<NumericAgg>::default();
64    cx.set_max_exponent(isize::from(NUMERIC_AGG_MAX_PRECISION - 1))
65        .unwrap();
66    cx.set_min_exponent(-isize::from(NUMERIC_AGG_MAX_PRECISION))
67        .unwrap();
68    cx
69});
70static U128_SPLITTER_DATUM: LazyLock<Numeric> = LazyLock::new(|| {
71    let mut cx = Numeric::context();
72    // 1 << 128
73    cx.parse("340282366920938463463374607431768211456").unwrap()
74});
75static U128_SPLITTER_AGG: LazyLock<NumericAgg> = LazyLock::new(|| {
76    let mut cx = NumericAgg::context();
77    // 1 << 128
78    cx.parse("340282366920938463463374607431768211456").unwrap()
79});
80
81/// Module to simplify serde'ing a `Numeric` through its string representation.
82pub mod str_serde {
83    use std::str::FromStr;
84
85    use serde::Deserialize;
86
87    use super::Numeric;
88
89    /// Deserializing a [`Numeric`] value from its `String` representation.
90    pub fn deserialize<'de, D>(deserializer: D) -> Result<Numeric, D::Error>
91    where
92        D: serde::Deserializer<'de>,
93    {
94        let buf = String::deserialize(deserializer)?;
95        Numeric::from_str(&buf).map_err(serde::de::Error::custom)
96    }
97}
98
99/// The `max_scale` of a [`SqlScalarType::Numeric`].
100///
101/// This newtype wrapper ensures that the scale is within the valid range.
102///
103/// [`SqlScalarType::Numeric`]: crate::SqlScalarType::Numeric
104#[derive(
105    Arbitrary,
106    Debug,
107    Clone,
108    Copy,
109    Eq,
110    PartialEq,
111    Ord,
112    PartialOrd,
113    Hash,
114    Serialize,
115    Deserialize,
116    MzReflect
117)]
118pub struct NumericMaxScale(pub(crate) u8);
119
120impl NumericMaxScale {
121    /// A max scale of zero.
122    pub const ZERO: NumericMaxScale = NumericMaxScale(0);
123
124    /// Consumes the newtype wrapper, returning the inner `u8`.
125    pub fn into_u8(self) -> u8 {
126        self.0
127    }
128}
129
130impl TryFrom<i64> for NumericMaxScale {
131    type Error = InvalidNumericMaxScaleError;
132
133    fn try_from(max_scale: i64) -> Result<Self, Self::Error> {
134        match u8::try_from(max_scale) {
135            Ok(max_scale) if max_scale <= NUMERIC_DATUM_MAX_PRECISION => {
136                Ok(NumericMaxScale(max_scale))
137            }
138            _ => Err(InvalidNumericMaxScaleError),
139        }
140    }
141}
142
143impl TryFrom<usize> for NumericMaxScale {
144    type Error = InvalidNumericMaxScaleError;
145
146    fn try_from(max_scale: usize) -> Result<Self, Self::Error> {
147        Self::try_from(i64::try_from(max_scale).map_err(|_| InvalidNumericMaxScaleError)?)
148    }
149}
150
151impl RustType<ProtoNumericMaxScale> for NumericMaxScale {
152    fn into_proto(&self) -> ProtoNumericMaxScale {
153        ProtoNumericMaxScale {
154            value: self.0.into_proto(),
155        }
156    }
157
158    fn from_proto(max_scale: ProtoNumericMaxScale) -> Result<Self, TryFromProtoError> {
159        Ok(NumericMaxScale(max_scale.value.into_rust()?))
160    }
161}
162
163impl RustType<ProtoOptionalNumericMaxScale> for Option<NumericMaxScale> {
164    fn into_proto(&self) -> ProtoOptionalNumericMaxScale {
165        ProtoOptionalNumericMaxScale {
166            value: self.into_proto(),
167        }
168    }
169
170    fn from_proto(max_scale: ProtoOptionalNumericMaxScale) -> Result<Self, TryFromProtoError> {
171        max_scale.value.into_rust()
172    }
173}
174
175/// The error returned when constructing a [`NumericMaxScale`] from an invalid
176/// value.
177#[derive(Debug, Clone)]
178pub struct InvalidNumericMaxScaleError;
179
180impl fmt::Display for InvalidNumericMaxScaleError {
181    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
182        write!(
183            f,
184            "scale for type numeric must be between 0 and {}",
185            NUMERIC_DATUM_MAX_PRECISION
186        )
187    }
188}
189
190impl Error for InvalidNumericMaxScaleError {}
191
192/// Traits to generalize converting [`Decimal`] values to and from their
193/// coefficients' two's complements.
194pub trait Dec<const N: usize> {
195    // The number of bytes required to represent the min/max value of a decimal
196    // using two's complement.
197    const TWOS_COMPLEMENT_BYTE_WIDTH: usize;
198    // Convenience method for generating appropriate default contexts.
199    fn context() -> Context<Decimal<N>>;
200    // Provides value to break decimal into units of `u128`s for binary
201    // encoding/decoding.
202    fn u128_splitter() -> &'static Decimal<N>;
203}
204
205impl Dec<NUMERIC_DATUM_WIDTH_USIZE> for Numeric {
206    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 17;
207    fn context() -> Context<Numeric> {
208        CX_DATUM.clone()
209    }
210    fn u128_splitter() -> &'static Numeric {
211        &U128_SPLITTER_DATUM
212    }
213}
214
215impl Dec<NUMERIC_AGG_WIDTH_USIZE> for NumericAgg {
216    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 33;
217    fn context() -> Context<NumericAgg> {
218        CX_AGG.clone()
219    }
220    fn u128_splitter() -> &'static NumericAgg {
221        &U128_SPLITTER_AGG
222    }
223}
224
225/// Returns a new context appropriate for operating on numeric datums.
226pub fn cx_datum() -> Context<Numeric> {
227    CX_DATUM.clone()
228}
229
230/// Returns a new context appropriate for operating on numeric aggregates.
231pub fn cx_agg() -> Context<NumericAgg> {
232    CX_AGG.clone()
233}
234
235fn twos_complement_be_to_u128(input: &[u8]) -> u128 {
236    assert!(input.len() <= 16);
237    let mut buf = [0; 16];
238    buf[16 - input.len()..16].copy_from_slice(input);
239    u128::from_be_bytes(buf)
240}
241
242/// Using negative binary numbers can require more digits of precision than
243/// [`Numeric`] offers, so we need to have the option to swap bytes' signs at the
244/// byte- rather than the library-level.
245fn negate_twos_complement_le<'a, I>(b: I)
246where
247    I: Iterator<Item = &'a mut u8>,
248{
249    let mut seen_first_one = false;
250    for i in b {
251        if seen_first_one {
252            *i = *i ^ 0xFF;
253        } else if *i > 0 {
254            seen_first_one = true;
255            if i == &0x80 {
256                continue;
257            }
258            let tz = i.trailing_zeros();
259            *i = *i ^ (0xFF << tz + 1);
260        }
261    }
262}
263
264/// Converts an [`Numeric`] into its big endian two's complement representation.
265pub fn numeric_to_twos_complement_be(
266    mut numeric: Numeric,
267) -> [u8; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH] {
268    let mut buf = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
269    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
270    // as zeroes so as to avoid erroring (encoding values is meant to be
271    // infallible) and retain downstream associativity/commutativity.
272    if numeric.is_special() {
273        return buf;
274    }
275
276    let mut cx = Numeric::context();
277
278    // Ensure `numeric` is a canonical coefficient.
279    if numeric.exponent() < 0 {
280        let s = Numeric::from(-numeric.exponent());
281        cx.scaleb(&mut numeric, &s);
282    }
283
284    numeric_to_twos_complement_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(
285        numeric, &mut cx, &mut buf,
286    );
287    buf
288}
289
290/// Converts an [`Numeric`] into a big endian two's complement representation where
291/// the encoded value has [`NUMERIC_AGG_MAX_PRECISION`] digits and a scale of
292/// [`NUMERIC_DATUM_MAX_PRECISION`].
293///
294/// This representation is appropriate to use in
295/// contexts requiring two's complement representation but `Numeric` values' scale
296/// isn't known, e.g. when working with columns with an explicitly defined
297/// scale.
298pub fn numeric_to_twos_complement_wide(
299    numeric: Numeric,
300) -> [u8; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH] {
301    let mut buf = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
302    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
303    // as zeroes so as to avoid erroring (encoding values is meant to be
304    // infallible) and retain downstream associativity/commutativity.
305    if numeric.is_special() {
306        return buf;
307    }
308    let mut cx = NumericAgg::context();
309    let mut d = cx.to_width(numeric);
310    let mut scaler = NumericAgg::from(NUMERIC_DATUM_MAX_PRECISION);
311    cx.neg(&mut scaler);
312    // Shape `d` so that its exponent is -NUMERIC_DATUM_MAX_PRECISION
313    cx.rescale(&mut d, &scaler);
314    // Adjust `d` so it is a canonical coefficient, i.e. its exact value can be
315    // recovered by setting its exponent to -39.
316    cx.abs(&mut scaler);
317    cx.scaleb(&mut d, &scaler);
318
319    numeric_to_twos_complement_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(d, &mut cx, &mut buf);
320    buf
321}
322
323fn numeric_to_twos_complement_inner<D: Dec<N>, const N: usize>(
324    mut d: Decimal<N>,
325    cx: &mut Context<Decimal<N>>,
326    buf: &mut [u8],
327) {
328    // Adjust negative values to be writable as series of `u128`.
329    let is_neg = if d.is_negative() {
330        cx.neg(&mut d);
331        true
332    } else {
333        false
334    };
335
336    // Values have all been made into canonical coefficients.
337    assert!(d.exponent() >= 0);
338
339    let mut buf_cursor = 0;
340    while !d.is_zero() {
341        let mut w = d.clone();
342        // Take the remainder; this represents one of our "units" to take the coefficient of, i.e. d & u128::MAX
343        cx.rem(&mut w, D::u128_splitter());
344
345        // Take the `u128` version of the coefficient, which will always be what
346        // we want given that we adjusted negative values to have an unsigned
347        // integer representation.
348        let c = w.coefficient::<u128>().unwrap();
349
350        // Determine the width of the coefficient we want to take, i.e. the full
351        // coefficient or a part of it to fill the buffer.
352        let e = std::cmp::min(buf_cursor + 16, D::TWOS_COMPLEMENT_BYTE_WIDTH);
353
354        // We're putting less significant bytes at index 0, which is little endian.
355        buf[buf_cursor..e].copy_from_slice(&c.to_le_bytes()[0..e - buf_cursor]);
356        // Advance cursor; ok that it will go past buffer on final + 1th iteration.
357        buf_cursor += 16;
358
359        // Take the quotient to represent the next unit, i.e. d >> 128
360        cx.div_integer(&mut d, D::u128_splitter());
361    }
362
363    if is_neg {
364        negate_twos_complement_le(buf.iter_mut());
365    }
366
367    // Convert from little endian to big endian.
368    buf.reverse();
369}
370
371pub fn twos_complement_be_to_numeric(
372    input: &mut [u8],
373    scale: u8,
374) -> Result<Numeric, anyhow::Error> {
375    let mut cx = cx_datum();
376    if input.len() <= 17 {
377        if let Ok(mut n) =
378            twos_complement_be_to_numeric_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(input)
379        {
380            n.set_exponent(-i32::from(scale));
381            return Ok(n);
382        }
383    }
384    // If bytes were invalid for narrower representation, try to use wider
385    // representation in case e.g. simply has more trailing zeroes.
386    let mut n = twos_complement_be_to_numeric_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(input)?;
387    // Exponent must be set before converting to `Numeric` width, otherwise values can overflow 39 dop.
388    n.set_exponent(-i32::from(scale));
389    let d = cx.to_width(n);
390    if cx.status().inexact() {
391        bail!("Value exceeds maximum numeric value")
392    }
393    Ok(d)
394}
395
396/// Parses a buffer of two's complement digits in big-endian order and converts
397/// them to [`Decimal<N>`].
398pub fn twos_complement_be_to_numeric_inner<D: Dec<N>, const N: usize>(
399    input: &mut [u8],
400) -> Result<Decimal<N>, anyhow::Error> {
401    let is_neg = if (input[0] & 0x80) != 0 {
402        // byte-level negate all negative values, guaranteeing all bytes are
403        // readable as unsigned.
404        negate_twos_complement_le(input.iter_mut().rev());
405        true
406    } else {
407        false
408    };
409
410    let head = input.len() % 16;
411    let i = twos_complement_be_to_u128(&input[0..head]);
412    let mut cx = D::context();
413    let mut d = cx.from_u128(i);
414
415    for c in input[head..].chunks(16) {
416        assert_eq!(c.len(), 16);
417        // essentially d << 128
418        cx.mul(&mut d, D::u128_splitter());
419        let i = twos_complement_be_to_u128(c);
420        let i = cx.from_u128(i);
421        cx.add(&mut d, &i);
422    }
423
424    if cx.status().inexact() {
425        bail!("Value exceeds maximum numeric value")
426    } else if cx.status().any() {
427        bail!("unexpected status {:?}", cx.status());
428    }
429    if is_neg {
430        cx.neg(&mut d);
431    }
432    Ok(d)
433}
434
435#[mz_ore::test]
436#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
437fn test_twos_complement_roundtrip() {
438    fn inner(s: &str) {
439        let mut cx = cx_datum();
440        let d = cx.parse(s).unwrap();
441        let scale = std::cmp::min(d.exponent(), 0).abs();
442        let mut b = numeric_to_twos_complement_be(d.clone());
443        let x = twos_complement_be_to_numeric(&mut b, u8::try_from(scale).unwrap()).unwrap();
444        assert_eq!(d, x);
445    }
446    inner("0");
447    inner("0.000000000000000000000000000000000012345");
448    inner("0.123456789012345678901234567890123456789");
449    inner("1.00000000000000000000000000000000000000");
450    inner("1");
451    inner("2");
452    inner("170141183460469231731687303715884105727");
453    inner("170141183460469231731687303715884105728");
454    inner("12345678901234567890.1234567890123456789");
455    inner("999999999999999999999999999999999999999");
456    inner("7e35");
457    inner("7e-35");
458    inner("-0.000000000000000000000000000000000012345");
459    inner("-0.12345678901234567890123456789012345678");
460    inner("-1.00000000000000000000000000000000000000");
461    inner("-1");
462    inner("-2");
463    inner("-170141183460469231731687303715884105727");
464    inner("-170141183460469231731687303715884105728");
465    inner("-12345678901234567890.1234567890123456789");
466    inner("-999999999999999999999999999999999999999");
467    inner("-7.2e35");
468    inner("-7.2e-35");
469}
470
471#[mz_ore::test]
472#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
473fn test_twos_comp_numeric_primitive() {
474    fn inner_inner<P>(i: P, i_be_bytes: &mut [u8])
475    where
476        P: Into<Numeric> + TryFrom<Numeric> + Eq + PartialEq + std::fmt::Debug + Copy,
477    {
478        let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
479        e[Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
480        let mut w = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
481        w[NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
482
483        let d: Numeric = i.into();
484
485        // Extend negative sign into most-significant bits
486        if d.is_negative() {
487            for i in e[..Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
488                *i = 0xFF;
489            }
490            for i in w[..NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
491                *i = 0xFF;
492            }
493        }
494
495        // Ensure decimal value's two's complement representation matches an
496        // extended version of `to_be_bytes`.
497        let d_be_bytes = numeric_to_twos_complement_be(d);
498        assert_eq!(
499            e, d_be_bytes,
500            "expected repr of {:?}, got {:?}",
501            e, d_be_bytes
502        );
503
504        // Ensure extended version of `to_be_bytes` generates same `i128`.
505        let e_numeric = twos_complement_be_to_numeric(&mut e, 0).unwrap();
506        let e_p: P = e_numeric
507            .try_into()
508            .unwrap_or_else(|_e| panic!("try_into failed"));
509        assert_eq!(i, e_p, "expected val of {:?}, got {:?}", i, e_p);
510
511        // Wide representation produces same result.
512        let w_numeric = twos_complement_be_to_numeric(&mut w, 0).unwrap();
513        let w_p: P = w_numeric
514            .try_into()
515            .unwrap_or_else(|_e| panic!("try_into failed"));
516        assert_eq!(i, w_p, "expected val of {:?}, got {:?}", i, e_p);
517
518        // Bytes do not need to be in `Numeric`-specific format
519        let p_numeric = twos_complement_be_to_numeric(i_be_bytes, 0).unwrap();
520        let p_p: P = p_numeric
521            .try_into()
522            .unwrap_or_else(|_e| panic!("try_into failed"));
523        assert_eq!(i, p_p, "expected val of {:?}, got {:?}", i, p_p);
524    }
525
526    fn inner_i32(i: i32) {
527        inner_inner(i, &mut i.to_be_bytes());
528    }
529
530    fn inner_i64(i: i64) {
531        inner_inner(i, &mut i.to_be_bytes());
532    }
533
534    // We need a wrapper around i128 to implement the same traits as the other
535    // primitive types. This is less code than a second implementation of the
536    // same test that takes unwrapped i128s.
537    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
538    struct FromableI128 {
539        i: i128,
540    }
541    impl From<i128> for FromableI128 {
542        fn from(i: i128) -> FromableI128 {
543            FromableI128 { i }
544        }
545    }
546    impl From<FromableI128> for Numeric {
547        fn from(n: FromableI128) -> Numeric {
548            Numeric::try_from(n.i).unwrap()
549        }
550    }
551    impl TryFrom<Numeric> for FromableI128 {
552        type Error = ();
553        fn try_from(n: Numeric) -> Result<FromableI128, Self::Error> {
554            match i128::try_from(n) {
555                Ok(i) => Ok(FromableI128 { i }),
556                Err(_) => Err(()),
557            }
558        }
559    }
560
561    fn inner_i128(i: i128) {
562        inner_inner(FromableI128::from(i), &mut i.to_be_bytes());
563    }
564
565    inner_i32(0);
566    inner_i32(1);
567    inner_i32(2);
568    inner_i32(-1);
569    inner_i32(-2);
570    inner_i32(i32::MAX);
571    inner_i32(i32::MIN);
572    inner_i32(i32::MAX / 7 + 7);
573    inner_i32(i32::MIN / 7 + 7);
574    inner_i64(0);
575    inner_i64(1);
576    inner_i64(2);
577    inner_i64(-1);
578    inner_i64(-2);
579    inner_i64(i64::MAX);
580    inner_i64(i64::MIN);
581    inner_i64(i64::MAX / 7 + 7);
582    inner_i64(i64::MIN / 7 + 7);
583    inner_i128(0);
584    inner_i128(1);
585    inner_i128(2);
586    inner_i128(-1);
587    inner_i128(-2);
588    inner_i128(i128::from(i64::MAX));
589    inner_i128(i128::from(i64::MIN));
590    inner_i128(i128::MAX);
591    inner_i128(i128::MIN);
592    inner_i128(i128::MAX / 7 + 7);
593    inner_i128(i128::MIN / 7 + 7);
594}
595
596#[mz_ore::test]
597#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
598fn test_twos_complement_to_numeric_fail() {
599    fn inner(b: &mut [u8]) {
600        let r = twos_complement_be_to_numeric(b, 0);
601        mz_ore::assert_err!(r);
602    }
603    // 17-byte signed digit's max value exceeds 39 digits of precision
604    let mut e = [0xFF; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
605    e[0] -= 0x80;
606    inner(&mut e);
607
608    // 1 << 17 * 8 exceeds exceeds 39 digits of precision
609    let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH + 1];
610    e[0] = 1;
611    inner(&mut e);
612}
613
614#[mz_ore::test]
615#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
616fn test_wide_twos_complement_roundtrip() {
617    fn inner(s: &str) {
618        let mut cx = cx_datum();
619        let d = cx.parse(s).unwrap();
620        let mut b = numeric_to_twos_complement_wide(d.clone());
621        let x = twos_complement_be_to_numeric(&mut b, NUMERIC_DATUM_MAX_PRECISION).unwrap();
622        assert_eq!(d, x);
623    }
624    inner("0");
625    inner("0.000000000000000000000000000000000012345");
626    inner("0.123456789012345678901234567890123456789");
627    inner("1.00000000000000000000000000000000000000");
628    inner("1");
629    inner("2");
630    inner("170141183460469231731687303715884105727");
631    inner("170141183460469231731687303715884105728");
632    inner("12345678901234567890.1234567890123456789");
633    inner("999999999999999999999999999999999999999");
634    inner("-0.000000000000000000000000000000000012345");
635    inner("-0.123456789012345678901234567890123456789");
636    inner("-1.00000000000000000000000000000000000000");
637    inner("-1");
638    inner("-2");
639    inner("-170141183460469231731687303715884105727");
640    inner("-170141183460469231731687303715884105728");
641    inner("-12345678901234567890.1234567890123456789");
642    inner("-999999999999999999999999999999999999999");
643}
644
645/// Returns `n`'s precision, i.e. the total number of digits represented by `n`
646/// in standard notation not including a zero in the "one's place" in (-1,1).
647pub fn get_precision<const N: usize>(n: &Decimal<N>) -> u32 {
648    let e = n.exponent();
649    if e >= 0 {
650        // Positive exponent
651        n.digits() + u32::try_from(e).unwrap()
652    } else {
653        // Negative exponent
654        let d = n.digits();
655        let e = u32::try_from(e.abs()).unwrap();
656        // Precision is...
657        // - d if decimal point splits numbers
658        // - e if e dominates number of digits
659        std::cmp::max(d, e)
660    }
661}
662
663/// Returns `n`'s scale, i.e. the number of digits used after the decimal point.
664pub fn get_scale(n: &Numeric) -> u32 {
665    let exp = n.exponent();
666    if exp >= 0 { 0 } else { exp.unsigned_abs() }
667}
668
669/// Ensures [`Numeric`] values are:
670/// - Within `Numeric`'s max precision ([`NUMERIC_DATUM_MAX_PRECISION`]), or errors if not.
671/// - Never possible but invalid representations (i.e. never -Nan or -0).
672///
673/// Should be called after any operation that can change an [`Numeric`]'s scale or
674/// generate negative values (except addition and subtraction).
675pub fn munge_numeric(n: &mut Numeric) -> Result<(), anyhow::Error> {
676    rescale_within_max_precision(n)?;
677    if (n.is_zero() || n.is_nan()) && n.is_negative() {
678        cx_datum().neg(n);
679    }
680    Ok(())
681}
682
683/// Rescale's `n` to fit within [`Numeric`]'s max precision or error if not
684/// possible.
685fn rescale_within_max_precision(n: &mut Numeric) -> Result<(), anyhow::Error> {
686    let current_precision = get_precision(n);
687    if current_precision > u32::from(NUMERIC_DATUM_MAX_PRECISION) {
688        if n.exponent() < 0 {
689            let precision_diff = current_precision - u32::from(NUMERIC_DATUM_MAX_PRECISION);
690            let current_scale = u8::try_from(get_scale(n))?;
691            let scale_diff = current_scale - u8::try_from(precision_diff).unwrap();
692            rescale(n, scale_diff)?;
693        } else {
694            bail!(
695                "numeric value {} exceed maximum precision {}",
696                n,
697                NUMERIC_DATUM_MAX_PRECISION
698            )
699        }
700    }
701    Ok(())
702}
703
704/// Rescale `n` as an `OrderedDecimal` with the described scale, or error if:
705/// - Rescaling exceeds max precision
706/// - `n` requires > [`NUMERIC_DATUM_MAX_PRECISION`] - `scale` digits of precision
707///   left of the decimal point
708pub fn rescale(n: &mut Numeric, scale: u8) -> Result<(), anyhow::Error> {
709    let mut cx = cx_datum();
710    cx.rescale(n, &Numeric::from(-i32::from(scale)));
711    if cx.status().invalid_operation() || get_precision(n) > u32::from(NUMERIC_DATUM_MAX_PRECISION)
712    {
713        bail!(
714            "numeric value {} exceed maximum precision {}",
715            n,
716            NUMERIC_DATUM_MAX_PRECISION
717        )
718    }
719    munge_numeric(n)?;
720
721    Ok(())
722}
723
724/// A type that can represent Real Numbers. Useful for interoperability between Numeric and
725/// floating point.
726pub trait DecimalLike:
727    From<u8>
728    + From<u16>
729    + From<u32>
730    + From<i8>
731    + From<i16>
732    + From<i32>
733    + From<f32>
734    + From<f64>
735    + std::ops::Add<Output = Self>
736    + std::ops::Sub<Output = Self>
737    + std::ops::Mul<Output = Self>
738    + std::ops::Div<Output = Self>
739{
740    /// Used to do value-to-value conversions while consuming the input value. Depending on the
741    /// implementation it may be potentially lossy.
742    fn lossy_from(i: i64) -> Self;
743}
744
745impl DecimalLike for f64 {
746    // No other known way to convert `i64` to `f64`.
747    #[allow(clippy::as_conversions)]
748    fn lossy_from(i: i64) -> Self {
749        i as f64
750    }
751}
752
753impl DecimalLike for Numeric {
754    fn lossy_from(i: i64) -> Self {
755        Numeric::from(i)
756    }
757}
758
759/// An encoded packed variant of [`Numeric`].
760///
761/// Unlike other "Packed" types we _DO NOT_ uphold the invariant that
762/// [`PackedNumeric`] sorts the same as [`Numeric`].
763#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
764pub struct PackedNumeric(pub [u8; 40]);
765
766impl FixedSizeCodec<Numeric> for PackedNumeric {
767    const SIZE: usize = 40;
768
769    fn as_bytes(&self) -> &[u8] {
770        &self.0
771    }
772
773    fn from_bytes(slice: &[u8]) -> Result<Self, String> {
774        let buf: [u8; Self::SIZE] = slice.try_into().map_err(|_| {
775            format!(
776                "size for PackedNumeric is {} bytes, got {}",
777                Self::SIZE,
778                slice.len()
779            )
780        })?;
781        Ok(PackedNumeric(buf))
782    }
783
784    fn from_value(val: Numeric) -> PackedNumeric {
785        let (digits, exponent, bits, lsu) = val.to_raw_parts();
786
787        let mut buf = [0u8; 40];
788
789        buf[0..4].copy_from_slice(&digits.to_le_bytes());
790        buf[4..8].copy_from_slice(&exponent.to_le_bytes());
791
792        for i in 0..13 {
793            buf[(i * 2) + 8..(i * 2) + 10].copy_from_slice(&lsu[i].to_le_bytes());
794        }
795
796        buf[34..35].copy_from_slice(&bits.to_le_bytes());
797
798        PackedNumeric(buf)
799    }
800
801    fn into_value(self) -> Numeric {
802        let digits: [u8; 4] = self.0[0..4].try_into().unwrap();
803        let digits = u32::from_le_bytes(digits);
804
805        let exponent: [u8; 4] = self.0[4..8].try_into().unwrap();
806        let exponent = i32::from_le_bytes(exponent);
807
808        let mut lsu = [0u16; 13];
809        for i in 0..13 {
810            let x: [u8; 2] = self.0[(i * 2) + 8..(i * 2) + 10].try_into().unwrap();
811            let x = u16::from_le_bytes(x);
812            lsu[i] = x;
813        }
814
815        let bits: [u8; 1] = self.0[34..35].try_into().unwrap();
816        let bits = u8::from_le_bytes(bits);
817
818        Numeric::from_raw_parts(digits, exponent, bits, lsu)
819    }
820}
821
822#[cfg(test)]
823mod tests {
824    use mz_ore::assert_ok;
825    use mz_proto::protobuf_roundtrip;
826    use proptest::prelude::*;
827
828    use crate::scalar::arb_numeric;
829
830    use super::*;
831
832    proptest! {
833        #[mz_ore::test]
834        fn numeric_max_scale_protobuf_roundtrip(expect in any::<NumericMaxScale>()) {
835            let actual = protobuf_roundtrip::<_, ProtoNumericMaxScale>(&expect);
836            assert_ok!(actual);
837            assert_eq!(actual.unwrap(), expect);
838        }
839
840        #[mz_ore::test]
841        fn optional_numeric_max_scale_protobuf_roundtrip(
842            expect in any::<Option<NumericMaxScale>>(),
843        ) {
844            let actual = protobuf_roundtrip::<_, ProtoOptionalNumericMaxScale>(&expect);
845            assert_ok!(actual);
846            assert_eq!(actual.unwrap(), expect);
847        }
848    }
849
850    #[mz_ore::test]
851    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
852    fn smoketest_packed_numeric_roundtrips() {
853        let og = PackedNumeric::from_value(Numeric::from(-42));
854        let bytes = og.as_bytes();
855        let rnd = PackedNumeric::from_bytes(bytes).expect("valid");
856        assert_eq!(og, rnd);
857
858        // Returns an error if the size of the slice is invalid.
859        mz_ore::assert_err!(PackedNumeric::from_bytes(&[0, 0, 0, 0]));
860    }
861
862    #[mz_ore::test]
863    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
864    fn proptest_packed_numeric_roundtrip() {
865        fn test(og: Numeric) {
866            let packed = PackedNumeric::from_value(og);
867            let rnd = packed.into_value();
868
869            if og.is_nan() && rnd.is_nan() {
870                return;
871            }
872            assert_eq!(og, rnd);
873        }
874
875        proptest!(|(num in arb_numeric())| {
876            test(num);
877        });
878    }
879
880    // Note: It's expected that this test will fail if you update the strategy
881    // for generating an arbitrary Numeric. In that case feel free to
882    // regenerate the snapshot.
883    #[mz_ore::test]
884    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
885    fn packed_numeric_stability() {
886        /// This is the seed [`proptest`] uses for their deterministic RNG. We
887        /// copy it here to prevent breaking this test if [`proptest`] changes.
888        const RNG_SEED: [u8; 32] = [
889            0xf4, 0x16, 0x16, 0x48, 0xc3, 0xac, 0x77, 0xac, 0x72, 0x20, 0x0b, 0xea, 0x99, 0x67,
890            0x2d, 0x6d, 0xca, 0x9f, 0x76, 0xaf, 0x1b, 0x09, 0x73, 0xa0, 0x59, 0x22, 0x6d, 0xc5,
891            0x46, 0x39, 0x1c, 0x4a,
892        ];
893
894        let rng = proptest::test_runner::TestRng::from_seed(
895            proptest::test_runner::RngAlgorithm::ChaCha,
896            &RNG_SEED,
897        );
898        // Generate a collection of Rows.
899        let config = proptest::test_runner::Config {
900            // We let the loop below drive how much data we generate.
901            cases: u32::MAX,
902            rng_algorithm: proptest::test_runner::RngAlgorithm::ChaCha,
903            ..Default::default()
904        };
905        let mut runner = proptest::test_runner::TestRunner::new_with_rng(config, rng);
906
907        let test_cases = 2_000;
908        let strat = arb_numeric();
909
910        let mut all_numerics = Vec::new();
911        for _ in 0..test_cases {
912            let value_tree = strat.new_tree(&mut runner).unwrap();
913            let numeric = value_tree.current();
914            let packed = PackedNumeric::from_value(numeric);
915            let hex_bytes = format!("{:x?}", packed.as_bytes());
916
917            all_numerics.push((numeric, hex_bytes));
918        }
919
920        insta::assert_debug_snapshot!(all_numerics);
921    }
922}
mz_repr/adt/numeric.rs

mz_repr/adt/
numeric.rs
