mz_repr/adt/
numeric.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Functions related to Materialize's numeric type, which is largely a wrapper
11//! around [`rust-dec`].
12//!
13//! [`rust-dec`]: https://github.com/MaterializeInc/rust-dec/
14
15use std::error::Error;
16use std::fmt;
17use std::sync::LazyLock;
18
19use anyhow::bail;
20use dec::{Context, Decimal};
21use mz_lowertest::MzReflect;
22use mz_ore::cast;
23use mz_persist_types::columnar::FixedSizeCodec;
24use mz_proto::{ProtoType, RustType, TryFromProtoError};
25#[cfg(any(test, feature = "proptest"))]
26use proptest_derive::Arbitrary;
27use serde::{Deserialize, Serialize};
28
29include!(concat!(env!("OUT_DIR"), "/mz_repr.adt.numeric.rs"));
30
31/// The number of internal decimal units in a [`Numeric`] value.
32pub const NUMERIC_DATUM_WIDTH: u8 = 13;
33
34/// The value of [`NUMERIC_DATUM_WIDTH`] as a [`u8`].
35pub const NUMERIC_DATUM_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_DATUM_WIDTH);
36
37/// The maximum number of digits expressable in a [`Numeric`] value.
38pub const NUMERIC_DATUM_MAX_PRECISION: u8 = NUMERIC_DATUM_WIDTH * 3;
39
40/// A numeric value.
41pub type Numeric = Decimal<NUMERIC_DATUM_WIDTH_USIZE>;
42
43/// The number of internal decimal units in a [`NumericAgg`] value.
44pub const NUMERIC_AGG_WIDTH: u8 = 27;
45
46/// The value of [`NUMERIC_AGG_WIDTH`] as a [`u8`].
47pub const NUMERIC_AGG_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_AGG_WIDTH);
48
49/// The maximum number of digits expressable in a [`NumericAgg`] value.
50pub const NUMERIC_AGG_MAX_PRECISION: u8 = NUMERIC_AGG_WIDTH * 3;
51
52/// A double-width version of [`Numeric`] for use in aggregations.
53pub type NumericAgg = Decimal<NUMERIC_AGG_WIDTH_USIZE>;
54
55static CX_DATUM: LazyLock<Context<Numeric>> = LazyLock::new(|| {
56    let mut cx = Context::<Numeric>::default();
57    cx.set_max_exponent(isize::from(NUMERIC_DATUM_MAX_PRECISION - 1))
58        .unwrap();
59    cx.set_min_exponent(-isize::from(NUMERIC_DATUM_MAX_PRECISION))
60        .unwrap();
61    cx
62});
63static CX_AGG: LazyLock<Context<NumericAgg>> = LazyLock::new(|| {
64    let mut cx = Context::<NumericAgg>::default();
65    cx.set_max_exponent(isize::from(NUMERIC_AGG_MAX_PRECISION - 1))
66        .unwrap();
67    cx.set_min_exponent(-isize::from(NUMERIC_AGG_MAX_PRECISION))
68        .unwrap();
69    cx
70});
71static U128_SPLITTER_DATUM: LazyLock<Numeric> = LazyLock::new(|| {
72    let mut cx = Numeric::context();
73    // 1 << 128
74    cx.parse("340282366920938463463374607431768211456").unwrap()
75});
76static U128_SPLITTER_AGG: LazyLock<NumericAgg> = LazyLock::new(|| {
77    let mut cx = NumericAgg::context();
78    // 1 << 128
79    cx.parse("340282366920938463463374607431768211456").unwrap()
80});
81
82/// Module to simplify serde'ing a `Numeric` through its string representation.
83pub mod str_serde {
84    use std::str::FromStr;
85
86    use serde::Deserialize;
87
88    use super::Numeric;
89
90    /// Deserializing a [`Numeric`] value from its `String` representation.
91    pub fn deserialize<'de, D>(deserializer: D) -> Result<Numeric, D::Error>
92    where
93        D: serde::Deserializer<'de>,
94    {
95        let buf = String::deserialize(deserializer)?;
96        Numeric::from_str(&buf).map_err(serde::de::Error::custom)
97    }
98}
99
100/// The `max_scale` of a [`SqlScalarType::Numeric`].
101///
102/// This newtype wrapper ensures that the scale is within the valid range.
103///
104/// [`SqlScalarType::Numeric`]: crate::SqlScalarType::Numeric
105#[derive(
106    Debug,
107    Clone,
108    Copy,
109    Eq,
110    PartialEq,
111    Ord,
112    PartialOrd,
113    Hash,
114    Serialize,
115    Deserialize,
116    MzReflect
117)]
118#[cfg_attr(any(test, feature = "proptest"), derive(Arbitrary))]
119pub struct NumericMaxScale(pub(crate) u8);
120
121impl NumericMaxScale {
122    /// A max scale of zero.
123    pub const ZERO: NumericMaxScale = NumericMaxScale(0);
124
125    /// Consumes the newtype wrapper, returning the inner `u8`.
126    pub fn into_u8(self) -> u8 {
127        self.0
128    }
129}
130
131impl TryFrom<i64> for NumericMaxScale {
132    type Error = InvalidNumericMaxScaleError;
133
134    fn try_from(max_scale: i64) -> Result<Self, Self::Error> {
135        match u8::try_from(max_scale) {
136            Ok(max_scale) if max_scale <= NUMERIC_DATUM_MAX_PRECISION => {
137                Ok(NumericMaxScale(max_scale))
138            }
139            _ => Err(InvalidNumericMaxScaleError),
140        }
141    }
142}
143
144impl TryFrom<usize> for NumericMaxScale {
145    type Error = InvalidNumericMaxScaleError;
146
147    fn try_from(max_scale: usize) -> Result<Self, Self::Error> {
148        Self::try_from(i64::try_from(max_scale).map_err(|_| InvalidNumericMaxScaleError)?)
149    }
150}
151
152impl RustType<ProtoNumericMaxScale> for NumericMaxScale {
153    fn into_proto(&self) -> ProtoNumericMaxScale {
154        ProtoNumericMaxScale {
155            value: self.0.into_proto(),
156        }
157    }
158
159    fn from_proto(max_scale: ProtoNumericMaxScale) -> Result<Self, TryFromProtoError> {
160        Ok(NumericMaxScale(max_scale.value.into_rust()?))
161    }
162}
163
164impl RustType<ProtoOptionalNumericMaxScale> for Option<NumericMaxScale> {
165    fn into_proto(&self) -> ProtoOptionalNumericMaxScale {
166        ProtoOptionalNumericMaxScale {
167            value: self.into_proto(),
168        }
169    }
170
171    fn from_proto(max_scale: ProtoOptionalNumericMaxScale) -> Result<Self, TryFromProtoError> {
172        max_scale.value.into_rust()
173    }
174}
175
176/// The error returned when constructing a [`NumericMaxScale`] from an invalid
177/// value.
178#[derive(Debug, Clone)]
179pub struct InvalidNumericMaxScaleError;
180
181impl fmt::Display for InvalidNumericMaxScaleError {
182    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
183        write!(
184            f,
185            "scale for type numeric must be between 0 and {}",
186            NUMERIC_DATUM_MAX_PRECISION
187        )
188    }
189}
190
191impl Error for InvalidNumericMaxScaleError {}
192
193/// Traits to generalize converting [`Decimal`] values to and from their
194/// coefficients' two's complements.
195pub trait Dec<const N: usize> {
196    // The number of bytes required to represent the min/max value of a decimal
197    // using two's complement.
198    const TWOS_COMPLEMENT_BYTE_WIDTH: usize;
199    // Convenience method for generating appropriate default contexts.
200    fn context() -> Context<Decimal<N>>;
201    // Provides value to break decimal into units of `u128`s for binary
202    // encoding/decoding.
203    fn u128_splitter() -> &'static Decimal<N>;
204}
205
206impl Dec<NUMERIC_DATUM_WIDTH_USIZE> for Numeric {
207    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 17;
208    fn context() -> Context<Numeric> {
209        CX_DATUM.clone()
210    }
211    fn u128_splitter() -> &'static Numeric {
212        &U128_SPLITTER_DATUM
213    }
214}
215
216impl Dec<NUMERIC_AGG_WIDTH_USIZE> for NumericAgg {
217    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 33;
218    fn context() -> Context<NumericAgg> {
219        CX_AGG.clone()
220    }
221    fn u128_splitter() -> &'static NumericAgg {
222        &U128_SPLITTER_AGG
223    }
224}
225
226/// Returns a new context appropriate for operating on numeric datums.
227pub fn cx_datum() -> Context<Numeric> {
228    CX_DATUM.clone()
229}
230
231/// Returns a new context appropriate for operating on numeric aggregates.
232pub fn cx_agg() -> Context<NumericAgg> {
233    CX_AGG.clone()
234}
235
236fn twos_complement_be_to_u128(input: &[u8]) -> u128 {
237    assert!(input.len() <= 16);
238    let mut buf = [0; 16];
239    buf[16 - input.len()..16].copy_from_slice(input);
240    u128::from_be_bytes(buf)
241}
242
243/// Using negative binary numbers can require more digits of precision than
244/// [`Numeric`] offers, so we need to have the option to swap bytes' signs at the
245/// byte- rather than the library-level.
246fn negate_twos_complement_le<'a, I>(b: I)
247where
248    I: Iterator<Item = &'a mut u8>,
249{
250    let mut seen_first_one = false;
251    for i in b {
252        if seen_first_one {
253            *i = *i ^ 0xFF;
254        } else if *i > 0 {
255            seen_first_one = true;
256            if i == &0x80 {
257                continue;
258            }
259            let tz = i.trailing_zeros();
260            *i = *i ^ (0xFF << tz + 1);
261        }
262    }
263}
264
265/// Converts an [`Numeric`] into its big endian two's complement representation.
266pub fn numeric_to_twos_complement_be(
267    mut numeric: Numeric,
268) -> [u8; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH] {
269    let mut buf = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
270    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
271    // as zeroes so as to avoid erroring (encoding values is meant to be
272    // infallible) and retain downstream associativity/commutativity.
273    if numeric.is_special() {
274        return buf;
275    }
276
277    let mut cx = Numeric::context();
278
279    // Ensure `numeric` is a canonical coefficient.
280    if numeric.exponent() < 0 {
281        let s = Numeric::from(-numeric.exponent());
282        cx.scaleb(&mut numeric, &s);
283    }
284
285    numeric_to_twos_complement_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(
286        numeric, &mut cx, &mut buf,
287    );
288    buf
289}
290
291/// Converts an [`Numeric`] into a big endian two's complement representation where
292/// the encoded value has [`NUMERIC_AGG_MAX_PRECISION`] digits and a scale of
293/// [`NUMERIC_DATUM_MAX_PRECISION`].
294///
295/// This representation is appropriate to use in
296/// contexts requiring two's complement representation but `Numeric` values' scale
297/// isn't known, e.g. when working with columns with an explicitly defined
298/// scale.
299pub fn numeric_to_twos_complement_wide(
300    numeric: Numeric,
301) -> [u8; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH] {
302    let mut buf = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
303    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
304    // as zeroes so as to avoid erroring (encoding values is meant to be
305    // infallible) and retain downstream associativity/commutativity.
306    if numeric.is_special() {
307        return buf;
308    }
309    let mut cx = NumericAgg::context();
310    let mut d = cx.to_width(numeric);
311    let mut scaler = NumericAgg::from(NUMERIC_DATUM_MAX_PRECISION);
312    cx.neg(&mut scaler);
313    // Shape `d` so that its exponent is -NUMERIC_DATUM_MAX_PRECISION
314    cx.rescale(&mut d, &scaler);
315    // Adjust `d` so it is a canonical coefficient, i.e. its exact value can be
316    // recovered by setting its exponent to -39.
317    cx.abs(&mut scaler);
318    cx.scaleb(&mut d, &scaler);
319
320    numeric_to_twos_complement_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(d, &mut cx, &mut buf);
321    buf
322}
323
324fn numeric_to_twos_complement_inner<D: Dec<N>, const N: usize>(
325    mut d: Decimal<N>,
326    cx: &mut Context<Decimal<N>>,
327    buf: &mut [u8],
328) {
329    // Adjust negative values to be writable as series of `u128`.
330    let is_neg = if d.is_negative() {
331        cx.neg(&mut d);
332        true
333    } else {
334        false
335    };
336
337    // Values have all been made into canonical coefficients.
338    assert!(d.exponent() >= 0);
339
340    let mut buf_cursor = 0;
341    while !d.is_zero() {
342        let mut w = d.clone();
343        // Take the remainder; this represents one of our "units" to take the coefficient of, i.e. d & u128::MAX
344        cx.rem(&mut w, D::u128_splitter());
345
346        // Take the `u128` version of the coefficient, which will always be what
347        // we want given that we adjusted negative values to have an unsigned
348        // integer representation.
349        let c = w.coefficient::<u128>().unwrap();
350
351        // Determine the width of the coefficient we want to take, i.e. the full
352        // coefficient or a part of it to fill the buffer.
353        let e = std::cmp::min(buf_cursor + 16, D::TWOS_COMPLEMENT_BYTE_WIDTH);
354
355        // We're putting less significant bytes at index 0, which is little endian.
356        buf[buf_cursor..e].copy_from_slice(&c.to_le_bytes()[0..e - buf_cursor]);
357        // Advance cursor; ok that it will go past buffer on final + 1th iteration.
358        buf_cursor += 16;
359
360        // Take the quotient to represent the next unit, i.e. d >> 128
361        cx.div_integer(&mut d, D::u128_splitter());
362    }
363
364    if is_neg {
365        negate_twos_complement_le(buf.iter_mut());
366    }
367
368    // Convert from little endian to big endian.
369    buf.reverse();
370}
371
372pub fn twos_complement_be_to_numeric(
373    input: &mut [u8],
374    scale: u8,
375) -> Result<Numeric, anyhow::Error> {
376    let mut cx = cx_datum();
377    if input.len() <= 17 {
378        if let Ok(mut n) =
379            twos_complement_be_to_numeric_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(input)
380        {
381            n.set_exponent(-i32::from(scale));
382            return Ok(n);
383        }
384    }
385    // If bytes were invalid for narrower representation, try to use wider
386    // representation in case e.g. simply has more trailing zeroes.
387    let mut n = twos_complement_be_to_numeric_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(input)?;
388    // Exponent must be set before converting to `Numeric` width, otherwise values can overflow 39 dop.
389    n.set_exponent(-i32::from(scale));
390    let d = cx.to_width(n);
391    if cx.status().inexact() {
392        bail!("Value exceeds maximum numeric value")
393    }
394    Ok(d)
395}
396
397/// Parses a buffer of two's complement digits in big-endian order and converts
398/// them to [`Decimal<N>`].
399pub fn twos_complement_be_to_numeric_inner<D: Dec<N>, const N: usize>(
400    input: &mut [u8],
401) -> Result<Decimal<N>, anyhow::Error> {
402    if input.is_empty() {
403        // An empty byte string is not a valid two's-complement integer. Our own
404        // encoder never emits one (zero is the single byte `0x00`), so this only
405        // arises from untrusted input. One example is an Avro `decimal` field
406        // whose unscaled value was encoded as zero-length `bytes`. Reject it
407        // rather than indexing `input[0]` below and panicking.
408        bail!("cannot parse a numeric value from an empty byte string");
409    }
410    let is_neg = if (input[0] & 0x80) != 0 {
411        // byte-level negate all negative values, guaranteeing all bytes are
412        // readable as unsigned.
413        negate_twos_complement_le(input.iter_mut().rev());
414        true
415    } else {
416        false
417    };
418
419    let head = input.len() % 16;
420    let i = twos_complement_be_to_u128(&input[0..head]);
421    let mut cx = D::context();
422    let mut d = cx.from_u128(i);
423
424    for c in input[head..].chunks(16) {
425        assert_eq!(c.len(), 16);
426        // essentially d << 128
427        cx.mul(&mut d, D::u128_splitter());
428        let i = twos_complement_be_to_u128(c);
429        let i = cx.from_u128(i);
430        cx.add(&mut d, &i);
431    }
432
433    if cx.status().inexact() {
434        bail!("Value exceeds maximum numeric value")
435    } else if cx.status().any() {
436        bail!("unexpected status {:?}", cx.status());
437    }
438    if is_neg {
439        cx.neg(&mut d);
440    }
441    Ok(d)
442}
443
444#[mz_ore::test]
445#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
446fn test_twos_complement_roundtrip() {
447    fn inner(s: &str) {
448        let mut cx = cx_datum();
449        let d = cx.parse(s).unwrap();
450        let scale = std::cmp::min(d.exponent(), 0).abs();
451        let mut b = numeric_to_twos_complement_be(d.clone());
452        let x = twos_complement_be_to_numeric(&mut b, u8::try_from(scale).unwrap()).unwrap();
453        assert_eq!(d, x);
454    }
455    inner("0");
456    inner("0.000000000000000000000000000000000012345");
457    inner("0.123456789012345678901234567890123456789");
458    inner("1.00000000000000000000000000000000000000");
459    inner("1");
460    inner("2");
461    inner("170141183460469231731687303715884105727");
462    inner("170141183460469231731687303715884105728");
463    inner("12345678901234567890.1234567890123456789");
464    inner("999999999999999999999999999999999999999");
465    inner("7e35");
466    inner("7e-35");
467    inner("-0.000000000000000000000000000000000012345");
468    inner("-0.12345678901234567890123456789012345678");
469    inner("-1.00000000000000000000000000000000000000");
470    inner("-1");
471    inner("-2");
472    inner("-170141183460469231731687303715884105727");
473    inner("-170141183460469231731687303715884105728");
474    inner("-12345678901234567890.1234567890123456789");
475    inner("-999999999999999999999999999999999999999");
476    inner("-7.2e35");
477    inner("-7.2e-35");
478}
479
480#[mz_ore::test]
481#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
482fn test_twos_complement_empty_is_error() {
483    // An empty byte string is not a valid two's-complement integer and used to
484    // panic with an out-of-bounds index. It only arrives from untrusted input
485    // (e.g. an Avro `decimal` encoded as zero-length `bytes`), so it must be a
486    // clean error, not a panic. Regression test for that fix.
487    for scale in [0u8, 1, 38] {
488        assert!(twos_complement_be_to_numeric(&mut [], scale).is_err());
489    }
490    assert!(
491        twos_complement_be_to_numeric_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(&mut []).is_err()
492    );
493}
494
495#[mz_ore::test]
496#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
497fn test_twos_comp_numeric_primitive() {
498    fn inner_inner<P>(i: P, i_be_bytes: &mut [u8])
499    where
500        P: Into<Numeric> + TryFrom<Numeric> + Eq + PartialEq + std::fmt::Debug + Copy,
501    {
502        let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
503        e[Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
504        let mut w = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
505        w[NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
506
507        let d: Numeric = i.into();
508
509        // Extend negative sign into most-significant bits
510        if d.is_negative() {
511            for i in e[..Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
512                *i = 0xFF;
513            }
514            for i in w[..NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
515                *i = 0xFF;
516            }
517        }
518
519        // Ensure decimal value's two's complement representation matches an
520        // extended version of `to_be_bytes`.
521        let d_be_bytes = numeric_to_twos_complement_be(d);
522        assert_eq!(
523            e, d_be_bytes,
524            "expected repr of {:?}, got {:?}",
525            e, d_be_bytes
526        );
527
528        // Ensure extended version of `to_be_bytes` generates same `i128`.
529        let e_numeric = twos_complement_be_to_numeric(&mut e, 0).unwrap();
530        let e_p: P = e_numeric
531            .try_into()
532            .unwrap_or_else(|_e| panic!("try_into failed"));
533        assert_eq!(i, e_p, "expected val of {:?}, got {:?}", i, e_p);
534
535        // Wide representation produces same result.
536        let w_numeric = twos_complement_be_to_numeric(&mut w, 0).unwrap();
537        let w_p: P = w_numeric
538            .try_into()
539            .unwrap_or_else(|_e| panic!("try_into failed"));
540        assert_eq!(i, w_p, "expected val of {:?}, got {:?}", i, e_p);
541
542        // Bytes do not need to be in `Numeric`-specific format
543        let p_numeric = twos_complement_be_to_numeric(i_be_bytes, 0).unwrap();
544        let p_p: P = p_numeric
545            .try_into()
546            .unwrap_or_else(|_e| panic!("try_into failed"));
547        assert_eq!(i, p_p, "expected val of {:?}, got {:?}", i, p_p);
548    }
549
550    fn inner_i32(i: i32) {
551        inner_inner(i, &mut i.to_be_bytes());
552    }
553
554    fn inner_i64(i: i64) {
555        inner_inner(i, &mut i.to_be_bytes());
556    }
557
558    // We need a wrapper around i128 to implement the same traits as the other
559    // primitive types. This is less code than a second implementation of the
560    // same test that takes unwrapped i128s.
561    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
562    struct FromableI128 {
563        i: i128,
564    }
565    impl From<i128> for FromableI128 {
566        fn from(i: i128) -> FromableI128 {
567            FromableI128 { i }
568        }
569    }
570    impl From<FromableI128> for Numeric {
571        fn from(n: FromableI128) -> Numeric {
572            Numeric::try_from(n.i).unwrap()
573        }
574    }
575    impl TryFrom<Numeric> for FromableI128 {
576        type Error = ();
577        fn try_from(n: Numeric) -> Result<FromableI128, Self::Error> {
578            match i128::try_from(n) {
579                Ok(i) => Ok(FromableI128 { i }),
580                Err(_) => Err(()),
581            }
582        }
583    }
584
585    fn inner_i128(i: i128) {
586        inner_inner(FromableI128::from(i), &mut i.to_be_bytes());
587    }
588
589    inner_i32(0);
590    inner_i32(1);
591    inner_i32(2);
592    inner_i32(-1);
593    inner_i32(-2);
594    inner_i32(i32::MAX);
595    inner_i32(i32::MIN);
596    inner_i32(i32::MAX / 7 + 7);
597    inner_i32(i32::MIN / 7 + 7);
598    inner_i64(0);
599    inner_i64(1);
600    inner_i64(2);
601    inner_i64(-1);
602    inner_i64(-2);
603    inner_i64(i64::MAX);
604    inner_i64(i64::MIN);
605    inner_i64(i64::MAX / 7 + 7);
606    inner_i64(i64::MIN / 7 + 7);
607    inner_i128(0);
608    inner_i128(1);
609    inner_i128(2);
610    inner_i128(-1);
611    inner_i128(-2);
612    inner_i128(i128::from(i64::MAX));
613    inner_i128(i128::from(i64::MIN));
614    inner_i128(i128::MAX);
615    inner_i128(i128::MIN);
616    inner_i128(i128::MAX / 7 + 7);
617    inner_i128(i128::MIN / 7 + 7);
618}
619
620#[mz_ore::test]
621#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
622fn test_twos_complement_to_numeric_fail() {
623    fn inner(b: &mut [u8]) {
624        let r = twos_complement_be_to_numeric(b, 0);
625        mz_ore::assert_err!(r);
626    }
627    // 17-byte signed digit's max value exceeds 39 digits of precision
628    let mut e = [0xFF; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
629    e[0] -= 0x80;
630    inner(&mut e);
631
632    // 1 << 17 * 8 exceeds exceeds 39 digits of precision
633    let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH + 1];
634    e[0] = 1;
635    inner(&mut e);
636}
637
638#[mz_ore::test]
639#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
640fn test_wide_twos_complement_roundtrip() {
641    fn inner(s: &str) {
642        let mut cx = cx_datum();
643        let d = cx.parse(s).unwrap();
644        let mut b = numeric_to_twos_complement_wide(d.clone());
645        let x = twos_complement_be_to_numeric(&mut b, NUMERIC_DATUM_MAX_PRECISION).unwrap();
646        assert_eq!(d, x);
647    }
648    inner("0");
649    inner("0.000000000000000000000000000000000012345");
650    inner("0.123456789012345678901234567890123456789");
651    inner("1.00000000000000000000000000000000000000");
652    inner("1");
653    inner("2");
654    inner("170141183460469231731687303715884105727");
655    inner("170141183460469231731687303715884105728");
656    inner("12345678901234567890.1234567890123456789");
657    inner("999999999999999999999999999999999999999");
658    inner("-0.000000000000000000000000000000000012345");
659    inner("-0.123456789012345678901234567890123456789");
660    inner("-1.00000000000000000000000000000000000000");
661    inner("-1");
662    inner("-2");
663    inner("-170141183460469231731687303715884105727");
664    inner("-170141183460469231731687303715884105728");
665    inner("-12345678901234567890.1234567890123456789");
666    inner("-999999999999999999999999999999999999999");
667}
668
669/// Returns `n`'s precision, i.e. the total number of digits represented by `n`
670/// in standard notation not including a zero in the "one's place" in (-1,1).
671pub fn get_precision<const N: usize>(n: &Decimal<N>) -> u32 {
672    let e = n.exponent();
673    if e >= 0 {
674        // Positive exponent
675        n.digits() + u32::try_from(e).unwrap()
676    } else {
677        // Negative exponent
678        let d = n.digits();
679        let e = u32::try_from(e.abs()).unwrap();
680        // Precision is...
681        // - d if decimal point splits numbers
682        // - e if e dominates number of digits
683        std::cmp::max(d, e)
684    }
685}
686
687/// Returns `n`'s scale, i.e. the number of digits used after the decimal point.
688pub fn get_scale(n: &Numeric) -> u32 {
689    let exp = n.exponent();
690    if exp >= 0 { 0 } else { exp.unsigned_abs() }
691}
692
693/// Ensures [`Numeric`] values are:
694/// - Within `Numeric`'s max precision ([`NUMERIC_DATUM_MAX_PRECISION`]), or errors if not.
695/// - Never possible but invalid representations (i.e. never -Nan or -0).
696///
697/// Should be called after any operation that can change an [`Numeric`]'s scale or
698/// generate negative values (except addition and subtraction).
699pub fn munge_numeric(n: &mut Numeric) -> Result<(), anyhow::Error> {
700    rescale_within_max_precision(n)?;
701    if (n.is_zero() || n.is_nan()) && n.is_negative() {
702        cx_datum().neg(n);
703    }
704    Ok(())
705}
706
707/// Rescale's `n` to fit within [`Numeric`]'s max precision or error if not
708/// possible.
709fn rescale_within_max_precision(n: &mut Numeric) -> Result<(), anyhow::Error> {
710    let current_precision = get_precision(n);
711    if current_precision > u32::from(NUMERIC_DATUM_MAX_PRECISION) {
712        if n.exponent() < 0 {
713            let precision_diff = current_precision - u32::from(NUMERIC_DATUM_MAX_PRECISION);
714            let current_scale = u8::try_from(get_scale(n))?;
715            let scale_diff = current_scale - u8::try_from(precision_diff).unwrap();
716            rescale(n, scale_diff)?;
717        } else {
718            bail!(
719                "numeric value {} exceed maximum precision {}",
720                n,
721                NUMERIC_DATUM_MAX_PRECISION
722            )
723        }
724    }
725    Ok(())
726}
727
728/// Rescale `n` as an `OrderedDecimal` with the described scale, or error if:
729/// - Rescaling exceeds max precision
730/// - `n` requires > [`NUMERIC_DATUM_MAX_PRECISION`] - `scale` digits of precision
731///   left of the decimal point
732pub fn rescale(n: &mut Numeric, scale: u8) -> Result<(), anyhow::Error> {
733    let mut cx = cx_datum();
734    cx.rescale(n, &Numeric::from(-i32::from(scale)));
735    if cx.status().invalid_operation() || get_precision(n) > u32::from(NUMERIC_DATUM_MAX_PRECISION)
736    {
737        bail!(
738            "numeric value {} exceed maximum precision {}",
739            n,
740            NUMERIC_DATUM_MAX_PRECISION
741        )
742    }
743    munge_numeric(n)?;
744
745    Ok(())
746}
747
748/// A type that can represent Real Numbers. Useful for interoperability between Numeric and
749/// floating point.
750pub trait DecimalLike:
751    From<u8>
752    + From<u16>
753    + From<u32>
754    + From<i8>
755    + From<i16>
756    + From<i32>
757    + From<f32>
758    + From<f64>
759    + std::ops::Add<Output = Self>
760    + std::ops::Sub<Output = Self>
761    + std::ops::Mul<Output = Self>
762    + std::ops::Div<Output = Self>
763{
764    /// Used to do value-to-value conversions while consuming the input value. Depending on the
765    /// implementation it may be potentially lossy.
766    fn lossy_from(i: i64) -> Self;
767}
768
769impl DecimalLike for f64 {
770    // No other known way to convert `i64` to `f64`.
771    #[allow(clippy::as_conversions)]
772    fn lossy_from(i: i64) -> Self {
773        i as f64
774    }
775}
776
777impl DecimalLike for Numeric {
778    fn lossy_from(i: i64) -> Self {
779        Numeric::from(i)
780    }
781}
782
783/// An encoded packed variant of [`Numeric`].
784///
785/// Unlike other "Packed" types we _DO NOT_ uphold the invariant that
786/// [`PackedNumeric`] sorts the same as [`Numeric`].
787#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
788pub struct PackedNumeric(pub [u8; 40]);
789
790impl FixedSizeCodec<Numeric> for PackedNumeric {
791    const SIZE: usize = 40;
792
793    fn as_bytes(&self) -> &[u8] {
794        &self.0
795    }
796
797    fn from_bytes(slice: &[u8]) -> Result<Self, String> {
798        let buf: [u8; Self::SIZE] = slice.try_into().map_err(|_| {
799            format!(
800                "size for PackedNumeric is {} bytes, got {}",
801                Self::SIZE,
802                slice.len()
803            )
804        })?;
805        Ok(PackedNumeric(buf))
806    }
807
808    fn from_value(val: Numeric) -> PackedNumeric {
809        let (digits, exponent, bits, lsu) = val.to_raw_parts();
810
811        let mut buf = [0u8; 40];
812
813        buf[0..4].copy_from_slice(&digits.to_le_bytes());
814        buf[4..8].copy_from_slice(&exponent.to_le_bytes());
815
816        for i in 0..13 {
817            buf[(i * 2) + 8..(i * 2) + 10].copy_from_slice(&lsu[i].to_le_bytes());
818        }
819
820        buf[34..35].copy_from_slice(&bits.to_le_bytes());
821
822        PackedNumeric(buf)
823    }
824
825    fn into_value(self) -> Numeric {
826        let digits: [u8; 4] = self.0[0..4].try_into().unwrap();
827        let digits = u32::from_le_bytes(digits);
828
829        let exponent: [u8; 4] = self.0[4..8].try_into().unwrap();
830        let exponent = i32::from_le_bytes(exponent);
831
832        let mut lsu = [0u16; 13];
833        for i in 0..13 {
834            let x: [u8; 2] = self.0[(i * 2) + 8..(i * 2) + 10].try_into().unwrap();
835            let x = u16::from_le_bytes(x);
836            lsu[i] = x;
837        }
838
839        let bits: [u8; 1] = self.0[34..35].try_into().unwrap();
840        let bits = u8::from_le_bytes(bits);
841
842        Numeric::from_raw_parts(digits, exponent, bits, lsu)
843    }
844}
845
846#[cfg(test)]
847mod tests {
848    use mz_ore::assert_ok;
849    use mz_proto::protobuf_roundtrip;
850    use proptest::prelude::*;
851
852    use crate::scalar::arb_numeric;
853
854    use super::*;
855
856    proptest! {
857        #[mz_ore::test]
858        fn numeric_max_scale_protobuf_roundtrip(expect in any::<NumericMaxScale>()) {
859            let actual = protobuf_roundtrip::<_, ProtoNumericMaxScale>(&expect);
860            assert_ok!(actual);
861            assert_eq!(actual.unwrap(), expect);
862        }
863
864        #[mz_ore::test]
865        fn optional_numeric_max_scale_protobuf_roundtrip(
866            expect in any::<Option<NumericMaxScale>>(),
867        ) {
868            let actual = protobuf_roundtrip::<_, ProtoOptionalNumericMaxScale>(&expect);
869            assert_ok!(actual);
870            assert_eq!(actual.unwrap(), expect);
871        }
872    }
873
874    #[mz_ore::test]
875    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
876    fn smoketest_packed_numeric_roundtrips() {
877        let og = PackedNumeric::from_value(Numeric::from(-42));
878        let bytes = og.as_bytes();
879        let rnd = PackedNumeric::from_bytes(bytes).expect("valid");
880        assert_eq!(og, rnd);
881
882        // Returns an error if the size of the slice is invalid.
883        mz_ore::assert_err!(PackedNumeric::from_bytes(&[0, 0, 0, 0]));
884    }
885
886    #[mz_ore::test]
887    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
888    fn proptest_packed_numeric_roundtrip() {
889        fn test(og: Numeric) {
890            let packed = PackedNumeric::from_value(og);
891            let rnd = packed.into_value();
892
893            if og.is_nan() && rnd.is_nan() {
894                return;
895            }
896            assert_eq!(og, rnd);
897        }
898
899        proptest!(|(num in arb_numeric())| {
900            test(num);
901        });
902    }
903
904    // Note: It's expected that this test will fail if you update the strategy
905    // for generating an arbitrary Numeric. In that case feel free to
906    // regenerate the snapshot.
907    #[mz_ore::test]
908    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
909    fn packed_numeric_stability() {
910        /// This is the seed [`proptest`] uses for their deterministic RNG. We
911        /// copy it here to prevent breaking this test if [`proptest`] changes.
912        const RNG_SEED: [u8; 32] = [
913            0xf4, 0x16, 0x16, 0x48, 0xc3, 0xac, 0x77, 0xac, 0x72, 0x20, 0x0b, 0xea, 0x99, 0x67,
914            0x2d, 0x6d, 0xca, 0x9f, 0x76, 0xaf, 0x1b, 0x09, 0x73, 0xa0, 0x59, 0x22, 0x6d, 0xc5,
915            0x46, 0x39, 0x1c, 0x4a,
916        ];
917
918        let rng = proptest::test_runner::TestRng::from_seed(
919            proptest::test_runner::RngAlgorithm::ChaCha,
920            &RNG_SEED,
921        );
922        // Generate a collection of Rows.
923        let config = proptest::test_runner::Config {
924            // We let the loop below drive how much data we generate.
925            cases: u32::MAX,
926            rng_algorithm: proptest::test_runner::RngAlgorithm::ChaCha,
927            ..Default::default()
928        };
929        let mut runner = proptest::test_runner::TestRunner::new_with_rng(config, rng);
930
931        let test_cases = 2_000;
932        let strat = arb_numeric();
933
934        let mut all_numerics = Vec::new();
935        for _ in 0..test_cases {
936            let value_tree = strat.new_tree(&mut runner).unwrap();
937            let numeric = value_tree.current();
938            let packed = PackedNumeric::from_value(numeric);
939            let hex_bytes = format!("{:x?}", packed.as_bytes());
940
941            all_numerics.push((numeric, hex_bytes));
942        }
943
944        insta::assert_debug_snapshot!(all_numerics);
945    }
946}
mz_repr/adt/numeric.rs

mz_repr/adt/
numeric.rs
