mz_repr/adt/
numeric.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Functions related to Materialize's numeric type, which is largely a wrapper
11//! around [`rust-dec`].
12//!
13//! [`rust-dec`]: https://github.com/MaterializeInc/rust-dec/
14
15use std::error::Error;
16use std::fmt;
17use std::sync::LazyLock;
18
19use anyhow::bail;
20use dec::{Context, Decimal};
21use mz_ore::cast;
22use mz_persist_types::columnar::FixedSizeCodec;
23use mz_proto::{ProtoType, RustType, TryFromProtoError};
24#[cfg(any(test, feature = "proptest"))]
25use proptest_derive::Arbitrary;
26use serde::{Deserialize, Serialize};
27
28include!(concat!(env!("OUT_DIR"), "/mz_repr.adt.numeric.rs"));
29
30/// The number of internal decimal units in a [`Numeric`] value.
31pub const NUMERIC_DATUM_WIDTH: u8 = 13;
32
33/// The value of [`NUMERIC_DATUM_WIDTH`] as a [`u8`].
34pub const NUMERIC_DATUM_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_DATUM_WIDTH);
35
36/// The maximum number of digits expressable in a [`Numeric`] value.
37pub const NUMERIC_DATUM_MAX_PRECISION: u8 = NUMERIC_DATUM_WIDTH * 3;
38
39/// A numeric value.
40pub type Numeric = Decimal<NUMERIC_DATUM_WIDTH_USIZE>;
41
42/// The number of internal decimal units in a [`NumericAgg`] value.
43pub const NUMERIC_AGG_WIDTH: u8 = 27;
44
45/// The value of [`NUMERIC_AGG_WIDTH`] as a [`u8`].
46pub const NUMERIC_AGG_WIDTH_USIZE: usize = cast::u8_to_usize(NUMERIC_AGG_WIDTH);
47
48/// The maximum number of digits expressable in a [`NumericAgg`] value.
49pub const NUMERIC_AGG_MAX_PRECISION: u8 = NUMERIC_AGG_WIDTH * 3;
50
51/// A double-width version of [`Numeric`] for use in aggregations.
52pub type NumericAgg = Decimal<NUMERIC_AGG_WIDTH_USIZE>;
53
54static CX_DATUM: LazyLock<Context<Numeric>> = LazyLock::new(|| {
55    let mut cx = Context::<Numeric>::default();
56    cx.set_max_exponent(isize::from(NUMERIC_DATUM_MAX_PRECISION - 1))
57        .unwrap();
58    cx.set_min_exponent(-isize::from(NUMERIC_DATUM_MAX_PRECISION))
59        .unwrap();
60    cx
61});
62static CX_AGG: LazyLock<Context<NumericAgg>> = LazyLock::new(|| {
63    let mut cx = Context::<NumericAgg>::default();
64    cx.set_max_exponent(isize::from(NUMERIC_AGG_MAX_PRECISION - 1))
65        .unwrap();
66    cx.set_min_exponent(-isize::from(NUMERIC_AGG_MAX_PRECISION))
67        .unwrap();
68    cx
69});
70static U128_SPLITTER_DATUM: LazyLock<Numeric> = LazyLock::new(|| {
71    let mut cx = Numeric::context();
72    // 1 << 128
73    cx.parse("340282366920938463463374607431768211456").unwrap()
74});
75static U128_SPLITTER_AGG: LazyLock<NumericAgg> = LazyLock::new(|| {
76    let mut cx = NumericAgg::context();
77    // 1 << 128
78    cx.parse("340282366920938463463374607431768211456").unwrap()
79});
80
81/// Module to simplify serde'ing a `Numeric` through its string representation.
82pub mod str_serde {
83    use std::str::FromStr;
84
85    use serde::Deserialize;
86
87    use super::Numeric;
88
89    /// Deserializing a [`Numeric`] value from its `String` representation.
90    pub fn deserialize<'de, D>(deserializer: D) -> Result<Numeric, D::Error>
91    where
92        D: serde::Deserializer<'de>,
93    {
94        let buf = String::deserialize(deserializer)?;
95        Numeric::from_str(&buf).map_err(serde::de::Error::custom)
96    }
97}
98
99/// The `max_scale` of a [`SqlScalarType::Numeric`].
100///
101/// This newtype wrapper ensures that the scale is within the valid range.
102///
103/// [`SqlScalarType::Numeric`]: crate::SqlScalarType::Numeric
104#[derive(
105    Debug,
106    Clone,
107    Copy,
108    Eq,
109    PartialEq,
110    Ord,
111    PartialOrd,
112    Hash,
113    Serialize,
114    Deserialize
115)]
116#[cfg_attr(any(test, feature = "proptest"), derive(Arbitrary))]
117pub struct NumericMaxScale(pub(crate) u8);
118
119impl NumericMaxScale {
120    /// A max scale of zero.
121    pub const ZERO: NumericMaxScale = NumericMaxScale(0);
122
123    /// Consumes the newtype wrapper, returning the inner `u8`.
124    pub fn into_u8(self) -> u8 {
125        self.0
126    }
127}
128
129impl TryFrom<i64> for NumericMaxScale {
130    type Error = InvalidNumericMaxScaleError;
131
132    fn try_from(max_scale: i64) -> Result<Self, Self::Error> {
133        match u8::try_from(max_scale) {
134            Ok(max_scale) if max_scale <= NUMERIC_DATUM_MAX_PRECISION => {
135                Ok(NumericMaxScale(max_scale))
136            }
137            _ => Err(InvalidNumericMaxScaleError),
138        }
139    }
140}
141
142impl TryFrom<usize> for NumericMaxScale {
143    type Error = InvalidNumericMaxScaleError;
144
145    fn try_from(max_scale: usize) -> Result<Self, Self::Error> {
146        Self::try_from(i64::try_from(max_scale).map_err(|_| InvalidNumericMaxScaleError)?)
147    }
148}
149
150impl RustType<ProtoNumericMaxScale> for NumericMaxScale {
151    fn into_proto(&self) -> ProtoNumericMaxScale {
152        ProtoNumericMaxScale {
153            value: self.0.into_proto(),
154        }
155    }
156
157    fn from_proto(max_scale: ProtoNumericMaxScale) -> Result<Self, TryFromProtoError> {
158        Ok(NumericMaxScale(max_scale.value.into_rust()?))
159    }
160}
161
162impl RustType<ProtoOptionalNumericMaxScale> for Option<NumericMaxScale> {
163    fn into_proto(&self) -> ProtoOptionalNumericMaxScale {
164        ProtoOptionalNumericMaxScale {
165            value: self.into_proto(),
166        }
167    }
168
169    fn from_proto(max_scale: ProtoOptionalNumericMaxScale) -> Result<Self, TryFromProtoError> {
170        max_scale.value.into_rust()
171    }
172}
173
174/// The error returned when constructing a [`NumericMaxScale`] from an invalid
175/// value.
176#[derive(Debug, Clone)]
177pub struct InvalidNumericMaxScaleError;
178
179impl fmt::Display for InvalidNumericMaxScaleError {
180    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
181        write!(
182            f,
183            "scale for type numeric must be between 0 and {}",
184            NUMERIC_DATUM_MAX_PRECISION
185        )
186    }
187}
188
189impl Error for InvalidNumericMaxScaleError {}
190
191/// Traits to generalize converting [`Decimal`] values to and from their
192/// coefficients' two's complements.
193pub trait Dec<const N: usize> {
194    // The number of bytes required to represent the min/max value of a decimal
195    // using two's complement.
196    const TWOS_COMPLEMENT_BYTE_WIDTH: usize;
197    // Convenience method for generating appropriate default contexts.
198    fn context() -> Context<Decimal<N>>;
199    // Provides value to break decimal into units of `u128`s for binary
200    // encoding/decoding.
201    fn u128_splitter() -> &'static Decimal<N>;
202}
203
204impl Dec<NUMERIC_DATUM_WIDTH_USIZE> for Numeric {
205    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 17;
206    fn context() -> Context<Numeric> {
207        CX_DATUM.clone()
208    }
209    fn u128_splitter() -> &'static Numeric {
210        &U128_SPLITTER_DATUM
211    }
212}
213
214impl Dec<NUMERIC_AGG_WIDTH_USIZE> for NumericAgg {
215    const TWOS_COMPLEMENT_BYTE_WIDTH: usize = 33;
216    fn context() -> Context<NumericAgg> {
217        CX_AGG.clone()
218    }
219    fn u128_splitter() -> &'static NumericAgg {
220        &U128_SPLITTER_AGG
221    }
222}
223
224/// Returns a new context appropriate for operating on numeric datums.
225pub fn cx_datum() -> Context<Numeric> {
226    CX_DATUM.clone()
227}
228
229/// Returns a new context appropriate for operating on numeric aggregates.
230pub fn cx_agg() -> Context<NumericAgg> {
231    CX_AGG.clone()
232}
233
234fn twos_complement_be_to_u128(input: &[u8]) -> u128 {
235    assert!(input.len() <= 16);
236    let mut buf = [0; 16];
237    buf[16 - input.len()..16].copy_from_slice(input);
238    u128::from_be_bytes(buf)
239}
240
241/// Using negative binary numbers can require more digits of precision than
242/// [`Numeric`] offers, so we need to have the option to swap bytes' signs at the
243/// byte- rather than the library-level.
244fn negate_twos_complement_le<'a, I>(b: I)
245where
246    I: Iterator<Item = &'a mut u8>,
247{
248    let mut seen_first_one = false;
249    for i in b {
250        if seen_first_one {
251            *i = *i ^ 0xFF;
252        } else if *i > 0 {
253            seen_first_one = true;
254            if i == &0x80 {
255                continue;
256            }
257            let tz = i.trailing_zeros();
258            *i = *i ^ (0xFF << tz + 1);
259        }
260    }
261}
262
263/// Converts an [`Numeric`] into its big endian two's complement representation.
264pub fn numeric_to_twos_complement_be(
265    mut numeric: Numeric,
266) -> [u8; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH] {
267    let mut buf = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
268    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
269    // as zeroes so as to avoid erroring (encoding values is meant to be
270    // infallible) and retain downstream associativity/commutativity.
271    if numeric.is_special() {
272        return buf;
273    }
274
275    let mut cx = Numeric::context();
276
277    // Ensure `numeric` is a canonical coefficient.
278    if numeric.exponent() < 0 {
279        let s = Numeric::from(-numeric.exponent());
280        cx.scaleb(&mut numeric, &s);
281    }
282
283    numeric_to_twos_complement_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(
284        numeric, &mut cx, &mut buf,
285    );
286    buf
287}
288
289/// Converts an [`Numeric`] into a big endian two's complement representation where
290/// the encoded value has [`NUMERIC_AGG_MAX_PRECISION`] digits and a scale of
291/// [`NUMERIC_DATUM_MAX_PRECISION`].
292///
293/// This representation is appropriate to use in
294/// contexts requiring two's complement representation but `Numeric` values' scale
295/// isn't known, e.g. when working with columns with an explicitly defined
296/// scale.
297pub fn numeric_to_twos_complement_wide(
298    numeric: Numeric,
299) -> [u8; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH] {
300    let mut buf = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
301    // Avro doesn't specify how to handle NaN/infinity, so we simply treat them
302    // as zeroes so as to avoid erroring (encoding values is meant to be
303    // infallible) and retain downstream associativity/commutativity.
304    if numeric.is_special() {
305        return buf;
306    }
307    let mut cx = NumericAgg::context();
308    let mut d = cx.to_width(numeric);
309    let mut scaler = NumericAgg::from(NUMERIC_DATUM_MAX_PRECISION);
310    cx.neg(&mut scaler);
311    // Shape `d` so that its exponent is -NUMERIC_DATUM_MAX_PRECISION
312    cx.rescale(&mut d, &scaler);
313    // Adjust `d` so it is a canonical coefficient, i.e. its exact value can be
314    // recovered by setting its exponent to -39.
315    cx.abs(&mut scaler);
316    cx.scaleb(&mut d, &scaler);
317
318    numeric_to_twos_complement_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(d, &mut cx, &mut buf);
319    buf
320}
321
322fn numeric_to_twos_complement_inner<D: Dec<N>, const N: usize>(
323    mut d: Decimal<N>,
324    cx: &mut Context<Decimal<N>>,
325    buf: &mut [u8],
326) {
327    // Adjust negative values to be writable as series of `u128`.
328    let is_neg = if d.is_negative() {
329        cx.neg(&mut d);
330        true
331    } else {
332        false
333    };
334
335    // Values have all been made into canonical coefficients.
336    assert!(d.exponent() >= 0);
337
338    let mut buf_cursor = 0;
339    while !d.is_zero() {
340        let mut w = d.clone();
341        // Take the remainder; this represents one of our "units" to take the coefficient of, i.e. d & u128::MAX
342        cx.rem(&mut w, D::u128_splitter());
343
344        // Take the `u128` version of the coefficient, which will always be what
345        // we want given that we adjusted negative values to have an unsigned
346        // integer representation.
347        let c = w.coefficient::<u128>().unwrap();
348
349        // Determine the width of the coefficient we want to take, i.e. the full
350        // coefficient or a part of it to fill the buffer.
351        let e = std::cmp::min(buf_cursor + 16, D::TWOS_COMPLEMENT_BYTE_WIDTH);
352
353        // We're putting less significant bytes at index 0, which is little endian.
354        buf[buf_cursor..e].copy_from_slice(&c.to_le_bytes()[0..e - buf_cursor]);
355        // Advance cursor; ok that it will go past buffer on final + 1th iteration.
356        buf_cursor += 16;
357
358        // Take the quotient to represent the next unit, i.e. d >> 128
359        cx.div_integer(&mut d, D::u128_splitter());
360    }
361
362    if is_neg {
363        negate_twos_complement_le(buf.iter_mut());
364    }
365
366    // Convert from little endian to big endian.
367    buf.reverse();
368}
369
370pub fn twos_complement_be_to_numeric(
371    input: &mut [u8],
372    scale: u8,
373) -> Result<Numeric, anyhow::Error> {
374    let mut cx = cx_datum();
375    if input.len() <= 17 {
376        if let Ok(mut n) =
377            twos_complement_be_to_numeric_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(input)
378        {
379            n.set_exponent(-i32::from(scale));
380            return Ok(n);
381        }
382    }
383    // If bytes were invalid for narrower representation, try to use wider
384    // representation in case e.g. simply has more trailing zeroes.
385    let mut n = twos_complement_be_to_numeric_inner::<NumericAgg, NUMERIC_AGG_WIDTH_USIZE>(input)?;
386    // Exponent must be set before converting to `Numeric` width, otherwise values can overflow 39 dop.
387    n.set_exponent(-i32::from(scale));
388    let d = cx.to_width(n);
389    if cx.status().inexact() {
390        bail!("Value exceeds maximum numeric value")
391    }
392    Ok(d)
393}
394
395/// Parses a buffer of two's complement digits in big-endian order and converts
396/// them to [`Decimal<N>`].
397pub fn twos_complement_be_to_numeric_inner<D: Dec<N>, const N: usize>(
398    input: &mut [u8],
399) -> Result<Decimal<N>, anyhow::Error> {
400    if input.is_empty() {
401        // An empty byte string is not a valid two's-complement integer. Our own
402        // encoder never emits one (zero is the single byte `0x00`), so this only
403        // arises from untrusted input. One example is an Avro `decimal` field
404        // whose unscaled value was encoded as zero-length `bytes`. Reject it
405        // rather than indexing `input[0]` below and panicking.
406        bail!("cannot parse a numeric value from an empty byte string");
407    }
408    let is_neg = if (input[0] & 0x80) != 0 {
409        // byte-level negate all negative values, guaranteeing all bytes are
410        // readable as unsigned.
411        negate_twos_complement_le(input.iter_mut().rev());
412        true
413    } else {
414        false
415    };
416
417    let head = input.len() % 16;
418    let i = twos_complement_be_to_u128(&input[0..head]);
419    let mut cx = D::context();
420    let mut d = cx.from_u128(i);
421
422    for c in input[head..].chunks(16) {
423        assert_eq!(c.len(), 16);
424        // essentially d << 128
425        cx.mul(&mut d, D::u128_splitter());
426        let i = twos_complement_be_to_u128(c);
427        let i = cx.from_u128(i);
428        cx.add(&mut d, &i);
429    }
430
431    if cx.status().inexact() {
432        bail!("Value exceeds maximum numeric value")
433    } else if cx.status().any() {
434        bail!("unexpected status {:?}", cx.status());
435    }
436    if is_neg {
437        cx.neg(&mut d);
438    }
439    Ok(d)
440}
441
442#[mz_ore::test]
443#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
444fn test_twos_complement_roundtrip() {
445    fn inner(s: &str) {
446        let mut cx = cx_datum();
447        let d = cx.parse(s).unwrap();
448        let scale = std::cmp::min(d.exponent(), 0).abs();
449        let mut b = numeric_to_twos_complement_be(d.clone());
450        let x = twos_complement_be_to_numeric(&mut b, u8::try_from(scale).unwrap()).unwrap();
451        assert_eq!(d, x);
452    }
453    inner("0");
454    inner("0.000000000000000000000000000000000012345");
455    inner("0.123456789012345678901234567890123456789");
456    inner("1.00000000000000000000000000000000000000");
457    inner("1");
458    inner("2");
459    inner("170141183460469231731687303715884105727");
460    inner("170141183460469231731687303715884105728");
461    inner("12345678901234567890.1234567890123456789");
462    inner("999999999999999999999999999999999999999");
463    inner("7e35");
464    inner("7e-35");
465    inner("-0.000000000000000000000000000000000012345");
466    inner("-0.12345678901234567890123456789012345678");
467    inner("-1.00000000000000000000000000000000000000");
468    inner("-1");
469    inner("-2");
470    inner("-170141183460469231731687303715884105727");
471    inner("-170141183460469231731687303715884105728");
472    inner("-12345678901234567890.1234567890123456789");
473    inner("-999999999999999999999999999999999999999");
474    inner("-7.2e35");
475    inner("-7.2e-35");
476}
477
478#[mz_ore::test]
479#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
480fn test_twos_complement_empty_is_error() {
481    // An empty byte string is not a valid two's-complement integer and used to
482    // panic with an out-of-bounds index. It only arrives from untrusted input
483    // (e.g. an Avro `decimal` encoded as zero-length `bytes`), so it must be a
484    // clean error, not a panic. Regression test for that fix.
485    for scale in [0u8, 1, 38] {
486        assert!(twos_complement_be_to_numeric(&mut [], scale).is_err());
487    }
488    assert!(
489        twos_complement_be_to_numeric_inner::<Numeric, NUMERIC_DATUM_WIDTH_USIZE>(&mut []).is_err()
490    );
491}
492
493#[mz_ore::test]
494#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
495fn test_twos_comp_numeric_primitive() {
496    fn inner_inner<P>(i: P, i_be_bytes: &mut [u8])
497    where
498        P: Into<Numeric> + TryFrom<Numeric> + Eq + PartialEq + std::fmt::Debug + Copy,
499    {
500        let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
501        e[Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
502        let mut w = [0; NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH];
503        w[NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()..].copy_from_slice(i_be_bytes);
504
505        let d: Numeric = i.into();
506
507        // Extend negative sign into most-significant bits
508        if d.is_negative() {
509            for i in e[..Numeric::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
510                *i = 0xFF;
511            }
512            for i in w[..NumericAgg::TWOS_COMPLEMENT_BYTE_WIDTH - i_be_bytes.len()].iter_mut() {
513                *i = 0xFF;
514            }
515        }
516
517        // Ensure decimal value's two's complement representation matches an
518        // extended version of `to_be_bytes`.
519        let d_be_bytes = numeric_to_twos_complement_be(d);
520        assert_eq!(
521            e, d_be_bytes,
522            "expected repr of {:?}, got {:?}",
523            e, d_be_bytes
524        );
525
526        // Ensure extended version of `to_be_bytes` generates same `i128`.
527        let e_numeric = twos_complement_be_to_numeric(&mut e, 0).unwrap();
528        let e_p: P = e_numeric
529            .try_into()
530            .unwrap_or_else(|_e| panic!("try_into failed"));
531        assert_eq!(i, e_p, "expected val of {:?}, got {:?}", i, e_p);
532
533        // Wide representation produces same result.
534        let w_numeric = twos_complement_be_to_numeric(&mut w, 0).unwrap();
535        let w_p: P = w_numeric
536            .try_into()
537            .unwrap_or_else(|_e| panic!("try_into failed"));
538        assert_eq!(i, w_p, "expected val of {:?}, got {:?}", i, e_p);
539
540        // Bytes do not need to be in `Numeric`-specific format
541        let p_numeric = twos_complement_be_to_numeric(i_be_bytes, 0).unwrap();
542        let p_p: P = p_numeric
543            .try_into()
544            .unwrap_or_else(|_e| panic!("try_into failed"));
545        assert_eq!(i, p_p, "expected val of {:?}, got {:?}", i, p_p);
546    }
547
548    fn inner_i32(i: i32) {
549        inner_inner(i, &mut i.to_be_bytes());
550    }
551
552    fn inner_i64(i: i64) {
553        inner_inner(i, &mut i.to_be_bytes());
554    }
555
556    // We need a wrapper around i128 to implement the same traits as the other
557    // primitive types. This is less code than a second implementation of the
558    // same test that takes unwrapped i128s.
559    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
560    struct FromableI128 {
561        i: i128,
562    }
563    impl From<i128> for FromableI128 {
564        fn from(i: i128) -> FromableI128 {
565            FromableI128 { i }
566        }
567    }
568    impl From<FromableI128> for Numeric {
569        fn from(n: FromableI128) -> Numeric {
570            Numeric::try_from(n.i).unwrap()
571        }
572    }
573    impl TryFrom<Numeric> for FromableI128 {
574        type Error = ();
575        fn try_from(n: Numeric) -> Result<FromableI128, Self::Error> {
576            match i128::try_from(n) {
577                Ok(i) => Ok(FromableI128 { i }),
578                Err(_) => Err(()),
579            }
580        }
581    }
582
583    fn inner_i128(i: i128) {
584        inner_inner(FromableI128::from(i), &mut i.to_be_bytes());
585    }
586
587    inner_i32(0);
588    inner_i32(1);
589    inner_i32(2);
590    inner_i32(-1);
591    inner_i32(-2);
592    inner_i32(i32::MAX);
593    inner_i32(i32::MIN);
594    inner_i32(i32::MAX / 7 + 7);
595    inner_i32(i32::MIN / 7 + 7);
596    inner_i64(0);
597    inner_i64(1);
598    inner_i64(2);
599    inner_i64(-1);
600    inner_i64(-2);
601    inner_i64(i64::MAX);
602    inner_i64(i64::MIN);
603    inner_i64(i64::MAX / 7 + 7);
604    inner_i64(i64::MIN / 7 + 7);
605    inner_i128(0);
606    inner_i128(1);
607    inner_i128(2);
608    inner_i128(-1);
609    inner_i128(-2);
610    inner_i128(i128::from(i64::MAX));
611    inner_i128(i128::from(i64::MIN));
612    inner_i128(i128::MAX);
613    inner_i128(i128::MIN);
614    inner_i128(i128::MAX / 7 + 7);
615    inner_i128(i128::MIN / 7 + 7);
616}
617
618#[mz_ore::test]
619#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
620fn test_twos_complement_to_numeric_fail() {
621    fn inner(b: &mut [u8]) {
622        let r = twos_complement_be_to_numeric(b, 0);
623        mz_ore::assert_err!(r);
624    }
625    // 17-byte signed digit's max value exceeds 39 digits of precision
626    let mut e = [0xFF; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH];
627    e[0] -= 0x80;
628    inner(&mut e);
629
630    // 1 << 17 * 8 exceeds exceeds 39 digits of precision
631    let mut e = [0; Numeric::TWOS_COMPLEMENT_BYTE_WIDTH + 1];
632    e[0] = 1;
633    inner(&mut e);
634}
635
636#[mz_ore::test]
637#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
638fn test_wide_twos_complement_roundtrip() {
639    fn inner(s: &str) {
640        let mut cx = cx_datum();
641        let d = cx.parse(s).unwrap();
642        let mut b = numeric_to_twos_complement_wide(d.clone());
643        let x = twos_complement_be_to_numeric(&mut b, NUMERIC_DATUM_MAX_PRECISION).unwrap();
644        assert_eq!(d, x);
645    }
646    inner("0");
647    inner("0.000000000000000000000000000000000012345");
648    inner("0.123456789012345678901234567890123456789");
649    inner("1.00000000000000000000000000000000000000");
650    inner("1");
651    inner("2");
652    inner("170141183460469231731687303715884105727");
653    inner("170141183460469231731687303715884105728");
654    inner("12345678901234567890.1234567890123456789");
655    inner("999999999999999999999999999999999999999");
656    inner("-0.000000000000000000000000000000000012345");
657    inner("-0.123456789012345678901234567890123456789");
658    inner("-1.00000000000000000000000000000000000000");
659    inner("-1");
660    inner("-2");
661    inner("-170141183460469231731687303715884105727");
662    inner("-170141183460469231731687303715884105728");
663    inner("-12345678901234567890.1234567890123456789");
664    inner("-999999999999999999999999999999999999999");
665}
666
667/// Returns `n`'s precision, i.e. the total number of digits represented by `n`
668/// in standard notation not including a zero in the "one's place" in (-1,1).
669pub fn get_precision<const N: usize>(n: &Decimal<N>) -> u32 {
670    let e = n.exponent();
671    if e >= 0 {
672        // Positive exponent
673        n.digits() + u32::try_from(e).unwrap()
674    } else {
675        // Negative exponent
676        let d = n.digits();
677        let e = u32::try_from(e.abs()).unwrap();
678        // Precision is...
679        // - d if decimal point splits numbers
680        // - e if e dominates number of digits
681        std::cmp::max(d, e)
682    }
683}
684
685/// Returns `n`'s scale, i.e. the number of digits used after the decimal point.
686pub fn get_scale(n: &Numeric) -> u32 {
687    let exp = n.exponent();
688    if exp >= 0 { 0 } else { exp.unsigned_abs() }
689}
690
691/// Ensures [`Numeric`] values are:
692/// - Within `Numeric`'s max precision ([`NUMERIC_DATUM_MAX_PRECISION`]), or errors if not.
693/// - Never possible but invalid representations (i.e. never -Nan or -0).
694///
695/// Should be called after any operation that can change an [`Numeric`]'s scale or
696/// generate negative values (except addition and subtraction).
697pub fn munge_numeric(n: &mut Numeric) -> Result<(), anyhow::Error> {
698    rescale_within_max_precision(n)?;
699    if (n.is_zero() || n.is_nan()) && n.is_negative() {
700        cx_datum().neg(n);
701    }
702    Ok(())
703}
704
705/// Rescale's `n` to fit within [`Numeric`]'s max precision or error if not
706/// possible.
707fn rescale_within_max_precision(n: &mut Numeric) -> Result<(), anyhow::Error> {
708    let current_precision = get_precision(n);
709    if current_precision > u32::from(NUMERIC_DATUM_MAX_PRECISION) {
710        if n.exponent() < 0 {
711            let precision_diff = current_precision - u32::from(NUMERIC_DATUM_MAX_PRECISION);
712            let current_scale = u8::try_from(get_scale(n))?;
713            let scale_diff = current_scale - u8::try_from(precision_diff).unwrap();
714            rescale(n, scale_diff)?;
715        } else {
716            bail!(
717                "numeric value {} exceed maximum precision {}",
718                n,
719                NUMERIC_DATUM_MAX_PRECISION
720            )
721        }
722    }
723    Ok(())
724}
725
726/// Rescale `n` as an `OrderedDecimal` with the described scale, or error if:
727/// - Rescaling exceeds max precision
728/// - `n` requires > [`NUMERIC_DATUM_MAX_PRECISION`] - `scale` digits of precision
729///   left of the decimal point
730pub fn rescale(n: &mut Numeric, scale: u8) -> Result<(), anyhow::Error> {
731    let mut cx = cx_datum();
732    cx.rescale(n, &Numeric::from(-i32::from(scale)));
733    if cx.status().invalid_operation() || get_precision(n) > u32::from(NUMERIC_DATUM_MAX_PRECISION)
734    {
735        bail!(
736            "numeric value {} exceed maximum precision {}",
737            n,
738            NUMERIC_DATUM_MAX_PRECISION
739        )
740    }
741    munge_numeric(n)?;
742
743    Ok(())
744}
745
746/// A type that can represent Real Numbers. Useful for interoperability between Numeric and
747/// floating point.
748pub trait DecimalLike:
749    From<u8>
750    + From<u16>
751    + From<u32>
752    + From<i8>
753    + From<i16>
754    + From<i32>
755    + From<f32>
756    + From<f64>
757    + std::ops::Add<Output = Self>
758    + std::ops::Sub<Output = Self>
759    + std::ops::Mul<Output = Self>
760    + std::ops::Div<Output = Self>
761{
762    /// Used to do value-to-value conversions while consuming the input value. Depending on the
763    /// implementation it may be potentially lossy.
764    fn lossy_from(i: i64) -> Self;
765}
766
767impl DecimalLike for f64 {
768    // No other known way to convert `i64` to `f64`.
769    #[allow(clippy::as_conversions)]
770    fn lossy_from(i: i64) -> Self {
771        i as f64
772    }
773}
774
775impl DecimalLike for Numeric {
776    fn lossy_from(i: i64) -> Self {
777        Numeric::from(i)
778    }
779}
780
781/// An encoded packed variant of [`Numeric`].
782///
783/// Unlike other "Packed" types we _DO NOT_ uphold the invariant that
784/// [`PackedNumeric`] sorts the same as [`Numeric`].
785#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
786pub struct PackedNumeric(pub [u8; 40]);
787
788impl FixedSizeCodec<Numeric> for PackedNumeric {
789    const SIZE: usize = 40;
790
791    fn as_bytes(&self) -> &[u8] {
792        &self.0
793    }
794
795    fn from_bytes(slice: &[u8]) -> Result<Self, String> {
796        let buf: [u8; Self::SIZE] = slice.try_into().map_err(|_| {
797            format!(
798                "size for PackedNumeric is {} bytes, got {}",
799                Self::SIZE,
800                slice.len()
801            )
802        })?;
803        Ok(PackedNumeric(buf))
804    }
805
806    fn from_value(val: Numeric) -> PackedNumeric {
807        let (digits, exponent, bits, lsu) = val.to_raw_parts();
808
809        let mut buf = [0u8; 40];
810
811        buf[0..4].copy_from_slice(&digits.to_le_bytes());
812        buf[4..8].copy_from_slice(&exponent.to_le_bytes());
813
814        for i in 0..13 {
815            buf[(i * 2) + 8..(i * 2) + 10].copy_from_slice(&lsu[i].to_le_bytes());
816        }
817
818        buf[34..35].copy_from_slice(&bits.to_le_bytes());
819
820        PackedNumeric(buf)
821    }
822
823    fn into_value(self) -> Numeric {
824        let digits: [u8; 4] = self.0[0..4].try_into().unwrap();
825        let digits = u32::from_le_bytes(digits);
826
827        let exponent: [u8; 4] = self.0[4..8].try_into().unwrap();
828        let exponent = i32::from_le_bytes(exponent);
829
830        let mut lsu = [0u16; 13];
831        for i in 0..13 {
832            let x: [u8; 2] = self.0[(i * 2) + 8..(i * 2) + 10].try_into().unwrap();
833            let x = u16::from_le_bytes(x);
834            lsu[i] = x;
835        }
836
837        let bits: [u8; 1] = self.0[34..35].try_into().unwrap();
838        let bits = u8::from_le_bytes(bits);
839
840        Numeric::from_raw_parts(digits, exponent, bits, lsu)
841    }
842}
843
844#[cfg(test)]
845mod tests {
846    use mz_ore::assert_ok;
847    use mz_proto::protobuf_roundtrip;
848    use proptest::prelude::*;
849
850    use crate::scalar::arb_numeric;
851
852    use super::*;
853
854    proptest! {
855        #[mz_ore::test]
856        fn numeric_max_scale_protobuf_roundtrip(expect in any::<NumericMaxScale>()) {
857            let actual = protobuf_roundtrip::<_, ProtoNumericMaxScale>(&expect);
858            assert_ok!(actual);
859            assert_eq!(actual.unwrap(), expect);
860        }
861
862        #[mz_ore::test]
863        fn optional_numeric_max_scale_protobuf_roundtrip(
864            expect in any::<Option<NumericMaxScale>>(),
865        ) {
866            let actual = protobuf_roundtrip::<_, ProtoOptionalNumericMaxScale>(&expect);
867            assert_ok!(actual);
868            assert_eq!(actual.unwrap(), expect);
869        }
870    }
871
872    #[mz_ore::test]
873    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
874    fn smoketest_packed_numeric_roundtrips() {
875        let og = PackedNumeric::from_value(Numeric::from(-42));
876        let bytes = og.as_bytes();
877        let rnd = PackedNumeric::from_bytes(bytes).expect("valid");
878        assert_eq!(og, rnd);
879
880        // Returns an error if the size of the slice is invalid.
881        mz_ore::assert_err!(PackedNumeric::from_bytes(&[0, 0, 0, 0]));
882    }
883
884    #[mz_ore::test]
885    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
886    fn proptest_packed_numeric_roundtrip() {
887        fn test(og: Numeric) {
888            let packed = PackedNumeric::from_value(og);
889            let rnd = packed.into_value();
890
891            if og.is_nan() && rnd.is_nan() {
892                return;
893            }
894            assert_eq!(og, rnd);
895        }
896
897        proptest!(|(num in arb_numeric())| {
898            test(num);
899        });
900    }
901
902    // Note: It's expected that this test will fail if you update the strategy
903    // for generating an arbitrary Numeric. In that case feel free to
904    // regenerate the snapshot.
905    #[mz_ore::test]
906    #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decNumberCopyNegate` on OS `linux`
907    fn packed_numeric_stability() {
908        /// This is the seed [`proptest`] uses for their deterministic RNG. We
909        /// copy it here to prevent breaking this test if [`proptest`] changes.
910        const RNG_SEED: [u8; 32] = [
911            0xf4, 0x16, 0x16, 0x48, 0xc3, 0xac, 0x77, 0xac, 0x72, 0x20, 0x0b, 0xea, 0x99, 0x67,
912            0x2d, 0x6d, 0xca, 0x9f, 0x76, 0xaf, 0x1b, 0x09, 0x73, 0xa0, 0x59, 0x22, 0x6d, 0xc5,
913            0x46, 0x39, 0x1c, 0x4a,
914        ];
915
916        let rng = proptest::test_runner::TestRng::from_seed(
917            proptest::test_runner::RngAlgorithm::ChaCha,
918            &RNG_SEED,
919        );
920        // Generate a collection of Rows.
921        let config = proptest::test_runner::Config {
922            // We let the loop below drive how much data we generate.
923            cases: u32::MAX,
924            rng_algorithm: proptest::test_runner::RngAlgorithm::ChaCha,
925            ..Default::default()
926        };
927        let mut runner = proptest::test_runner::TestRunner::new_with_rng(config, rng);
928
929        let test_cases = 2_000;
930        let strat = arb_numeric();
931
932        let mut all_numerics = Vec::new();
933        for _ in 0..test_cases {
934            let value_tree = strat.new_tree(&mut runner).unwrap();
935            let numeric = value_tree.current();
936            let packed = PackedNumeric::from_value(numeric);
937            let hex_bytes = format!("{:x?}", packed.as_bytes());
938
939            all_numerics.push((numeric, hex_bytes));
940        }
941
942        insta::assert_debug_snapshot!(all_numerics);
943    }
944}
mz_repr/adt/numeric.rs

mz_repr/adt/
numeric.rs
