parquet/record/
api.rs

Help
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Contains Row enum that is used to represent record in Rust.
19
20use std::fmt;
21
22use chrono::{TimeZone, Utc};
23use half::f16;
24use num::traits::Float;
25use num_bigint::{BigInt, Sign};
26
27use crate::basic::{ConvertedType, LogicalType, Type as PhysicalType};
28use crate::data_type::{ByteArray, Decimal, Int96};
29use crate::errors::{ParquetError, Result};
30use crate::schema::types::ColumnDescPtr;
31
32#[cfg(any(feature = "json", test))]
33use serde_json::Value;
34
35/// Macro as a shortcut to generate 'not yet implemented' panic error.
36macro_rules! nyi {
37    ($column_descr:ident, $value:ident) => {{
38        unimplemented!(
39            "Conversion for physical type {}, converted type {}, value {:?}",
40            $column_descr.physical_type(),
41            $column_descr.converted_type(),
42            $value
43        );
44    }};
45}
46
47/// `Row` represents a nested Parquet record.
48#[derive(Clone, Debug, PartialEq)]
49pub struct Row {
50    fields: Vec<(String, Field)>,
51}
52
53#[allow(clippy::len_without_is_empty)]
54impl Row {
55    /// Get the number of fields in this row.
56    pub fn len(&self) -> usize {
57        self.fields.len()
58    }
59
60    /// Move columns data out of the row. Useful to avoid internal data cloning.
61    ///
62    /// # Example
63    ///
64    /// ```no_run
65    /// use std::fs::File;
66    /// use parquet::record::Row;
67    /// use parquet::file::reader::{FileReader, SerializedFileReader};
68    ///
69    /// let file = File::open("/path/to/file").unwrap();
70    /// let reader = SerializedFileReader::new(file).unwrap();
71    /// let row: Row = reader.get_row_iter(None).unwrap().next().unwrap().unwrap();
72    /// let columns = row.into_columns();
73    /// println!("row columns: {:?}", columns);
74    ///
75    /// ```
76    pub fn into_columns(self) -> Vec<(String, Field)> {
77        self.fields
78    }
79
80    /// Get an iterator to go through all columns in the row.
81    ///
82    /// # Example
83    ///
84    /// ```no_run
85    /// use std::fs::File;
86    /// use parquet::record::Row;
87    /// use parquet::file::reader::{FileReader, SerializedFileReader};
88    ///
89    /// let file = File::open("/path/to/file").unwrap();
90    /// let reader = SerializedFileReader::new(file).unwrap();
91    /// let row: Row = reader.get_row_iter(None).unwrap().next().unwrap().unwrap();
92    /// for (idx, (name, field)) in row.get_column_iter().enumerate() {
93    ///     println!("column index: {}, column name: {}, column value: {}", idx, name, field);
94    /// }
95    /// ```
96    pub fn get_column_iter(&self) -> RowColumnIter {
97        RowColumnIter {
98            fields: &self.fields,
99            curr: 0,
100            count: self.fields.len(),
101        }
102    }
103
104    /// Converts the row into a JSON object.
105    #[cfg(any(feature = "json", test))]
106    pub fn to_json_value(&self) -> Value {
107        Value::Object(
108            self.fields
109                .iter()
110                .map(|(key, field)| (key.to_owned(), field.to_json_value()))
111                .collect(),
112        )
113    }
114}
115
116/// `RowColumnIter` represents an iterator over column names and values in a Row.
117pub struct RowColumnIter<'a> {
118    fields: &'a Vec<(String, Field)>,
119    curr: usize,
120    count: usize,
121}
122
123impl<'a> Iterator for RowColumnIter<'a> {
124    type Item = (&'a String, &'a Field);
125
126    fn next(&mut self) -> Option<Self::Item> {
127        let idx = self.curr;
128        if idx >= self.count {
129            return None;
130        }
131        self.curr += 1;
132        Some((&self.fields[idx].0, &self.fields[idx].1))
133    }
134}
135
136/// Trait for type-safe convenient access to fields within a Row.
137pub trait RowAccessor {
138    /// Try to get a boolean value at the given index.
139    fn get_bool(&self, i: usize) -> Result<bool>;
140    /// Try to get a byte value at the given index.
141    fn get_byte(&self, i: usize) -> Result<i8>;
142    /// Try to get a short value at the given index.
143    fn get_short(&self, i: usize) -> Result<i16>;
144    /// Try to get a int value at the given index.
145    fn get_int(&self, i: usize) -> Result<i32>;
146    /// Try to get a long value at the given index.
147    fn get_long(&self, i: usize) -> Result<i64>;
148    /// Try to get a ubyte value at the given index.
149    fn get_ubyte(&self, i: usize) -> Result<u8>;
150    /// Try to get a ushort value at the given index.
151    fn get_ushort(&self, i: usize) -> Result<u16>;
152    /// Try to get a uint value at the given index.
153    fn get_uint(&self, i: usize) -> Result<u32>;
154    /// Try to get a ulong value at the given index.
155    fn get_ulong(&self, i: usize) -> Result<u64>;
156    /// Try to get a float16 value at the given index.
157    fn get_float16(&self, i: usize) -> Result<f16>;
158    /// Try to get a float value at the given index.
159    fn get_float(&self, i: usize) -> Result<f32>;
160    /// Try to get a double value at the given index.
161    fn get_double(&self, i: usize) -> Result<f64>;
162    /// Try to get a date value at the given index.
163    fn get_timestamp_millis(&self, i: usize) -> Result<i64>;
164    /// Try to get a date value at the given index.
165    fn get_timestamp_micros(&self, i: usize) -> Result<i64>;
166    /// Try to get a decimal value at the given index.
167    fn get_decimal(&self, i: usize) -> Result<&Decimal>;
168    /// Try to get a string value at the given index.
169    fn get_string(&self, i: usize) -> Result<&String>;
170    /// Try to get a bytes value at the given index.
171    fn get_bytes(&self, i: usize) -> Result<&ByteArray>;
172    /// Try to get a group value at the given index.
173    fn get_group(&self, i: usize) -> Result<&Row>;
174    /// Try to get a list value at the given index.
175    fn get_list(&self, i: usize) -> Result<&List>;
176    /// Try to get a map value at the given index.
177    fn get_map(&self, i: usize) -> Result<&Map>;
178}
179
180/// Trait for formatting fields within a Row.
181///
182/// # Examples
183///
184/// ```
185/// use std::fs::File;
186/// use std::path::Path;
187/// use parquet::record::Row;
188/// use parquet::record::RowFormatter;
189/// use parquet::file::reader::{FileReader, SerializedFileReader};
190///
191/// if let Ok(file) = File::open(&Path::new("test.parquet")) {
192///     let reader = SerializedFileReader::new(file).unwrap();
193///     let row = reader.get_row_iter(None).unwrap().next().unwrap().unwrap();
194///     println!("column 0: {}, column 1: {}", row.fmt(0), row.fmt(1));
195/// }
196/// ```
197///
198pub trait RowFormatter {
199    /// The method to format a field at the given index.
200    fn fmt(&self, i: usize) -> &dyn fmt::Display;
201}
202
203/// Macro to generate type-safe get_xxx methods for primitive types,
204/// e.g. `get_bool`, `get_short`.
205macro_rules! row_primitive_accessor {
206    ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
207        fn $METHOD(&self, i: usize) -> Result<$TY> {
208            match self.fields[i].1 {
209                Field::$VARIANT(v) => Ok(v),
210                _ => Err(general_err!(
211                    "Cannot access {} as {}",
212                    self.fields[i].1.get_type_name(),
213                    stringify!($VARIANT)
214                )),
215            }
216        }
217    };
218}
219
220/// Macro to generate type-safe get_xxx methods for reference types,
221/// e.g. `get_list`, `get_map`.
222macro_rules! row_complex_accessor {
223    ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
224        fn $METHOD(&self, i: usize) -> Result<&$TY> {
225            match self.fields[i].1 {
226                Field::$VARIANT(ref v) => Ok(v),
227                _ => Err(general_err!(
228                    "Cannot access {} as {}",
229                    self.fields[i].1.get_type_name(),
230                    stringify!($VARIANT)
231                )),
232            }
233        }
234    };
235}
236
237impl RowFormatter for Row {
238    /// Get Display reference for a given field.
239    fn fmt(&self, i: usize) -> &dyn fmt::Display {
240        &self.fields[i].1
241    }
242}
243
244impl RowAccessor for Row {
245    row_primitive_accessor!(get_bool, Bool, bool);
246
247    row_primitive_accessor!(get_byte, Byte, i8);
248
249    row_primitive_accessor!(get_short, Short, i16);
250
251    row_primitive_accessor!(get_int, Int, i32);
252
253    row_primitive_accessor!(get_long, Long, i64);
254
255    row_primitive_accessor!(get_ubyte, UByte, u8);
256
257    row_primitive_accessor!(get_ushort, UShort, u16);
258
259    row_primitive_accessor!(get_uint, UInt, u32);
260
261    row_primitive_accessor!(get_ulong, ULong, u64);
262
263    row_primitive_accessor!(get_float16, Float16, f16);
264
265    row_primitive_accessor!(get_float, Float, f32);
266
267    row_primitive_accessor!(get_double, Double, f64);
268
269    row_primitive_accessor!(get_timestamp_millis, TimestampMillis, i64);
270
271    row_primitive_accessor!(get_timestamp_micros, TimestampMicros, i64);
272
273    row_complex_accessor!(get_decimal, Decimal, Decimal);
274
275    row_complex_accessor!(get_string, Str, String);
276
277    row_complex_accessor!(get_bytes, Bytes, ByteArray);
278
279    row_complex_accessor!(get_group, Group, Row);
280
281    row_complex_accessor!(get_list, ListInternal, List);
282
283    row_complex_accessor!(get_map, MapInternal, Map);
284}
285
286/// Constructs a `Row` from the list of `fields` and returns it.
287#[inline]
288pub fn make_row(fields: Vec<(String, Field)>) -> Row {
289    Row { fields }
290}
291
292impl fmt::Display for Row {
293    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
294        write!(f, "{{")?;
295        for (i, (key, value)) in self.fields.iter().enumerate() {
296            key.fmt(f)?;
297            write!(f, ": ")?;
298            value.fmt(f)?;
299            if i < self.fields.len() - 1 {
300                write!(f, ", ")?;
301            }
302        }
303        write!(f, "}}")
304    }
305}
306
307/// `List` represents a list which contains an array of elements.
308#[derive(Clone, Debug, PartialEq)]
309pub struct List {
310    elements: Vec<Field>,
311}
312
313#[allow(clippy::len_without_is_empty)]
314impl List {
315    /// Get the number of fields in this row
316    pub fn len(&self) -> usize {
317        self.elements.len()
318    }
319
320    /// Get the reference to the elements in this list
321    pub fn elements(&self) -> &[Field] {
322        self.elements.as_slice()
323    }
324}
325
326/// Constructs a `List` from the list of `fields` and returns it.
327#[inline]
328pub fn make_list(elements: Vec<Field>) -> List {
329    List { elements }
330}
331
332/// Trait for type-safe access of an index for a `List`.
333/// Note that the get_XXX methods do not do bound checking.
334pub trait ListAccessor {
335    /// Try getting a `boolean` value at the given index.
336    fn get_bool(&self, i: usize) -> Result<bool>;
337    /// Try getting a `byte` value at the given index.
338    fn get_byte(&self, i: usize) -> Result<i8>;
339    /// Try getting an `i16` value at the given index.
340    fn get_short(&self, i: usize) -> Result<i16>;
341    /// Try getting an `i32` value at the given index.
342    fn get_int(&self, i: usize) -> Result<i32>;
343    /// Try getting an `i64` value at the given index.
344    fn get_long(&self, i: usize) -> Result<i64>;
345    /// Try getting a `u8` value at the given index.
346    fn get_ubyte(&self, i: usize) -> Result<u8>;
347    /// Try getting a `u16` value at the given index.
348    fn get_ushort(&self, i: usize) -> Result<u16>;
349    /// Try getting a `u32` value at the given index.
350    fn get_uint(&self, i: usize) -> Result<u32>;
351    /// Try getting a `u64` value at the given index.
352    fn get_ulong(&self, i: usize) -> Result<u64>;
353    /// Try getting a `f16` value at the given index.
354    fn get_float16(&self, i: usize) -> Result<f16>;
355    /// Try getting a `f32` value at the given index.
356    fn get_float(&self, i: usize) -> Result<f32>;
357    /// Try getting a `f64` value at the given index.
358    fn get_double(&self, i: usize) -> Result<f64>;
359    /// Try getting a `timestamp` as milliseconds value
360    /// encoded as `i64` at the given index.
361    fn get_timestamp_millis(&self, i: usize) -> Result<i64>;
362    /// Try getting a `timestamp` as microseconds value
363    /// encoded as `i64` at the given index.
364    fn get_timestamp_micros(&self, i: usize) -> Result<i64>;
365    /// Try getting a `decimal` value at the given index.
366    fn get_decimal(&self, i: usize) -> Result<&Decimal>;
367    /// Try getting a `string` value at the given index.
368    fn get_string(&self, i: usize) -> Result<&String>;
369    /// Try getting a `bytes` value at the given index.
370    fn get_bytes(&self, i: usize) -> Result<&ByteArray>;
371    /// Try getting a `group` value at the given index.
372    fn get_group(&self, i: usize) -> Result<&Row>;
373    /// Try getting a `list` value at the given index.
374    fn get_list(&self, i: usize) -> Result<&List>;
375    /// Try getting a `map` value at the given index.
376    fn get_map(&self, i: usize) -> Result<&Map>;
377}
378
379/// Macro to generate type-safe get_xxx methods for primitive types,
380/// e.g. get_bool, get_short
381macro_rules! list_primitive_accessor {
382    ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
383        fn $METHOD(&self, i: usize) -> Result<$TY> {
384            match self.elements[i] {
385                Field::$VARIANT(v) => Ok(v),
386                _ => Err(general_err!(
387                    "Cannot access {} as {}",
388                    self.elements[i].get_type_name(),
389                    stringify!($VARIANT)
390                )),
391            }
392        }
393    };
394}
395
396/// Macro to generate type-safe get_xxx methods for reference types
397/// e.g. get_list, get_map
398macro_rules! list_complex_accessor {
399    ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
400        fn $METHOD(&self, i: usize) -> Result<&$TY> {
401            match self.elements[i] {
402                Field::$VARIANT(ref v) => Ok(v),
403                _ => Err(general_err!(
404                    "Cannot access {} as {}",
405                    self.elements[i].get_type_name(),
406                    stringify!($VARIANT)
407                )),
408            }
409        }
410    };
411}
412
413impl ListAccessor for List {
414    list_primitive_accessor!(get_bool, Bool, bool);
415
416    list_primitive_accessor!(get_byte, Byte, i8);
417
418    list_primitive_accessor!(get_short, Short, i16);
419
420    list_primitive_accessor!(get_int, Int, i32);
421
422    list_primitive_accessor!(get_long, Long, i64);
423
424    list_primitive_accessor!(get_ubyte, UByte, u8);
425
426    list_primitive_accessor!(get_ushort, UShort, u16);
427
428    list_primitive_accessor!(get_uint, UInt, u32);
429
430    list_primitive_accessor!(get_ulong, ULong, u64);
431
432    list_primitive_accessor!(get_float16, Float16, f16);
433
434    list_primitive_accessor!(get_float, Float, f32);
435
436    list_primitive_accessor!(get_double, Double, f64);
437
438    list_primitive_accessor!(get_timestamp_millis, TimestampMillis, i64);
439
440    list_primitive_accessor!(get_timestamp_micros, TimestampMicros, i64);
441
442    list_complex_accessor!(get_decimal, Decimal, Decimal);
443
444    list_complex_accessor!(get_string, Str, String);
445
446    list_complex_accessor!(get_bytes, Bytes, ByteArray);
447
448    list_complex_accessor!(get_group, Group, Row);
449
450    list_complex_accessor!(get_list, ListInternal, List);
451
452    list_complex_accessor!(get_map, MapInternal, Map);
453}
454
455/// `Map` represents a map which contains a list of key->value pairs.
456#[derive(Clone, Debug, PartialEq)]
457pub struct Map {
458    entries: Vec<(Field, Field)>,
459}
460
461#[allow(clippy::len_without_is_empty)]
462impl Map {
463    /// Get the number of fields in this row
464    pub fn len(&self) -> usize {
465        self.entries.len()
466    }
467
468    /// Get the reference to the key-value pairs in this map
469    pub fn entries(&self) -> &[(Field, Field)] {
470        self.entries.as_slice()
471    }
472}
473
474/// Constructs a `Map` from the list of `entries` and returns it.
475#[inline]
476pub fn make_map(entries: Vec<(Field, Field)>) -> Map {
477    Map { entries }
478}
479
480/// Trait for type-safe access of an index for a `Map`
481pub trait MapAccessor {
482    /// Get the keys of the map.
483    fn get_keys<'a>(&'a self) -> Box<dyn ListAccessor + 'a>;
484    /// Get the values of the map.
485    fn get_values<'a>(&'a self) -> Box<dyn ListAccessor + 'a>;
486}
487
488struct MapList<'a> {
489    elements: Vec<&'a Field>,
490}
491
492/// Macro to generate type-safe get_xxx methods for primitive types,
493/// e.g. get_bool, get_short
494macro_rules! map_list_primitive_accessor {
495    ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
496        fn $METHOD(&self, i: usize) -> Result<$TY> {
497            match self.elements[i] {
498                Field::$VARIANT(v) => Ok(*v),
499                _ => Err(general_err!(
500                    "Cannot access {} as {}",
501                    self.elements[i].get_type_name(),
502                    stringify!($VARIANT)
503                )),
504            }
505        }
506    };
507}
508
509impl ListAccessor for MapList<'_> {
510    map_list_primitive_accessor!(get_bool, Bool, bool);
511
512    map_list_primitive_accessor!(get_byte, Byte, i8);
513
514    map_list_primitive_accessor!(get_short, Short, i16);
515
516    map_list_primitive_accessor!(get_int, Int, i32);
517
518    map_list_primitive_accessor!(get_long, Long, i64);
519
520    map_list_primitive_accessor!(get_ubyte, UByte, u8);
521
522    map_list_primitive_accessor!(get_ushort, UShort, u16);
523
524    map_list_primitive_accessor!(get_uint, UInt, u32);
525
526    map_list_primitive_accessor!(get_ulong, ULong, u64);
527
528    map_list_primitive_accessor!(get_float16, Float16, f16);
529
530    map_list_primitive_accessor!(get_float, Float, f32);
531
532    map_list_primitive_accessor!(get_double, Double, f64);
533
534    map_list_primitive_accessor!(get_timestamp_millis, TimestampMillis, i64);
535
536    map_list_primitive_accessor!(get_timestamp_micros, TimestampMicros, i64);
537
538    list_complex_accessor!(get_decimal, Decimal, Decimal);
539
540    list_complex_accessor!(get_string, Str, String);
541
542    list_complex_accessor!(get_bytes, Bytes, ByteArray);
543
544    list_complex_accessor!(get_group, Group, Row);
545
546    list_complex_accessor!(get_list, ListInternal, List);
547
548    list_complex_accessor!(get_map, MapInternal, Map);
549}
550
551impl MapAccessor for Map {
552    fn get_keys<'a>(&'a self) -> Box<dyn ListAccessor + 'a> {
553        let map_list = MapList {
554            elements: self.entries.iter().map(|v| &v.0).collect(),
555        };
556        Box::new(map_list)
557    }
558
559    fn get_values<'a>(&'a self) -> Box<dyn ListAccessor + 'a> {
560        let map_list = MapList {
561            elements: self.entries.iter().map(|v| &v.1).collect(),
562        };
563        Box::new(map_list)
564    }
565}
566
567/// API to represent a single field in a `Row`.
568#[derive(Clone, Debug, PartialEq)]
569pub enum Field {
570    // Primitive types
571    /// Null value.
572    Null,
573    /// Boolean value (`true`, `false`).
574    Bool(bool),
575    /// Signed integer INT_8.
576    Byte(i8),
577    /// Signed integer INT_16.
578    Short(i16),
579    /// Signed integer INT_32.
580    Int(i32),
581    /// Signed integer INT_64.
582    Long(i64),
583    /// Unsigned integer UINT_8.
584    UByte(u8),
585    /// Unsigned integer UINT_16.
586    UShort(u16),
587    /// Unsigned integer UINT_32.
588    UInt(u32),
589    /// Unsigned integer UINT_64.
590    ULong(u64),
591    /// IEEE 16-bit floating point value.
592    Float16(f16),
593    /// IEEE 32-bit floating point value.
594    Float(f32),
595    /// IEEE 64-bit floating point value.
596    Double(f64),
597    /// Decimal value.
598    Decimal(Decimal),
599    /// UTF-8 encoded character string.
600    Str(String),
601    /// General binary value.
602    Bytes(ByteArray),
603    /// Date without a time of day, stores the number of days from the
604    /// Unix epoch, 1 January 1970.
605    Date(i32),
606    /// Milliseconds from the Unix epoch, 1 January 1970.
607    TimestampMillis(i64),
608    /// Microseconds from the Unix epoch, 1 January 1970.
609    TimestampMicros(i64),
610
611    // ----------------------------------------------------------------------
612    // Complex types
613    /// Struct, child elements are tuples of field-value pairs.
614    Group(Row),
615    /// List of elements.
616    ListInternal(List),
617    /// List of key-value pairs.
618    MapInternal(Map),
619}
620
621impl Field {
622    /// Get the type name.
623    fn get_type_name(&self) -> &'static str {
624        match *self {
625            Field::Null => "Null",
626            Field::Bool(_) => "Bool",
627            Field::Byte(_) => "Byte",
628            Field::Short(_) => "Short",
629            Field::Int(_) => "Int",
630            Field::Long(_) => "Long",
631            Field::UByte(_) => "UByte",
632            Field::UShort(_) => "UShort",
633            Field::UInt(_) => "UInt",
634            Field::ULong(_) => "ULong",
635            Field::Float16(_) => "Float16",
636            Field::Float(_) => "Float",
637            Field::Double(_) => "Double",
638            Field::Decimal(_) => "Decimal",
639            Field::Date(_) => "Date",
640            Field::Str(_) => "Str",
641            Field::Bytes(_) => "Bytes",
642            Field::TimestampMillis(_) => "TimestampMillis",
643            Field::TimestampMicros(_) => "TimestampMicros",
644            Field::Group(_) => "Group",
645            Field::ListInternal(_) => "ListInternal",
646            Field::MapInternal(_) => "MapInternal",
647        }
648    }
649
650    /// Determines if this Row represents a primitive value.
651    pub fn is_primitive(&self) -> bool {
652        !matches!(
653            *self,
654            Field::Group(_) | Field::ListInternal(_) | Field::MapInternal(_)
655        )
656    }
657
658    /// Converts Parquet BOOLEAN type with logical type into `bool` value.
659    #[inline]
660    pub fn convert_bool(_descr: &ColumnDescPtr, value: bool) -> Self {
661        Field::Bool(value)
662    }
663
664    /// Converts Parquet INT32 type with converted type into `i32` value.
665    #[inline]
666    pub fn convert_int32(descr: &ColumnDescPtr, value: i32) -> Self {
667        match descr.converted_type() {
668            ConvertedType::INT_8 => Field::Byte(value as i8),
669            ConvertedType::INT_16 => Field::Short(value as i16),
670            ConvertedType::INT_32 | ConvertedType::NONE => Field::Int(value),
671            ConvertedType::UINT_8 => Field::UByte(value as u8),
672            ConvertedType::UINT_16 => Field::UShort(value as u16),
673            ConvertedType::UINT_32 => Field::UInt(value as u32),
674            ConvertedType::DATE => Field::Date(value),
675            ConvertedType::DECIMAL => Field::Decimal(Decimal::from_i32(
676                value,
677                descr.type_precision(),
678                descr.type_scale(),
679            )),
680            _ => nyi!(descr, value),
681        }
682    }
683
684    /// Converts Parquet INT64 type with converted type into `i64` value.
685    #[inline]
686    pub fn convert_int64(descr: &ColumnDescPtr, value: i64) -> Self {
687        match descr.converted_type() {
688            ConvertedType::INT_64 | ConvertedType::NONE => Field::Long(value),
689            ConvertedType::UINT_64 => Field::ULong(value as u64),
690            ConvertedType::TIMESTAMP_MILLIS => Field::TimestampMillis(value),
691            ConvertedType::TIMESTAMP_MICROS => Field::TimestampMicros(value),
692            ConvertedType::DECIMAL => Field::Decimal(Decimal::from_i64(
693                value,
694                descr.type_precision(),
695                descr.type_scale(),
696            )),
697            _ => nyi!(descr, value),
698        }
699    }
700
701    /// Converts Parquet INT96 (nanosecond timestamps) type and logical type into
702    /// `Timestamp` value.
703    #[inline]
704    pub fn convert_int96(_descr: &ColumnDescPtr, value: Int96) -> Self {
705        Field::TimestampMillis(value.to_i64())
706    }
707
708    /// Converts Parquet FLOAT type with logical type into `f32` value.
709    #[inline]
710    pub fn convert_float(_descr: &ColumnDescPtr, value: f32) -> Self {
711        Field::Float(value)
712    }
713
714    /// Converts Parquet DOUBLE type with converted type into `f64` value.
715    #[inline]
716    pub fn convert_double(_descr: &ColumnDescPtr, value: f64) -> Self {
717        Field::Double(value)
718    }
719
720    /// Converts Parquet BYTE_ARRAY type with converted type into a UTF8
721    /// string, decimal, float16, or an array of bytes.
722    #[inline]
723    pub fn convert_byte_array(descr: &ColumnDescPtr, value: ByteArray) -> Result<Self> {
724        let field = match descr.physical_type() {
725            PhysicalType::BYTE_ARRAY => match descr.converted_type() {
726                ConvertedType::UTF8 | ConvertedType::ENUM | ConvertedType::JSON => {
727                    let value = String::from_utf8(value.data().to_vec()).map_err(|e| {
728                        general_err!(
729                            "Error reading BYTE_ARRAY as String. Bytes: {:?} Error: {:?}",
730                            value.data(),
731                            e
732                        )
733                    })?;
734                    Field::Str(value)
735                }
736                ConvertedType::BSON | ConvertedType::NONE => Field::Bytes(value),
737                ConvertedType::DECIMAL => Field::Decimal(Decimal::from_bytes(
738                    value,
739                    descr.type_precision(),
740                    descr.type_scale(),
741                )),
742                _ => nyi!(descr, value),
743            },
744            PhysicalType::FIXED_LEN_BYTE_ARRAY => match descr.converted_type() {
745                ConvertedType::DECIMAL => Field::Decimal(Decimal::from_bytes(
746                    value,
747                    descr.type_precision(),
748                    descr.type_scale(),
749                )),
750                ConvertedType::NONE if descr.logical_type() == Some(LogicalType::Float16) => {
751                    if value.len() != 2 {
752                        return Err(general_err!(
753                            "Error reading FIXED_LEN_BYTE_ARRAY as FLOAT16. Length must be 2, got {}",
754                            value.len()
755                        ));
756                    }
757                    let bytes = [value.data()[0], value.data()[1]];
758                    Field::Float16(f16::from_le_bytes(bytes))
759                }
760                ConvertedType::NONE => Field::Bytes(value),
761                _ => nyi!(descr, value),
762            },
763            _ => nyi!(descr, value),
764        };
765        Ok(field)
766    }
767
768    /// Converts the Parquet field into a JSON [`Value`].
769    #[cfg(any(feature = "json", test))]
770    pub fn to_json_value(&self) -> Value {
771        use base64::prelude::BASE64_STANDARD;
772        use base64::Engine;
773
774        match &self {
775            Field::Null => Value::Null,
776            Field::Bool(b) => Value::Bool(*b),
777            Field::Byte(n) => Value::Number(serde_json::Number::from(*n)),
778            Field::Short(n) => Value::Number(serde_json::Number::from(*n)),
779            Field::Int(n) => Value::Number(serde_json::Number::from(*n)),
780            Field::Long(n) => Value::Number(serde_json::Number::from(*n)),
781            Field::UByte(n) => Value::Number(serde_json::Number::from(*n)),
782            Field::UShort(n) => Value::Number(serde_json::Number::from(*n)),
783            Field::UInt(n) => Value::Number(serde_json::Number::from(*n)),
784            Field::ULong(n) => Value::Number(serde_json::Number::from(*n)),
785            Field::Float16(n) => serde_json::Number::from_f64(f64::from(*n))
786                .map(Value::Number)
787                .unwrap_or(Value::Null),
788            Field::Float(n) => serde_json::Number::from_f64(f64::from(*n))
789                .map(Value::Number)
790                .unwrap_or(Value::Null),
791            Field::Double(n) => serde_json::Number::from_f64(*n)
792                .map(Value::Number)
793                .unwrap_or(Value::Null),
794            Field::Decimal(n) => Value::String(convert_decimal_to_string(n)),
795            Field::Str(s) => Value::String(s.to_owned()),
796            Field::Bytes(b) => Value::String(BASE64_STANDARD.encode(b.data())),
797            Field::Date(d) => Value::String(convert_date_to_string(*d)),
798            Field::TimestampMillis(ts) => Value::String(convert_timestamp_millis_to_string(*ts)),
799            Field::TimestampMicros(ts) => Value::String(convert_timestamp_micros_to_string(*ts)),
800            Field::Group(row) => row.to_json_value(),
801            Field::ListInternal(fields) => {
802                Value::Array(fields.elements.iter().map(|f| f.to_json_value()).collect())
803            }
804            Field::MapInternal(map) => Value::Object(
805                map.entries
806                    .iter()
807                    .map(|(key_field, value_field)| {
808                        let key_val = key_field.to_json_value();
809                        let key_str = key_val
810                            .as_str()
811                            .map(|s| s.to_owned())
812                            .unwrap_or_else(|| key_val.to_string());
813                        (key_str, value_field.to_json_value())
814                    })
815                    .collect(),
816            ),
817        }
818    }
819}
820
821impl fmt::Display for Field {
822    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
823        match *self {
824            Field::Null => write!(f, "null"),
825            Field::Bool(value) => write!(f, "{value}"),
826            Field::Byte(value) => write!(f, "{value}"),
827            Field::Short(value) => write!(f, "{value}"),
828            Field::Int(value) => write!(f, "{value}"),
829            Field::Long(value) => write!(f, "{value}"),
830            Field::UByte(value) => write!(f, "{value}"),
831            Field::UShort(value) => write!(f, "{value}"),
832            Field::UInt(value) => write!(f, "{value}"),
833            Field::ULong(value) => write!(f, "{value}"),
834            Field::Float16(value) => {
835                if !value.is_finite() {
836                    write!(f, "{value}")
837                } else if value.trunc() == value {
838                    write!(f, "{value}.0")
839                } else {
840                    write!(f, "{value}")
841                }
842            }
843            Field::Float(value) => {
844                if !(1e-15..=1e19).contains(&value) {
845                    write!(f, "{value:E}")
846                } else if value.trunc() == value {
847                    write!(f, "{value}.0")
848                } else {
849                    write!(f, "{value}")
850                }
851            }
852            Field::Double(value) => {
853                if !(1e-15..=1e19).contains(&value) {
854                    write!(f, "{value:E}")
855                } else if value.trunc() == value {
856                    write!(f, "{value}.0")
857                } else {
858                    write!(f, "{value}")
859                }
860            }
861            Field::Decimal(ref value) => {
862                write!(f, "{}", convert_decimal_to_string(value))
863            }
864            Field::Str(ref value) => write!(f, "\"{value}\""),
865            Field::Bytes(ref value) => write!(f, "{:?}", value.data()),
866            Field::Date(value) => write!(f, "{}", convert_date_to_string(value)),
867            Field::TimestampMillis(value) => {
868                write!(f, "{}", convert_timestamp_millis_to_string(value))
869            }
870            Field::TimestampMicros(value) => {
871                write!(f, "{}", convert_timestamp_micros_to_string(value))
872            }
873            Field::Group(ref fields) => write!(f, "{fields}"),
874            Field::ListInternal(ref list) => {
875                let elems = &list.elements;
876                write!(f, "[")?;
877                for (i, field) in elems.iter().enumerate() {
878                    field.fmt(f)?;
879                    if i < elems.len() - 1 {
880                        write!(f, ", ")?;
881                    }
882                }
883                write!(f, "]")
884            }
885            Field::MapInternal(ref map) => {
886                let entries = &map.entries;
887                write!(f, "{{")?;
888                for (i, (key, value)) in entries.iter().enumerate() {
889                    key.fmt(f)?;
890                    write!(f, " -> ")?;
891                    value.fmt(f)?;
892                    if i < entries.len() - 1 {
893                        write!(f, ", ")?;
894                    }
895                }
896                write!(f, "}}")
897            }
898        }
899    }
900}
901
902/// Helper method to convert Parquet date into a string.
903/// Input `value` is a number of days since the epoch in UTC.
904/// Date is displayed in local timezone.
905#[inline]
906fn convert_date_to_string(value: i32) -> String {
907    static NUM_SECONDS_IN_DAY: i64 = 60 * 60 * 24;
908    let dt = Utc
909        .timestamp_opt(value as i64 * NUM_SECONDS_IN_DAY, 0)
910        .unwrap();
911    format!("{}", dt.format("%Y-%m-%d"))
912}
913
914/// Helper method to convert Parquet timestamp into a string.
915/// Input `value` is a number of seconds since the epoch in UTC.
916/// Datetime is displayed in local timezone.
917#[inline]
918fn convert_timestamp_secs_to_string(value: i64) -> String {
919    let dt = Utc.timestamp_opt(value, 0).unwrap();
920    format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"))
921}
922
923/// Helper method to convert Parquet timestamp into a string.
924/// Input `value` is a number of milliseconds since the epoch in UTC.
925/// Datetime is displayed in local timezone.
926#[inline]
927fn convert_timestamp_millis_to_string(value: i64) -> String {
928    convert_timestamp_secs_to_string(value / 1000)
929}
930
931/// Helper method to convert Parquet timestamp into a string.
932/// Input `value` is a number of microseconds since the epoch in UTC.
933/// Datetime is displayed in local timezone.
934#[inline]
935fn convert_timestamp_micros_to_string(value: i64) -> String {
936    convert_timestamp_secs_to_string(value / 1000000)
937}
938
939/// Helper method to convert Parquet decimal into a string.
940/// We assert that `scale >= 0` and `precision > scale`, but this will be enforced
941/// when constructing Parquet schema.
942#[inline]
943fn convert_decimal_to_string(decimal: &Decimal) -> String {
944    assert!(decimal.scale() >= 0 && decimal.precision() > decimal.scale());
945
946    // Specify as signed bytes to resolve sign as part of conversion.
947    let num = BigInt::from_signed_bytes_be(decimal.data());
948
949    // Offset of the first digit in a string.
950    let negative = i32::from(num.sign() == Sign::Minus);
951    let mut num_str = num.to_string();
952    let mut point = num_str.len() as i32 - decimal.scale() - negative;
953
954    // Convert to string form without scientific notation.
955    if point <= 0 {
956        // Zeros need to be prepended to the unscaled value.
957        while point < 0 {
958            num_str.insert(negative as usize, '0');
959            point += 1;
960        }
961        num_str.insert_str(negative as usize, "0.");
962    } else {
963        // No zeroes need to be prepended to the unscaled value, simply insert decimal
964        // point.
965        num_str.insert((point + negative) as usize, '.');
966    }
967
968    num_str
969}
970
971#[cfg(test)]
972#[allow(clippy::many_single_char_names)]
973mod tests {
974    use super::*;
975
976    use std::f64::consts::PI;
977    use std::sync::Arc;
978
979    use crate::schema::types::{ColumnDescriptor, ColumnPath, PrimitiveTypeBuilder};
980
981    /// Creates test column descriptor based on provided type parameters.
982    macro_rules! make_column_descr {
983        ($physical_type:expr, $logical_type:expr) => {{
984            let tpe = PrimitiveTypeBuilder::new("col", $physical_type)
985                .with_converted_type($logical_type)
986                .build()
987                .unwrap();
988            Arc::new(ColumnDescriptor::new(
989                Arc::new(tpe),
990                0,
991                0,
992                ColumnPath::from("col"),
993            ))
994        }};
995        ($physical_type:expr, $logical_type:expr, $len:expr, $prec:expr, $scale:expr) => {{
996            let tpe = PrimitiveTypeBuilder::new("col", $physical_type)
997                .with_converted_type($logical_type)
998                .with_length($len)
999                .with_precision($prec)
1000                .with_scale($scale)
1001                .build()
1002                .unwrap();
1003            Arc::new(ColumnDescriptor::new(
1004                Arc::new(tpe),
1005                0,
1006                0,
1007                ColumnPath::from("col"),
1008            ))
1009        }};
1010    }
1011
1012    #[test]
1013    fn test_row_convert_bool() {
1014        // BOOLEAN value does not depend on logical type
1015        let descr = make_column_descr![PhysicalType::BOOLEAN, ConvertedType::NONE];
1016
1017        let row = Field::convert_bool(&descr, true);
1018        assert_eq!(row, Field::Bool(true));
1019
1020        let row = Field::convert_bool(&descr, false);
1021        assert_eq!(row, Field::Bool(false));
1022    }
1023
1024    #[test]
1025    fn test_row_convert_int32() {
1026        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::INT_8];
1027        let row = Field::convert_int32(&descr, 111);
1028        assert_eq!(row, Field::Byte(111));
1029
1030        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::INT_16];
1031        let row = Field::convert_int32(&descr, 222);
1032        assert_eq!(row, Field::Short(222));
1033
1034        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::INT_32];
1035        let row = Field::convert_int32(&descr, 333);
1036        assert_eq!(row, Field::Int(333));
1037
1038        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::UINT_8];
1039        let row = Field::convert_int32(&descr, -1);
1040        assert_eq!(row, Field::UByte(255));
1041
1042        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::UINT_16];
1043        let row = Field::convert_int32(&descr, 256);
1044        assert_eq!(row, Field::UShort(256));
1045
1046        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::UINT_32];
1047        let row = Field::convert_int32(&descr, 1234);
1048        assert_eq!(row, Field::UInt(1234));
1049
1050        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::NONE];
1051        let row = Field::convert_int32(&descr, 444);
1052        assert_eq!(row, Field::Int(444));
1053
1054        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::DATE];
1055        let row = Field::convert_int32(&descr, 14611);
1056        assert_eq!(row, Field::Date(14611));
1057
1058        let descr = make_column_descr![PhysicalType::INT32, ConvertedType::DECIMAL, 0, 8, 2];
1059        let row = Field::convert_int32(&descr, 444);
1060        assert_eq!(row, Field::Decimal(Decimal::from_i32(444, 8, 2)));
1061    }
1062
1063    #[test]
1064    fn test_row_convert_int64() {
1065        let descr = make_column_descr![PhysicalType::INT64, ConvertedType::INT_64];
1066        let row = Field::convert_int64(&descr, 1111);
1067        assert_eq!(row, Field::Long(1111));
1068
1069        let descr = make_column_descr![PhysicalType::INT64, ConvertedType::UINT_64];
1070        let row = Field::convert_int64(&descr, 78239823);
1071        assert_eq!(row, Field::ULong(78239823));
1072
1073        let descr = make_column_descr![PhysicalType::INT64, ConvertedType::TIMESTAMP_MILLIS];
1074        let row = Field::convert_int64(&descr, 1541186529153);
1075        assert_eq!(row, Field::TimestampMillis(1541186529153));
1076
1077        let descr = make_column_descr![PhysicalType::INT64, ConvertedType::TIMESTAMP_MICROS];
1078        let row = Field::convert_int64(&descr, 1541186529153123);
1079        assert_eq!(row, Field::TimestampMicros(1541186529153123));
1080
1081        let descr = make_column_descr![PhysicalType::INT64, ConvertedType::NONE];
1082        let row = Field::convert_int64(&descr, 2222);
1083        assert_eq!(row, Field::Long(2222));
1084
1085        let descr = make_column_descr![PhysicalType::INT64, ConvertedType::DECIMAL, 0, 8, 2];
1086        let row = Field::convert_int64(&descr, 3333);
1087        assert_eq!(row, Field::Decimal(Decimal::from_i64(3333, 8, 2)));
1088    }
1089
1090    #[test]
1091    fn test_row_convert_int96() {
1092        // INT96 value does not depend on logical type
1093        let descr = make_column_descr![PhysicalType::INT96, ConvertedType::NONE];
1094
1095        let value = Int96::from(vec![0, 0, 2454923]);
1096        let row = Field::convert_int96(&descr, value);
1097        assert_eq!(row, Field::TimestampMillis(1238544000000));
1098
1099        let value = Int96::from(vec![4165425152, 13, 2454923]);
1100        let row = Field::convert_int96(&descr, value);
1101        assert_eq!(row, Field::TimestampMillis(1238544060000));
1102    }
1103
1104    #[test]
1105    fn test_row_convert_float() {
1106        // FLOAT value does not depend on logical type
1107        let descr = make_column_descr![PhysicalType::FLOAT, ConvertedType::NONE];
1108        let row = Field::convert_float(&descr, 2.31);
1109        assert_eq!(row, Field::Float(2.31));
1110    }
1111
1112    #[test]
1113    fn test_row_convert_double() {
1114        // DOUBLE value does not depend on logical type
1115        let descr = make_column_descr![PhysicalType::DOUBLE, ConvertedType::NONE];
1116        let row = Field::convert_double(&descr, 1.56);
1117        assert_eq!(row, Field::Double(1.56));
1118    }
1119
1120    #[test]
1121    fn test_row_convert_byte_array() {
1122        // UTF8
1123        let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::UTF8];
1124        let value = ByteArray::from(vec![b'A', b'B', b'C', b'D']);
1125        let row = Field::convert_byte_array(&descr, value);
1126        assert_eq!(row.unwrap(), Field::Str("ABCD".to_string()));
1127
1128        // ENUM
1129        let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::ENUM];
1130        let value = ByteArray::from(vec![b'1', b'2', b'3']);
1131        let row = Field::convert_byte_array(&descr, value);
1132        assert_eq!(row.unwrap(), Field::Str("123".to_string()));
1133
1134        // JSON
1135        let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::JSON];
1136        let value = ByteArray::from(vec![b'{', b'"', b'a', b'"', b':', b'1', b'}']);
1137        let row = Field::convert_byte_array(&descr, value);
1138        assert_eq!(row.unwrap(), Field::Str("{\"a\":1}".to_string()));
1139
1140        // NONE
1141        let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::NONE];
1142        let value = ByteArray::from(vec![1, 2, 3, 4, 5]);
1143        let row = Field::convert_byte_array(&descr, value.clone());
1144        assert_eq!(row.unwrap(), Field::Bytes(value));
1145
1146        // BSON
1147        let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::BSON];
1148        let value = ByteArray::from(vec![1, 2, 3, 4, 5]);
1149        let row = Field::convert_byte_array(&descr, value.clone());
1150        assert_eq!(row.unwrap(), Field::Bytes(value));
1151
1152        // DECIMAL
1153        let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::DECIMAL, 0, 8, 2];
1154        let value = ByteArray::from(vec![207, 200]);
1155        let row = Field::convert_byte_array(&descr, value.clone());
1156        assert_eq!(
1157            row.unwrap(),
1158            Field::Decimal(Decimal::from_bytes(value, 8, 2))
1159        );
1160
1161        // DECIMAL (FIXED_LEN_BYTE_ARRAY)
1162        let descr = make_column_descr![
1163            PhysicalType::FIXED_LEN_BYTE_ARRAY,
1164            ConvertedType::DECIMAL,
1165            8,
1166            17,
1167            5
1168        ];
1169        let value = ByteArray::from(vec![0, 0, 0, 0, 0, 4, 147, 224]);
1170        let row = Field::convert_byte_array(&descr, value.clone());
1171        assert_eq!(
1172            row.unwrap(),
1173            Field::Decimal(Decimal::from_bytes(value, 17, 5))
1174        );
1175
1176        // FLOAT16
1177        let descr = {
1178            let tpe = PrimitiveTypeBuilder::new("col", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1179                .with_logical_type(Some(LogicalType::Float16))
1180                .with_length(2)
1181                .build()
1182                .unwrap();
1183            Arc::new(ColumnDescriptor::new(
1184                Arc::new(tpe),
1185                0,
1186                0,
1187                ColumnPath::from("col"),
1188            ))
1189        };
1190        let value = ByteArray::from(f16::PI);
1191        let row = Field::convert_byte_array(&descr, value.clone());
1192        assert_eq!(row.unwrap(), Field::Float16(f16::PI));
1193
1194        // NONE (FIXED_LEN_BYTE_ARRAY)
1195        let descr = make_column_descr![
1196            PhysicalType::FIXED_LEN_BYTE_ARRAY,
1197            ConvertedType::NONE,
1198            6,
1199            0,
1200            0
1201        ];
1202        let value = ByteArray::from(vec![1, 2, 3, 4, 5, 6]);
1203        let row = Field::convert_byte_array(&descr, value.clone());
1204        assert_eq!(row.unwrap(), Field::Bytes(value));
1205    }
1206
1207    #[test]
1208    fn test_convert_date_to_string() {
1209        fn check_date_conversion(y: u32, m: u32, d: u32) {
1210            let datetime = chrono::NaiveDate::from_ymd_opt(y as i32, m, d)
1211                .unwrap()
1212                .and_hms_opt(0, 0, 0)
1213                .unwrap();
1214            let dt = Utc.from_utc_datetime(&datetime);
1215            let res = convert_date_to_string((dt.timestamp() / 60 / 60 / 24) as i32);
1216            let exp = format!("{}", dt.format("%Y-%m-%d"));
1217            assert_eq!(res, exp);
1218        }
1219
1220        check_date_conversion(1969, 12, 31);
1221        check_date_conversion(2010, 1, 2);
1222        check_date_conversion(2014, 5, 1);
1223        check_date_conversion(2016, 2, 29);
1224        check_date_conversion(2017, 9, 12);
1225        check_date_conversion(2018, 3, 31);
1226    }
1227
1228    #[test]
1229    fn test_convert_timestamp_millis_to_string() {
1230        fn check_datetime_conversion(y: u32, m: u32, d: u32, h: u32, mi: u32, s: u32) {
1231            let datetime = chrono::NaiveDate::from_ymd_opt(y as i32, m, d)
1232                .unwrap()
1233                .and_hms_opt(h, mi, s)
1234                .unwrap();
1235            let dt = Utc.from_utc_datetime(&datetime);
1236            let res = convert_timestamp_millis_to_string(dt.timestamp_millis());
1237            let exp = format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"));
1238            assert_eq!(res, exp);
1239        }
1240
1241        check_datetime_conversion(1969, 9, 10, 1, 2, 3);
1242        check_datetime_conversion(2010, 1, 2, 13, 12, 54);
1243        check_datetime_conversion(2011, 1, 3, 8, 23, 1);
1244        check_datetime_conversion(2012, 4, 5, 11, 6, 32);
1245        check_datetime_conversion(2013, 5, 12, 16, 38, 0);
1246        check_datetime_conversion(2014, 11, 28, 21, 15, 12);
1247    }
1248
1249    #[test]
1250    fn test_convert_timestamp_micros_to_string() {
1251        fn check_datetime_conversion(y: u32, m: u32, d: u32, h: u32, mi: u32, s: u32) {
1252            let datetime = chrono::NaiveDate::from_ymd_opt(y as i32, m, d)
1253                .unwrap()
1254                .and_hms_opt(h, mi, s)
1255                .unwrap();
1256            let dt = Utc.from_utc_datetime(&datetime);
1257            let res = convert_timestamp_micros_to_string(dt.timestamp_micros());
1258            let exp = format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"));
1259            assert_eq!(res, exp);
1260        }
1261
1262        check_datetime_conversion(1969, 9, 10, 1, 2, 3);
1263        check_datetime_conversion(2010, 1, 2, 13, 12, 54);
1264        check_datetime_conversion(2011, 1, 3, 8, 23, 1);
1265        check_datetime_conversion(2012, 4, 5, 11, 6, 32);
1266        check_datetime_conversion(2013, 5, 12, 16, 38, 0);
1267        check_datetime_conversion(2014, 11, 28, 21, 15, 12);
1268    }
1269
1270    #[test]
1271    fn test_convert_float16_to_string() {
1272        assert_eq!(format!("{}", Field::Float16(f16::ONE)), "1.0");
1273        assert_eq!(format!("{}", Field::Float16(f16::PI)), "3.140625");
1274        assert_eq!(format!("{}", Field::Float16(f16::MAX)), "65504.0");
1275        assert_eq!(format!("{}", Field::Float16(f16::NAN)), "NaN");
1276        assert_eq!(format!("{}", Field::Float16(f16::INFINITY)), "inf");
1277        assert_eq!(format!("{}", Field::Float16(f16::NEG_INFINITY)), "-inf");
1278        assert_eq!(format!("{}", Field::Float16(f16::ZERO)), "0.0");
1279        assert_eq!(format!("{}", Field::Float16(f16::NEG_ZERO)), "-0.0");
1280    }
1281
1282    #[test]
1283    fn test_convert_float_to_string() {
1284        assert_eq!(format!("{}", Field::Float(1.0)), "1.0");
1285        assert_eq!(format!("{}", Field::Float(9.63)), "9.63");
1286        assert_eq!(format!("{}", Field::Float(1e-15)), "0.000000000000001");
1287        assert_eq!(format!("{}", Field::Float(1e-16)), "1E-16");
1288        assert_eq!(format!("{}", Field::Float(1e19)), "10000000000000000000.0");
1289        assert_eq!(format!("{}", Field::Float(1e20)), "1E20");
1290        assert_eq!(format!("{}", Field::Float(1.7976931E30)), "1.7976931E30");
1291        assert_eq!(format!("{}", Field::Float(-1.7976931E30)), "-1.7976931E30");
1292    }
1293
1294    #[test]
1295    fn test_convert_double_to_string() {
1296        assert_eq!(format!("{}", Field::Double(1.0)), "1.0");
1297        assert_eq!(format!("{}", Field::Double(9.63)), "9.63");
1298        assert_eq!(format!("{}", Field::Double(1e-15)), "0.000000000000001");
1299        assert_eq!(format!("{}", Field::Double(1e-16)), "1E-16");
1300        assert_eq!(format!("{}", Field::Double(1e19)), "10000000000000000000.0");
1301        assert_eq!(format!("{}", Field::Double(1e20)), "1E20");
1302        assert_eq!(
1303            format!("{}", Field::Double(1.79769313486E308)),
1304            "1.79769313486E308"
1305        );
1306        assert_eq!(
1307            format!("{}", Field::Double(-1.79769313486E308)),
1308            "-1.79769313486E308"
1309        );
1310    }
1311
1312    #[test]
1313    fn test_convert_decimal_to_string() {
1314        // Helper method to compare decimal
1315        fn check_decimal(bytes: Vec<u8>, precision: i32, scale: i32, res: &str) {
1316            let decimal = Decimal::from_bytes(ByteArray::from(bytes), precision, scale);
1317            assert_eq!(convert_decimal_to_string(&decimal), res);
1318        }
1319
1320        // This example previously used to fail in some engines
1321        check_decimal(
1322            vec![0, 0, 0, 0, 0, 0, 0, 0, 13, 224, 182, 179, 167, 100, 0, 0],
1323            38,
1324            18,
1325            "1.000000000000000000",
1326        );
1327        check_decimal(
1328            vec![
1329                249, 233, 247, 16, 185, 192, 202, 223, 215, 165, 192, 166, 67, 72,
1330            ],
1331            36,
1332            28,
1333            "-12344.0242342304923409234234293432",
1334        );
1335        check_decimal(vec![0, 0, 0, 0, 0, 4, 147, 224], 17, 5, "3.00000");
1336        check_decimal(vec![0, 0, 0, 0, 1, 201, 195, 140], 18, 2, "300000.12");
1337        check_decimal(vec![207, 200], 10, 2, "-123.44");
1338        check_decimal(vec![207, 200], 10, 8, "-0.00012344");
1339    }
1340
1341    #[test]
1342    fn test_row_display() {
1343        // Primitive types
1344        assert_eq!(format!("{}", Field::Null), "null");
1345        assert_eq!(format!("{}", Field::Bool(true)), "true");
1346        assert_eq!(format!("{}", Field::Bool(false)), "false");
1347        assert_eq!(format!("{}", Field::Byte(1)), "1");
1348        assert_eq!(format!("{}", Field::Short(2)), "2");
1349        assert_eq!(format!("{}", Field::Int(3)), "3");
1350        assert_eq!(format!("{}", Field::Long(4)), "4");
1351        assert_eq!(format!("{}", Field::UByte(1)), "1");
1352        assert_eq!(format!("{}", Field::UShort(2)), "2");
1353        assert_eq!(format!("{}", Field::UInt(3)), "3");
1354        assert_eq!(format!("{}", Field::ULong(4)), "4");
1355        assert_eq!(format!("{}", Field::Float16(f16::E)), "2.71875");
1356        assert_eq!(format!("{}", Field::Float(5.0)), "5.0");
1357        assert_eq!(format!("{}", Field::Float(5.1234)), "5.1234");
1358        assert_eq!(format!("{}", Field::Double(6.0)), "6.0");
1359        assert_eq!(format!("{}", Field::Double(6.1234)), "6.1234");
1360        assert_eq!(format!("{}", Field::Str("abc".to_string())), "\"abc\"");
1361        assert_eq!(
1362            format!("{}", Field::Bytes(ByteArray::from(vec![1, 2, 3]))),
1363            "[1, 2, 3]"
1364        );
1365        assert_eq!(
1366            format!("{}", Field::Date(14611)),
1367            convert_date_to_string(14611)
1368        );
1369        assert_eq!(
1370            format!("{}", Field::TimestampMillis(1262391174000)),
1371            convert_timestamp_millis_to_string(1262391174000)
1372        );
1373        assert_eq!(
1374            format!("{}", Field::TimestampMicros(1262391174000000)),
1375            convert_timestamp_micros_to_string(1262391174000000)
1376        );
1377        assert_eq!(
1378            format!("{}", Field::Decimal(Decimal::from_i32(4, 8, 2))),
1379            convert_decimal_to_string(&Decimal::from_i32(4, 8, 2))
1380        );
1381
1382        // Complex types
1383        let fields = vec![
1384            ("x".to_string(), Field::Null),
1385            ("Y".to_string(), Field::Int(2)),
1386            ("z".to_string(), Field::Float(3.1)),
1387            ("a".to_string(), Field::Str("abc".to_string())),
1388        ];
1389        let row = Field::Group(make_row(fields));
1390        assert_eq!(format!("{row}"), "{x: null, Y: 2, z: 3.1, a: \"abc\"}");
1391
1392        let row = Field::ListInternal(make_list(vec![
1393            Field::Int(2),
1394            Field::Int(1),
1395            Field::Null,
1396            Field::Int(12),
1397        ]));
1398        assert_eq!(format!("{row}"), "[2, 1, null, 12]");
1399
1400        let row = Field::MapInternal(make_map(vec![
1401            (Field::Int(1), Field::Float(1.2)),
1402            (Field::Int(2), Field::Float(4.5)),
1403            (Field::Int(3), Field::Float(2.3)),
1404        ]));
1405        assert_eq!(format!("{row}"), "{1 -> 1.2, 2 -> 4.5, 3 -> 2.3}");
1406    }
1407
1408    #[test]
1409    fn test_is_primitive() {
1410        // primitives
1411        assert!(Field::Null.is_primitive());
1412        assert!(Field::Bool(true).is_primitive());
1413        assert!(Field::Bool(false).is_primitive());
1414        assert!(Field::Byte(1).is_primitive());
1415        assert!(Field::Short(2).is_primitive());
1416        assert!(Field::Int(3).is_primitive());
1417        assert!(Field::Long(4).is_primitive());
1418        assert!(Field::UByte(1).is_primitive());
1419        assert!(Field::UShort(2).is_primitive());
1420        assert!(Field::UInt(3).is_primitive());
1421        assert!(Field::ULong(4).is_primitive());
1422        assert!(Field::Float16(f16::E).is_primitive());
1423        assert!(Field::Float(5.0).is_primitive());
1424        assert!(Field::Float(5.1234).is_primitive());
1425        assert!(Field::Double(6.0).is_primitive());
1426        assert!(Field::Double(6.1234).is_primitive());
1427        assert!(Field::Str("abc".to_string()).is_primitive());
1428        assert!(Field::Bytes(ByteArray::from(vec![1, 2, 3])).is_primitive());
1429        assert!(Field::TimestampMillis(12345678).is_primitive());
1430        assert!(Field::TimestampMicros(12345678901).is_primitive());
1431        assert!(Field::Decimal(Decimal::from_i32(4, 8, 2)).is_primitive());
1432
1433        // complex types
1434        assert!(!Field::Group(make_row(vec![
1435            ("x".to_string(), Field::Null),
1436            ("Y".to_string(), Field::Int(2)),
1437            ("z".to_string(), Field::Float(3.1)),
1438            ("a".to_string(), Field::Str("abc".to_string()))
1439        ]))
1440        .is_primitive());
1441
1442        assert!(!Field::ListInternal(make_list(vec![
1443            Field::Int(2),
1444            Field::Int(1),
1445            Field::Null,
1446            Field::Int(12)
1447        ]))
1448        .is_primitive());
1449
1450        assert!(!Field::MapInternal(make_map(vec![
1451            (Field::Int(1), Field::Float(1.2)),
1452            (Field::Int(2), Field::Float(4.5)),
1453            (Field::Int(3), Field::Float(2.3))
1454        ]))
1455        .is_primitive());
1456    }
1457
1458    #[test]
1459    fn test_row_primitive_field_fmt() {
1460        // Primitives types
1461        let row = make_row(vec![
1462            ("00".to_string(), Field::Null),
1463            ("01".to_string(), Field::Bool(false)),
1464            ("02".to_string(), Field::Byte(3)),
1465            ("03".to_string(), Field::Short(4)),
1466            ("04".to_string(), Field::Int(5)),
1467            ("05".to_string(), Field::Long(6)),
1468            ("06".to_string(), Field::UByte(7)),
1469            ("07".to_string(), Field::UShort(8)),
1470            ("08".to_string(), Field::UInt(9)),
1471            ("09".to_string(), Field::ULong(10)),
1472            ("10".to_string(), Field::Float(11.1)),
1473            ("11".to_string(), Field::Double(12.1)),
1474            ("12".to_string(), Field::Str("abc".to_string())),
1475            (
1476                "13".to_string(),
1477                Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5])),
1478            ),
1479            ("14".to_string(), Field::Date(14611)),
1480            ("15".to_string(), Field::TimestampMillis(1262391174000)),
1481            ("16".to_string(), Field::TimestampMicros(1262391174000000)),
1482            ("17".to_string(), Field::Decimal(Decimal::from_i32(4, 7, 2))),
1483            ("18".to_string(), Field::Float16(f16::PI)),
1484        ]);
1485
1486        assert_eq!("null", format!("{}", row.fmt(0)));
1487        assert_eq!("false", format!("{}", row.fmt(1)));
1488        assert_eq!("3", format!("{}", row.fmt(2)));
1489        assert_eq!("4", format!("{}", row.fmt(3)));
1490        assert_eq!("5", format!("{}", row.fmt(4)));
1491        assert_eq!("6", format!("{}", row.fmt(5)));
1492        assert_eq!("7", format!("{}", row.fmt(6)));
1493        assert_eq!("8", format!("{}", row.fmt(7)));
1494        assert_eq!("9", format!("{}", row.fmt(8)));
1495        assert_eq!("10", format!("{}", row.fmt(9)));
1496        assert_eq!("11.1", format!("{}", row.fmt(10)));
1497        assert_eq!("12.1", format!("{}", row.fmt(11)));
1498        assert_eq!("\"abc\"", format!("{}", row.fmt(12)));
1499        assert_eq!("[1, 2, 3, 4, 5]", format!("{}", row.fmt(13)));
1500        assert_eq!(convert_date_to_string(14611), format!("{}", row.fmt(14)));
1501        assert_eq!(
1502            convert_timestamp_millis_to_string(1262391174000),
1503            format!("{}", row.fmt(15))
1504        );
1505        assert_eq!(
1506            convert_timestamp_micros_to_string(1262391174000000),
1507            format!("{}", row.fmt(16))
1508        );
1509        assert_eq!("0.04", format!("{}", row.fmt(17)));
1510        assert_eq!("3.140625", format!("{}", row.fmt(18)));
1511    }
1512
1513    #[test]
1514    fn test_row_complex_field_fmt() {
1515        // Complex types
1516        let row = make_row(vec![
1517            (
1518                "00".to_string(),
1519                Field::Group(make_row(vec![
1520                    ("x".to_string(), Field::Null),
1521                    ("Y".to_string(), Field::Int(2)),
1522                ])),
1523            ),
1524            (
1525                "01".to_string(),
1526                Field::ListInternal(make_list(vec![
1527                    Field::Int(2),
1528                    Field::Int(1),
1529                    Field::Null,
1530                    Field::Int(12),
1531                ])),
1532            ),
1533            (
1534                "02".to_string(),
1535                Field::MapInternal(make_map(vec![
1536                    (Field::Int(1), Field::Float(1.2)),
1537                    (Field::Int(2), Field::Float(4.5)),
1538                    (Field::Int(3), Field::Float(2.3)),
1539                ])),
1540            ),
1541        ]);
1542
1543        assert_eq!("{x: null, Y: 2}", format!("{}", row.fmt(0)));
1544        assert_eq!("[2, 1, null, 12]", format!("{}", row.fmt(1)));
1545        assert_eq!("{1 -> 1.2, 2 -> 4.5, 3 -> 2.3}", format!("{}", row.fmt(2)));
1546    }
1547
1548    #[test]
1549    fn test_row_primitive_accessors() {
1550        // primitives
1551        let row = make_row(vec![
1552            ("a".to_string(), Field::Null),
1553            ("b".to_string(), Field::Bool(false)),
1554            ("c".to_string(), Field::Byte(3)),
1555            ("d".to_string(), Field::Short(4)),
1556            ("e".to_string(), Field::Int(5)),
1557            ("f".to_string(), Field::Long(6)),
1558            ("g".to_string(), Field::UByte(3)),
1559            ("h".to_string(), Field::UShort(4)),
1560            ("i".to_string(), Field::UInt(5)),
1561            ("j".to_string(), Field::ULong(6)),
1562            ("k".to_string(), Field::Float(7.1)),
1563            ("l".to_string(), Field::Double(8.1)),
1564            ("m".to_string(), Field::Str("abc".to_string())),
1565            (
1566                "n".to_string(),
1567                Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5])),
1568            ),
1569            ("o".to_string(), Field::Decimal(Decimal::from_i32(4, 7, 2))),
1570            ("p".to_string(), Field::Float16(f16::from_f32(9.1))),
1571        ]);
1572
1573        assert!(!row.get_bool(1).unwrap());
1574        assert_eq!(3, row.get_byte(2).unwrap());
1575        assert_eq!(4, row.get_short(3).unwrap());
1576        assert_eq!(5, row.get_int(4).unwrap());
1577        assert_eq!(6, row.get_long(5).unwrap());
1578        assert_eq!(3, row.get_ubyte(6).unwrap());
1579        assert_eq!(4, row.get_ushort(7).unwrap());
1580        assert_eq!(5, row.get_uint(8).unwrap());
1581        assert_eq!(6, row.get_ulong(9).unwrap());
1582        assert!((7.1 - row.get_float(10).unwrap()).abs() < f32::EPSILON);
1583        assert!((8.1 - row.get_double(11).unwrap()).abs() < f64::EPSILON);
1584        assert_eq!("abc", row.get_string(12).unwrap());
1585        assert_eq!(5, row.get_bytes(13).unwrap().len());
1586        assert_eq!(7, row.get_decimal(14).unwrap().precision());
1587        assert!((f16::from_f32(9.1) - row.get_float16(15).unwrap()).abs() < f16::EPSILON);
1588    }
1589
1590    #[test]
1591    fn test_row_primitive_invalid_accessors() {
1592        // primitives
1593        let row = make_row(vec![
1594            ("a".to_string(), Field::Null),
1595            ("b".to_string(), Field::Bool(false)),
1596            ("c".to_string(), Field::Byte(3)),
1597            ("d".to_string(), Field::Short(4)),
1598            ("e".to_string(), Field::Int(5)),
1599            ("f".to_string(), Field::Long(6)),
1600            ("g".to_string(), Field::UByte(3)),
1601            ("h".to_string(), Field::UShort(4)),
1602            ("i".to_string(), Field::UInt(5)),
1603            ("j".to_string(), Field::ULong(6)),
1604            ("k".to_string(), Field::Float(7.1)),
1605            ("l".to_string(), Field::Double(8.1)),
1606            ("m".to_string(), Field::Str("abc".to_string())),
1607            (
1608                "n".to_string(),
1609                Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5])),
1610            ),
1611            ("o".to_string(), Field::Decimal(Decimal::from_i32(4, 7, 2))),
1612            ("p".to_string(), Field::Float16(f16::from_f32(9.1))),
1613        ]);
1614
1615        for i in 0..row.len() {
1616            assert!(row.get_group(i).is_err());
1617        }
1618    }
1619
1620    #[test]
1621    fn test_row_complex_accessors() {
1622        let row = make_row(vec![
1623            (
1624                "a".to_string(),
1625                Field::Group(make_row(vec![
1626                    ("x".to_string(), Field::Null),
1627                    ("Y".to_string(), Field::Int(2)),
1628                ])),
1629            ),
1630            (
1631                "b".to_string(),
1632                Field::ListInternal(make_list(vec![
1633                    Field::Int(2),
1634                    Field::Int(1),
1635                    Field::Null,
1636                    Field::Int(12),
1637                ])),
1638            ),
1639            (
1640                "c".to_string(),
1641                Field::MapInternal(make_map(vec![
1642                    (Field::Int(1), Field::Float(1.2)),
1643                    (Field::Int(2), Field::Float(4.5)),
1644                    (Field::Int(3), Field::Float(2.3)),
1645                ])),
1646            ),
1647        ]);
1648
1649        assert_eq!(2, row.get_group(0).unwrap().len());
1650        assert_eq!(4, row.get_list(1).unwrap().len());
1651        assert_eq!(3, row.get_map(2).unwrap().len());
1652    }
1653
1654    #[test]
1655    fn test_row_complex_invalid_accessors() {
1656        let row = make_row(vec![
1657            (
1658                "a".to_string(),
1659                Field::Group(make_row(vec![
1660                    ("x".to_string(), Field::Null),
1661                    ("Y".to_string(), Field::Int(2)),
1662                ])),
1663            ),
1664            (
1665                "b".to_string(),
1666                Field::ListInternal(make_list(vec![
1667                    Field::Int(2),
1668                    Field::Int(1),
1669                    Field::Null,
1670                    Field::Int(12),
1671                ])),
1672            ),
1673            (
1674                "c".to_string(),
1675                Field::MapInternal(make_map(vec![
1676                    (Field::Int(1), Field::Float(1.2)),
1677                    (Field::Int(2), Field::Float(4.5)),
1678                    (Field::Int(3), Field::Float(2.3)),
1679                ])),
1680            ),
1681        ]);
1682
1683        assert_eq!(
1684            row.get_float(0).unwrap_err().to_string(),
1685            "Parquet error: Cannot access Group as Float"
1686        );
1687        assert_eq!(
1688            row.get_float(1).unwrap_err().to_string(),
1689            "Parquet error: Cannot access ListInternal as Float"
1690        );
1691        assert_eq!(
1692            row.get_float(2).unwrap_err().to_string(),
1693            "Parquet error: Cannot access MapInternal as Float",
1694        );
1695    }
1696
1697    #[test]
1698    fn test_list_primitive_accessors() {
1699        // primitives
1700        let list = make_list(vec![Field::Bool(false)]);
1701        assert!(!list.get_bool(0).unwrap());
1702
1703        let list = make_list(vec![Field::Byte(3), Field::Byte(4)]);
1704        assert_eq!(4, list.get_byte(1).unwrap());
1705
1706        let list = make_list(vec![Field::Short(4), Field::Short(5), Field::Short(6)]);
1707        assert_eq!(6, list.get_short(2).unwrap());
1708
1709        let list = make_list(vec![Field::Int(5)]);
1710        assert_eq!(5, list.get_int(0).unwrap());
1711
1712        let list = make_list(vec![Field::Long(6), Field::Long(7)]);
1713        assert_eq!(7, list.get_long(1).unwrap());
1714
1715        let list = make_list(vec![Field::UByte(3), Field::UByte(4)]);
1716        assert_eq!(4, list.get_ubyte(1).unwrap());
1717
1718        let list = make_list(vec![Field::UShort(4), Field::UShort(5), Field::UShort(6)]);
1719        assert_eq!(6, list.get_ushort(2).unwrap());
1720
1721        let list = make_list(vec![Field::UInt(5)]);
1722        assert_eq!(5, list.get_uint(0).unwrap());
1723
1724        let list = make_list(vec![Field::ULong(6), Field::ULong(7)]);
1725        assert_eq!(7, list.get_ulong(1).unwrap());
1726
1727        let list = make_list(vec![Field::Float16(f16::PI)]);
1728        assert!((f16::PI - list.get_float16(0).unwrap()).abs() < f16::EPSILON);
1729
1730        let list = make_list(vec![
1731            Field::Float(8.1),
1732            Field::Float(9.2),
1733            Field::Float(10.3),
1734        ]);
1735        assert!((10.3 - list.get_float(2).unwrap()).abs() < f32::EPSILON);
1736
1737        let list = make_list(vec![Field::Double(PI)]);
1738        assert!((PI - list.get_double(0).unwrap()).abs() < f64::EPSILON);
1739
1740        let list = make_list(vec![Field::Str("abc".to_string())]);
1741        assert_eq!(&"abc".to_string(), list.get_string(0).unwrap());
1742
1743        let list = make_list(vec![Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5]))]);
1744        assert_eq!(&[1, 2, 3, 4, 5], list.get_bytes(0).unwrap().data());
1745
1746        let list = make_list(vec![Field::Decimal(Decimal::from_i32(4, 5, 2))]);
1747        assert_eq!(&[0, 0, 0, 4], list.get_decimal(0).unwrap().data());
1748    }
1749
1750    #[test]
1751    fn test_list_primitive_invalid_accessors() {
1752        // primitives
1753        let list = make_list(vec![Field::Bool(false)]);
1754        assert!(list.get_byte(0).is_err());
1755
1756        let list = make_list(vec![Field::Byte(3), Field::Byte(4)]);
1757        assert!(list.get_short(1).is_err());
1758
1759        let list = make_list(vec![Field::Short(4), Field::Short(5), Field::Short(6)]);
1760        assert!(list.get_int(2).is_err());
1761
1762        let list = make_list(vec![Field::Int(5)]);
1763        assert!(list.get_long(0).is_err());
1764
1765        let list = make_list(vec![Field::Long(6), Field::Long(7)]);
1766        assert!(list.get_float(1).is_err());
1767
1768        let list = make_list(vec![Field::UByte(3), Field::UByte(4)]);
1769        assert!(list.get_short(1).is_err());
1770
1771        let list = make_list(vec![Field::UShort(4), Field::UShort(5), Field::UShort(6)]);
1772        assert!(list.get_int(2).is_err());
1773
1774        let list = make_list(vec![Field::UInt(5)]);
1775        assert!(list.get_long(0).is_err());
1776
1777        let list = make_list(vec![Field::ULong(6), Field::ULong(7)]);
1778        assert!(list.get_float(1).is_err());
1779
1780        let list = make_list(vec![Field::Float16(f16::PI)]);
1781        assert!(list.get_string(0).is_err());
1782
1783        let list = make_list(vec![
1784            Field::Float(8.1),
1785            Field::Float(9.2),
1786            Field::Float(10.3),
1787        ]);
1788        assert!(list.get_double(2).is_err());
1789
1790        let list = make_list(vec![Field::Double(PI)]);
1791        assert!(list.get_string(0).is_err());
1792
1793        let list = make_list(vec![Field::Str("abc".to_string())]);
1794        assert!(list.get_bytes(0).is_err());
1795
1796        let list = make_list(vec![Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5]))]);
1797        assert!(list.get_bool(0).is_err());
1798
1799        let list = make_list(vec![Field::Decimal(Decimal::from_i32(4, 5, 2))]);
1800        assert!(list.get_bool(0).is_err());
1801    }
1802
1803    #[test]
1804    fn test_list_complex_accessors() {
1805        let list = make_list(vec![Field::Group(make_row(vec![
1806            ("x".to_string(), Field::Null),
1807            ("Y".to_string(), Field::Int(2)),
1808        ]))]);
1809        assert_eq!(2, list.get_group(0).unwrap().len());
1810
1811        let list = make_list(vec![Field::ListInternal(make_list(vec![
1812            Field::Int(2),
1813            Field::Int(1),
1814            Field::Null,
1815            Field::Int(12),
1816        ]))]);
1817        assert_eq!(4, list.get_list(0).unwrap().len());
1818
1819        let list = make_list(vec![Field::MapInternal(make_map(vec![
1820            (Field::Int(1), Field::Float(1.2)),
1821            (Field::Int(2), Field::Float(4.5)),
1822            (Field::Int(3), Field::Float(2.3)),
1823        ]))]);
1824        assert_eq!(3, list.get_map(0).unwrap().len());
1825    }
1826
1827    #[test]
1828    fn test_list_complex_invalid_accessors() {
1829        let list = make_list(vec![Field::Group(make_row(vec![
1830            ("x".to_string(), Field::Null),
1831            ("Y".to_string(), Field::Int(2)),
1832        ]))]);
1833        assert_eq!(
1834            list.get_float(0).unwrap_err().to_string(),
1835            "Parquet error: Cannot access Group as Float"
1836        );
1837
1838        let list = make_list(vec![Field::ListInternal(make_list(vec![
1839            Field::Int(2),
1840            Field::Int(1),
1841            Field::Null,
1842            Field::Int(12),
1843        ]))]);
1844        assert_eq!(
1845            list.get_float(0).unwrap_err().to_string(),
1846            "Parquet error: Cannot access ListInternal as Float"
1847        );
1848
1849        let list = make_list(vec![Field::MapInternal(make_map(vec![
1850            (Field::Int(1), Field::Float(1.2)),
1851            (Field::Int(2), Field::Float(4.5)),
1852            (Field::Int(3), Field::Float(2.3)),
1853        ]))]);
1854        assert_eq!(
1855            list.get_float(0).unwrap_err().to_string(),
1856            "Parquet error: Cannot access MapInternal as Float",
1857        );
1858    }
1859
1860    #[test]
1861    fn test_map_accessors() {
1862        // a map from int to string
1863        let map = make_map(vec![
1864            (Field::Int(1), Field::Str("a".to_string())),
1865            (Field::Int(2), Field::Str("b".to_string())),
1866            (Field::Int(3), Field::Str("c".to_string())),
1867            (Field::Int(4), Field::Str("d".to_string())),
1868            (Field::Int(5), Field::Str("e".to_string())),
1869        ]);
1870
1871        assert_eq!(5, map.len());
1872        for i in 0..5 {
1873            assert_eq!((i + 1) as i32, map.get_keys().get_int(i).unwrap());
1874            assert_eq!(
1875                &((i as u8 + b'a') as char).to_string(),
1876                map.get_values().get_string(i).unwrap()
1877            );
1878        }
1879    }
1880
1881    #[test]
1882    fn test_to_json_value() {
1883        assert_eq!(Field::Null.to_json_value(), Value::Null);
1884        assert_eq!(Field::Bool(true).to_json_value(), Value::Bool(true));
1885        assert_eq!(Field::Bool(false).to_json_value(), Value::Bool(false));
1886        assert_eq!(
1887            Field::Byte(1).to_json_value(),
1888            Value::Number(serde_json::Number::from(1))
1889        );
1890        assert_eq!(
1891            Field::Short(2).to_json_value(),
1892            Value::Number(serde_json::Number::from(2))
1893        );
1894        assert_eq!(
1895            Field::Int(3).to_json_value(),
1896            Value::Number(serde_json::Number::from(3))
1897        );
1898        assert_eq!(
1899            Field::Long(4).to_json_value(),
1900            Value::Number(serde_json::Number::from(4))
1901        );
1902        assert_eq!(
1903            Field::UByte(1).to_json_value(),
1904            Value::Number(serde_json::Number::from(1))
1905        );
1906        assert_eq!(
1907            Field::UShort(2).to_json_value(),
1908            Value::Number(serde_json::Number::from(2))
1909        );
1910        assert_eq!(
1911            Field::UInt(3).to_json_value(),
1912            Value::Number(serde_json::Number::from(3))
1913        );
1914        assert_eq!(
1915            Field::ULong(4).to_json_value(),
1916            Value::Number(serde_json::Number::from(4))
1917        );
1918        assert_eq!(
1919            Field::Float16(f16::from_f32(5.0)).to_json_value(),
1920            Value::Number(serde_json::Number::from_f64(5.0).unwrap())
1921        );
1922        assert_eq!(
1923            Field::Float(5.0).to_json_value(),
1924            Value::Number(serde_json::Number::from_f64(5.0).unwrap())
1925        );
1926        assert_eq!(
1927            Field::Float(5.1234).to_json_value(),
1928            Value::Number(serde_json::Number::from_f64(5.1234_f32 as f64).unwrap())
1929        );
1930        assert_eq!(
1931            Field::Double(6.0).to_json_value(),
1932            Value::Number(serde_json::Number::from_f64(6.0).unwrap())
1933        );
1934        assert_eq!(
1935            Field::Double(6.1234).to_json_value(),
1936            Value::Number(serde_json::Number::from_f64(6.1234).unwrap())
1937        );
1938        assert_eq!(
1939            Field::Str("abc".to_string()).to_json_value(),
1940            Value::String(String::from("abc"))
1941        );
1942        assert_eq!(
1943            Field::Decimal(Decimal::from_i32(4, 8, 2)).to_json_value(),
1944            Value::String(String::from("0.04"))
1945        );
1946        assert_eq!(
1947            Field::Bytes(ByteArray::from(vec![1, 2, 3])).to_json_value(),
1948            Value::String(String::from("AQID"))
1949        );
1950        assert_eq!(
1951            Field::TimestampMillis(12345678).to_json_value(),
1952            Value::String("1970-01-01 03:25:45 +00:00".to_string())
1953        );
1954        assert_eq!(
1955            Field::TimestampMicros(12345678901).to_json_value(),
1956            Value::String(convert_timestamp_micros_to_string(12345678901))
1957        );
1958
1959        let fields = vec![
1960            ("X".to_string(), Field::Int(1)),
1961            ("Y".to_string(), Field::Double(2.2)),
1962            ("Z".to_string(), Field::Str("abc".to_string())),
1963        ];
1964        let row = Field::Group(make_row(fields));
1965        assert_eq!(
1966            row.to_json_value(),
1967            serde_json::json!({"X": 1, "Y": 2.2, "Z": "abc"})
1968        );
1969
1970        let row = Field::ListInternal(make_list(vec![Field::Int(1), Field::Int(12), Field::Null]));
1971        let array = vec![
1972            Value::Number(serde_json::Number::from(1)),
1973            Value::Number(serde_json::Number::from(12)),
1974            Value::Null,
1975        ];
1976        assert_eq!(row.to_json_value(), Value::Array(array));
1977
1978        let row = Field::MapInternal(make_map(vec![
1979            (Field::Str("k1".to_string()), Field::Double(1.2)),
1980            (Field::Str("k2".to_string()), Field::Double(3.4)),
1981            (Field::Str("k3".to_string()), Field::Double(4.5)),
1982        ]));
1983        assert_eq!(
1984            row.to_json_value(),
1985            serde_json::json!({"k1": 1.2, "k2": 3.4, "k3": 4.5})
1986        );
1987    }
1988}
1989
1990#[cfg(test)]
1991#[allow(clippy::many_single_char_names)]
1992mod api_tests {
1993    use super::{make_list, make_map, make_row};
1994    use crate::record::Field;
1995
1996    #[test]
1997    fn test_field_visibility() {
1998        let row = make_row(vec![(
1999            "a".to_string(),
2000            Field::Group(make_row(vec![
2001                ("x".to_string(), Field::Null),
2002                ("Y".to_string(), Field::Int(2)),
2003            ])),
2004        )]);
2005
2006        match row.get_column_iter().next() {
2007            Some(column) => {
2008                assert_eq!("a", column.0);
2009                match column.1 {
2010                    Field::Group(r) => {
2011                        assert_eq!(
2012                            &make_row(vec![
2013                                ("x".to_string(), Field::Null),
2014                                ("Y".to_string(), Field::Int(2)),
2015                            ]),
2016                            r
2017                        );
2018                    }
2019                    _ => panic!("Expected the first column to be Field::Group"),
2020                }
2021            }
2022            None => panic!("Expected at least one column"),
2023        }
2024    }
2025
2026    #[test]
2027    fn test_list_element_access() {
2028        let expected = vec![
2029            Field::Int(1),
2030            Field::Group(make_row(vec![
2031                ("x".to_string(), Field::Null),
2032                ("Y".to_string(), Field::Int(2)),
2033            ])),
2034        ];
2035
2036        let list = make_list(expected.clone());
2037        assert_eq!(expected.as_slice(), list.elements());
2038    }
2039
2040    #[test]
2041    fn test_map_entry_access() {
2042        let expected = vec![
2043            (Field::Str("one".to_owned()), Field::Int(1)),
2044            (Field::Str("two".to_owned()), Field::Int(2)),
2045        ];
2046
2047        let map = make_map(expected.clone());
2048        assert_eq!(expected.as_slice(), map.entries());
2049    }
2050}
parquet/record/api.rs

parquet/record/
api.rs
