parquet/
thrift.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Custom thrift definitions
19
20pub use thrift::protocol::TCompactOutputProtocol;
21use thrift::protocol::{TInputProtocol, TOutputProtocol};
22
23/// Reads and writes the struct to Thrift protocols.
24///
25/// Unlike [`thrift::protocol::TSerializable`] this uses generics instead of trait objects
26pub trait TSerializable: Sized {
27    /// Reads the struct from the input Thrift protocol
28    fn read_from_in_protocol<T: TInputProtocol>(i_prot: &mut T) -> thrift::Result<Self>;
29    /// Writes the struct to the output Thrift protocol
30    fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()>;
31}
32
33#[cfg(test)]
34mod tests {
35    use crate::{
36        basic::Type,
37        file::page_index::{column_index::ColumnIndexMetaData, index_reader::decode_column_index},
38    };
39
40    #[test]
41    #[allow(deprecated)]
42    fn legacy_statistics_reject_oversized_binary_before_allocation() {
43        use crate::{format::Statistics, thrift::TSerializable};
44        use thrift::{Error, ProtocolErrorKind, protocol::TCompactInputProtocol};
45
46        // Field 1 (max), binary length 100 MiB + 1, and no payload. The length
47        // must be rejected before allocating the claimed buffer or reading it.
48        let bytes = [0x18, 0x81, 0x80, 0x80, 0x32];
49        let mut input = TCompactInputProtocol::new(bytes.as_slice());
50        let error = Statistics::read_from_in_protocol(&mut input).unwrap_err();
51        assert!(
52            matches!(error, Error::Protocol(error) if error.kind == ProtocolErrorKind::SizeLimit)
53        );
54    }
55
56    #[test]
57    #[allow(deprecated)]
58    fn legacy_statistics_compact_protocol_roundtrip() {
59        use crate::{format::Statistics, thrift::TSerializable};
60        use thrift::protocol::{TCompactInputProtocol, TCompactOutputProtocol};
61
62        let expected = Statistics {
63            min_value: Some(vec![0, 1, 255]),
64            max_value: Some(vec![255, 0, 1]),
65            null_count: Some(7),
66            ..Default::default()
67        };
68        let mut bytes = Vec::new();
69        expected
70            .write_to_out_protocol(&mut TCompactOutputProtocol::new(&mut bytes))
71            .unwrap();
72        let actual =
73            Statistics::read_from_in_protocol(&mut TCompactInputProtocol::new(bytes.as_slice()))
74                .unwrap();
75        assert_eq!(actual, expected);
76    }
77
78    #[test]
79    pub fn read_boolean_list_field_type() {
80        // Boolean collection type encoded as 0x01, as used by this crate when writing.
81        // Values encoded as 1 (true) or 2 (false) as in the current version of the thrift
82        // documentation.
83        let bytes = vec![
84            0x19, 0x21, 2, 1, 0x19, 0x28, 1, 0, 0, 0x19, 0x28, 1, 1, 0, 0x15, 0, 0,
85        ];
86        let index = decode_column_index(&bytes, Type::BOOLEAN).unwrap();
87
88        let index = match index {
89            ColumnIndexMetaData::BOOLEAN(index) => index,
90            _ => panic!("expected boolean column index"),
91        };
92
93        // should be false, true
94        assert!(!index.is_null_page(0));
95        assert!(index.is_null_page(1));
96        assert!(!index.min_value(0).unwrap()); // min is false
97        assert!(index.max_value(0).unwrap()); // max is true
98        assert!(index.min_value(1).is_none());
99        assert!(index.max_value(1).is_none());
100    }
101
102    #[test]
103    pub fn read_boolean_list_alternative_encoding() {
104        // Boolean collection type encoded as 0x02, as allowed by the spec.
105        // Values encoded as 1 (true) or 0 (false) as before the thrift documentation change on 2024-12-13.
106        let bytes = vec![
107            0x19, 0x22, 0, 1, 0x19, 0x28, 1, 0, 0, 0x19, 0x28, 1, 1, 0, 0x15, 0, 0,
108        ];
109        let index = decode_column_index(&bytes, Type::BOOLEAN).unwrap();
110
111        let index = match index {
112            ColumnIndexMetaData::BOOLEAN(index) => index,
113            _ => panic!("expected boolean column index"),
114        };
115
116        // should be false, true
117        assert!(!index.is_null_page(0));
118        assert!(index.is_null_page(1));
119        assert!(!index.min_value(0).unwrap()); // min is false
120        assert!(index.max_value(0).unwrap()); // max is true
121        assert!(index.min_value(1).is_none());
122        assert!(index.max_value(1).is_none());
123    }
124}
parquet/thrift.rs

parquet/
thrift.rs
