mz_persist/mnt/build/debug/build/mz-persist-a60b618bb57e1ffe/out/
mz_persist.gen.persist.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
// This file is @generated by prost-build.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoU64Antichain {
    #[prost(uint64, repeated, tag = "1")]
    pub elements: ::prost::alloc::vec::Vec<u64>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoU64Description {
    #[prost(message, optional, tag = "1")]
    pub lower: ::core::option::Option<ProtoU64Antichain>,
    #[prost(message, optional, tag = "2")]
    pub upper: ::core::option::Option<ProtoU64Antichain>,
    #[prost(message, optional, tag = "3")]
    pub since: ::core::option::Option<ProtoU64Antichain>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoBatchPartInline {
    #[prost(enumeration = "ProtoBatchFormat", tag = "1")]
    pub format: i32,
    /// TraceBatchParts can contain partial data for a given trace batch, and so
    /// this desc means only that the records contained in this part have to
    /// abide by the constraints in the description. There may be other parts
    /// for the same trace batch with the same description. However, there will
    /// be only one trace batch with the same description and index.
    #[prost(message, optional, tag = "2")]
    pub desc: ::core::option::Option<ProtoU64Description>,
    #[prost(uint64, tag = "3")]
    pub index: u64,
    /// Optional metadata for the `format`.
    #[prost(oneof = "proto_batch_part_inline::FormatMetadata", tags = "4")]
    pub format_metadata: ::core::option::Option<proto_batch_part_inline::FormatMetadata>,
}
/// Nested message and enum types in `ProtoBatchPartInline`.
pub mod proto_batch_part_inline {
    /// Optional metadata for the `format`.
    #[derive(Clone, Copy, PartialEq, ::prost::Oneof)]
    pub enum FormatMetadata {
        /// Metadata for the structured format with `\[(K, V, T, D, K_S, V_S)\]`.
        #[prost(uint64, tag = "4")]
        StructuredMigration(u64),
    }
}
#[derive(serde::Serialize)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoColumnarRecords {
    #[prost(uint64, tag = "1")]
    pub len: u64,
    #[prost(int32, repeated, tag = "2")]
    pub key_offsets: ::prost::alloc::vec::Vec<i32>,
    #[prost(bytes = "bytes", tag = "3")]
    pub key_data: ::prost::bytes::Bytes,
    #[prost(int32, repeated, tag = "4")]
    pub val_offsets: ::prost::alloc::vec::Vec<i32>,
    #[prost(bytes = "bytes", tag = "5")]
    pub val_data: ::prost::bytes::Bytes,
    #[prost(int64, repeated, tag = "6")]
    pub timestamps: ::prost::alloc::vec::Vec<i64>,
    #[prost(int64, repeated, tag = "7")]
    pub diffs: ::prost::alloc::vec::Vec<i64>,
    #[prost(message, optional, tag = "8")]
    pub key_structured: ::core::option::Option<
        ::mz_persist_types::arrow::ProtoArrayData,
    >,
    #[prost(message, optional, tag = "9")]
    pub val_structured: ::core::option::Option<
        ::mz_persist_types::arrow::ProtoArrayData,
    >,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
#[repr(i32)]
pub enum ProtoBatchFormat {
    Unknown = 0,
    /// Arrow, which we'd use for the local blob cache if we use it, gets a
    /// structure like `\[(K, V, T, D)\]` so that we could mmap it and use it
    /// directly as our in-mem batches (which have this structure).
    ArrowKvtd = 1,
    /// We have more flexibility with Parquet. Initially we'll start with the
    /// same `\[(K, V, T, D)\]` as our in-mem batches. Another option would be
    /// something like `\[(K, [(V, [(T, D)\])])]`, which would only store each
    /// key's and each val's data once (this is similar to the
    /// \[differential_dataflow::trace::layers::Trie\] structure of
    /// \[differential_dataflow::trace::implementations::ord::OrdValBatch\]).
    ///
    /// Which is better probably comes down to how much duplication we expect of
    /// keys and vals in a batch as well as how big the batches are (the trie
    /// structure introduces more columns, so has some amount of overhead).
    ///
    /// For unsealed batches, we have a better chance of duplicates than trace,
    /// but we probably don't want to pay the cost of converting between the
    /// in-mem `\[(K, V, T, D)\]` representation and anything else (to keep the hot
    /// path clean). Unsealed batches are also likely to be our smallest. For
    /// this reason, they'll probably always stay as ParquetKvtd.
    ///
    /// For trace batches, we consolidate them before writing them out, so we're
    /// guaranteed to get nothing from the V level of the trie. For duplicate
    /// keys, we'll probably get a good amount of benefit from column specific
    /// compression, and I'd like to exhaust that direction first before dealing
    /// with a trie-like column structure.
    ParquetKvtd = 2,
    /// Parquet format that understands the structure of the inner data. See the
    /// `metadata` field on `ProtoBatchPartInline` for more information about
    /// how this data is structured in Parquet.
    ///
    /// For example, the initial use of this format will contain the columns
    /// `\[(K, V, T, D, K_S, V_S)\]` where `K_S` and `V_S` are structured versions
    /// of the opaque data stored in `K` and `V`, respectively. Eventually we'll
    /// stop writing `K` and `V` and migrate entirely to `K_S` and `V_S`.
    ParquetStructured = 3,
}
impl ProtoBatchFormat {
    /// String value of the enum field names used in the ProtoBuf definition.
    ///
    /// The values are not transformed in any way and thus are considered stable
    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
    pub fn as_str_name(&self) -> &'static str {
        match self {
            Self::Unknown => "Unknown",
            Self::ArrowKvtd => "ArrowKVTD",
            Self::ParquetKvtd => "ParquetKvtd",
            Self::ParquetStructured => "ParquetStructured",
        }
    }
    /// Creates an enum from field names used in the ProtoBuf definition.
    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
        match value {
            "Unknown" => Some(Self::Unknown),
            "ArrowKVTD" => Some(Self::ArrowKvtd),
            "ParquetKvtd" => Some(Self::ParquetKvtd),
            "ParquetStructured" => Some(Self::ParquetStructured),
            _ => None,
        }
    }
}