mz_catalog_protos/
lib.rs

1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! All of the protobuf types we durably persist in the Catalog.
11//!
12//! These exist outside of the `mz_catalog` crate because the code generated by
13//! [`prost`] for the protobuf files, takes a considerable (>40s) amount of
14//! time to `build` and even `check`, yet the protobuf files rarely change. So
15//! moving them into a separate crate allows folks to iterate on the
16//! `mz_catalog` crate without having to pay the cost of re-compiling these
17//! protobuf files.
18
19pub mod audit_log;
20pub mod serialization;
21
22/// The current version of the `Catalog`.
23///
24/// We will initialize new `Catalog`s with this version, and migrate existing `Catalog`s to this
25/// version. Whenever the `Catalog` changes, e.g. the protobufs we serialize in the `Catalog`
26/// change, we need to bump this version.
27pub const CATALOG_VERSION: u64 = 77;
28
29/// The minimum `Catalog` version number that we support migrating from.
30///
31/// After bumping this we can delete the old migrations.
32pub const MIN_CATALOG_VERSION: u64 = 67;
33
34macro_rules! proto_objects {
35    ( $( $x:ident ),* ) => {
36        paste::paste! {
37            $(
38                pub mod [<objects_ $x>] {
39                    include!(concat!(env!("OUT_DIR"), "/objects_", stringify!($x), ".rs"));
40                }
41            )*
42            pub mod objects {
43                include!(concat!(env!("OUT_DIR"), "/objects.rs"));
44            }
45        }
46    };
47}
48
49proto_objects!(v67, v68, v69, v70, v71, v72, v73, v74, v75, v76, v77);
50
51#[cfg(test)]
52mod tests {
53    use std::collections::BTreeSet;
54    use std::fs;
55    use std::io::{BufRead, BufReader};
56
57    use crate::{CATALOG_VERSION, MIN_CATALOG_VERSION};
58
59    // Note: Feel free to update this path if the protos move.
60    const PROTO_DIRECTORY: &str = "protos";
61
62    #[mz_ore::test]
63    fn test_assert_snapshots_exist() {
64        // Get all of the files in the snapshot directory, with the `.proto` extension.
65        let mut filenames: BTreeSet<_> = fs::read_dir(PROTO_DIRECTORY)
66            .expect("failed to read protos dir")
67            .map(|entry| entry.expect("failed to read dir entry").file_name())
68            .map(|filename| filename.to_str().expect("utf8").to_string())
69            .filter(|filename| filename.ends_with("proto"))
70            .collect();
71
72        // Assert objects.proto exists.
73        assert!(filenames.remove("objects.proto"));
74
75        // Assert snapshots exist for all of the versions we support.
76        for version in MIN_CATALOG_VERSION..=CATALOG_VERSION {
77            let filename = format!("objects_v{version}.proto");
78            assert!(
79                filenames.remove(&filename),
80                "Missing snapshot for v{version}."
81            );
82        }
83
84        // Common case. Check to make sure the user bumped the CATALOG_VERSION.
85        if !filenames.is_empty()
86            && filenames.remove(&format!("objects_v{}.proto", CATALOG_VERSION + 1))
87        {
88            panic!(
89                "Found snapshot for v{}, please also bump `CATALOG_VERSION`.",
90                CATALOG_VERSION + 1
91            )
92        }
93
94        // Assert there aren't any extra snapshots.
95        assert!(
96            filenames.is_empty(),
97            "Found snapshots for unsupported catalog versions {filenames:?}.\nIf you just increased `MIN_CATALOG_VERSION`, then please delete the old snapshots. If you created a new snapshot, please bump `CATALOG_VERSION`."
98        );
99    }
100
101    #[mz_ore::test]
102    fn test_assert_current_snapshot() {
103        // Read the content from both files.
104        let current = fs::File::open(format!("{PROTO_DIRECTORY}/objects.proto"))
105            .map(BufReader::new)
106            .expect("read current");
107        let snapshot = fs::File::open(format!(
108            "{PROTO_DIRECTORY}/objects_v{CATALOG_VERSION}.proto"
109        ))
110        .map(BufReader::new)
111        .expect("read snapshot");
112
113        // Read in all of the lines so we can compare the content of †he files.
114        let current: Vec<_> = current
115            .lines()
116            .map(|r| r.expect("failed to read line from current"))
117            // Filter out the package name, since we expect that to be different.
118            .filter(|line| line != "package objects;")
119            .collect();
120        let snapshot: Vec<_> = snapshot
121            .lines()
122            .map(|r| r.expect("failed to read line from current"))
123            // Filter out the package name, since we expect that to be different.
124            .filter(|line| line != &format!("package objects_v{CATALOG_VERSION};"))
125            .collect();
126
127        // Note: objects.proto and objects_v<CATALOG_VERSION>.proto should be exactly the same. The
128        // reason being, when bumping the catalog to the next version, CATALOG_VERSION + 1, we need a
129        // snapshot to migrate _from_, which should be a snapshot of how the protos are today.
130        // Hence why the two files should be exactly the same.
131        similar_asserts::assert_eq!(current, snapshot);
132    }
133}
mz_catalog_protos/lib.rs

mz_catalog_protos/
lib.rs
