mz_persist_proc/
lib.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
// Copyright Materialize, Inc. and contributors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License in the LICENSE file at the
// root of this repository, or online at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Internal utility proc-macros for persist.
//!
//! Note: This is separate from the `mz_persist_client` crate because
//! `proc-macro` crates are only allowed to export procedural macros and nothing
//! else.

use proc_macro::TokenStream;
use proc_macro2::TokenStream as TokenStream2;
use quote::quote;
use syn::{parse_macro_input, ItemFn, ReturnType};

/// Persist wrapper around the `test` macro.
///
/// The wrapper automatically runs the test with various interesting
/// configurations.
#[proc_macro_attribute]
pub fn test(attr: TokenStream, item: TokenStream) -> TokenStream {
    test_impl(attr, item)
}

/// Implementation for the `#[mz_persist_proc::test]` macro.
fn test_impl(attr: TokenStream, item: TokenStream) -> TokenStream {
    let args = TokenStream2::from(attr);
    let item = parse_macro_input!(item as ItemFn);

    let attrs = &item.attrs;
    let async_ = &item.sig.asyncness;
    let await_ = if async_.is_some() {
        quote! {.await}
    } else {
        quote! {}
    };
    let inputs = &item.sig.inputs;
    let body = &item.block;
    let test_name = &item.sig.ident;

    // Note that Rust does not allow us to have a test function with
    // #[should_panic] that has a non-unit return value.
    let ret = match &item.sig.output {
        ReturnType::Default => quote! {},
        ReturnType::Type(_, type_) => quote! {-> #type_},
    };

    quote! {
        #[::mz_ore::test(
            #args
        )]
        #(#attrs)*
        #async_ fn #test_name() #ret {
            #async_ fn test_impl(#inputs) #ret {
              #body
            }

            let dyncfgs = [
                {
                    // Inline writes disabled
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_inline_writes_single_max_bytes", ::mz_dyncfg::ConfigVal::Usize(0));
                    x.add_dynamic("persist_inline_writes_total_max_bytes", ::mz_dyncfg::ConfigVal::Usize(0));
                    x
                },
                {
                    // Inline writes enabled
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_inline_writes_single_max_bytes", ::mz_dyncfg::ConfigVal::Usize(4 * 1024));
                    x.add_dynamic("persist_inline_writes_total_max_bytes", ::mz_dyncfg::ConfigVal::Usize(1024 * 1024));
                    x
                },
                {
                    // Stress inline writes backpressure
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_inline_writes_single_max_bytes", ::mz_dyncfg::ConfigVal::Usize(4 * 1024));
                    x.add_dynamic("persist_inline_writes_total_max_bytes", ::mz_dyncfg::ConfigVal::Usize(0));
                    x
                },
                {
                    // Enable new compaction tracking / claiming
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_record_compactions", ::mz_dyncfg::ConfigVal::Bool(true));
                    x.add_dynamic("persist_claim_unclaimed_compactions", ::mz_dyncfg::ConfigVal::Bool(true));
                    x
                },
                {
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_record_schema_id", ::mz_dyncfg::ConfigVal::Bool(true));
                    x
                },
                {
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_batch_columnar_format", ::mz_dyncfg::ConfigVal::String("both_v2".into()));
                    x.add_dynamic("persist_batch_columnar_format_percent", ::mz_dyncfg::ConfigVal::Usize(100));
                    x.add_dynamic("persist_part_decode_format", ::mz_dyncfg::ConfigVal::String("arrow".into()));
                    x
                },
                {
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_batch_columnar_format", ::mz_dyncfg::ConfigVal::String("both_v2".into()));
                    x.add_dynamic("persist_batch_columnar_format_percent", ::mz_dyncfg::ConfigVal::Usize(100));
                    x
                },
                {
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_batch_columnar_format", ::mz_dyncfg::ConfigVal::String("both_v2".into()));
                    x.add_dynamic("persist_batch_columnar_format_percent", ::mz_dyncfg::ConfigVal::Usize(100));
                    x.add_dynamic("persist_batch_structured_key_lower_len", ::mz_dyncfg::ConfigVal::Usize(256));
                    x.add_dynamic("persist_batch_structured_order", ::mz_dyncfg::ConfigVal::Bool(true));
                    x
                },
                {
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_encoding_enable_dictionary", ::mz_dyncfg::ConfigVal::Bool(true));
                    x
                },
                {
                    let mut x = ::mz_dyncfg::ConfigUpdates::default();
                    x.add_dynamic("persist_batch_max_run_len", ::mz_dyncfg::ConfigVal::Usize(4));
                    x
                },
            ];

            for (idx, dyncfgs) in dyncfgs.into_iter().enumerate() {
                let debug = dyncfgs.updates.iter().map(|(name, val)| {
                    format!(" {}={:?}", name, val.val.clone().unwrap())
                }).collect::<String>();
                eprintln!("mz_persist_proc::test {}{}", idx, debug);
                test_impl(dyncfgs)#await_
            }
          }
    }
    .into()
}