mz_sql/
lib.rs

Help
1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! SQL-dataflow translation.
11//!
12//! There are two main parts of the SQL–dataflow translation process:
13//!
14//!   * **Purification** eliminates any external state from a SQL AST. It is an
15//!     asynchronous process that may make network calls to external services.
16//!     The input and output of purification is a SQL AST.
17//!
18//!   * **Planning** converts a purified AST to a [`Plan`], which describes an
19//!     action that the system should take to effect the results of the query.
20//!     Planning is a fast, pure function that always produces the same plan for
21//!     a given input.
22//!
23//! # Details
24//!
25//! The purification step is, to our knowledge, unique to Materialize. In other
26//! SQL databases, there is no concept of purifying a statement before planning
27//! it. The reason for this difference is that in Materialize SQL statements can
28//! depend on external state: local files, Confluent Schema Registries, etc.
29//!
30//! Presently only `CREATE SOURCE` statements can depend on external state,
31//! though this could change in the future. Consider, for example:
32//!
33//! ```sql
34//! CREATE SOURCE ... FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://csr:8081'
35//! ```
36//!
37//! The shape of the created source is dependent on the Avro schema that is
38//! stored in the schema registry running at `csr:8081`.
39//!
40//! This is problematic, because we need planning to be a pure function of its
41//! input. Why?
42//!
43//!   * Planning locks the catalog while it operates. Therefore it needs to be
44//!     fast, because only one SQL query can be planned at a time. Depending on
45//!     external state while holding a lock on the catalog would be seriously
46//!     detrimental to the latency of other queries running on the system.
47//!
48//!   * The catalog persists SQL ASTs across restarts of Materialize. If those
49//!     ASTs depend on external state, then changes to that external state could
50//!     corrupt Materialize's catalog.
51//!
52//! Purification is the escape hatch. It is a transformation from SQL AST to SQL
53//! AST that "inlines" any external state. For example, we purify the schema
54//! above by fetching the schema from the schema registry and inlining it.
55//!
56//! ```sql
57//! CREATE SOURCE ... FORMAT AVRO USING SCHEMA '{"name": "foo", "fields": [...]}'
58//! ```
59//!
60//! Importantly, purification cannot hold its reference to the catalog across an
61//! await point. That means it can run in its own Tokio task so that it does not
62//! block any other SQL commands on the server.
63//!
64//! [`Plan`]: crate::plan::Plan
65
66#![warn(missing_debug_implementations)]
67
68macro_rules! bail_unsupported {
69    ($feature:expr) => {
70        return Err(crate::plan::error::PlanError::Unsupported {
71            feature: $feature.to_string(),
72            discussion_no: None,
73        }
74        .into())
75    };
76    ($discussion_no:expr, $feature:expr) => {
77        return Err(crate::plan::error::PlanError::Unsupported {
78            feature: $feature.to_string(),
79            discussion_no: Some($discussion_no),
80        }
81        .into())
82    };
83}
84
85macro_rules! bail_never_supported {
86    ($feature:expr, $docs:expr, $details:expr) => {
87        return Err(crate::plan::error::PlanError::NeverSupported {
88            feature: $feature.to_string(),
89            documentation_link: Some($docs.to_string()),
90            details: Some($details.to_string()),
91        }
92        .into())
93    };
94    ($feature:expr, $docs:expr) => {
95        return Err(crate::plan::error::PlanError::NeverSupported {
96            feature: $feature.to_string(),
97            documentation_link: Some($docs.to_string()),
98            details: None,
99        }
100        .into())
101    };
102    ($feature:expr) => {
103        return Err(crate::plan::error::PlanError::NeverSupported {
104            feature: $feature.to_string(),
105            documentation_link: None,
106            details: None,
107        }
108        .into())
109    };
110}
111
112// TODO(benesch): delete these macros once we use structured errors everywhere.
113macro_rules! sql_bail {
114    ($($e:expr),* $(,)?) => {
115        return Err(sql_err!($($e),*))
116    }
117}
118macro_rules! sql_err {
119    ($($e:expr),* $(,)?) => {
120        crate::plan::error::PlanError::Unstructured(format!($($e),*))
121    }
122}
123
124pub const DEFAULT_SCHEMA: &str = "public";
125
126/// The number of concurrent requests we allow at once for webhook sources.
127pub const WEBHOOK_CONCURRENCY_LIMIT: usize = 500;
128
129pub mod ast;
130pub mod catalog;
131pub mod func;
132pub mod kafka_util;
133pub mod names;
134#[macro_use]
135pub mod normalize;
136pub mod optimizer_metrics;
137pub mod parse;
138pub mod plan;
139pub mod pure;
140pub mod rbac;
141pub mod session;
mz_sql/lib.rs

mz_sql/
lib.rs
