mz_storage/source.rs
1// Copyright Materialize, Inc. and contributors. All rights reserved.
2//
3// Use of this software is governed by the Business Source License
4// included in the LICENSE file.
5//
6// As of the Change Date specified in that file, in accordance with
7// the Business Source License, use of this software will be governed
8// by the Apache License, Version 2.0.
9
10//! Types related to the creation of dataflow raw sources.
11//!
12//! Raw sources are streams (currently, Timely streams) of data directly
13//! produced by the upstream service. The main export of this module is
14//! [`create_raw_source`], which turns
15//! [`RawSourceCreationConfigs`](RawSourceCreationConfig) and
16//! [`SourceConnections`](mz_storage_types::sources::SourceConnection)
17//! implementations into the aforementioned streams.
18//!
19//! The full source, which is the _differential_ stream that represents the
20//! actual object created by a `CREATE SOURCE` statement, is created by
21//! composing [`create_raw_source`] with decoding,
22//! [`SourceEnvelope`](mz_storage_types::sources::SourceEnvelope) rendering, and
23//! more.
24
25// https://github.com/tokio-rs/prost/issues/237
26#![allow(missing_docs)]
27
28use crate::source::types::SourceMessage;
29
30mod probe;
31pub(crate) mod reclock;
32mod source_reader_pipeline;
33mod statistics;
34pub mod types;
35
36pub mod generator;
37mod kafka;
38mod mysql;
39mod postgres;
40mod sql_server;
41
42pub use kafka::KafkaSourceReader;
43pub use source_reader_pipeline::{
44 RawSourceCreationConfig, SourceExportCreationConfig, create_raw_source,
45};