mz_storage/source/

postgres.rs

// Copyright Materialize, Inc. and contributors. All rights reserved.
//
// Use of this software is governed by the Business Source License
// included in the LICENSE file.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0.

//! Code to render the ingestion dataflow of a [`PostgresSourceConnection`]. The dataflow consists
//! of multiple operators in order to take advantage of all the available workers.
//!
//! # Snapshot
//!
//! One part of the dataflow deals with snapshotting the tables involved in the ingestion. Each
//! table is partitioned across all workers using PostgreSQL's `ctid` column to identify row
//! ranges. Each worker fetches its assigned range using a `COPY` query with ctid filtering,
//! enabling parallel snapshotting of large tables.
//!
//! For all tables that ended up being snapshotted the snapshot reader also emits a rewind request
//! to the replication reader which will ensure that the requested portion of the replication
//! stream is subtracted from the snapshot.
//!
//! See the [snapshot] module for more information on the snapshot strategy.
//!
//! # Replication
//!
//! The other part of the dataflow deals with reading the logical replication slot, which must
//! happen from a single worker. The minimum amount of processing is performed from that worker
//! and the data is then distributed among all workers for decoding.
//!
//! See the [replication] module for more information on the replication strategy.
//!
//! # Error handling
//!
//! There are two kinds of errors that can happen during ingestion that are represented as two
//! separate error types:
//!
//! [`DefiniteError`]s are errors that happen during processing of a specific
//! collection record at a specific LSN. These are the only errors that can ever end up in the
//! error collection of a subsource.
//!
//! Transient errors are any errors that can happen for reasons that are unrelated to the data
//! itself. This could be authentication failures, connection failures, etc. The only operators
//! that can emit such errors are the `TableReader` and the `ReplicationReader` operators, which
//! are the ones that talk to the external world. Both of these operators are built with the
//! `AsyncOperatorBuilder::build_fallible` method which allows transient errors to be propagated
//! upwards with the standard `?` operator without risking downgrading the capability and producing
//! bogus frontiers.
//!
//! The error streams from both of those operators are published to the source status and also
//! trigger a restart of the dataflow.
//!
//! ```text
//!    ┏━━━━━━━━━━━━━━┓
//!    ┃    table     ┃
//!    ┃    reader    ┃
//!    ┗━┯━━━━━━━━━━┯━┛
//!      │          │rewind
//!      │          │requests
//!      │          ╰────╮
//!      │             ┏━v━━━━━━━━━━━┓
//!      │             ┃ replication ┃
//!      │             ┃   reader    ┃
//!      │             ┗━┯━━━━━━━━━┯━┛
//!  COPY│           slot│         │
//!  data│           data│         │
//! ┏━━━━v━━━━━┓ ┏━━━━━━━v━━━━━┓   │
//! ┃  COPY    ┃ ┃ replication ┃   │
//! ┃ decoder  ┃ ┃   decoder   ┃   │
//! ┗━━━━┯━━━━━┛ ┗━━━━━┯━━━━━━━┛   │
//!      │snapshot     │replication│
//!      │updates      │updates    │
//!      ╰────╮    ╭───╯           │
//!          ╭┴────┴╮              │
//!          │concat│              │
//!          ╰──┬───╯              │
//!             │ data             │progress
//!             │ outputs          │output
//!             │ (one per export) │
//!             v                  v
//! ```
//!
//! Both the snapshot and the replication operators emit one output stream per source export, and
//! the streams of each export are concatenated pairwise.

use std::collections::BTreeMap;
use std::rc::Rc;
use std::time::Duration;

use itertools::Itertools as _;
use mz_expr::EvalError;
use mz_ore::error::ErrorExt;
use mz_postgres_util::desc::PostgresTableDesc;
use mz_postgres_util::schema_change::SchemaChangeError;
use mz_postgres_util::{
    Client, PostgresError, Sql, query_opt, simple_query, simple_query_opt, sql,
};
use mz_repr::{Datum, GlobalId, Row};
use mz_storage_types::errors::{DataflowError, SourceError, SourceErrorDetails};
use mz_storage_types::sources::casts::StorageScalarExpr;
use mz_storage_types::sources::postgres::CastType;
use mz_storage_types::sources::{
    MzOffset, PostgresSourceConnection, SourceExport, SourceExportDetails, SourceTimestamp,
};
use mz_timely_util::builder_async::PressOnDropButton;
use serde::{Deserialize, Serialize};
use timely::dataflow::operators::Concat;
use timely::dataflow::operators::vec::{Map, ToStream};
use timely::dataflow::{Scope, StreamVec};
use timely::progress::{Antichain, Timestamp};
use tokio_postgres::SimpleQueryRow;
use tokio_postgres::error::SqlState;
use tokio_postgres::types::PgLsn;

use crate::healthcheck::{HealthStatusMessage, HealthStatusUpdate, StatusNamespace};
use crate::source::types::{Probe, ResumeUppers, SourceRender, StackedCollection};
use crate::source::{RawSourceCreationConfig, SourceMessage};

mod replication;
mod snapshot;

impl SourceRender for PostgresSourceConnection {
    type Time = MzOffset;

    const STATUS_NAMESPACE: StatusNamespace = StatusNamespace::Postgres;

    /// Render the ingestion dataflow. This function only connects things together and contains no
    /// actual processing logic.
    fn render<'scope>(
        self,
        scope: Scope<'scope, MzOffset>,
        config: &RawSourceCreationConfig,
        resume_uppers: impl futures::Stream<Item = ResumeUppers<MzOffset>> + 'static,
        _start_signal: impl std::future::Future<Output = ()> + 'static,
    ) -> (
        BTreeMap<
            GlobalId,
            StackedCollection<'scope, MzOffset, Result<SourceMessage, DataflowError>>,
        >,
        StreamVec<'scope, MzOffset, HealthStatusMessage>,
        StreamVec<'scope, MzOffset, Probe<MzOffset>>,
        Vec<PressOnDropButton>,
    ) {
        // Collect the source outputs that we will be exporting into a per-table map.
        let mut table_info = BTreeMap::new();
        for (idx, (id, export)) in config.source_exports.iter().enumerate() {
            let SourceExport {
                details,
                storage_metadata: _,
                data_config: _,
            } = export;
            let details = match details {
                SourceExportDetails::Postgres(details) => details,
                // This is an export that doesn't need any data output to it.
                SourceExportDetails::None => continue,
                _ => panic!("unexpected source export details: {:?}", details),
            };
            let desc = details.table.clone();
            let casts = details.column_casts.clone();
            let resume_upper = Antichain::from_iter(
                config
                    .source_resume_uppers
                    .get(id)
                    .expect("all source exports must be present in source resume uppers")
                    .iter()
                    .map(MzOffset::decode_row),
            );
            let output = SourceOutputInfo {
                desc,
                projection: None,
                casts,
                resume_upper,
                export_id: id.clone(),
                initial_lsn: details.initial_lsn.unwrap_or_else(MzOffset::minimum),
            };
            table_info
                .entry(output.desc.oid)
                .or_insert_with(BTreeMap::new)
                .insert(idx, output);
        }

        let metrics = config.metrics.get_postgres_source_metrics(config.id);

        let (snapshot_updates, rewinds, slot_ready, snapshot_err, snapshot_token) =
            snapshot::render(
                scope.clone(),
                config.clone(),
                self.clone(),
                table_info.clone(),
                metrics.snapshot_metrics.clone(),
            );

        let (repl_updates, probe_stream, repl_err, repl_token) = replication::render(
            scope.clone(),
            config.clone(),
            self,
            table_info,
            rewinds,
            slot_ready,
            resume_uppers,
            metrics,
        );

        // Both operators produce one stream per export, in output index order, which matches the
        // iteration order of `source_exports`.
        let mut data_collections = BTreeMap::new();
        for ((id, snapshot_updates), repl_updates) in config
            .source_exports
            .keys()
            .zip_eq(snapshot_updates)
            .zip_eq(repl_updates)
        {
            data_collections.insert(*id, snapshot_updates.concat(repl_updates));
        }

        let export_ids = config.source_exports.keys().copied();
        let health_init = export_ids
            .map(Some)
            .chain(std::iter::once(None))
            .map(|id| HealthStatusMessage {
                id,
                namespace: Self::STATUS_NAMESPACE,
                update: HealthStatusUpdate::Running,
            })
            .collect::<Vec<_>>()
            .to_stream(scope);

        // N.B. Note that we don't check ssh tunnel statuses here. We could, but immediately on
        // restart we are going to set the status to an ssh error correctly, so we don't do this
        // extra work.
        let errs = snapshot_err.concat(repl_err).map(move |err| {
            // This update will cause the dataflow to restart
            let err_string = err.display_with_causes().to_string();
            let hint = match &err {
                ReplicationError::Definite(err) => err.hint(),
                ReplicationError::Transient(_) => None,
            };
            let update = HealthStatusUpdate::halting(err_string.clone(), hint);

            let namespace = match err {
                ReplicationError::Transient(err)
                    if matches!(
                        &*err,
                        TransientError::PostgresError(PostgresError::Ssh(_))
                            | TransientError::PostgresError(PostgresError::SshIo(_))
                    ) =>
                {
                    StatusNamespace::Ssh
                }
                _ => Self::STATUS_NAMESPACE,
            };

            HealthStatusMessage {
                id: None,
                namespace: namespace.clone(),
                update,
            }
        });

        let health = health_init.concat(errs);

        (
            data_collections,
            health,
            probe_stream,
            vec![snapshot_token, repl_token],
        )
    }
}

#[derive(Clone, Debug)]
struct SourceOutputInfo {
    /// The expected upstream schema of this output.
    desc: PostgresTableDesc,
    /// A projection of the upstream columns into the columns expected by this output. This field
    /// is recalculated every time we observe an upstream schema change. On dataflow initialization
    /// this field is None since we haven't yet observed any schemas.
    projection: Option<Vec<usize>>,
    casts: Vec<(CastType, StorageScalarExpr)>,
    resume_upper: Antichain<MzOffset>,
    export_id: GlobalId,
    /// An upper bound on the LSN whose upstream schema `desc` describes, read during purification
    /// once `desc` was in hand. Outputs created before this was recorded fall back to
    /// [`MzOffset::minimum`], which ignores nothing.
    initial_lsn: MzOffset,
}

impl SourceOutputInfo {
    /// Whether this output must skip replication messages committed at `commit_lsn`.
    ///
    /// Such messages fall outside the range this output describes. Its snapshot is taken at an LSN
    /// at or after `initial_lsn`, so the rewind that subtracts the replication stream from that
    /// snapshot stops at the same point. They also need not line up with `desc`, since the
    /// upstream schema may have changed between them and `initial_lsn`.
    ///
    /// The comparison is strict because `initial_lsn` is an upper. Everything committed when it
    /// was read is strictly below it, so a transaction landing exactly on it is one this output
    /// has to ingest.
    fn ignores(&self, commit_lsn: MzOffset) -> bool {
        commit_lsn < self.initial_lsn
    }
}

#[derive(Clone, Debug, thiserror::Error)]
pub enum ReplicationError {
    #[error(transparent)]
    Transient(#[from] Rc<TransientError>),
    #[error(transparent)]
    Definite(#[from] Rc<DefiniteError>),
}

/// A transient error that never ends up in the collection of a specific table.
#[derive(Debug, thiserror::Error)]
pub enum TransientError {
    #[error("replication slot mysteriously missing")]
    MissingReplicationSlot,
    #[error(
        "slot overcompacted. Requested LSN {requested_lsn} but only LSNs >= {available_lsn} are available"
    )]
    OvercompactedReplicationSlot {
        requested_lsn: MzOffset,
        available_lsn: MzOffset,
    },
    #[error("replication slot already exists")]
    ReplicationSlotAlreadyExists,
    #[error(
        "creating replication slot {slot} timed out after {timeout:?}. Slot creation waits for \
         in-progress upstream transactions to end"
    )]
    ReplicationSlotCreationTimeout { slot: String, timeout: Duration },
    #[error("stream ended prematurely")]
    ReplicationEOF,
    #[error("unexpected replication message")]
    UnknownReplicationMessage,
    #[error("unexpected logical replication message")]
    UnknownLogicalReplicationMessage,
    #[error("received replication event outside of transaction")]
    BareTransactionEvent,
    #[error("lsn mismatch between BEGIN and COMMIT")]
    InvalidTransaction,
    #[error("BEGIN within existing BEGIN stream")]
    NestedTransaction,
    #[error("recoverable errors should crash the process during snapshots")]
    SyntheticError,
    #[error("sql client error")]
    SQLClient(#[from] tokio_postgres::Error),
    #[error(transparent)]
    PostgresError(#[from] PostgresError),
    #[error(transparent)]
    Generic(#[from] anyhow::Error),
}

/// A definite error that always ends up in the collection of a specific table.
#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)]
pub enum DefiniteError {
    #[error("slot compacted past snapshot point. snapshot consistent point={0} resume_lsn={1}")]
    SlotCompactedPastResumePoint(MzOffset, MzOffset),
    #[error("table was truncated")]
    TableTruncated,
    #[error("table was dropped")]
    TableDropped,
    #[error("publication {0:?} does not exist")]
    PublicationDropped(String),
    #[error("replication slot has been invalidated because it exceeded the maximum reserved size")]
    InvalidReplicationSlot,
    #[error("unexpected number of columns while parsing COPY output")]
    MissingColumn,
    #[error("failed to parse COPY protocol")]
    InvalidCopyInput,
    #[error(
        "unsupported action: database restored from point-in-time backup. Expected timeline ID {expected} but got {actual}"
    )]
    InvalidTimelineId { expected: u64, actual: u64 },
    #[error(
        "unsupported action: upstream went back in time. Expected a snapshot at or after LSN {initial_lsn} but the snapshot was taken at {snapshot_lsn}"
    )]
    InvalidSnapshotLsn {
        initial_lsn: MzOffset,
        snapshot_lsn: MzOffset,
    },
    #[error(
        "unsupported action: upstream physical replica status changed (e.g. a physical replica was promoted to a primary). Expected pg_is_in_recovery()={expected} but got {actual}"
    )]
    InvalidPhysicalReplica { expected: bool, actual: bool },
    #[error(
        "TOASTed value missing from old row. Did you forget to set REPLICA IDENTITY to FULL for your table?"
    )]
    MissingToast,
    #[error(
        "old row missing from replication stream. Did you forget to set REPLICA IDENTITY to FULL for your table?"
    )]
    DefaultReplicaIdentity,
    #[error(transparent)]
    IncompatibleSchema(SchemaChangeError),
    #[error("invalid UTF8 string: {0:?}")]
    InvalidUTF8(Vec<u8>),
    #[error("failed to cast raw column: {0}")]
    CastError(#[source] EvalError),
    #[error("unexpected binary data in replication stream")]
    UnexpectedBinaryData,
}

impl DefiniteError {
    fn hint(&self) -> Option<String> {
        match self {
            DefiniteError::IncompatibleSchema(err) => err.hint(),
            _ => None,
        }
    }
}

impl From<DefiniteError> for DataflowError {
    fn from(err: DefiniteError) -> Self {
        let m = err.to_string().into();
        DataflowError::SourceError(Box::new(SourceError {
            hint: err.hint().map(Into::into),
            error: match &err {
                DefiniteError::SlotCompactedPastResumePoint(_, _) => SourceErrorDetails::Other(m),
                DefiniteError::TableTruncated => SourceErrorDetails::Other(m),
                DefiniteError::TableDropped => SourceErrorDetails::Other(m),
                DefiniteError::PublicationDropped(_) => SourceErrorDetails::Initialization(m),
                DefiniteError::InvalidReplicationSlot => SourceErrorDetails::Initialization(m),
                DefiniteError::MissingColumn => SourceErrorDetails::Other(m),
                DefiniteError::InvalidCopyInput => SourceErrorDetails::Other(m),
                DefiniteError::InvalidTimelineId { .. } => SourceErrorDetails::Initialization(m),
                DefiniteError::InvalidSnapshotLsn { .. } => SourceErrorDetails::Initialization(m),
                DefiniteError::InvalidPhysicalReplica { .. } => {
                    SourceErrorDetails::Initialization(m)
                }
                DefiniteError::MissingToast => SourceErrorDetails::Other(m),
                DefiniteError::DefaultReplicaIdentity => SourceErrorDetails::Other(m),
                DefiniteError::IncompatibleSchema(_) => SourceErrorDetails::Other(m),
                DefiniteError::InvalidUTF8(_) => SourceErrorDetails::Other(m),
                DefiniteError::CastError(_) => SourceErrorDetails::Other(m),
                DefiniteError::UnexpectedBinaryData => SourceErrorDetails::Other(m),
            },
        }))
    }
}

async fn ensure_replication_slot(
    client: &Client,
    slot: &str,
    timeout: Duration,
) -> Result<(), TransientError> {
    let query = sql!(
        "CREATE_REPLICATION_SLOT {} LOGICAL \"pgoutput\" NOEXPORT_SNAPSHOT",
        Sql::ident(slot)
    );
    match create_replication_slot(client, slot, query, timeout).await {
        Ok(_) => Ok(()),
        Err(TransientError::ReplicationSlotAlreadyExists) => {
            tracing::trace!(%slot, "replication slot already existed");
            Ok(())
        }
        Err(err) => Err(err),
    }
}

/// Runs the `CREATE_REPLICATION_SLOT` command `query` for `slot`, failing with
/// [`TransientError::ReplicationSlotCreationTimeout`] once the command has waited `timeout` for
/// in-progress upstream transactions to end.
///
/// A failure inside a transaction leaves that transaction aborted.
async fn create_replication_slot(
    client: &Client,
    slot: &str,
    query: Sql,
    timeout: Duration,
) -> Result<Option<SimpleQueryRow>, TransientError> {
    // Slot creation blocks on the transaction locks of in-progress transactions, which
    // `lock_timeout` bounds. `statement_timeout` does not apply: the server arms it for SQL
    // statements only, never for replication commands. Abandoning the command client-side does not
    // help either: a walsender blocked on a transaction lock does not notice the closed connection
    // and the `CREATE` command remains on the upstream.
    //
    // NOTE: `lock_timeout` only detects locks of concurrent sessions on the same server, not
    // on upstream servers of physical replicas.
    set_timeout(client, "lock_timeout", timeout).await?;
    let result = simple_query_opt(client, query).await;
    // Inside a transaction a failed command also fails the reset, and rolling the transaction back
    // reverts the `SET`. The command's error is the one to report.
    let reset = simple_query(client, sql!("RESET lock_timeout")).await;
    let row = match result {
        Ok(row) => row,
        Err(PostgresError::Postgres(err)) if err.code() == Some(&SqlState::DUPLICATE_OBJECT) => {
            return Err(TransientError::ReplicationSlotAlreadyExists);
        }
        Err(PostgresError::Postgres(err)) if err.code() == Some(&SqlState::LOCK_NOT_AVAILABLE) => {
            return Err(TransientError::ReplicationSlotCreationTimeout {
                slot: slot.to_owned(),
                timeout,
            });
        }
        Err(err) => return Err(err.into()),
    };
    reset?;
    Ok(row)
}

/// Sets the session's timeout `parameter` (for example `statement_timeout`) to `timeout`.
async fn set_timeout(
    client: &Client,
    parameter: &'static str,
    timeout: Duration,
) -> Result<(), TransientError> {
    // A unitless value is interpreted as milliseconds, and the server rejects values above
    // `i32::MAX` (about 24.8 days) rather than clamping them.
    // https://www.postgresql.org/docs/current/runtime-config-client.html
    let millis = i32::try_from(timeout.as_millis()).unwrap_or(i32::MAX);
    let query = sql!("SET {} = {}", Sql::new(parameter), Sql::from(millis));
    simple_query(client, query).await?;
    Ok(())
}

/// The state of a replication slot.
struct SlotMetadata {
    /// The process ID of the session using this slot if the slot is currently actively being used.
    /// None if inactive.
    active_pid: Option<i32>,
    /// The address (LSN) up to which the logical slot's consumer has confirmed receiving data.
    /// Data corresponding to the transactions committed before this LSN is not available anymore.
    confirmed_flush_lsn: MzOffset,
    /// Whether the slot can no longer be read from, which PostgreSQL reports as a `wal_status` of
    /// `lost`.
    invalidated: bool,
}

/// Fetches the state of `slot`, waiting for it to finish initializing.
async fn fetch_slot_metadata(
    client: &Client,
    slot: &str,
    interval: Duration,
) -> Result<SlotMetadata, TransientError> {
    loop {
        // `wal_status` exists on PostgreSQL 13 and later, which are also the only versions that
        // invalidate slots. Reading it through `to_jsonb` yields NULL on earlier versions instead
        // of failing the query.
        let Some(row) = query_opt(
            &**client,
            sql!(
                "SELECT active_pid, confirmed_flush_lsn, \
                    to_jsonb(s) ->> 'wal_status' = 'lost' AS invalidated \
                 FROM pg_replication_slots s WHERE slot_name = $1"
            ),
            &[&slot],
        )
        .await?
        else {
            return Err(TransientError::MissingReplicationSlot);
        };

        match row.get::<_, Option<PgLsn>>("confirmed_flush_lsn") {
            // For postgres, `confirmed_flush_lsn` means that the slot is able to produce
            // all transactions that happen at tx_lsn >= confirmed_flush_lsn. Therefore this value
            // already has "upper" semantics.
            Some(lsn) => {
                return Ok(SlotMetadata {
                    confirmed_flush_lsn: MzOffset::from(lsn),
                    active_pid: row.get("active_pid"),
                    invalidated: row.get::<_, Option<bool>>("invalidated").unwrap_or(false),
                });
            }
            // It can happen that confirmed_flush_lsn is NULL as the slot initializes
            // This could probably be a `tokio::time::interval`, but its only is called twice,
            // so its fine like this.
            None => tokio::time::sleep(interval).await,
        };
    }
}

// Ensures that the table with oid `oid` and expected schema `expected_schema` is still compatible
// with the current upstream schema `upstream_info`.
fn verify_schema(
    oid: u32,
    info: &SourceOutputInfo,
    upstream_info: &BTreeMap<u32, PostgresTableDesc>,
) -> Result<(), DefiniteError> {
    let current_desc = upstream_info.get(&oid).ok_or(DefiniteError::TableDropped)?;

    let allow_oids_to_change_by_col_num = info
        .desc
        .columns
        .iter()
        .zip_eq(info.casts.iter())
        .flat_map(|(col, (cast_type, _))| match cast_type {
            CastType::Text => Some(col.col_num),
            CastType::Natural => None,
        })
        .collect();

    match info
        .desc
        .determine_compatibility(current_desc, &allow_oids_to_change_by_col_num)
    {
        Ok(()) => Ok(()),
        Err(err) => Err(DefiniteError::IncompatibleSchema(err)),
    }
}

/// Casts a text row into the target types
fn cast_row(
    casts: &[(CastType, StorageScalarExpr)],
    datums: &[Datum<'_>],
    row: &mut Row,
) -> Result<(), DefiniteError> {
    let arena = mz_repr::RowArena::new();
    let mut packer = row.packer();
    for (_, column_cast) in casts {
        let datum = column_cast
            .eval(datums, &arena)
            .map_err(DefiniteError::CastError)?;
        packer.push(datum);
    }
    Ok(())
}

/// Converts raw bytes that are expected to be UTF8 encoded into a `Datum::String`
fn decode_utf8_text(bytes: &[u8]) -> Result<Datum<'_>, DefiniteError> {
    match std::str::from_utf8(bytes) {
        Ok(text) => Ok(Datum::String(text)),
        Err(_) => Err(DefiniteError::InvalidUTF8(bytes.to_vec())),
    }
}