Struct mz_storage_operators::s3_oneshot_sink::parquet::ParquetFile

struct ParquetFile {
    builder: ArrowBuilder,
    writer: ArrowWriter<Vec<u8>>,
    uploader: S3MultiPartUploader,
    arrow_builder_buffer_bytes: u64,
    row_group_size: u64,
    desc: Arc<RelationDesc>,
}

Helper to tie the lifecycle of the ArrowBuilder, ArrowWriter, and S3MultiPartUploader together for a single parquet file.

Fields

builder: ArrowBuilder

The active arrow builder.

writer: ArrowWriter<Vec<u8>>

uploader: S3MultiPartUploader

arrow_builder_buffer_bytes: u64

row_group_size: u64

desc: Arc<RelationDesc>

Implementations

impl ParquetFile

async fn new( bucket: String, key: String, desc: Arc<RelationDesc>, sdk_config: Arc<SdkConfig>, arrow_builder_buffer_bytes: u64, row_group_size: u64, part_size_limit: u64, ) -> Result<Self, Error>

fn add_row(&mut self, row: &Row) -> Result<(), Error>

async fn finish(self) -> Result<CompletedUpload, Error>

Flush the current arrow builder, the parquet writer, and the uploader.

fn flush_builder(&mut self) -> Result<(), Error>

Flush the current arrow builder to the parquet writer then flushes the writer’s buffer to the uploader which may trigger an upload.

fn size_estimate(&self) -> u64

Returns an approximate size estimate of the file being written.