1use std::collections::VecDeque;
11use std::fmt::Display;
12use std::iter;
13use std::ops::RangeInclusive;
14use std::sync::LazyLock;
15use std::time::Duration;
16
17use chrono::NaiveDate;
18use dec::{Context as DecimalContext, OrderedDecimal};
19use mz_ore::now::NowFn;
20use mz_repr::adt::date::Date;
21use mz_repr::adt::numeric::{self, DecimalLike, Numeric};
22use mz_repr::{Datum, Diff, Row};
23use mz_storage_types::sources::MzOffset;
24use mz_storage_types::sources::load_generator::{
25 Event, Generator, GeneratorUpdate, LoadGeneratorOutput, TpchView,
26};
27use rand_8::distributions::{Alphanumeric, DistString};
28use rand_8::rngs::StdRng;
29use rand_8::seq::SliceRandom;
30use rand_8::{Rng, SeedableRng};
31
32#[derive(Clone, Debug)]
33pub struct Tpch {
34 pub count_supplier: i64,
35 pub count_part: i64,
36 pub count_customer: i64,
37 pub count_orders: i64,
38 pub count_clerk: i64,
39 pub tick: Duration,
40}
41
42impl Generator for Tpch {
43 fn by_seed(
44 &self,
45 _: NowFn,
46 seed: Option<u64>,
47 _resume_offset: MzOffset,
48 ) -> Box<dyn Iterator<Item = (LoadGeneratorOutput, Event<Option<MzOffset>, (Row, Diff)>)>> {
49 self.updates(seed, 0, 1)
50 }
51
52 fn by_seed_for_worker(
53 &self,
54 _: NowFn,
55 seed: Option<u64>,
56 _resume_offset: MzOffset,
57 worker_index: usize,
58 worker_count: usize,
59 ) -> Box<dyn Iterator<Item = GeneratorUpdate>> {
60 self.updates(seed, worker_index, worker_count)
61 }
62}
63
64impl Tpch {
65 fn updates(
69 &self,
70 seed: Option<u64>,
71 worker_index: usize,
72 worker_count: usize,
73 ) -> Box<dyn Iterator<Item = GeneratorUpdate>> {
74 let ticks = !self.tick.is_zero() && worker_index == 0;
79 let owns = move |key: usize| key % worker_count == worker_index;
80
81 let mut rng = StdRng::seed_from_u64(seed.unwrap_or_default());
86 let mut ctx = Context {
87 tpch: self.clone(),
88 decimal_one: Numeric::from(1),
89 decimal_neg_one: Numeric::from(-1),
90 cx: numeric::cx_datum(),
91 text_string_source: Alphanumeric.sample_string(&mut rng, 3 << 20),
93 row_buffer: Row::default(),
94 };
95
96 let count_nation: i64 = NATIONS.len().try_into().unwrap();
97 let count_region: i64 = REGIONS.len().try_into().unwrap();
98
99 let mut rows = (0..ctx.tpch.count_supplier)
100 .map(|i| (TpchView::Supplier, i))
101 .chain((1..=ctx.tpch.count_part).map(|i| (TpchView::Part, i)))
102 .chain((1..=ctx.tpch.count_customer).map(|i| (TpchView::Customer, i)))
103 .chain((1..=ctx.tpch.count_orders).map(|i| (TpchView::Orders, i)))
104 .chain((0..count_nation).map(|i| (TpchView::Nation, i)))
105 .chain((0..count_region).map(|i| (TpchView::Region, i)))
106 .peekable();
107
108 let mut pending = VecDeque::new();
111
112 let mut active_orders = Vec::new();
115
116 let mut offset = 0;
117 let mut row = Row::default();
118 let updates = iter::from_fn(move || {
121 if let Some((output, event)) = pending.pop_front() {
122 return Some(Some((LoadGeneratorOutput::Tpch(output), event)));
123 }
124 if let Some((output, key)) = rows.next() {
125 let key_usize = usize::try_from(key).expect("key known to be non-negative");
126 let owned = owns(key_usize);
127 let row = match output {
128 TpchView::Supplier => {
129 let nation = rng.gen_range(0..count_nation);
130 row.packer().extend([
131 Datum::Int64(key),
132 Datum::String(&pad_nine("Supplier", key)),
133 Datum::String(&v_string(&mut rng, 10, 40)), Datum::Int64(nation),
135 Datum::String(&phone(&mut rng, nation)),
136 Datum::Numeric(decimal(&mut rng, &mut ctx.cx, -999_99, 9_999_99, 100)), Datum::String(text_string(&mut rng, &ctx.text_string_source, 25, 100)),
139 ]);
140 Some(row.clone())
141 }
142 TpchView::Part => {
143 let name: String = PARTNAMES
144 .choose_multiple(&mut rng, 5)
145 .cloned()
146 .collect::<Vec<_>>()
147 .join(" ");
148 let m = rng.gen_range(1..=5);
149 let n = rng.gen_range(1..=5);
150 for _ in 1..=4 {
151 let suppkey = (key
152 + (rng.gen_range(0..=3)
153 * ((ctx.tpch.count_supplier / 4)
154 + (key - 1) / ctx.tpch.count_supplier)))
155 % ctx.tpch.count_supplier
156 + 1;
157 row.packer().extend([
158 Datum::Int64(key),
159 Datum::Int64(suppkey),
160 Datum::Int32(rng.gen_range(1..=9_999)), Datum::Numeric(decimal(&mut rng, &mut ctx.cx, 1_00, 1_000_00, 100)), Datum::String(text_string(
163 &mut rng,
164 &ctx.text_string_source,
165 49,
166 198,
167 )),
168 ]);
169 if !owned {
170 continue;
171 }
172 pending.push_back((
173 TpchView::Partsupp,
174 Event::Message(MzOffset::from(offset), (row.clone(), Diff::ONE)),
175 ));
176 }
177 row.packer().extend([
178 Datum::Int64(key),
179 Datum::String(&name),
180 Datum::String(&format!("Manufacturer#{m}")),
181 Datum::String(&format!("Brand#{m}{n}")),
182 Datum::String(&syllables(&mut rng, TYPES)),
183 Datum::Int32(rng.gen_range(1..=50)), Datum::String(&syllables(&mut rng, CONTAINERS)),
185 Datum::Numeric(partkey_retailprice(key)),
186 Datum::String(text_string(&mut rng, &ctx.text_string_source, 49, 198)),
187 ]);
188 Some(row.clone())
189 }
190 TpchView::Customer => {
191 let nation = rng.gen_range(0..count_nation);
192 row.packer().extend([
193 Datum::Int64(key),
194 Datum::String(&pad_nine("Customer", key)),
195 Datum::String(&v_string(&mut rng, 10, 40)), Datum::Int64(nation),
197 Datum::String(&phone(&mut rng, nation)),
198 Datum::Numeric(decimal(&mut rng, &mut ctx.cx, -999_99, 9_999_99, 100)), Datum::String(SEGMENTS.choose(&mut rng).unwrap()),
200 Datum::String(text_string(&mut rng, &ctx.text_string_source, 29, 116)),
201 ]);
202 Some(row.clone())
203 }
204 TpchView::Orders => 'order: {
205 let seed = rng.r#gen();
206 if ticks {
207 active_orders.push((key, seed));
208 }
209 if !owned {
210 break 'order None;
211 }
212 let (order, lineitems) = ctx.order_row(seed, key);
213 for row in lineitems {
214 pending.push_back((
215 TpchView::Lineitem,
216 Event::Message(MzOffset::from(offset), (row, Diff::ONE)),
217 ));
218 }
219 Some(order)
220 }
221 TpchView::Nation => {
222 let (name, region) = NATIONS[key_usize];
223 row.packer().extend([
224 Datum::Int64(key),
225 Datum::String(name),
226 Datum::Int64(region),
227 Datum::String(text_string(&mut rng, &ctx.text_string_source, 31, 114)),
228 ]);
229 Some(row.clone())
230 }
231 TpchView::Region => {
232 row.packer().extend([
233 Datum::Int64(key),
234 Datum::String(REGIONS[key_usize]),
235 Datum::String(text_string(&mut rng, &ctx.text_string_source, 31, 115)),
236 ]);
237 Some(row.clone())
238 }
239 _ => unreachable!("{output:?}"),
240 };
241
242 if let Some(row) = row.filter(|_| owned) {
243 pending.push_back((
244 output,
245 Event::Message(MzOffset::from(offset), (row, Diff::ONE)),
246 ));
247 }
248 if rows.peek().is_none() {
249 offset += 1;
250 pending.push_back((output, Event::Progress(Some(MzOffset::from(offset)))));
251 }
252 } else {
253 if !ticks {
254 return None;
255 }
256 let idx = rng.gen_range(0..active_orders.len());
257 let (key, old_seed) = active_orders.swap_remove(idx);
258 let (old_order, old_lineitems) = ctx.order_row(old_seed, key);
259 for row in old_lineitems {
263 pending.push_back((
264 TpchView::Lineitem,
265 Event::Message(MzOffset::from(offset), (row, Diff::MINUS_ONE)),
266 ));
267 }
268 let new_seed = rng.r#gen();
269 let (new_order, new_lineitems) = ctx.order_row(new_seed, key);
270 for row in new_lineitems {
271 pending.push_back((
272 TpchView::Lineitem,
273 Event::Message(MzOffset::from(offset), (row, Diff::ONE)),
274 ));
275 }
276 pending.push_back((
277 TpchView::Orders,
278 Event::Message(MzOffset::from(offset), (old_order, Diff::MINUS_ONE)),
279 ));
280 pending.push_back((
281 TpchView::Orders,
282 Event::Message(MzOffset::from(offset), (new_order, Diff::ONE)),
283 ));
284 offset += 1;
285 pending.push_back((
286 TpchView::Orders,
287 Event::Progress(Some(MzOffset::from(offset))),
288 ));
289 active_orders.push((key, new_seed));
290 }
291 Some(
292 pending
293 .pop_front()
294 .map(|(output, event)| (LoadGeneratorOutput::Tpch(output), event)),
295 )
296 });
297 Box::new(updates.flatten())
298 }
299}
300
301struct Context {
302 tpch: Tpch,
303 decimal_one: Numeric,
304 decimal_neg_one: Numeric,
305 cx: DecimalContext<Numeric>,
306 text_string_source: String,
307 row_buffer: Row,
308}
309
310impl Context {
311 fn order_row(&mut self, seed: u64, key: i64) -> (Row, Vec<Row>) {
315 let mut rng = StdRng::seed_from_u64(seed);
316 let key = order_key(key);
317 let custkey = loop {
318 let custkey = rng.gen_range(1..=self.tpch.count_customer);
319 if custkey % 3 != 0 {
320 break custkey;
321 }
322 };
323 let orderdate = date(&mut rng, &*START_DATE, 1..=*ORDER_END_DAYS);
324 let mut totalprice = Numeric::lossy_from(0);
325 let mut orderstatus = None;
326 let lineitem_count = rng.gen_range(1..=7);
327 let mut lineitems = Vec::with_capacity(lineitem_count);
328
329 for linenumber in 1..=lineitem_count {
330 let partkey = rng.gen_range(1..=self.tpch.count_part);
331 let suppkey = (partkey
332 + (rng.gen_range(0..=3)
333 * ((self.tpch.count_supplier / 4) + (partkey - 1) / self.tpch.count_supplier)))
334 % self.tpch.count_supplier
335 + 1;
336 let quantity = Numeric::from(rng.gen_range(1..=50));
337 let mut extendedprice = quantity;
338 self.cx
339 .mul(&mut extendedprice, &partkey_retailprice(partkey).0);
340 let mut discount = decimal(&mut rng, &mut self.cx, 0, 8, 100);
341 let mut tax = decimal(&mut rng, &mut self.cx, 0, 10, 100);
342 let shipdate = date(&mut rng, &orderdate, 1..=121);
343 let receiptdate = date(&mut rng, &shipdate, 1..=30);
344 let linestatus = if shipdate > *CURRENT_DATE { "O" } else { "F" };
345 self.row_buffer.packer().extend([
346 Datum::Int64(key),
347 Datum::Int64(partkey),
348 Datum::Int64(suppkey),
349 Datum::Int32(linenumber.try_into().expect("must fit")),
350 Datum::Numeric(OrderedDecimal(quantity)),
351 Datum::Numeric(OrderedDecimal(extendedprice)),
352 Datum::Numeric(discount),
353 Datum::Numeric(tax),
354 Datum::String(if receiptdate <= *CURRENT_DATE {
355 ["R", "A"].choose(&mut rng).unwrap()
356 } else {
357 "N"
358 }), Datum::String(linestatus),
360 Datum::Date(shipdate),
361 Datum::Date(date(&mut rng, &orderdate, 30..=90)), Datum::Date(receiptdate),
363 Datum::String(INSTRUCTIONS.choose(&mut rng).unwrap()),
364 Datum::String(MODES.choose(&mut rng).unwrap()),
365 Datum::String(text_string(&mut rng, &self.text_string_source, 10, 43)),
366 ]);
367 let row = self.row_buffer.clone();
368 self.cx.add(&mut tax.0, &self.decimal_one);
370 self.cx.sub(&mut discount.0, &self.decimal_neg_one);
371 self.cx.abs(&mut discount.0);
372 self.cx.mul(&mut extendedprice, &tax.0);
373 self.cx.mul(&mut extendedprice, &discount.0);
374 self.cx.add(&mut totalprice, &extendedprice);
375 if let Some(status) = orderstatus {
376 if status != linestatus {
377 orderstatus = Some("P");
378 }
379 } else {
380 orderstatus = Some(linestatus);
381 }
382 lineitems.push(row);
383 }
384
385 self.cx.rescale(&mut totalprice, &Numeric::from(-2));
386
387 self.row_buffer.packer().extend([
388 Datum::Int64(key),
389 Datum::Int64(custkey),
390 Datum::String(orderstatus.unwrap()),
391 Datum::Numeric(OrderedDecimal(totalprice)),
392 Datum::Date(orderdate),
393 Datum::String(PRIORITIES.choose(&mut rng).unwrap()),
394 Datum::String(&pad_nine("Clerk", rng.gen_range(1..=self.tpch.count_clerk))),
395 Datum::Int32(0), Datum::String(text_string(&mut rng, &self.text_string_source, 19, 78)),
397 ]);
398 let order = self.row_buffer.clone();
399 (order, lineitems)
400 }
401}
402
403fn partkey_retailprice(key: i64) -> OrderedDecimal<Numeric> {
404 let price = (90000 + ((key / 10) % 20001) + 100 * (key % 1000)) / 100;
405 OrderedDecimal(Numeric::from(price))
406}
407
408fn pad_nine<S: Display>(prefix: &str, s: S) -> String {
409 format!("{}#{s:09}", prefix)
410}
411
412pub static START_DATE: LazyLock<Date> =
413 LazyLock::new(|| Date::try_from(NaiveDate::from_ymd_opt(1992, 1, 1).unwrap()).unwrap());
414pub static CURRENT_DATE: LazyLock<Date> =
415 LazyLock::new(|| Date::try_from(NaiveDate::from_ymd_opt(1995, 6, 17).unwrap()).unwrap());
416pub static END_DATE: LazyLock<Date> =
417 LazyLock::new(|| Date::try_from(NaiveDate::from_ymd_opt(1998, 12, 31).unwrap()).unwrap());
418pub static ORDER_END_DAYS: LazyLock<i32> = LazyLock::new(|| *END_DATE - *START_DATE - 151);
419
420fn text_string<'a, R: Rng + ?Sized>(
421 rng: &mut R,
422 source: &'a str,
423 min: usize,
424 max: usize,
425) -> &'a str {
426 let start = rng.gen_range(0..=(source.len() - max));
427 let len = rng.gen_range(min..=max);
428 &source[start..(start + len)]
429}
430
431fn date<R: Rng + ?Sized>(rng: &mut R, start: &Date, days: RangeInclusive<i32>) -> Date {
432 let days = rng.gen_range(days);
433 start.checked_add(days).expect("must fit")
434}
435
436fn order_key(mut i: i64) -> i64 {
438 const SPARSE_BITS: usize = 2;
439 const SPARSE_KEEP: usize = 3;
440 let low_bits = i & ((1 << SPARSE_KEEP) - 1);
441 i >>= SPARSE_KEEP;
442 i <<= SPARSE_BITS;
443 i <<= SPARSE_KEEP;
446 i += low_bits;
447 i
448}
449
450fn syllables<R: Rng + ?Sized>(rng: &mut R, syllables: &[&[&str]]) -> String {
451 let mut s = String::new();
452 for (i, syllable) in syllables.iter().enumerate() {
453 if i > 0 {
454 s.push(' ');
455 }
456 s.push_str(syllable.choose(rng).unwrap());
457 }
458 s
459}
460
461fn decimal<R: Rng + ?Sized>(
462 rng: &mut R,
463 cx: &mut dec::Context<Numeric>,
464 min: i64,
465 max: i64,
466 div: i64,
467) -> OrderedDecimal<Numeric> {
468 let n = rng.gen_range(min..=max);
469 let mut n = Numeric::lossy_from(n);
470 cx.div(&mut n, &Numeric::lossy_from(div));
471 OrderedDecimal(n)
472}
473
474fn phone<R: Rng + ?Sized>(rng: &mut R, nation: i64) -> String {
475 let mut s = String::with_capacity(15);
476 s.push_str(&(nation + 10).to_string());
477 s.push('-');
478 s.push_str(&rng.gen_range(100..=999).to_string());
479 s.push('-');
480 s.push_str(&rng.gen_range(100..=999).to_string());
481 s.push('-');
482 s.push_str(&rng.gen_range(1000..=9999).to_string());
483 s
484}
485
486fn v_string<R: Rng + ?Sized>(rng: &mut R, min: usize, max: usize) -> String {
487 const ALPHABET: [char; 64] = [
488 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
489 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
490 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
491 '3', '4', '5', '6', '7', '8', '9', '0', ',', ' ',
492 ];
493 let take = rng.gen_range(min..=max);
494 let mut s = String::with_capacity(take);
495 for _ in 0..take {
496 s.push(*ALPHABET.choose(rng).unwrap());
497 }
498 s
499}
500
501const INSTRUCTIONS: &[&str] = &[
502 "DELIVER IN PERSON",
503 "COLLECT COD",
504 "NONE",
505 "TAKE BACK RETURN",
506];
507
508const MODES: &[&str] = &["REG AIR", "AIR", "RAIL", "SHIP", "TRUCK", "MAIL", "FOB"];
509
510const PARTNAMES: &[&str] = &[
511 "almond",
512 "antique",
513 "aquamarine",
514 "azure",
515 "beige",
516 "bisque",
517 "black",
518 "blanched",
519 "blue",
520 "blush",
521 "brown",
522 "burlywood",
523 "burnished",
524 "chartreuse",
525 "chiffon",
526 "chocolate",
527 "coral",
528 "cornflower",
529 "cornsilk",
530 "cream",
531 "cyan",
532 "dark",
533 "deep",
534 "dim",
535 "dodger",
536 "drab",
537 "firebrick",
538 "floral",
539 "forest",
540 "frosted",
541 "gainsboro",
542 "ghost",
543 "goldenrod",
544 "green",
545 "grey",
546 "honeydew",
547 "hot",
548 "indian",
549 "ivory",
550 "khaki",
551 "lace",
552 "lavender",
553 "lawn",
554 "lemon",
555 "light",
556 "lime",
557 "linen",
558 "magenta",
559 "maroon",
560 "medium",
561 "metallic",
562 "midnight",
563 "mint",
564 "misty",
565 "moccasin",
566 "navajo",
567 "navy",
568 "olive",
569 "orange",
570 "orchid",
571 "pale",
572 "papaya",
573 "peach",
574 "peru",
575 "pink",
576 "plum",
577 "powder",
578 "puff",
579 "purple",
580 "red",
581 "rose",
582 "rosy",
583 "royal",
584 "saddle",
585 "salmon",
586 "sandy",
587 "seashell",
588 "sienna",
589 "sky",
590 "slate",
591 "smoke",
592 "snow",
593 "spring",
594 "steel",
595 "tan",
596 "thistle",
597 "tomato",
598 "turquoise",
599 "violet",
600 "wheat",
601 "white",
602 "yellow",
603];
604
605const PRIORITIES: &[&str] = &["1-URGENT", "2-HIGH", "3-MEDIUM", "4-NOT SPECIFIED"];
606
607const TYPES: &[&[&str]] = &[
608 &["STANDARD", "SMALL", "MEDIUM", "LARGE", "ECONOMY", "PROMO"],
609 &["ANODIZED", "BURNISHED", "PLATED", "POLISHED", "BRUSHED"],
610 &["TIN", "NICKEL", "BRASS", "STEEL", "COPPER"],
611];
612
613const CONTAINERS: &[&[&str]] = &[
614 &["SM", "MED", "JUMBO", "WRAP"],
615 &["BOX", "BAG", "JAR", "PKG", "PACK", "CAN", "DRUM"],
616];
617
618const SEGMENTS: &[&str] = &[
619 "AUTOMOBILE",
620 "BUILDING",
621 "FURNITURE",
622 "MACHINERY",
623 "HOUSEHOLD",
624];
625
626const REGIONS: &[&str] = &["AFRICA", "AMERICA", "ASIA", "EUROPE", "MIDDLE EAST"];
627
628const NATIONS: &[(&str, i64)] = &[
629 ("ALGERIA", 0),
630 ("ARGENTINA", 1),
631 ("BRAZIL", 1),
632 ("CANADA", 1),
633 ("EGYPT", 4),
634 ("ETHIOPIA", 0),
635 ("FRANCE", 3),
636 ("GERMANY", 3),
637 ("INDIA", 2),
638 ("INDONESIA", 2),
639 ("IRAN", 4),
640 ("IRAQ", 4),
641 ("JAPAN", 2),
642 ("JORDAN", 4),
643 ("KENYA", 0),
644 ("MOROCCO", 0),
645 ("MOZAMBIQUE", 0),
646 ("PERU", 1),
647 ("CHINA", 2),
648 ("ROMANIA", 3),
649 ("SAUDI ARABIA", 4),
650 ("VIETNAM", 2),
651 ("RUSSIA", 3),
652 ("UNITED KINGDOM", 3),
653 ("UNITED STATES", 1),
654];
655
656#[cfg(test)]
657mod tests;