columnar/
string.rs

1use alloc::{vec::Vec, string::String, string::ToString, boxed::Box};
2use super::{Clear, Columnar, Container, Len, Index, IndexAs, Push, Borrow};
3
4/// A stand-in for `Vec<String>`.
5///
6/// The reference type for `Strings` is `&[u8]` rather than `&str` to remove utf8 validation
7/// from the critical path of reads. You get to make the call about whether and how you'd like
8/// to manage this validation. The `copy_from` and `into_owned` methods panic on invalid data.
9#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
10#[derive(Copy, Clone, Debug, Default, PartialEq)]
11pub struct Strings<BC = Vec<u64>, VC = Vec<u8>> {
12    /// Bounds container; provides indexed access to offsets.
13    pub bounds: BC,
14    /// Values container; provides slice access to bytes.
15    pub values: VC,
16}
17
18impl Columnar for String {
19    #[inline(always)]
20    fn copy_from<'a>(&mut self, other: crate::Ref<'a, Self>) {
21        self.clear();
22        self.push_str(core::str::from_utf8(other).expect("invalid utf8 in Strings column"));
23    }
24    #[inline(always)]
25    fn into_owned<'a>(other: crate::Ref<'a, Self>) -> Self {
26        core::str::from_utf8(other).expect("invalid utf8 in Strings column").to_string()
27    }
28    type Container = Strings;
29}
30
31impl Columnar for Box<str> {
32    #[inline(always)]
33    fn copy_from<'a>(&mut self, other: crate::Ref<'a, Self>) {
34        let mut s = String::from(core::mem::take(self));
35        s.clear();
36        s.push_str(core::str::from_utf8(other).expect("invalid utf8 in Strings column"));
37        *self = s.into_boxed_str();
38    }
39    #[inline(always)]
40    fn into_owned<'a>(other: crate::Ref<'a, Self>) -> Self {
41        Self::from(core::str::from_utf8(other).expect("invalid utf8 in Strings column"))
42    }
43    type Container = Strings;
44}
45
46impl<BC: crate::common::BorrowIndexAs<u64>> Borrow for Strings<BC, Vec<u8>> {
47    type Ref<'a> = &'a [u8];
48    type Borrowed<'a> = Strings<BC::Borrowed<'a>, &'a [u8]> where BC: 'a;
49    #[inline(always)]
50    fn borrow<'a>(&'a self) -> Self::Borrowed<'a> {
51        Strings {
52            bounds: self.bounds.borrow(),
53            values: self.values.borrow(),
54        }
55    }
56    #[inline(always)]
57    fn reborrow<'c, 'a: 'c>(thing: Self::Borrowed<'a>) -> Self::Borrowed<'c> where BC: 'a {
58        Strings {
59            bounds: BC::reborrow(thing.bounds),
60            values: thing.values,
61        }
62    }
63    #[inline(always)]
64    fn reborrow_ref<'b, 'a: 'b>(thing: Self::Ref<'a>) -> Self::Ref<'b> where Self: 'a { thing }
65}
66
67impl<BC: crate::common::PushIndexAs<u64>> Container for Strings<BC, Vec<u8>> {
68    #[inline(always)]
69    fn extend_from_self(&mut self, other: Self::Borrowed<'_>, range: core::ops::Range<usize>) {
70        if !range.is_empty() {
71            // Imported bounds will be relative to this starting offset.
72            let values_len = self.values.len() as u64;
73
74            // Push all bytes that we can, all at once.
75            let other_lower = if range.start == 0 { 0 } else { other.bounds.index_as(range.start-1) };
76            let other_upper = other.bounds.index_as(range.end-1);
77            self.values.extend_from_self(other.values, other_lower as usize .. other_upper as usize);
78
79            // Each bound needs to be shifted by `values_len - other_lower`.
80            if values_len == other_lower {
81                self.bounds.extend_from_self(other.bounds, range);
82            }
83            else {
84                for index in range {
85                    let shifted = other.bounds.index_as(index) - other_lower + values_len;
86                    self.bounds.push(&shifted)
87                }
88            }
89        }
90    }
91
92    fn reserve_for<'a, I>(&mut self, selves: I) where Self: 'a, I: Iterator<Item = Self::Borrowed<'a>> + Clone {
93        self.bounds.reserve_for(selves.clone().map(|x| x.bounds));
94        self.values.reserve_for(selves.map(|x| x.values));
95    }
96
97}
98
99impl<'a, BC: crate::AsBytes<'a>, VC: crate::AsBytes<'a>> crate::AsBytes<'a> for Strings<BC, VC> {
100    const SLICE_COUNT: usize = BC::SLICE_COUNT + VC::SLICE_COUNT;
101    #[inline]
102    fn get_byte_slice(&self, index: usize) -> (u64, &'a [u8]) {
103        debug_assert!(index < Self::SLICE_COUNT);
104        if index < BC::SLICE_COUNT {
105            self.bounds.get_byte_slice(index)
106        } else {
107            self.values.get_byte_slice(index - BC::SLICE_COUNT)
108        }
109    }
110}
111impl<'a, BC: crate::FromBytes<'a>, VC: crate::FromBytes<'a>> crate::FromBytes<'a> for Strings<BC, VC> {
112    const SLICE_COUNT: usize = BC::SLICE_COUNT + VC::SLICE_COUNT;
113    #[inline(always)]
114    fn from_bytes(bytes: &mut impl Iterator<Item=&'a [u8]>) -> Self {
115        Self {
116            bounds: crate::FromBytes::from_bytes(bytes),
117            values: crate::FromBytes::from_bytes(bytes),
118        }
119    }
120    #[inline(always)]
121    fn from_store(store: &crate::bytes::indexed::DecodedStore<'a>, offset: &mut usize) -> Self {
122        Self {
123            bounds: BC::from_store(store, offset),
124            values: VC::from_store(store, offset),
125        }
126    }
127    #[inline(always)]
128    fn element_sizes(sizes: &mut Vec<usize>) -> Result<(), String> {
129        BC::element_sizes(sizes)?;
130        VC::element_sizes(sizes)?;
131        Ok(())
132    }
133}
134
135impl<BC: Len, VC> Len for Strings<BC, VC> {
136    #[inline(always)] fn len(&self) -> usize { self.bounds.len() }
137}
138
139impl<'a, BC: Len+IndexAs<u64>> Strings<BC, &'a [u8]> {
140    /// Returns the `index`-th string as `&str`, validating UTF-8.
141    ///
142    /// This is a convenience wrapper around [`Index::get`] (which returns `&[u8]`)
143    /// for callers who need `&str`. The UTF-8 validation has a measurable cost;
144    /// use `get` directly if you can work with `&[u8]`.
145    #[inline(always)]
146    pub fn get_str(&self, index: usize) -> &'a str {
147        core::str::from_utf8(self.get(index)).expect("invalid utf8 in Strings column")
148    }
149}
150
151impl<'a, BC: Len+IndexAs<u64>> Index for Strings<BC, &'a [u8]> {
152    type Ref = &'a [u8];
153    #[inline(always)] fn get(&self, index: usize) -> Self::Ref {
154        let lower = if index == 0 { 0 } else { self.bounds.index_as(index - 1) };
155        let upper = self.bounds.index_as(index);
156        let lower: usize = lower.try_into().expect("bounds must fit in `usize`");
157        let upper: usize = upper.try_into().expect("bounds must fit in `usize`");
158        &self.values[lower .. upper]
159    }
160}
161impl<'a, BC: Len+IndexAs<u64>> Index for &'a Strings<BC, Vec<u8>> {
162    type Ref = &'a [u8];
163    #[inline(always)] fn get(&self, index: usize) -> Self::Ref {
164        let lower = if index == 0 { 0 } else { self.bounds.index_as(index - 1) };
165        let upper = self.bounds.index_as(index);
166        let lower: usize = lower.try_into().expect("bounds must fit in `usize`");
167        let upper: usize = upper.try_into().expect("bounds must fit in `usize`");
168        &self.values[lower .. upper]
169    }
170}
171
172// This is a simpler implementation, but it leads to a performance regression
173// for Strings and str because it loses access to `Vec::extend_from_slice`.
174//
175// impl<BC: Push<u64>, D: std::fmt::Display> Push<D> for Strings<BC> {
176//     #[inline(always)]
177//     fn push(&mut self, item: D) {
178//         use std::io::Write;
179//         write!(self.values, "{}", item).unwrap();
180//         self.bounds.push(self.values.len() as u64);
181//     }
182// }
183
184impl<BC: for<'a> Push<&'a u64>> Push<&[u8]> for Strings<BC> {
185    #[inline(always)] fn push(&mut self, item: &[u8]) {
186        self.values.extend_from_slice(item);
187        self.bounds.push(&(self.values.len() as u64));
188    }
189}
190impl<BC: for<'a> Push<&'a u64>> Push<&String> for Strings<BC> {
191    #[inline(always)] fn push(&mut self, item: &String) {
192        self.values.extend_from_slice(item.as_bytes());
193        self.bounds.push(&(self.values.len() as u64));
194    }
195}
196impl<BC: for<'a> Push<&'a u64>> Push<&str> for Strings<BC> {
197    #[inline]
198    fn push(&mut self, item: &str) {
199        self.values.extend_from_slice(item.as_bytes());
200        self.bounds.push(&(self.values.len() as u64));
201    }
202}
203impl<BC: for<'a> Push<&'a u64>> Push<&Box<str>> for Strings<BC> {
204    #[inline]
205    fn push(&mut self, item: &Box<str>) {
206        self.values.extend_from_slice(item.as_bytes());
207        self.bounds.push(&(self.values.len() as u64));
208    }
209}
210impl<'a, BC: for<'b> Push<&'b u64>> Push<core::fmt::Arguments<'a>> for Strings<BC> {
211    #[inline]
212    fn push(&mut self, item: core::fmt::Arguments<'a>) {
213        // Use core::fmt::Write via a wrapper to avoid requiring std::io::Write.
214        struct VecWriter<'a>(&'a mut alloc::vec::Vec<u8>);
215        impl core::fmt::Write for VecWriter<'_> {
216            fn write_str(&mut self, s: &str) -> core::fmt::Result {
217                self.0.extend_from_slice(s.as_bytes());
218                Ok(())
219            }
220        }
221        core::fmt::Write::write_fmt(&mut VecWriter(&mut self.values), item).expect("write_fmt failed");
222        self.bounds.push(&(self.values.len() as u64));
223    }
224}
225impl<'a, 'b, BC: for<'c> Push<&'c u64>> Push<&'b core::fmt::Arguments<'a>> for Strings<BC> {
226    #[inline]
227    fn push(&mut self, item: &'b core::fmt::Arguments<'a>) {
228        self.push(*item);
229    }
230}
231impl<BC: Clear, VC: Clear> Clear for Strings<BC, VC> {
232    #[inline(always)]
233    fn clear(&mut self) {
234        self.bounds.clear();
235        self.values.clear();
236    }
237}
238
columnar/string.rs

columnar/
string.rs
