winnow/stream/
token.rs

Help
1use crate::error::Needed;
2use crate::lib::std::iter::Enumerate;
3use crate::lib::std::slice::Iter;
4use crate::stream::Checkpoint;
5use crate::stream::Compare;
6use crate::stream::CompareResult;
7use crate::stream::Location;
8use crate::stream::Offset;
9#[cfg(feature = "unstable-recover")]
10#[cfg(feature = "std")]
11use crate::stream::Recover;
12use crate::stream::SliceLen;
13use crate::stream::Stream;
14use crate::stream::StreamIsPartial;
15use crate::stream::UpdateSlice;
16
17/// Specialized input for parsing lexed tokens
18///
19/// Helpful impls
20/// - Any `PartialEq` type (e.g. a `TokenKind` or `&str`) can be used with
21///   [`literal`][crate::token::literal]
22/// - A `PartialEq` for `&str` allows for using `&str` as a parser for tokens
23///
24/// See also [Lexing and Parsing][crate::_topic::lexing].
25#[derive(Debug, Copy, Clone, PartialEq, Eq)]
26pub struct TokenSlice<'t, T> {
27    initial: &'t [T],
28    input: &'t [T],
29}
30
31impl<'t, T> TokenSlice<'t, T>
32where
33    T: crate::lib::std::fmt::Debug + Clone,
34{
35    /// Make a stream to parse tokens
36    #[inline]
37    pub fn new(input: &'t [T]) -> Self {
38        Self {
39            initial: input,
40            input,
41        }
42    }
43
44    /// Reset the stream to the start
45    ///
46    /// This is useful for formats that encode a graph with addresses relative to the start of the
47    /// input.
48    #[doc(alias = "fseek")]
49    #[inline]
50    pub fn reset_to_start(&mut self) {
51        let start = self.initial.checkpoint();
52        self.input.reset(&start);
53    }
54
55    /// Iterate over consumed tokens starting with the last emitted
56    ///
57    /// This is intended to help build up appropriate context when reporting errors.
58    #[inline]
59    pub fn previous_tokens(&self) -> impl Iterator<Item = &'t T> {
60        let offset = self.input.offset_from(&self.initial);
61        self.initial[0..offset].iter().rev()
62    }
63}
64
65/// Track locations by implementing [`Location`] on the Token.
66impl<T> TokenSlice<'_, T>
67where
68    T: Location,
69{
70    #[inline(always)]
71    fn previous_token_end(&self) -> Option<usize> {
72        let index = self.input.offset_from(&self.initial);
73        index
74            .checked_sub(1)
75            .map(|i| self.initial[i].previous_token_end())
76    }
77
78    #[inline(always)]
79    fn current_token_start(&self) -> Option<usize> {
80        self.input.first().map(|t| t.current_token_start())
81    }
82}
83
84impl<T> Default for TokenSlice<'_, T>
85where
86    T: crate::lib::std::fmt::Debug + Clone,
87{
88    fn default() -> Self {
89        Self::new(&[])
90    }
91}
92
93impl<T> crate::lib::std::ops::Deref for TokenSlice<'_, T> {
94    type Target = [T];
95
96    fn deref(&self) -> &Self::Target {
97        self.input
98    }
99}
100
101impl<T> SliceLen for TokenSlice<'_, T> {
102    #[inline(always)]
103    fn slice_len(&self) -> usize {
104        self.input.slice_len()
105    }
106}
107
108impl<'t, T> Stream for TokenSlice<'t, T>
109where
110    T: crate::lib::std::fmt::Debug + Clone,
111{
112    type Token = &'t T;
113    type Slice = &'t [T];
114
115    type IterOffsets = Enumerate<Iter<'t, T>>;
116
117    type Checkpoint = Checkpoint<&'t [T], Self>;
118
119    #[inline(always)]
120    fn iter_offsets(&self) -> Self::IterOffsets {
121        self.input.iter().enumerate()
122    }
123    #[inline(always)]
124    fn eof_offset(&self) -> usize {
125        self.input.eof_offset()
126    }
127
128    #[inline(always)]
129    fn next_token(&mut self) -> Option<Self::Token> {
130        let (token, next) = self.input.split_first()?;
131        self.input = next;
132        Some(token)
133    }
134
135    #[inline(always)]
136    fn peek_token(&self) -> Option<Self::Token> {
137        self.input.first()
138    }
139
140    #[inline(always)]
141    fn offset_for<P>(&self, predicate: P) -> Option<usize>
142    where
143        P: Fn(Self::Token) -> bool,
144    {
145        self.input.iter().position(predicate)
146    }
147    #[inline(always)]
148    fn offset_at(&self, tokens: usize) -> Result<usize, Needed> {
149        self.input.offset_at(tokens)
150    }
151    #[inline(always)]
152    fn next_slice(&mut self, offset: usize) -> Self::Slice {
153        self.input.next_slice(offset)
154    }
155    #[inline(always)]
156    unsafe fn next_slice_unchecked(&mut self, offset: usize) -> Self::Slice {
157        // SAFETY: Passing up invariants
158        unsafe { self.input.next_slice_unchecked(offset) }
159    }
160    #[inline(always)]
161    fn peek_slice(&self, offset: usize) -> Self::Slice {
162        self.input.peek_slice(offset)
163    }
164    #[inline(always)]
165    unsafe fn peek_slice_unchecked(&self, offset: usize) -> Self::Slice {
166        // SAFETY: Passing up invariants
167        unsafe { self.input.peek_slice_unchecked(offset) }
168    }
169
170    #[inline(always)]
171    fn checkpoint(&self) -> Self::Checkpoint {
172        Checkpoint::<_, Self>::new(self.input)
173    }
174    #[inline(always)]
175    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
176        self.input = checkpoint.inner;
177    }
178
179    #[inline(always)]
180    fn raw(&self) -> &dyn crate::lib::std::fmt::Debug {
181        &self.input
182    }
183}
184
185impl<T> Location for TokenSlice<'_, T>
186where
187    T: Location,
188{
189    #[inline(always)]
190    fn previous_token_end(&self) -> usize {
191        self.previous_token_end()
192            .or_else(|| self.current_token_start())
193            .unwrap_or(0)
194    }
195    #[inline(always)]
196    fn current_token_start(&self) -> usize {
197        self.current_token_start()
198            .or_else(|| self.previous_token_end())
199            .unwrap_or(0)
200    }
201}
202
203#[cfg(feature = "unstable-recover")]
204#[cfg(feature = "std")]
205impl<T, E> Recover<E> for TokenSlice<'_, T>
206where
207    T: crate::lib::std::fmt::Debug + Clone,
208{
209    #[inline(always)]
210    fn record_err(
211        &mut self,
212        _token_start: &Self::Checkpoint,
213        _err_start: &Self::Checkpoint,
214        err: E,
215    ) -> Result<(), E> {
216        Err(err)
217    }
218
219    /// Report whether the [`Stream`] can save off errors for recovery
220    #[inline(always)]
221    fn is_recovery_supported() -> bool {
222        false
223    }
224}
225
226impl<'t, T> StreamIsPartial for TokenSlice<'t, T>
227where
228    T: crate::lib::std::fmt::Debug + Clone,
229{
230    type PartialState = <&'t [T] as StreamIsPartial>::PartialState;
231
232    #[inline]
233    fn complete(&mut self) -> Self::PartialState {
234        #![allow(clippy::semicolon_if_nothing_returned)]
235        self.input.complete()
236    }
237
238    #[inline]
239    fn restore_partial(&mut self, state: Self::PartialState) {
240        self.input.restore_partial(state);
241    }
242
243    #[inline(always)]
244    fn is_partial_supported() -> bool {
245        <&[T] as StreamIsPartial>::is_partial_supported()
246    }
247
248    #[inline(always)]
249    fn is_partial(&self) -> bool {
250        self.input.is_partial()
251    }
252}
253
254impl<T> Offset for TokenSlice<'_, T>
255where
256    T: crate::lib::std::fmt::Debug + Clone,
257{
258    #[inline(always)]
259    fn offset_from(&self, other: &Self) -> usize {
260        self.offset_from(&other.checkpoint())
261    }
262}
263
264impl<T> Offset<<TokenSlice<'_, T> as Stream>::Checkpoint> for TokenSlice<'_, T>
265where
266    T: crate::lib::std::fmt::Debug + Clone,
267{
268    #[inline(always)]
269    fn offset_from(&self, other: &<TokenSlice<'_, T> as Stream>::Checkpoint) -> usize {
270        self.checkpoint().offset_from(other)
271    }
272}
273
274impl<T, O> Compare<O> for TokenSlice<'_, T>
275where
276    T: PartialEq<O> + Eq,
277{
278    #[inline]
279    fn compare(&self, t: O) -> CompareResult {
280        if let Some(token) = self.first() {
281            if *token == t {
282                CompareResult::Ok(1)
283            } else {
284                CompareResult::Error
285            }
286        } else {
287            CompareResult::Incomplete
288        }
289    }
290}
291
292impl<T> UpdateSlice for TokenSlice<'_, T>
293where
294    T: crate::lib::std::fmt::Debug + Clone,
295{
296    #[inline(always)]
297    fn update_slice(mut self, inner: Self::Slice) -> Self {
298        self.input = <&[T] as UpdateSlice>::update_slice(self.input, inner);
299        self
300    }
301}
winnow/stream/token.rs

winnow/stream/
token.rs
