protobuf_native/
compiler.rs

Help
1// Copyright Materialize, Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License in the LICENSE file at the
6// root of this repository, or online at
7//
8//     http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16//! Implementation of the Protocol Buffer compiler.
17//!
18//! This module contains code for parsing .proto files and generating code based
19//! on them. It is particularly useful when you need to deal with arbitrary
20//! Protobuf messages at runtime.
21
22use std::collections::HashSet;
23use std::error::Error;
24use std::fmt;
25use std::marker::PhantomData;
26use std::marker::PhantomPinned;
27use std::mem;
28use std::path::Path;
29use std::pin::Pin;
30
31use cxx::let_cxx_string;
32
33use crate::internal::{unsafe_ffi_conversions, CInt, ProtobufPath};
34use crate::io::DynZeroCopyInputStream;
35use crate::{DescriptorDatabase, FileDescriptorProto, FileDescriptorSet, OperationFailedError};
36
37#[cxx::bridge(namespace = "protobuf_native::compiler")]
38pub(crate) mod ffi {
39    #[derive(Debug)]
40    struct FileLoadError {
41        filename: String,
42        line: i64,
43        column: i64,
44        message: String,
45        warning: bool,
46    }
47
48    unsafe extern "C++" {
49        include!("protobuf-native/src/compiler.h");
50        include!("protobuf-native/src/internal.h");
51
52        #[namespace = "protobuf_native::internal"]
53        type CInt = crate::internal::CInt;
54
55        #[namespace = "absl"]
56        type string_view<'a> = crate::internal::StringView<'a>;
57
58        #[namespace = "google::protobuf"]
59        type FileDescriptorProto = crate::ffi::FileDescriptorProto;
60
61        #[namespace = "google::protobuf::io"]
62        type ZeroCopyInputStream = crate::io::ffi::ZeroCopyInputStream;
63
64        type SimpleErrorCollector;
65        fn NewSimpleErrorCollector() -> *mut SimpleErrorCollector;
66        unsafe fn DeleteSimpleErrorCollector(collector: *mut SimpleErrorCollector);
67        fn Errors(self: Pin<&mut SimpleErrorCollector>) -> Pin<&mut CxxVector<FileLoadError>>;
68
69        #[namespace = "google::protobuf::compiler"]
70        type MultiFileErrorCollector;
71        fn RecordError(
72            self: Pin<&mut MultiFileErrorCollector>,
73            filename: string_view,
74            line: CInt,
75            column: CInt,
76            message: string_view,
77        );
78        fn RecordWarning(
79            self: Pin<&mut MultiFileErrorCollector>,
80            filename: string_view,
81            line: CInt,
82            column: CInt,
83            message: string_view,
84        );
85
86        #[namespace = "google::protobuf::compiler"]
87        type SourceTree;
88        fn Open(self: Pin<&mut SourceTree>, filename: string_view) -> *mut ZeroCopyInputStream;
89        fn SourceTreeGetLastErrorMessage(source_tree: Pin<&mut SourceTree>) -> String;
90
91        #[namespace = "google::protobuf::compiler"]
92        type SourceTreeDescriptorDatabase;
93        unsafe fn NewSourceTreeDescriptorDatabase(
94            source_tree: *mut SourceTree,
95        ) -> *mut SourceTreeDescriptorDatabase;
96        unsafe fn DeleteSourceTreeDescriptorDatabase(
97            source_tree: *mut SourceTreeDescriptorDatabase,
98        );
99        unsafe fn FindFileByName(
100            self: Pin<&mut SourceTreeDescriptorDatabase>,
101            filename: &CxxString,
102            output: *mut FileDescriptorProto,
103        ) -> bool;
104        unsafe fn RecordErrorsTo(
105            self: Pin<&mut SourceTreeDescriptorDatabase>,
106            error_collector: *mut MultiFileErrorCollector,
107        );
108
109        type VirtualSourceTree;
110        fn NewVirtualSourceTree() -> *mut VirtualSourceTree;
111        unsafe fn DeleteVirtualSourceTree(tree: *mut VirtualSourceTree);
112        fn AddFile(self: Pin<&mut VirtualSourceTree>, filename: string_view, contents: Vec<u8>);
113
114        #[namespace = "google::protobuf::compiler"]
115        type DiskSourceTree;
116        fn NewDiskSourceTree() -> *mut DiskSourceTree;
117        unsafe fn DeleteDiskSourceTree(tree: *mut DiskSourceTree);
118        fn MapPath(
119            self: Pin<&mut DiskSourceTree>,
120            virtual_path: string_view,
121            disk_path: string_view,
122        );
123    }
124}
125
126/// If the importer encounters problems while trying to import the proto files,
127/// it reports them to a `MultiFileErrorCollector`.
128pub trait MultiFileErrorCollector: multi_file_error_collector::Sealed {
129    /// Adds an error message to the error collector at the specified position.
130    ///
131    /// Line and column numbers are zero-based. A line number of -1 indicates
132    /// an error with the entire file (e.g., "not found").
133    fn add_error(self: Pin<&mut Self>, filename: &str, line: i32, column: i32, message: &str) {
134        self.upcast_mut().RecordError(
135            filename.into(),
136            CInt::expect_from(line),
137            CInt::expect_from(column),
138            message.into(),
139        )
140    }
141
142    /// Adds a warning to the error collector at the specified position.
143    ///
144    /// See the documentation for [`add_error`] for details on the meaning of
145    /// the `line` and `column` parameters.
146    ///
147    /// [`add_error`]: MultiFileErrorCollector::add_error
148    fn add_warning(self: Pin<&mut Self>, filename: &str, line: i32, column: i32, message: &str) {
149        self.upcast_mut().RecordWarning(
150            filename.into(),
151            CInt::expect_from(line),
152            CInt::expect_from(column),
153            message.into(),
154        )
155    }
156}
157
158mod multi_file_error_collector {
159    use std::pin::Pin;
160
161    use super::ffi;
162
163    pub trait Sealed {
164        fn upcast(&self) -> &ffi::MultiFileErrorCollector;
165        fn upcast_mut(self: Pin<&mut Self>) -> Pin<&mut ffi::MultiFileErrorCollector>;
166        unsafe fn upcast_mut_ptr(self: Pin<&mut Self>) -> *mut ffi::MultiFileErrorCollector {
167            self.upcast_mut().get_unchecked_mut() as *mut _
168        }
169    }
170}
171
172/// A simple implementation of [`MultiFileErrorCollector`] that records errors
173/// in memory for later retrieval.
174pub struct SimpleErrorCollector {
175    _opaque: PhantomPinned,
176}
177
178impl Drop for SimpleErrorCollector {
179    fn drop(&mut self) {
180        unsafe { ffi::DeleteSimpleErrorCollector(self.as_ffi_mut_ptr_unpinned()) }
181    }
182}
183
184impl SimpleErrorCollector {
185    /// Creates a new simple error collector.
186    pub fn new() -> Pin<Box<SimpleErrorCollector>> {
187        let collector = ffi::NewSimpleErrorCollector();
188        unsafe { Self::from_ffi_owned(collector) }
189    }
190
191    unsafe_ffi_conversions!(ffi::SimpleErrorCollector);
192}
193
194impl<'a> Iterator for Pin<&'a mut SimpleErrorCollector> {
195    type Item = FileLoadError;
196
197    fn next(&mut self) -> Option<FileLoadError> {
198        self.as_mut().as_ffi_mut().Errors().pop().map(Into::into)
199    }
200}
201
202impl MultiFileErrorCollector for SimpleErrorCollector {}
203
204impl multi_file_error_collector::Sealed for SimpleErrorCollector {
205    fn upcast(&self) -> &ffi::MultiFileErrorCollector {
206        unsafe { mem::transmute(self) }
207    }
208
209    fn upcast_mut(self: Pin<&mut Self>) -> Pin<&mut ffi::MultiFileErrorCollector> {
210        unsafe { mem::transmute(self) }
211    }
212}
213
214/// An implementation of `DescriptorDatabase` which loads files from a
215/// `SourceTree` and parses them.
216///
217/// Note: This class does not implement `FindFileContainingSymbol` or
218/// `FindFileContainingExtension`; these will always return false.
219pub struct SourceTreeDescriptorDatabase<'a> {
220    _opaque: PhantomPinned,
221    _lifetime: PhantomData<&'a ()>,
222}
223
224impl<'a> Drop for SourceTreeDescriptorDatabase<'a> {
225    fn drop(&mut self) {
226        unsafe { ffi::DeleteSourceTreeDescriptorDatabase(self.as_ffi_mut_ptr_unpinned()) }
227    }
228}
229
230impl<'a> SourceTreeDescriptorDatabase<'a> {
231    /// Constructs a new descriptor database for the provided source tree.
232    pub fn new(
233        source_tree: Pin<&'a mut dyn SourceTree>,
234    ) -> Pin<Box<SourceTreeDescriptorDatabase<'a>>> {
235        let db = unsafe { ffi::NewSourceTreeDescriptorDatabase(source_tree.upcast_mut_ptr()) };
236        unsafe { Self::from_ffi_owned(db) }
237    }
238
239    /// Instructs the source tree descriptor database to report any parse errors
240    /// to the given [`MultiFileErrorCollector`].
241    ///
242    /// This should b ecalled before parsing.
243    pub fn record_errors_to(
244        self: Pin<&mut Self>,
245        error_collector: Pin<&'a mut dyn MultiFileErrorCollector>,
246    ) {
247        unsafe {
248            self.as_ffi_mut()
249                .RecordErrorsTo(error_collector.upcast_mut_ptr())
250        }
251    }
252
253    /// Builds a file descriptor set containing all file descriptor protos
254    /// reachable from the specified roots.
255    pub fn build_file_descriptor_set<P>(
256        mut self: Pin<&mut Self>,
257        roots: &[P],
258    ) -> Result<Pin<Box<FileDescriptorSet>>, OperationFailedError>
259    where
260        P: AsRef<Path>,
261    {
262        let mut out = FileDescriptorSet::new();
263        let mut seen = HashSet::new();
264        let mut stack = vec![];
265        for root in roots {
266            let root = root.as_ref();
267            stack.push(self.as_mut().find_file_by_name(root)?);
268            seen.insert(ProtobufPath::from(root).as_bytes().to_vec());
269        }
270        while let Some(file) = stack.pop() {
271            out.as_mut().add_file().copy_from(&file);
272            for i in 0..file.dependency_size() {
273                let dep_path = ProtobufPath::from(file.dependency(i));
274                if !seen.contains(dep_path.as_bytes()) {
275                    let dep = self
276                        .as_mut()
277                        .find_file_by_name(dep_path.as_path().as_ref())?;
278                    stack.push(dep);
279                    seen.insert(dep_path.as_bytes().to_vec());
280                }
281            }
282        }
283        Ok(out)
284    }
285
286    unsafe_ffi_conversions!(ffi::SourceTreeDescriptorDatabase);
287}
288
289impl<'a> DescriptorDatabase for SourceTreeDescriptorDatabase<'a> {
290    fn find_file_by_name(
291        self: Pin<&mut Self>,
292        filename: &Path,
293    ) -> Result<Pin<Box<FileDescriptorProto>>, OperationFailedError> {
294        let mut fd = FileDescriptorProto::new();
295        let_cxx_string!(filename = ProtobufPath::from(filename).as_bytes());
296        if unsafe {
297            self.as_ffi_mut()
298                .FindFileByName(&filename, fd.as_mut().as_ffi_mut_ptr())
299        } {
300            Ok(fd)
301        } else {
302            Err(OperationFailedError)
303        }
304    }
305}
306
307/// Abstract interface which represents a directory tree containing .proto
308/// files.
309///
310/// Used by the default implementation of `Importer` to resolve import
311/// statements. Most users will probably want to use the `DiskSourceTree`
312/// implementation.
313///
314/// This trait is sealed and cannot be implemented outside of this crate.
315pub trait SourceTree: source_tree::Sealed {
316    /// Opens the given file and return a stream that reads it.
317    ///
318    /// The filename must be a path relative to the root of the source tree and
319    /// must not contain "." or ".." components.
320    fn open<'a>(
321        self: Pin<&'a mut Self>,
322        filename: &Path,
323    ) -> Result<Pin<Box<DynZeroCopyInputStream<'a>>>, FileOpenError> {
324        let filename = ProtobufPath::from(filename);
325        let mut source_tree = self.upcast_mut();
326        let stream = source_tree.as_mut().Open(filename.into());
327        if stream.is_null() {
328            Err(FileOpenError(ffi::SourceTreeGetLastErrorMessage(
329                source_tree,
330            )))
331        } else {
332            Ok(unsafe { DynZeroCopyInputStream::from_ffi_owned(stream) })
333        }
334    }
335}
336
337mod source_tree {
338    use std::pin::Pin;
339
340    use super::ffi;
341
342    pub trait Sealed {
343        fn upcast(&self) -> &ffi::SourceTree;
344        fn upcast_mut(self: Pin<&mut Self>) -> Pin<&mut ffi::SourceTree>;
345        unsafe fn upcast_mut_ptr(self: Pin<&mut Self>) -> *mut ffi::SourceTree {
346            self.upcast_mut().get_unchecked_mut() as *mut _
347        }
348    }
349}
350
351/// An implementation of `SourceTree` which stores files in memory.
352pub struct VirtualSourceTree {
353    _opaque: PhantomPinned,
354}
355
356impl Drop for VirtualSourceTree {
357    fn drop(&mut self) {
358        unsafe { ffi::DeleteVirtualSourceTree(self.as_ffi_mut_ptr_unpinned()) }
359    }
360}
361
362impl VirtualSourceTree {
363    /// Creates a new virtual source tree.
364    pub fn new() -> Pin<Box<VirtualSourceTree>> {
365        let tree = ffi::NewVirtualSourceTree();
366        unsafe { Self::from_ffi_owned(tree) }
367    }
368
369    /// Adds a file to the source tree with the specified name and contents.
370    pub fn add_file(self: Pin<&mut Self>, filename: &Path, contents: Vec<u8>) {
371        let filename = ProtobufPath::from(filename);
372        self.as_ffi_mut().AddFile(filename.into(), contents)
373    }
374
375    unsafe_ffi_conversions!(ffi::VirtualSourceTree);
376}
377
378impl SourceTree for VirtualSourceTree {}
379
380impl source_tree::Sealed for VirtualSourceTree {
381    fn upcast(&self) -> &ffi::SourceTree {
382        unsafe { mem::transmute(self) }
383    }
384
385    fn upcast_mut(self: Pin<&mut Self>) -> Pin<&mut ffi::SourceTree> {
386        unsafe { mem::transmute(self) }
387    }
388}
389
390/// An implementation of `SourceTree` which loads files from locations on disk.
391///
392/// Multiple mappings can be set up to map locations in the `DiskSourceTree` to
393/// locations in the physical filesystem.
394pub struct DiskSourceTree {
395    _opaque: PhantomPinned,
396}
397
398impl Drop for DiskSourceTree {
399    fn drop(&mut self) {
400        unsafe { ffi::DeleteDiskSourceTree(self.as_ffi_mut_ptr_unpinned()) }
401    }
402}
403
404impl DiskSourceTree {
405    /// Creates a new disk source tree.
406    pub fn new() -> Pin<Box<DiskSourceTree>> {
407        let tree = ffi::NewDiskSourceTree();
408        unsafe { Self::from_ffi_owned(tree) }
409    }
410
411    /// Maps a path on disk to a location in the source tree.
412    ///
413    /// The path may be either a file or a directory. If it is a directory, the
414    /// entire tree under it will be mapped to the given virtual location. To
415    /// map a directory to the root of the source tree, pass an empty string for
416    /// `virtual_path`.
417    ///
418    /// If multiple mapped paths apply when opening a file, they will be
419    /// searched in order. For example, if you do:
420    ///
421    /// ```
422    /// use std::path::Path;
423    /// use protobuf_native::compiler::DiskSourceTree;
424    ///
425    /// let mut source_tree = DiskSourceTree::new();
426    /// source_tree.as_mut().map_path(Path::new("bar"), Path::new("foo/bar"));
427    /// source_tree.as_mut().map_path(Path::new(""), Path::new("baz"));
428    /// ```
429    ///
430    /// and then you do:
431    ///
432    /// ```
433    /// # use std::path::Path;
434    /// # use std::pin::Pin;
435    /// # use protobuf_native::compiler::{SourceTree, DiskSourceTree};
436    /// # fn f(mut source_tree: Pin<&mut DiskSourceTree>) {
437    /// source_tree.open(Path::new("bar/qux"));
438    /// # }
439    /// ```
440    ///
441    /// the `DiskSourceTree` will first try to open foo/bar/qux, then
442    /// baz/bar/qux, returning the first one that opens successfully.
443    ///
444    /// `disk_path` may be an absolute path or relative to the current directory,
445    /// just like a path you'd pass to [`File::open`].
446    ///
447    /// [`File::open`]: std::fs::File::open
448    pub fn map_path(self: Pin<&mut Self>, virtual_path: &Path, disk_path: &Path) {
449        let virtual_path = ProtobufPath::from(virtual_path);
450        let disk_path = ProtobufPath::from(disk_path);
451        self.as_ffi_mut()
452            .MapPath(virtual_path.into(), disk_path.into())
453    }
454
455    unsafe_ffi_conversions!(ffi::DiskSourceTree);
456}
457
458impl SourceTree for DiskSourceTree {}
459
460impl source_tree::Sealed for DiskSourceTree {
461    fn upcast(&self) -> &ffi::SourceTree {
462        unsafe { mem::transmute(self) }
463    }
464
465    fn upcast_mut(self: Pin<&mut Self>) -> Pin<&mut ffi::SourceTree> {
466        unsafe { mem::transmute(self) }
467    }
468}
469
470/// An error occurred while opening a file.
471#[derive(Debug, Clone, Eq, PartialEq, Hash)]
472pub struct FileOpenError(String);
473
474impl fmt::Display for FileOpenError {
475    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
476        // The underlying error is descriptive enough in all cases to not
477        // warrant any additional context.
478        f.write_str(&self.0)
479    }
480}
481
482impl Error for FileOpenError {}
483
484/// Describes the severity of a [`FileLoadError`].
485#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
486pub enum Severity {
487    /// A true error.
488    Error,
489    /// An informational warning.
490    Warning,
491}
492
493impl fmt::Display for Severity {
494    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
495        match self {
496            Severity::Warning => f.write_str("warning"),
497            Severity::Error => f.write_str("error"),
498        }
499    }
500}
501
502/// Describes the location at which a [`FileLoadError`] occurred.
503#[derive(Debug, Clone, PartialEq, Eq, Hash)]
504pub struct Location {
505    /// The 1-based line number.
506    pub line: i64,
507    /// The 1-based column number.
508    pub column: i64,
509}
510
511/// An error occured while loading a file.
512#[derive(Debug, Clone, PartialEq, Eq, Hash)]
513
514pub struct FileLoadError {
515    /// The name of the file which failed to load.
516    pub filename: String,
517    /// A message describing the cause of the error.
518    pub message: String,
519    /// The severity of the error.
520    pub severity: Severity,
521    /// The specific location at which the error occurred, if applicable.
522    pub location: Option<Location>,
523}
524
525impl From<ffi::FileLoadError> for FileLoadError {
526    fn from(ffi: ffi::FileLoadError) -> FileLoadError {
527        let location = (ffi.line >= 0).then(|| Location {
528            line: ffi.line + 1,
529            column: ffi.column + 1,
530        });
531        FileLoadError {
532            filename: ffi.filename,
533            message: ffi.message,
534            severity: if ffi.warning {
535                Severity::Warning
536            } else {
537                Severity::Error
538            },
539            location,
540        }
541    }
542}
543
544impl fmt::Display for FileLoadError {
545    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
546        write!(f, "{}:", self.filename)?;
547        if let Some(location) = &self.location {
548            write!(f, "{}:{}:", location.line, location.column)?;
549        }
550        write!(f, " {}: {}", self.severity, self.message)
551    }
552}
protobuf_native/compiler.rs

protobuf_native/
compiler.rs
