proptest/test_runner/failure_persistence/
file.rs

1//-
2// Copyright 2017, 2018, 2019 The proptest developers
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use core::any::Any;
11use core::fmt::Debug;
12use std::borrow::{Cow, ToOwned};
13use std::boxed::Box;
14use std::env;
15use std::fs;
16use std::io::{self, BufRead, Write};
17use std::path::{Path, PathBuf};
18use std::string::{String, ToString};
19use std::sync::RwLock;
20use std::vec::Vec;
21
22use self::FileFailurePersistence::*;
23use crate::test_runner::failure_persistence::{
24    FailurePersistence, PersistedSeed,
25};
26
27/// Describes how failing test cases are persisted.
28///
29/// Note that file names in this enum are `&str` rather than `&Path` since
30/// constant functions are not yet in Rust stable as of 2017-12-16.
31///
32/// In all cases, if a derived path references a directory which does not yet
33/// exist, proptest will attempt to create all necessary parent directories.
34#[derive(Clone, Copy, Debug, PartialEq)]
35pub enum FileFailurePersistence {
36    /// Completely disables persistence of failing test cases.
37    ///
38    /// This is semantically equivalent to `Direct("/dev/null")` on Unix and
39    /// `Direct("NUL")` on Windows (though it is internally handled by simply
40    /// not doing any I/O).
41    Off,
42    /// The path of the source file under test is traversed up the directory tree
43    /// until a directory containing a file named `lib.rs` or `main.rs` is found.
44    /// A sibling to that directory with the name given by the string in this
45    /// configuration is created, and a file with the same name and path relative
46    /// to the source directory, but with the extension changed to `.txt`, is used.
47    ///
48    /// For example, given a source path of
49    /// `/home/jsmith/code/project/src/foo/bar.rs` and a configuration of
50    /// `SourceParallel("proptest-regressions")` (the default), assuming the
51    /// `src` directory has a `lib.rs` or `main.rs`, the resulting file would
52    /// be `/home/jsmith/code/project/proptest-regressions/foo/bar.txt`.
53    ///
54    /// If no `lib.rs` or `main.rs` can be found, a warning is printed and this
55    /// behaves like `WithSource`.
56    ///
57    /// If no source file has been configured, a warning is printed and this
58    /// behaves like `Off`.
59    SourceParallel(&'static str),
60    /// Failures are persisted in a file with the same path as the source file
61    /// under test, but the extension is changed to the string given in this
62    /// configuration.
63    ///
64    /// For example, given a source path of
65    /// `/home/jsmith/code/project/src/foo/bar.rs` and a configuration of
66    /// `WithSource("regressions")`, the resulting path would be
67    /// `/home/jsmith/code/project/src/foo/bar.regressions`.
68    WithSource(&'static str),
69    /// The string given in this option is directly used as a file path without
70    /// any further processing.
71    Direct(&'static str),
72    #[doc(hidden)]
73    #[allow(missing_docs)]
74    _NonExhaustive,
75}
76
77impl Default for FileFailurePersistence {
78    fn default() -> Self {
79        SourceParallel("proptest-regressions")
80    }
81}
82
83impl FailurePersistence for FileFailurePersistence {
84    fn load_persisted_failures2(
85        &self,
86        source_file: Option<&'static str>,
87    ) -> Vec<PersistedSeed> {
88        let p = self.resolve(
89            source_file
90                .and_then(|s| absolutize_source_file(Path::new(s)))
91                .as_ref()
92                .map(|cow| &**cow),
93        );
94
95        let path: Option<&PathBuf> = p.as_ref();
96        let result: io::Result<Vec<PersistedSeed>> = path.map_or_else(
97            || Ok(vec![]),
98            |path| {
99                // .ok() instead of .unwrap() so we don't propagate panics here
100                let _lock = PERSISTENCE_LOCK.read().ok();
101                io::BufReader::new(fs::File::open(path)?)
102                    .lines()
103                    .enumerate()
104                    .filter_map(|(lineno, line)| match line {
105                        Err(err) => Some(Err(err)),
106                        Ok(line) => parse_seed_line(line, path, lineno).map(Ok),
107                    })
108                    .collect()
109            },
110        );
111
112        unwrap_or!(result, err => {
113            if io::ErrorKind::NotFound != err.kind() {
114                eprintln!(
115                    "proptest: failed to open {}: {}",
116                    &path.map(|x| &**x)
117                        .unwrap_or_else(|| Path::new("??"))
118                        .display(),
119                    err
120                );
121            }
122            vec![]
123        })
124    }
125
126    fn save_persisted_failure2(
127        &mut self,
128        source_file: Option<&'static str>,
129        seed: PersistedSeed,
130        shrunken_value: &dyn Debug,
131    ) {
132        let path = self.resolve(source_file.map(Path::new));
133        if let Some(path) = path {
134            // .ok() instead of .unwrap() so we don't propagate panics here
135            let _lock = PERSISTENCE_LOCK.write().ok();
136            let is_new = !path.is_file();
137
138            let mut to_write = Vec::<u8>::new();
139            if is_new {
140                write_header(&mut to_write)
141                    .expect("proptest: couldn't write header.");
142            }
143
144            write_seed_line(&mut to_write, &seed, shrunken_value)
145                .expect("proptest: couldn't write seed line.");
146
147            if let Err(e) = write_seed_data_to_file(&path, &to_write) {
148                eprintln!(
149                    "proptest: failed to append to {}: {}",
150                    path.display(),
151                    e
152                );
153            } else {
154                eprintln!(
155                    "proptest: Saving this and future failures in {}\n\
156                     proptest: If this test was run on a CI system, you may \
157                     wish to add the following line to your copy of the file.{}\n\
158                     {}",
159                    path.display(),
160                    if is_new { " (You may need to create it.)" } else { "" },
161                    seed);
162            }
163        }
164    }
165
166    fn box_clone(&self) -> Box<dyn FailurePersistence> {
167        Box::new(*self)
168    }
169
170    fn eq(&self, other: &dyn FailurePersistence) -> bool {
171        other
172            .as_any()
173            .downcast_ref::<Self>()
174            .map_or(false, |x| x == self)
175    }
176
177    fn as_any(&self) -> &dyn Any {
178        self
179    }
180}
181
182/// Ensure that the source file to use for resolving the location of the persisted
183/// failing cases file is absolute.
184///
185/// The source location can only be used if it is absolute. If `source` is
186/// not an absolute path, an attempt will be made to determine the absolute
187/// path based on the current working directory and its parents. If no
188/// absolute path can be determined, a warning will be printed and proptest
189/// will continue as if this function had never been called.
190///
191/// See [`FileFailurePersistence`](enum.FileFailurePersistence.html) for details on
192/// how this value is used once it is made absolute.
193///
194/// This is normally called automatically by the `proptest!` macro, which
195/// passes `file!()`.
196///
197fn absolutize_source_file<'a>(source: &'a Path) -> Option<Cow<'a, Path>> {
198    absolutize_source_file_with_cwd(env::current_dir, source)
199}
200
201fn absolutize_source_file_with_cwd<'a>(
202    getcwd: impl FnOnce() -> io::Result<PathBuf>,
203    source: &'a Path,
204) -> Option<Cow<'a, Path>> {
205    if source.is_absolute() {
206        // On Unix, `file!()` is absolute. In these cases, we can use
207        // that path directly.
208        Some(Cow::Borrowed(source))
209    } else {
210        // On Windows, `file!()` is relative to the crate root, but the
211        // test is not generally run with the crate root as the working
212        // directory, so the path is not directly usable. However, the
213        // working directory is almost always a subdirectory of the crate
214        // root, so pop directories off until pushing the source onto the
215        // directory results in a path that refers to an existing file.
216        // Once we find such a path, we can use that.
217        //
218        // If we can't figure out an absolute path, print a warning and act
219        // as if no source had been given.
220        match getcwd() {
221            Ok(mut cwd) => loop {
222                let joined = cwd.join(source);
223                if joined.is_file() {
224                    break Some(Cow::Owned(joined));
225                }
226
227                if !cwd.pop() {
228                    eprintln!(
229                        "proptest: Failed to find absolute path of \
230                         source file '{:?}'. Ensure the test is \
231                         being run from somewhere within the crate \
232                         directory hierarchy.",
233                        source
234                    );
235                    break None;
236                }
237            },
238
239            Err(e) => {
240                eprintln!(
241                    "proptest: Failed to determine current \
242                     directory, so the relative source path \
243                     '{:?}' cannot be resolved: {}",
244                    source, e
245                );
246                None
247            }
248        }
249    }
250}
251
252fn parse_seed_line(
253    mut line: String,
254    path: &Path,
255    lineno: usize,
256) -> Option<PersistedSeed> {
257    // Remove anything after and including '#':
258    if let Some(comment_start) = line.find('#') {
259        line.truncate(comment_start);
260    }
261
262    if line.len() > 0 {
263        let ret = line.parse::<PersistedSeed>().ok();
264        if !ret.is_some() {
265            eprintln!(
266                "proptest: {}:{}: unparsable line, ignoring",
267                path.display(),
268                lineno + 1
269            );
270        }
271        return ret;
272    }
273
274    None
275}
276
277fn write_seed_line(
278    buf: &mut Vec<u8>,
279    seed: &PersistedSeed,
280    shrunken_value: &dyn Debug,
281) -> io::Result<()> {
282    // Write the seed itself
283    write!(buf, "{}", seed.to_string())?;
284
285    // Write out comment:
286    let debug_start = buf.len();
287    write!(buf, " # shrinks to {:?}", shrunken_value)?;
288
289    // Ensure there are no newlines in the debug output
290    for byte in &mut buf[debug_start..] {
291        if b'\n' == *byte || b'\r' == *byte {
292            *byte = b' ';
293        }
294    }
295
296    buf.push(b'\n');
297
298    Ok(())
299}
300
301fn write_header(buf: &mut Vec<u8>) -> io::Result<()> {
302    writeln!(
303        buf,
304        "\
305# Seeds for failure cases proptest has generated in the past. It is
306# automatically read and these particular cases re-run before any
307# novel cases are generated.
308#
309# It is recommended to check this file in to source control so that
310# everyone who runs the test benefits from these saved cases."
311    )
312}
313
314fn write_seed_data_to_file(dst: &Path, data: &[u8]) -> io::Result<()> {
315    if let Some(parent) = dst.parent() {
316        fs::create_dir_all(parent)?;
317    }
318
319    let mut options = fs::OpenOptions::new();
320    options.append(true).create(true);
321    let mut out = options.open(dst)?;
322    out.write_all(data)?;
323
324    Ok(())
325}
326
327impl FileFailurePersistence {
328    /// Given the nominal source path, determine the location of the failure
329    /// persistence file, if any.
330    pub(super) fn resolve(&self, source: Option<&Path>) -> Option<PathBuf> {
331        let source = source.and_then(absolutize_source_file);
332
333        match *self {
334            Off => None,
335
336            SourceParallel(sibling) => match source {
337                Some(source_path) => {
338                    let mut dir = Cow::into_owned(source_path.clone());
339                    let mut found = false;
340                    while dir.pop() {
341                        if dir.join("lib.rs").is_file()
342                            || dir.join("main.rs").is_file()
343                        {
344                            found = true;
345                            break;
346                        }
347                    }
348
349                    if !found {
350                        eprintln!(
351                            "proptest: FileFailurePersistence::SourceParallel set, \
352                             but failed to find lib.rs or main.rs"
353                        );
354                        WithSource(sibling).resolve(Some(&*source_path))
355                    } else {
356                        let suffix = source_path
357                            .strip_prefix(&dir)
358                            .expect("parent of source is not a prefix of it?")
359                            .to_owned();
360                        let mut result = dir;
361                        // If we've somehow reached the root, or someone gave
362                        // us a relative path that we've exhausted, just accept
363                        // creating a subdirectory instead.
364                        let _ = result.pop();
365                        result.push(sibling);
366                        result.push(&suffix);
367                        result.set_extension("txt");
368                        Some(result)
369                    }
370                }
371                None => {
372                    eprintln!(
373                        "proptest: FileFailurePersistence::SourceParallel set, \
374                         but no source file known"
375                    );
376                    None
377                }
378            },
379
380            WithSource(extension) => match source {
381                Some(source_path) => {
382                    let mut result = Cow::into_owned(source_path);
383                    result.set_extension(extension);
384                    Some(result)
385                }
386
387                None => {
388                    eprintln!(
389                        "proptest: FileFailurePersistence::WithSource set, \
390                         but no source file known"
391                    );
392                    None
393                }
394            },
395
396            Direct(path) => Some(Path::new(path).to_owned()),
397
398            _NonExhaustive => {
399                panic!("FailurePersistence set to _NonExhaustive")
400            }
401        }
402    }
403}
404
405/// Used to guard access to the persistence file(s) so that a single
406/// process will not step on its own toes.
407///
408/// We don't have much protecting us should two separate process try to
409/// write to the same file at once (depending on how atomic append mode is
410/// on the OS), but this should be extremely rare.
411static PERSISTENCE_LOCK: RwLock<()> = RwLock::new(());
412
413#[cfg(test)]
414mod tests {
415    use super::*;
416
417    struct TestPaths {
418        crate_root: &'static Path,
419        src_file: PathBuf,
420        subdir_file: PathBuf,
421        misplaced_file: PathBuf,
422    }
423
424    static TEST_PATHS: std::sync::LazyLock<TestPaths> = std::sync::LazyLock::new(|| {
425        let crate_root = Path::new(env!("CARGO_MANIFEST_DIR"));
426        let lib_root = crate_root.join("src");
427        let src_subdir = lib_root.join("strategy");
428        let src_file = lib_root.join("foo.rs");
429        let subdir_file = src_subdir.join("foo.rs");
430        let misplaced_file = crate_root.join("foo.rs");
431        TestPaths {
432            crate_root,
433            src_file,
434            subdir_file,
435            misplaced_file,
436        }
437    });
438
439    #[test]
440    fn persistence_file_location_resolved_correctly() {
441        // If off, there is never a file
442        assert_eq!(None, Off.resolve(None));
443        assert_eq!(None, Off.resolve(Some(&TEST_PATHS.subdir_file)));
444
445        // For direct, we don't care about the source file, and instead always
446        // use whatever is in the config.
447        assert_eq!(
448            Some(Path::new("bar.txt").to_owned()),
449            Direct("bar.txt").resolve(None)
450        );
451        assert_eq!(
452            Some(Path::new("bar.txt").to_owned()),
453            Direct("bar.txt").resolve(Some(&TEST_PATHS.subdir_file))
454        );
455
456        // For WithSource, only the extension changes, but we get nothing if no
457        // source file was configured.
458        // Accounting for the way absolute paths work on Windows would be more
459        // complex, so for now don't test that case.
460        #[cfg(unix)]
461        fn absolute_path_case() {
462            assert_eq!(
463                Some(Path::new("/foo/bar.ext").to_owned()),
464                WithSource("ext").resolve(Some(Path::new("/foo/bar.rs")))
465            );
466        }
467        #[cfg(not(unix))]
468        fn absolute_path_case() {}
469        absolute_path_case();
470        assert_eq!(None, WithSource("ext").resolve(None));
471
472        // For SourceParallel, we make a sibling directory tree and change the
473        // extensions to .txt ...
474        assert_eq!(
475            Some(TEST_PATHS.crate_root.join("sib").join("foo.txt")),
476            SourceParallel("sib").resolve(Some(&TEST_PATHS.src_file))
477        );
478        assert_eq!(
479            Some(
480                TEST_PATHS
481                    .crate_root
482                    .join("sib")
483                    .join("strategy")
484                    .join("foo.txt")
485            ),
486            SourceParallel("sib").resolve(Some(&TEST_PATHS.subdir_file))
487        );
488        // ... but if we can't find lib.rs / main.rs, give up and set the
489        // extension instead ...
490        assert_eq!(
491            Some(TEST_PATHS.crate_root.join("foo.sib")),
492            SourceParallel("sib").resolve(Some(&TEST_PATHS.misplaced_file))
493        );
494        // ... and if no source is configured, we do nothing
495        assert_eq!(None, SourceParallel("ext").resolve(None));
496    }
497
498    #[test]
499    fn relative_source_files_absolutified() {
500        const TEST_RUNNER_PATH: &[&str] = &["src", "test_runner", "mod.rs"];
501        static TEST_RUNNER_RELATIVE: std::sync::LazyLock<PathBuf> = std::sync::LazyLock::new(|| TEST_RUNNER_PATH.iter().collect());
502        const CARGO_DIR: &str = env!("CARGO_MANIFEST_DIR");
503
504        let expected = ::std::iter::once(CARGO_DIR)
505            .chain(TEST_RUNNER_PATH.iter().map(|s| *s))
506            .collect::<PathBuf>();
507
508        // Running from crate root
509        assert_eq!(
510            &*expected,
511            absolutize_source_file_with_cwd(
512                || Ok(Path::new(CARGO_DIR).to_owned()),
513                &*TEST_RUNNER_RELATIVE
514            )
515            .unwrap()
516        );
517
518        // Running from test subdirectory
519        assert_eq!(
520            &*expected,
521            absolutize_source_file_with_cwd(
522                || Ok(Path::new(CARGO_DIR).join("target")),
523                &*TEST_RUNNER_RELATIVE
524            )
525            .unwrap()
526        );
527    }
528}
proptest/test_runner/failure_persistence/file.rs

proptest/test_runner/failure_persistence/
file.rs
