criterion/
report.rs

Help
1#[cfg(feature = "csv_output")]
2use crate::csv_report::FileCsvReport;
3use crate::stats::bivariate::regression::Slope;
4use crate::stats::univariate::outliers::tukey::LabeledSample;
5use crate::{html::Html, stats::bivariate::Data};
6
7use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates};
8use crate::format;
9use crate::measurement::ValueFormatter;
10use crate::stats::univariate::Sample;
11use crate::stats::Distribution;
12use crate::{PlotConfiguration, Throughput};
13use anes::{Attribute, ClearLine, Color, ResetAttributes, SetAttribute, SetForegroundColor};
14use std::cmp;
15use std::collections::HashSet;
16use std::fmt;
17use std::io::stderr;
18use std::io::Write;
19use std::path::{Path, PathBuf};
20
21const MAX_DIRECTORY_NAME_LEN: usize = 64;
22const MAX_TITLE_LEN: usize = 100;
23
24pub(crate) struct ComparisonData {
25    pub p_value: f64,
26    pub t_distribution: Distribution<f64>,
27    pub t_value: f64,
28    pub relative_estimates: ChangeEstimates,
29    pub relative_distributions: ChangeDistributions,
30    pub significance_threshold: f64,
31    pub noise_threshold: f64,
32    pub base_iter_counts: Vec<f64>,
33    pub base_sample_times: Vec<f64>,
34    pub base_avg_times: Vec<f64>,
35    pub base_estimates: Estimates,
36}
37
38pub(crate) struct MeasurementData<'a> {
39    pub data: Data<'a, f64, f64>,
40    pub avg_times: LabeledSample<'a, f64>,
41    pub absolute_estimates: Estimates,
42    pub distributions: Distributions,
43    pub comparison: Option<ComparisonData>,
44    pub throughput: Option<Throughput>,
45}
46impl<'a> MeasurementData<'a> {
47    pub fn iter_counts(&self) -> &Sample<f64> {
48        self.data.x()
49    }
50
51    #[cfg(feature = "csv_output")]
52    pub fn sample_times(&self) -> &Sample<f64> {
53        self.data.y()
54    }
55}
56
57#[derive(Debug, Clone, Copy, Eq, PartialEq)]
58pub enum ValueType {
59    Bytes,
60    Elements,
61    Value,
62}
63
64#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
65pub struct BenchmarkId {
66    pub group_id: String,
67    pub function_id: Option<String>,
68    pub value_str: Option<String>,
69    pub throughput: Option<Throughput>,
70    full_id: String,
71    directory_name: String,
72    title: String,
73}
74
75fn truncate_to_character_boundary(s: &mut String, max_len: usize) {
76    let mut boundary = cmp::min(max_len, s.len());
77    while !s.is_char_boundary(boundary) {
78        boundary -= 1;
79    }
80    s.truncate(boundary);
81}
82
83pub fn make_filename_safe(string: &str) -> String {
84    let mut string = string.replace(
85        &['?', '"', '/', '\\', '*', '<', '>', ':', '|', '^'][..],
86        "_",
87    );
88
89    // Truncate to last character boundary before max length...
90    truncate_to_character_boundary(&mut string, MAX_DIRECTORY_NAME_LEN);
91
92    if cfg!(target_os = "windows") {
93        {
94            string = string
95                // On Windows, spaces in the end of the filename are ignored and will be trimmed.
96                //
97                // Without trimming ourselves, creating a directory `dir ` will silently create
98                // `dir` instead, but then operations on files like `dir /file` will fail.
99                //
100                // Also note that it's important to do this *after* trimming to MAX_DIRECTORY_NAME_LEN,
101                // otherwise it can trim again to a name with a trailing space.
102                .trim_end()
103                // On Windows, file names are not case-sensitive, so lowercase everything.
104                .to_lowercase();
105        }
106    }
107
108    string
109}
110
111impl BenchmarkId {
112    pub fn new(
113        group_id: String,
114        function_id: Option<String>,
115        value_str: Option<String>,
116        throughput: Option<Throughput>,
117    ) -> BenchmarkId {
118        let full_id = match (&function_id, &value_str) {
119            (Some(func), Some(val)) => format!("{}/{}/{}", group_id, func, val),
120            (Some(func), &None) => format!("{}/{}", group_id, func),
121            (&None, Some(val)) => format!("{}/{}", group_id, val),
122            (&None, &None) => group_id.clone(),
123        };
124
125        let mut title = full_id.clone();
126        truncate_to_character_boundary(&mut title, MAX_TITLE_LEN);
127        if title != full_id {
128            title.push_str("...");
129        }
130
131        let directory_name = match (&function_id, &value_str) {
132            (Some(func), Some(val)) => format!(
133                "{}/{}/{}",
134                make_filename_safe(&group_id),
135                make_filename_safe(func),
136                make_filename_safe(val)
137            ),
138            (Some(func), &None) => format!(
139                "{}/{}",
140                make_filename_safe(&group_id),
141                make_filename_safe(func)
142            ),
143            (&None, Some(val)) => format!(
144                "{}/{}",
145                make_filename_safe(&group_id),
146                make_filename_safe(val)
147            ),
148            (&None, &None) => make_filename_safe(&group_id),
149        };
150
151        BenchmarkId {
152            group_id,
153            function_id,
154            value_str,
155            throughput,
156            full_id,
157            directory_name,
158            title,
159        }
160    }
161
162    pub fn id(&self) -> &str {
163        &self.full_id
164    }
165
166    pub fn as_title(&self) -> &str {
167        &self.title
168    }
169
170    pub fn as_directory_name(&self) -> &str {
171        &self.directory_name
172    }
173
174    pub fn as_number(&self) -> Option<f64> {
175        match self.throughput {
176            Some(Throughput::Bytes(n))
177            | Some(Throughput::Elements(n))
178            | Some(Throughput::BytesDecimal(n)) => Some(n as f64),
179            None => self
180                .value_str
181                .as_ref()
182                .and_then(|string| string.parse::<f64>().ok()),
183        }
184    }
185
186    pub fn value_type(&self) -> Option<ValueType> {
187        match self.throughput {
188            Some(Throughput::Bytes(_)) => Some(ValueType::Bytes),
189            Some(Throughput::BytesDecimal(_)) => Some(ValueType::Bytes),
190            Some(Throughput::Elements(_)) => Some(ValueType::Elements),
191            None => self
192                .value_str
193                .as_ref()
194                .and_then(|string| string.parse::<f64>().ok())
195                .map(|_| ValueType::Value),
196        }
197    }
198
199    pub fn ensure_directory_name_unique(&mut self, existing_directories: &HashSet<String>) {
200        if !existing_directories.contains(self.as_directory_name()) {
201            return;
202        }
203
204        let mut counter = 2;
205        loop {
206            let new_dir_name = format!("{}_{}", self.as_directory_name(), counter);
207            if !existing_directories.contains(&new_dir_name) {
208                self.directory_name = new_dir_name;
209                return;
210            }
211            counter += 1;
212        }
213    }
214
215    pub fn ensure_title_unique(&mut self, existing_titles: &HashSet<String>) {
216        if !existing_titles.contains(self.as_title()) {
217            return;
218        }
219
220        let mut counter = 2;
221        loop {
222            let new_title = format!("{} #{}", self.as_title(), counter);
223            if !existing_titles.contains(&new_title) {
224                self.title = new_title;
225                return;
226            }
227            counter += 1;
228        }
229    }
230}
231impl fmt::Display for BenchmarkId {
232    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
233        f.write_str(self.as_title())
234    }
235}
236impl fmt::Debug for BenchmarkId {
237    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
238        fn format_opt(opt: &Option<String>) -> String {
239            match *opt {
240                Some(ref string) => format!("\"{}\"", string),
241                None => "None".to_owned(),
242            }
243        }
244
245        write!(
246            f,
247            "BenchmarkId {{ group_id: \"{}\", function_id: {}, value_str: {}, throughput: {:?} }}",
248            self.group_id,
249            format_opt(&self.function_id),
250            format_opt(&self.value_str),
251            self.throughput,
252        )
253    }
254}
255
256pub struct ReportContext {
257    pub output_directory: PathBuf,
258    pub plot_config: PlotConfiguration,
259}
260impl ReportContext {
261    pub fn report_path<P: AsRef<Path> + ?Sized>(&self, id: &BenchmarkId, file_name: &P) -> PathBuf {
262        let mut path = self.output_directory.clone();
263        path.push(id.as_directory_name());
264        path.push("report");
265        path.push(file_name);
266        path
267    }
268}
269
270pub(crate) trait Report {
271    fn test_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
272    fn test_pass(&self, _id: &BenchmarkId, _context: &ReportContext) {}
273
274    fn benchmark_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
275    fn profile(&self, _id: &BenchmarkId, _context: &ReportContext, _profile_ns: f64) {}
276    fn warmup(&self, _id: &BenchmarkId, _context: &ReportContext, _warmup_ns: f64) {}
277    fn terminated(&self, _id: &BenchmarkId, _context: &ReportContext) {}
278    fn analysis(&self, _id: &BenchmarkId, _context: &ReportContext) {}
279    fn measurement_start(
280        &self,
281        _id: &BenchmarkId,
282        _context: &ReportContext,
283        _sample_count: u64,
284        _estimate_ns: f64,
285        _iter_count: u64,
286    ) {
287    }
288    fn measurement_complete(
289        &self,
290        _id: &BenchmarkId,
291        _context: &ReportContext,
292        _measurements: &MeasurementData<'_>,
293        _formatter: &dyn ValueFormatter,
294    ) {
295    }
296    fn summarize(
297        &self,
298        _context: &ReportContext,
299        _all_ids: &[BenchmarkId],
300        _formatter: &dyn ValueFormatter,
301    ) {
302    }
303    fn final_summary(&self, _context: &ReportContext) {}
304    fn group_separator(&self) {}
305}
306
307pub(crate) struct Reports {
308    pub(crate) cli_enabled: bool,
309    pub(crate) cli: CliReport,
310    pub(crate) bencher_enabled: bool,
311    pub(crate) bencher: BencherReport,
312    pub(crate) csv_enabled: bool,
313    pub(crate) html: Option<Html>,
314}
315macro_rules! reports_impl {
316    (fn $name:ident(&self, $($argn:ident: $argt:ty),*)) => {
317        fn $name(&self, $($argn: $argt),* ) {
318            if self.cli_enabled {
319                self.cli.$name($($argn),*);
320            }
321            if self.bencher_enabled {
322                self.bencher.$name($($argn),*);
323            }
324            #[cfg(feature = "csv_output")]
325            if self.csv_enabled {
326                FileCsvReport.$name($($argn),*);
327            }
328            if let Some(reporter) = &self.html {
329                reporter.$name($($argn),*);
330            }
331        }
332    };
333}
334
335impl Report for Reports {
336    reports_impl!(fn test_start(&self, id: &BenchmarkId, context: &ReportContext));
337    reports_impl!(fn test_pass(&self, id: &BenchmarkId, context: &ReportContext));
338    reports_impl!(fn benchmark_start(&self, id: &BenchmarkId, context: &ReportContext));
339    reports_impl!(fn profile(&self, id: &BenchmarkId, context: &ReportContext, profile_ns: f64));
340    reports_impl!(fn warmup(&self, id: &BenchmarkId, context: &ReportContext, warmup_ns: f64));
341    reports_impl!(fn terminated(&self, id: &BenchmarkId, context: &ReportContext));
342    reports_impl!(fn analysis(&self, id: &BenchmarkId, context: &ReportContext));
343    reports_impl!(fn measurement_start(
344        &self,
345        id: &BenchmarkId,
346        context: &ReportContext,
347        sample_count: u64,
348        estimate_ns: f64,
349        iter_count: u64
350    ));
351    reports_impl!(
352    fn measurement_complete(
353        &self,
354        id: &BenchmarkId,
355        context: &ReportContext,
356        measurements: &MeasurementData<'_>,
357        formatter: &dyn ValueFormatter
358    ));
359    reports_impl!(
360    fn summarize(
361        &self,
362        context: &ReportContext,
363        all_ids: &[BenchmarkId],
364        formatter: &dyn ValueFormatter
365    ));
366
367    reports_impl!(fn final_summary(&self, context: &ReportContext));
368    reports_impl!(fn group_separator(&self, ));
369}
370
371#[derive(Debug, Clone, Copy, Eq, PartialEq)]
372pub(crate) enum CliVerbosity {
373    Quiet,
374    Normal,
375    Verbose,
376}
377
378pub(crate) struct CliReport {
379    pub enable_text_overwrite: bool,
380    pub enable_text_coloring: bool,
381    pub verbosity: CliVerbosity,
382}
383impl CliReport {
384    pub fn new(
385        enable_text_overwrite: bool,
386        enable_text_coloring: bool,
387        verbosity: CliVerbosity,
388    ) -> CliReport {
389        CliReport {
390            enable_text_overwrite,
391            enable_text_coloring,
392            verbosity,
393        }
394    }
395
396    fn text_overwrite(&self) {
397        if self.enable_text_overwrite {
398            eprint!("\r{}", ClearLine::All)
399        }
400    }
401
402    // Passing a String is the common case here.
403    #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))]
404    fn print_overwritable(&self, s: String) {
405        if self.enable_text_overwrite {
406            eprint!("{}", s);
407            stderr().flush().unwrap();
408        } else {
409            eprintln!("{}", s);
410        }
411    }
412
413    fn with_color(&self, color: Color, s: &str) -> String {
414        if self.enable_text_coloring {
415            format!("{}{}{}", SetForegroundColor(color), s, ResetAttributes)
416        } else {
417            String::from(s)
418        }
419    }
420
421    fn green(&self, s: &str) -> String {
422        self.with_color(Color::DarkGreen, s)
423    }
424
425    fn yellow(&self, s: &str) -> String {
426        self.with_color(Color::DarkYellow, s)
427    }
428
429    fn red(&self, s: &str) -> String {
430        self.with_color(Color::DarkRed, s)
431    }
432
433    fn bold(&self, s: String) -> String {
434        if self.enable_text_coloring {
435            format!("{}{}{}", SetAttribute(Attribute::Bold), s, ResetAttributes)
436        } else {
437            s
438        }
439    }
440
441    fn faint(&self, s: String) -> String {
442        if self.enable_text_coloring {
443            format!("{}{}{}", SetAttribute(Attribute::Faint), s, ResetAttributes)
444        } else {
445            s
446        }
447    }
448
449    pub fn outliers(&self, sample: &LabeledSample<'_, f64>) {
450        let (los, lom, _, him, his) = sample.count();
451        let noutliers = los + lom + him + his;
452        let sample_size = sample.len();
453
454        if noutliers == 0 {
455            return;
456        }
457
458        let percent = |n: usize| 100. * n as f64 / sample_size as f64;
459
460        println!(
461            "{}",
462            self.yellow(&format!(
463                "Found {} outliers among {} measurements ({:.2}%)",
464                noutliers,
465                sample_size,
466                percent(noutliers)
467            ))
468        );
469
470        let print = |n, label| {
471            if n != 0 {
472                println!("  {} ({:.2}%) {}", n, percent(n), label);
473            }
474        };
475
476        print(los, "low severe");
477        print(lom, "low mild");
478        print(him, "high mild");
479        print(his, "high severe");
480    }
481}
482impl Report for CliReport {
483    fn test_start(&self, id: &BenchmarkId, _: &ReportContext) {
484        println!("Testing {}", id);
485    }
486    fn test_pass(&self, _: &BenchmarkId, _: &ReportContext) {
487        println!("Success");
488    }
489
490    fn benchmark_start(&self, id: &BenchmarkId, _: &ReportContext) {
491        self.print_overwritable(format!("Benchmarking {}", id));
492    }
493
494    fn profile(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
495        self.text_overwrite();
496        self.print_overwritable(format!(
497            "Benchmarking {}: Profiling for {}",
498            id,
499            format::time(warmup_ns)
500        ));
501    }
502
503    fn warmup(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
504        self.text_overwrite();
505        self.print_overwritable(format!(
506            "Benchmarking {}: Warming up for {}",
507            id,
508            format::time(warmup_ns)
509        ));
510    }
511
512    fn terminated(&self, id: &BenchmarkId, _: &ReportContext) {
513        self.text_overwrite();
514        println!("Benchmarking {}: Complete (Analysis Disabled)", id);
515    }
516
517    fn analysis(&self, id: &BenchmarkId, _: &ReportContext) {
518        self.text_overwrite();
519        self.print_overwritable(format!("Benchmarking {}: Analyzing", id));
520    }
521
522    fn measurement_start(
523        &self,
524        id: &BenchmarkId,
525        _: &ReportContext,
526        sample_count: u64,
527        estimate_ns: f64,
528        iter_count: u64,
529    ) {
530        self.text_overwrite();
531        let iter_string = if matches!(self.verbosity, CliVerbosity::Verbose) {
532            format!("{} iterations", iter_count)
533        } else {
534            format::iter_count(iter_count)
535        };
536
537        self.print_overwritable(format!(
538            "Benchmarking {}: Collecting {} samples in estimated {} ({})",
539            id,
540            sample_count,
541            format::time(estimate_ns),
542            iter_string
543        ));
544    }
545
546    fn measurement_complete(
547        &self,
548        id: &BenchmarkId,
549        _: &ReportContext,
550        meas: &MeasurementData<'_>,
551        formatter: &dyn ValueFormatter,
552    ) {
553        self.text_overwrite();
554
555        let typical_estimate = &meas.absolute_estimates.typical();
556
557        {
558            let mut id = id.as_title().to_owned();
559
560            if id.len() > 23 {
561                println!("{}", self.green(&id));
562                id.clear();
563            }
564            let id_len = id.len();
565
566            println!(
567                "{}{}time:   [{} {} {}]",
568                self.green(&id),
569                " ".repeat(24 - id_len),
570                self.faint(
571                    formatter.format_value(typical_estimate.confidence_interval.lower_bound)
572                ),
573                self.bold(formatter.format_value(typical_estimate.point_estimate)),
574                self.faint(
575                    formatter.format_value(typical_estimate.confidence_interval.upper_bound)
576                )
577            );
578        }
579
580        if let Some(ref throughput) = meas.throughput {
581            println!(
582                "{}thrpt:  [{} {} {}]",
583                " ".repeat(24),
584                self.faint(formatter.format_throughput(
585                    throughput,
586                    typical_estimate.confidence_interval.upper_bound
587                )),
588                self.bold(formatter.format_throughput(throughput, typical_estimate.point_estimate)),
589                self.faint(formatter.format_throughput(
590                    throughput,
591                    typical_estimate.confidence_interval.lower_bound
592                )),
593            )
594        }
595
596        if !matches!(self.verbosity, CliVerbosity::Quiet) {
597            if let Some(ref comp) = meas.comparison {
598                let different_mean = comp.p_value < comp.significance_threshold;
599                let mean_est = &comp.relative_estimates.mean;
600                let point_estimate = mean_est.point_estimate;
601                let mut point_estimate_str = format::change(point_estimate, true);
602                // The change in throughput is related to the change in timing. Reducing the timing by
603                // 50% increases the throughput by 100%.
604                let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
605                let mut thrpt_point_estimate_str =
606                    format::change(to_thrpt_estimate(point_estimate), true);
607                let explanation_str: String;
608
609                if !different_mean {
610                    explanation_str = "No change in performance detected.".to_owned();
611                } else {
612                    let comparison = compare_to_threshold(mean_est, comp.noise_threshold);
613                    match comparison {
614                        ComparisonResult::Improved => {
615                            point_estimate_str = self.green(&self.bold(point_estimate_str));
616                            thrpt_point_estimate_str =
617                                self.green(&self.bold(thrpt_point_estimate_str));
618                            explanation_str =
619                                format!("Performance has {}.", self.green("improved"));
620                        }
621                        ComparisonResult::Regressed => {
622                            point_estimate_str = self.red(&self.bold(point_estimate_str));
623                            thrpt_point_estimate_str =
624                                self.red(&self.bold(thrpt_point_estimate_str));
625                            explanation_str = format!("Performance has {}.", self.red("regressed"));
626                        }
627                        ComparisonResult::NonSignificant => {
628                            explanation_str = "Change within noise threshold.".to_owned();
629                        }
630                    }
631                }
632
633                if meas.throughput.is_some() {
634                    println!("{}change:", " ".repeat(17));
635
636                    println!(
637                        "{}time:   [{} {} {}] (p = {:.2} {} {:.2})",
638                        " ".repeat(24),
639                        self.faint(format::change(
640                            mean_est.confidence_interval.lower_bound,
641                            true
642                        )),
643                        point_estimate_str,
644                        self.faint(format::change(
645                            mean_est.confidence_interval.upper_bound,
646                            true
647                        )),
648                        comp.p_value,
649                        if different_mean { "<" } else { ">" },
650                        comp.significance_threshold
651                    );
652                    println!(
653                        "{}thrpt:  [{} {} {}]",
654                        " ".repeat(24),
655                        self.faint(format::change(
656                            to_thrpt_estimate(mean_est.confidence_interval.upper_bound),
657                            true
658                        )),
659                        thrpt_point_estimate_str,
660                        self.faint(format::change(
661                            to_thrpt_estimate(mean_est.confidence_interval.lower_bound),
662                            true
663                        )),
664                    );
665                } else {
666                    println!(
667                        "{}change: [{} {} {}] (p = {:.2} {} {:.2})",
668                        " ".repeat(24),
669                        self.faint(format::change(
670                            mean_est.confidence_interval.lower_bound,
671                            true
672                        )),
673                        point_estimate_str,
674                        self.faint(format::change(
675                            mean_est.confidence_interval.upper_bound,
676                            true
677                        )),
678                        comp.p_value,
679                        if different_mean { "<" } else { ">" },
680                        comp.significance_threshold
681                    );
682                }
683
684                println!("{}{}", " ".repeat(24), explanation_str);
685            }
686        }
687
688        if !matches!(self.verbosity, CliVerbosity::Quiet) {
689            self.outliers(&meas.avg_times);
690        }
691
692        if matches!(self.verbosity, CliVerbosity::Verbose) {
693            let format_short_estimate = |estimate: &Estimate| -> String {
694                format!(
695                    "[{} {}]",
696                    formatter.format_value(estimate.confidence_interval.lower_bound),
697                    formatter.format_value(estimate.confidence_interval.upper_bound)
698                )
699            };
700
701            let data = &meas.data;
702            if let Some(slope_estimate) = meas.absolute_estimates.slope.as_ref() {
703                println!(
704                    "{:<7}{} {:<15}[{:0.7} {:0.7}]",
705                    "slope",
706                    format_short_estimate(slope_estimate),
707                    "R^2",
708                    Slope(slope_estimate.confidence_interval.lower_bound).r_squared(data),
709                    Slope(slope_estimate.confidence_interval.upper_bound).r_squared(data),
710                );
711            }
712            println!(
713                "{:<7}{} {:<15}{}",
714                "mean",
715                format_short_estimate(&meas.absolute_estimates.mean),
716                "std. dev.",
717                format_short_estimate(&meas.absolute_estimates.std_dev),
718            );
719            println!(
720                "{:<7}{} {:<15}{}",
721                "median",
722                format_short_estimate(&meas.absolute_estimates.median),
723                "med. abs. dev.",
724                format_short_estimate(&meas.absolute_estimates.median_abs_dev),
725            );
726        }
727    }
728
729    fn group_separator(&self) {
730        println!();
731    }
732}
733
734pub struct BencherReport;
735impl Report for BencherReport {
736    fn measurement_start(
737        &self,
738        id: &BenchmarkId,
739        _context: &ReportContext,
740        _sample_count: u64,
741        _estimate_ns: f64,
742        _iter_count: u64,
743    ) {
744        print!("test {} ... ", id);
745    }
746
747    fn measurement_complete(
748        &self,
749        _id: &BenchmarkId,
750        _: &ReportContext,
751        meas: &MeasurementData<'_>,
752        formatter: &dyn ValueFormatter,
753    ) {
754        let mut values = [
755            meas.absolute_estimates.median.point_estimate,
756            meas.absolute_estimates.std_dev.point_estimate,
757        ];
758        let unit = formatter.scale_for_machines(&mut values);
759
760        println!(
761            "bench: {:>11} {}/iter (+/- {})",
762            format::integer(values[0]),
763            unit,
764            format::integer(values[1])
765        );
766    }
767
768    fn group_separator(&self) {
769        println!();
770    }
771}
772
773enum ComparisonResult {
774    Improved,
775    Regressed,
776    NonSignificant,
777}
778
779fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult {
780    let ci = &estimate.confidence_interval;
781    let lb = ci.lower_bound;
782    let ub = ci.upper_bound;
783
784    if lb < -noise && ub < -noise {
785        ComparisonResult::Improved
786    } else if lb > noise && ub > noise {
787        ComparisonResult::Regressed
788    } else {
789        ComparisonResult::NonSignificant
790    }
791}
792
793#[cfg(test)]
794mod test {
795    use super::*;
796
797    #[test]
798    fn test_make_filename_safe_replaces_characters() {
799        let input = "?/\\*\"";
800        let safe = make_filename_safe(input);
801        assert_eq!("_____", &safe);
802    }
803
804    #[test]
805    fn test_make_filename_safe_truncates_long_strings() {
806        let input = "this is a very long string. it is too long to be safe as a directory name, and so it needs to be truncated. what a long string this is.";
807        let safe = make_filename_safe(input);
808        assert!(input.len() > MAX_DIRECTORY_NAME_LEN);
809        assert_eq!(&input[0..MAX_DIRECTORY_NAME_LEN], &safe);
810    }
811
812    #[test]
813    fn test_make_filename_safe_respects_character_boundaries() {
814        let input = "✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓";
815        let safe = make_filename_safe(input);
816        assert!(safe.len() < MAX_DIRECTORY_NAME_LEN);
817    }
818
819    #[test]
820    fn test_benchmark_id_make_directory_name_unique() {
821        let existing_id = BenchmarkId::new(
822            "group".to_owned(),
823            Some("function".to_owned()),
824            Some("value".to_owned()),
825            None,
826        );
827        let mut directories = HashSet::new();
828        directories.insert(existing_id.as_directory_name().to_owned());
829
830        let mut new_id = existing_id.clone();
831        new_id.ensure_directory_name_unique(&directories);
832        assert_eq!("group/function/value_2", new_id.as_directory_name());
833        directories.insert(new_id.as_directory_name().to_owned());
834
835        new_id = existing_id;
836        new_id.ensure_directory_name_unique(&directories);
837        assert_eq!("group/function/value_3", new_id.as_directory_name());
838        directories.insert(new_id.as_directory_name().to_owned());
839    }
840    #[test]
841    fn test_benchmark_id_make_long_directory_name_unique() {
842        let long_name = (0..MAX_DIRECTORY_NAME_LEN).map(|_| 'a').collect::<String>();
843        let existing_id = BenchmarkId::new(long_name, None, None, None);
844        let mut directories = HashSet::new();
845        directories.insert(existing_id.as_directory_name().to_owned());
846
847        let mut new_id = existing_id.clone();
848        new_id.ensure_directory_name_unique(&directories);
849        assert_ne!(existing_id.as_directory_name(), new_id.as_directory_name());
850    }
851}
criterion/report.rs

criterion/
report.rs
