criterion/html/

mod.rs

use crate::report::{make_filename_safe, BenchmarkId, MeasurementData, Report, ReportContext};
use crate::stats::bivariate::regression::Slope;

use crate::estimate::Estimate;
use crate::format;
use crate::fs;
use crate::measurement::ValueFormatter;
use crate::plot::{PlotContext, PlotData, Plotter};
use crate::SavedSample;
use criterion_plot::Size;
use serde::Serialize;
use std::cell::RefCell;
use std::cmp::Ordering;
use std::collections::{BTreeSet, HashMap};
use std::path::{Path, PathBuf};
use tinytemplate::TinyTemplate;

const THUMBNAIL_SIZE: Option<Size> = Some(Size(450, 300));

fn debug_context<S: Serialize>(path: &Path, context: &S) {
    if crate::debug_enabled() {
        let mut context_path = PathBuf::from(path);
        context_path.set_extension("json");
        println!("Writing report context to {:?}", context_path);
        let result = fs::save(context, &context_path);
        if let Err(e) = result {
            error!("Failed to write report context debug output: {}", e);
        }
    }
}

#[derive(Serialize)]
struct Context {
    title: String,
    confidence: String,

    thumbnail_width: usize,
    thumbnail_height: usize,

    slope: Option<ConfidenceInterval>,
    r2: ConfidenceInterval,
    mean: ConfidenceInterval,
    std_dev: ConfidenceInterval,
    median: ConfidenceInterval,
    mad: ConfidenceInterval,
    throughput: Option<ConfidenceInterval>,

    additional_plots: Vec<Plot>,

    comparison: Option<Comparison>,
}

#[derive(Serialize)]
struct IndividualBenchmark {
    name: String,
    path: String,
    regression_exists: bool,
}
impl IndividualBenchmark {
    fn from_id(
        output_directory: &Path,
        path_prefix: &str,
        id: &BenchmarkId,
    ) -> IndividualBenchmark {
        let mut regression_path = PathBuf::from(output_directory);
        regression_path.push(id.as_directory_name());
        regression_path.push("report");
        regression_path.push("regression.svg");

        IndividualBenchmark {
            name: id.as_title().to_owned(),
            path: format!("{}/{}", path_prefix, id.as_directory_name()),
            regression_exists: regression_path.is_file(),
        }
    }
}

#[derive(Serialize)]
struct SummaryContext {
    group_id: String,

    thumbnail_width: usize,
    thumbnail_height: usize,

    violin_plot: Option<String>,
    line_chart: Option<String>,

    benchmarks: Vec<IndividualBenchmark>,
}

#[derive(Serialize)]
struct ConfidenceInterval {
    lower: String,
    upper: String,
    point: String,
}

#[derive(Serialize)]
struct Plot {
    name: String,
    url: String,
}
impl Plot {
    fn new(name: &str, url: &str) -> Plot {
        Plot {
            name: name.to_owned(),
            url: url.to_owned(),
        }
    }
}

#[derive(Serialize)]
struct Comparison {
    p_value: String,
    inequality: String,
    significance_level: String,
    explanation: String,

    change: ConfidenceInterval,
    thrpt_change: Option<ConfidenceInterval>,
    additional_plots: Vec<Plot>,
}

fn if_exists(output_directory: &Path, path: &Path) -> Option<String> {
    let report_path = path.join("report/index.html");
    if PathBuf::from(output_directory).join(&report_path).is_file() {
        Some(report_path.to_string_lossy().to_string())
    } else {
        None
    }
}
#[derive(Serialize, Debug)]
struct ReportLink<'a> {
    name: &'a str,
    path: Option<String>,
}
impl<'a> ReportLink<'a> {
    // TODO: Would be nice if I didn't have to keep making these components filename-safe.
    fn group(output_directory: &Path, group_id: &'a str) -> ReportLink<'a> {
        let path = PathBuf::from(make_filename_safe(group_id));

        ReportLink {
            name: group_id,
            path: if_exists(output_directory, &path),
        }
    }

    fn function(output_directory: &Path, group_id: &str, function_id: &'a str) -> ReportLink<'a> {
        let mut path = PathBuf::from(make_filename_safe(group_id));
        path.push(make_filename_safe(function_id));

        ReportLink {
            name: function_id,
            path: if_exists(output_directory, &path),
        }
    }

    fn value(output_directory: &Path, group_id: &str, value_str: &'a str) -> ReportLink<'a> {
        let mut path = PathBuf::from(make_filename_safe(group_id));
        path.push(make_filename_safe(value_str));

        ReportLink {
            name: value_str,
            path: if_exists(output_directory, &path),
        }
    }

    fn individual(output_directory: &Path, id: &'a BenchmarkId) -> ReportLink<'a> {
        let path = PathBuf::from(id.as_directory_name());
        ReportLink {
            name: id.as_title(),
            path: if_exists(output_directory, &path),
        }
    }
}

#[derive(Serialize)]
struct BenchmarkValueGroup<'a> {
    value: Option<ReportLink<'a>>,
    benchmarks: Vec<ReportLink<'a>>,
}

#[derive(Serialize)]
struct BenchmarkGroup<'a> {
    group_report: ReportLink<'a>,

    function_ids: Option<Vec<ReportLink<'a>>>,
    values: Option<Vec<ReportLink<'a>>>,

    individual_links: Vec<BenchmarkValueGroup<'a>>,
}
impl<'a> BenchmarkGroup<'a> {
    fn new(output_directory: &Path, ids: &[&'a BenchmarkId]) -> BenchmarkGroup<'a> {
        let group_id = &ids[0].group_id;
        let group_report = ReportLink::group(output_directory, group_id);

        let mut function_ids = Vec::with_capacity(ids.len());
        let mut values = Vec::with_capacity(ids.len());
        let mut individual_links = HashMap::with_capacity(ids.len());

        for id in ids.iter() {
            let function_id = id.function_id.as_deref();
            let value = id.value_str.as_deref();

            let individual_link = ReportLink::individual(output_directory, id);

            function_ids.push(function_id);
            values.push(value);

            individual_links.insert((function_id, value), individual_link);
        }

        fn parse_opt(os: &Option<&str>) -> Option<f64> {
            os.and_then(|s| s.parse::<f64>().ok())
        }

        // If all of the value strings can be parsed into a number, sort/dedupe
        // numerically. Otherwise sort lexicographically.
        if values.iter().all(|os| parse_opt(os).is_some()) {
            values.sort_unstable_by(|v1, v2| {
                let num1 = parse_opt(v1);
                let num2 = parse_opt(v2);

                num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
            });
            values.dedup_by_key(|os| parse_opt(os).unwrap());
        } else {
            values.sort_unstable();
            values.dedup();
        }

        // Sort and dedupe functions by name.
        function_ids.sort_unstable();
        function_ids.dedup();

        let mut value_groups = Vec::with_capacity(values.len());
        for value in values.iter() {
            let row = function_ids
                .iter()
                .filter_map(|f| individual_links.remove(&(*f, *value)))
                .collect::<Vec<_>>();
            value_groups.push(BenchmarkValueGroup {
                value: value.map(|s| ReportLink::value(output_directory, group_id, s)),
                benchmarks: row,
            });
        }

        let function_ids = function_ids
            .into_iter()
            .map(|os| os.map(|s| ReportLink::function(output_directory, group_id, s)))
            .collect::<Option<Vec<_>>>();
        let values = values
            .into_iter()
            .map(|os| os.map(|s| ReportLink::value(output_directory, group_id, s)))
            .collect::<Option<Vec<_>>>();

        BenchmarkGroup {
            group_report,
            function_ids,
            values,
            individual_links: value_groups,
        }
    }
}

#[derive(Serialize)]
struct IndexContext<'a> {
    groups: Vec<BenchmarkGroup<'a>>,
}

pub struct Html {
    templates: TinyTemplate<'static>,
    plotter: RefCell<Box<dyn Plotter>>,
}
impl Html {
    pub(crate) fn new(plotter: Box<dyn Plotter>) -> Html {
        let mut templates = TinyTemplate::new();
        templates
            .add_template("report_link", include_str!("report_link.html.tt"))
            .expect("Unable to parse report_link template.");
        templates
            .add_template("index", include_str!("index.html.tt"))
            .expect("Unable to parse index template.");
        templates
            .add_template("benchmark_report", include_str!("benchmark_report.html.tt"))
            .expect("Unable to parse benchmark_report template");
        templates
            .add_template("summary_report", include_str!("summary_report.html.tt"))
            .expect("Unable to parse summary_report template");

        let plotter = RefCell::new(plotter);
        Html { templates, plotter }
    }
}
impl Report for Html {
    fn measurement_complete(
        &self,
        id: &BenchmarkId,
        report_context: &ReportContext,
        measurements: &MeasurementData<'_>,
        formatter: &dyn ValueFormatter,
    ) {
        try_else_return!({
            let mut report_dir = report_context.output_directory.clone();
            report_dir.push(id.as_directory_name());
            report_dir.push("report");
            fs::mkdirp(&report_dir)
        });

        let typical_estimate = &measurements.absolute_estimates.typical();

        let time_interval = |est: &Estimate| -> ConfidenceInterval {
            ConfidenceInterval {
                lower: formatter.format_value(est.confidence_interval.lower_bound),
                point: formatter.format_value(est.point_estimate),
                upper: formatter.format_value(est.confidence_interval.upper_bound),
            }
        };

        let data = measurements.data;

        elapsed! {
            "Generating plots",
            self.generate_plots(id, report_context, formatter, measurements)
        }

        let mut additional_plots = vec![
            Plot::new("Typical", "typical.svg"),
            Plot::new("Mean", "mean.svg"),
            Plot::new("Std. Dev.", "SD.svg"),
            Plot::new("Median", "median.svg"),
            Plot::new("MAD", "MAD.svg"),
        ];
        if measurements.absolute_estimates.slope.is_some() {
            additional_plots.push(Plot::new("Slope", "slope.svg"));
        }

        let throughput = measurements
            .throughput
            .as_ref()
            .map(|thr| ConfidenceInterval {
                lower: formatter
                    .format_throughput(thr, typical_estimate.confidence_interval.upper_bound),
                upper: formatter
                    .format_throughput(thr, typical_estimate.confidence_interval.lower_bound),
                point: formatter.format_throughput(thr, typical_estimate.point_estimate),
            });

        let context = Context {
            title: id.as_title().to_owned(),
            confidence: format!(
                "{:.2}",
                typical_estimate.confidence_interval.confidence_level
            ),

            thumbnail_width: THUMBNAIL_SIZE.unwrap().0,
            thumbnail_height: THUMBNAIL_SIZE.unwrap().1,

            slope: measurements
                .absolute_estimates
                .slope
                .as_ref()
                .map(time_interval),
            mean: time_interval(&measurements.absolute_estimates.mean),
            median: time_interval(&measurements.absolute_estimates.median),
            mad: time_interval(&measurements.absolute_estimates.median_abs_dev),
            std_dev: time_interval(&measurements.absolute_estimates.std_dev),
            throughput,

            r2: ConfidenceInterval {
                lower: format!(
                    "{:0.7}",
                    Slope(typical_estimate.confidence_interval.lower_bound).r_squared(&data)
                ),
                upper: format!(
                    "{:0.7}",
                    Slope(typical_estimate.confidence_interval.upper_bound).r_squared(&data)
                ),
                point: format!(
                    "{:0.7}",
                    Slope(typical_estimate.point_estimate).r_squared(&data)
                ),
            },

            additional_plots,

            comparison: self.comparison(measurements),
        };

        let mut report_path = report_context.output_directory.clone();
        report_path.push(id.as_directory_name());
        report_path.push("report");
        report_path.push("index.html");
        debug_context(&report_path, &context);

        let text = self
            .templates
            .render("benchmark_report", &context)
            .expect("Failed to render benchmark report template");
        try_else_return!(fs::save_string(&text, &report_path));
    }

    fn summarize(
        &self,
        context: &ReportContext,
        all_ids: &[BenchmarkId],
        formatter: &dyn ValueFormatter,
    ) {
        let all_ids = all_ids
            .iter()
            .filter(|id| {
                let id_dir = context.output_directory.join(id.as_directory_name());
                fs::is_dir(&id_dir)
            })
            .collect::<Vec<_>>();
        if all_ids.is_empty() {
            return;
        }

        let group_id = all_ids[0].group_id.clone();

        let data = self.load_summary_data(&context.output_directory, &all_ids);

        let mut function_ids = BTreeSet::new();
        let mut value_strs = Vec::with_capacity(all_ids.len());
        for id in all_ids {
            if let Some(ref function_id) = id.function_id {
                function_ids.insert(function_id);
            }
            if let Some(ref value_str) = id.value_str {
                value_strs.push(value_str);
            }
        }

        fn try_parse(s: &str) -> Option<f64> {
            s.parse::<f64>().ok()
        }

        // If all of the value strings can be parsed into a number, sort/dedupe
        // numerically. Otherwise sort lexicographically.
        if value_strs.iter().all(|os| try_parse(os).is_some()) {
            value_strs.sort_unstable_by(|v1, v2| {
                let num1 = try_parse(v1);
                let num2 = try_parse(v2);

                num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
            });
            value_strs.dedup_by_key(|os| try_parse(os).unwrap());
        } else {
            value_strs.sort_unstable();
            value_strs.dedup();
        }

        for function_id in function_ids {
            let samples_with_function: Vec<_> = data
                .iter()
                .by_ref()
                .filter(|&&(id, _)| id.function_id.as_ref() == Some(function_id))
                .collect();

            if samples_with_function.len() > 1 {
                let subgroup_id =
                    BenchmarkId::new(group_id.clone(), Some(function_id.clone()), None, None);

                self.generate_summary(
                    &subgroup_id,
                    &samples_with_function,
                    context,
                    formatter,
                    false,
                );
            }
        }

        for value_str in value_strs {
            let samples_with_value: Vec<_> = data
                .iter()
                .by_ref()
                .filter(|&&(id, _)| id.value_str.as_ref() == Some(value_str))
                .collect();

            if samples_with_value.len() > 1 {
                let subgroup_id =
                    BenchmarkId::new(group_id.clone(), None, Some(value_str.clone()), None);

                self.generate_summary(&subgroup_id, &samples_with_value, context, formatter, false);
            }
        }

        let mut all_data = data.iter().by_ref().collect::<Vec<_>>();
        // First sort the ids/data by value.
        // If all of the value strings can be parsed into a number, sort/dedupe
        // numerically. Otherwise sort lexicographically.
        let all_values_numeric = all_data
            .iter()
            .all(|(id, _)| id.value_str.as_deref().and_then(try_parse).is_some());
        if all_values_numeric {
            all_data.sort_unstable_by(|(a, _), (b, _)| {
                let num1 = a.value_str.as_deref().and_then(try_parse);
                let num2 = b.value_str.as_deref().and_then(try_parse);

                num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
            });
        } else {
            all_data.sort_unstable_by_key(|(id, _)| id.value_str.as_ref());
        }
        // Next, sort the ids/data by function name. This results in a sorting priority of
        // function name, then value. This one has to be a stable sort.
        all_data.sort_by_key(|(id, _)| id.function_id.as_ref());

        self.generate_summary(
            &BenchmarkId::new(group_id, None, None, None),
            &all_data,
            context,
            formatter,
            true,
        );
        self.plotter.borrow_mut().wait();
    }

    fn final_summary(&self, report_context: &ReportContext) {
        let output_directory = &report_context.output_directory;
        if !fs::is_dir(&output_directory) {
            return;
        }

        let mut found_ids = try_else_return!(fs::list_existing_benchmarks(&output_directory));
        found_ids.sort_unstable_by_key(|id| id.id().to_owned());

        // Group IDs by group id
        let mut id_groups: HashMap<&str, Vec<&BenchmarkId>> = HashMap::new();
        for id in found_ids.iter() {
            id_groups
                .entry(&id.group_id)
                .or_insert_with(Vec::new)
                .push(id);
        }

        let mut groups = id_groups
            .into_values()
            .map(|group| BenchmarkGroup::new(output_directory, &group))
            .collect::<Vec<BenchmarkGroup<'_>>>();
        groups.sort_unstable_by_key(|g| g.group_report.name);

        try_else_return!(fs::mkdirp(&output_directory.join("report")));

        let report_path = output_directory.join("report").join("index.html");

        let context = IndexContext { groups };

        debug_context(&report_path, &context);

        let text = self
            .templates
            .render("index", &context)
            .expect("Failed to render index template");
        try_else_return!(fs::save_string(&text, &report_path,));
    }
}
impl Html {
    fn comparison(&self, measurements: &MeasurementData<'_>) -> Option<Comparison> {
        if let Some(ref comp) = measurements.comparison {
            let different_mean = comp.p_value < comp.significance_threshold;
            let mean_est = &comp.relative_estimates.mean;
            let explanation_str: String;

            if !different_mean {
                explanation_str = "No change in performance detected.".to_owned();
            } else {
                let comparison = compare_to_threshold(mean_est, comp.noise_threshold);
                match comparison {
                    ComparisonResult::Improved => {
                        explanation_str = "Performance has improved.".to_owned();
                    }
                    ComparisonResult::Regressed => {
                        explanation_str = "Performance has regressed.".to_owned();
                    }
                    ComparisonResult::NonSignificant => {
                        explanation_str = "Change within noise threshold.".to_owned();
                    }
                }
            }

            let comp = Comparison {
                p_value: format!("{:.2}", comp.p_value),
                inequality: (if different_mean { "<" } else { ">" }).to_owned(),
                significance_level: format!("{:.2}", comp.significance_threshold),
                explanation: explanation_str,

                change: ConfidenceInterval {
                    point: format::change(mean_est.point_estimate, true),
                    lower: format::change(mean_est.confidence_interval.lower_bound, true),
                    upper: format::change(mean_est.confidence_interval.upper_bound, true),
                },

                thrpt_change: measurements.throughput.as_ref().map(|_| {
                    let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
                    ConfidenceInterval {
                        point: format::change(to_thrpt_estimate(mean_est.point_estimate), true),
                        lower: format::change(
                            to_thrpt_estimate(mean_est.confidence_interval.lower_bound),
                            true,
                        ),
                        upper: format::change(
                            to_thrpt_estimate(mean_est.confidence_interval.upper_bound),
                            true,
                        ),
                    }
                }),

                additional_plots: vec![
                    Plot::new("Change in mean", "change/mean.svg"),
                    Plot::new("Change in median", "change/median.svg"),
                    Plot::new("T-Test", "change/t-test.svg"),
                ],
            };
            Some(comp)
        } else {
            None
        }
    }

    fn generate_plots(
        &self,
        id: &BenchmarkId,
        context: &ReportContext,
        formatter: &dyn ValueFormatter,
        measurements: &MeasurementData<'_>,
    ) {
        let plot_ctx = PlotContext {
            id,
            context,
            size: None,
            is_thumbnail: false,
        };

        let plot_data = PlotData {
            measurements,
            formatter,
            comparison: None,
        };

        let plot_ctx_small = plot_ctx.thumbnail(true).size(THUMBNAIL_SIZE);

        self.plotter.borrow_mut().pdf(plot_ctx, plot_data);
        self.plotter.borrow_mut().pdf(plot_ctx_small, plot_data);
        if measurements.absolute_estimates.slope.is_some() {
            self.plotter.borrow_mut().regression(plot_ctx, plot_data);
            self.plotter
                .borrow_mut()
                .regression(plot_ctx_small, plot_data);
        } else {
            self.plotter
                .borrow_mut()
                .iteration_times(plot_ctx, plot_data);
            self.plotter
                .borrow_mut()
                .iteration_times(plot_ctx_small, plot_data);
        }

        self.plotter
            .borrow_mut()
            .abs_distributions(plot_ctx, plot_data);

        if let Some(ref comp) = measurements.comparison {
            try_else_return!({
                let mut change_dir = context.output_directory.clone();
                change_dir.push(id.as_directory_name());
                change_dir.push("report");
                change_dir.push("change");
                fs::mkdirp(&change_dir)
            });

            try_else_return!({
                let mut both_dir = context.output_directory.clone();
                both_dir.push(id.as_directory_name());
                both_dir.push("report");
                both_dir.push("both");
                fs::mkdirp(&both_dir)
            });

            let comp_data = plot_data.comparison(comp);

            self.plotter.borrow_mut().pdf(plot_ctx, comp_data);
            self.plotter.borrow_mut().pdf(plot_ctx_small, comp_data);
            if measurements.absolute_estimates.slope.is_some()
                && comp.base_estimates.slope.is_some()
            {
                self.plotter.borrow_mut().regression(plot_ctx, comp_data);
                self.plotter
                    .borrow_mut()
                    .regression(plot_ctx_small, comp_data);
            } else {
                self.plotter
                    .borrow_mut()
                    .iteration_times(plot_ctx, comp_data);
                self.plotter
                    .borrow_mut()
                    .iteration_times(plot_ctx_small, comp_data);
            }
            self.plotter.borrow_mut().t_test(plot_ctx, comp_data);
            self.plotter
                .borrow_mut()
                .rel_distributions(plot_ctx, comp_data);
        }

        self.plotter.borrow_mut().wait();
    }

    fn load_summary_data<'a>(
        &self,
        output_directory: &Path,
        all_ids: &[&'a BenchmarkId],
    ) -> Vec<(&'a BenchmarkId, Vec<f64>)> {
        all_ids
            .iter()
            .filter_map(|id| {
                let entry = output_directory.join(id.as_directory_name()).join("new");

                let SavedSample { iters, times, .. } =
                    try_else_return!(fs::load(&entry.join("sample.json")), || None);
                let avg_times = iters
                    .into_iter()
                    .zip(times.into_iter())
                    .map(|(iters, time)| time / iters)
                    .collect::<Vec<_>>();

                Some((*id, avg_times))
            })
            .collect::<Vec<_>>()
    }

    fn generate_summary(
        &self,
        id: &BenchmarkId,
        data: &[&(&BenchmarkId, Vec<f64>)],
        report_context: &ReportContext,
        formatter: &dyn ValueFormatter,
        full_summary: bool,
    ) {
        let plot_ctx = PlotContext {
            id,
            context: report_context,
            size: None,
            is_thumbnail: false,
        };

        try_else_return!(
            {
                let mut report_dir = report_context.output_directory.clone();
                report_dir.push(id.as_directory_name());
                report_dir.push("report");
                fs::mkdirp(&report_dir)
            },
            || {}
        );

        self.plotter.borrow_mut().violin(plot_ctx, formatter, data);

        let value_types: Vec<_> = data.iter().map(|&&(id, _)| id.value_type()).collect();
        let mut line_path = None;

        if value_types.iter().all(|x| x == &value_types[0]) {
            if let Some(value_type) = value_types[0] {
                let values: Vec<_> = data.iter().map(|&&(id, _)| id.as_number()).collect();
                if values.iter().any(|x| x != &values[0]) {
                    self.plotter
                        .borrow_mut()
                        .line_comparison(plot_ctx, formatter, data, value_type);
                    line_path = Some(plot_ctx.line_comparison_path());
                }
            }
        }

        let path_prefix = if full_summary { "../.." } else { "../../.." };
        let benchmarks = data
            .iter()
            .map(|&&(id, _)| {
                IndividualBenchmark::from_id(&report_context.output_directory, path_prefix, id)
            })
            .collect();

        let context = SummaryContext {
            group_id: id.as_title().to_owned(),

            thumbnail_width: THUMBNAIL_SIZE.unwrap().0,
            thumbnail_height: THUMBNAIL_SIZE.unwrap().1,

            violin_plot: Some(plot_ctx.violin_path().to_string_lossy().into_owned()),
            line_chart: line_path.map(|p| p.to_string_lossy().into_owned()),

            benchmarks,
        };

        let mut report_path = report_context.output_directory.clone();
        report_path.push(id.as_directory_name());
        report_path.push("report");
        report_path.push("index.html");
        debug_context(&report_path, &context);

        let text = self
            .templates
            .render("summary_report", &context)
            .expect("Failed to render summary report template");
        try_else_return!(fs::save_string(&text, &report_path,), || {});
    }
}

enum ComparisonResult {
    Improved,
    Regressed,
    NonSignificant,
}

fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult {
    let ci = &estimate.confidence_interval;
    let lb = ci.lower_bound;
    let ub = ci.upper_bound;

    if lb < -noise && ub < -noise {
        ComparisonResult::Improved
    } else if lb > noise && ub > noise {
        ComparisonResult::Regressed
    } else {
        ComparisonResult::NonSignificant
    }
}