nova-cli/src/bench/diff.rs - main - nv-lang/nova

nova-cli/src/bench/diff.rs
432 строки17 KB
Evgeniy Golovin
feat(57.H.1): multi-group geomean в bench diff (benchstat-style)
17 май 2026, 21:49
17 май 2026, 21:4921b587b
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Plan 57 L5 — `nova bench diff`. Welch's t-test pairwise compare с
//! geomean aggregate + reproducibility check.

use std::fmt::Write as FmtWrite;
use std::path::Path;

use anyhow::{anyhow, Result};
use serde_json::Value;

use super::schema::{AnalyzedBench, RunResultParsed};
use super::stats::{welch_t_test, geomean};
use super::report::fmt_duration;

#[derive(Debug, Clone, Copy)]
pub enum DiffFormat {
    Terminal,
    Markdown,
    Json,
}

impl DiffFormat {
    pub fn parse(s: &str) -> Result<Self> {
        match s {
            "terminal" | "text" => Ok(DiffFormat::Terminal),
            "md" | "markdown"   => Ok(DiffFormat::Markdown),
            "json"              => Ok(DiffFormat::Json),
            _ => Err(anyhow!("unknown diff format: {} (expected: terminal|markdown|json)", s)),
        }
    }
}

#[derive(Debug, Clone)]
pub struct DiffRow {
    pub name: String,
    pub baseline_median_ns: Option<f64>,
    pub new_median_ns: Option<f64>,
    pub delta_pct: Option<f64>,    // (new - base) / base * 100
    pub p_value: Option<f64>,      // Welch's t-test p
    pub n_baseline: usize,
    pub n_new: usize,
}

pub fn compare(baseline_path: &Path, new_path: &Path, format: DiffFormat) -> Result<i32> {
    let p = load_pair(baseline_path, new_path)?;
    let (output, exit) = match format {
        DiffFormat::Terminal => (terminal_format(&p.rows, &p.compat_warnings), 0),
        DiffFormat::Markdown => (markdown_format(&p.rows, &p.compat_warnings), 0),
        DiffFormat::Json => (json_format(&p.rows, &p.compat_warnings)?, 0),
    };
    print!("{}", output);
    Ok(exit)
}

/// Loaded + analyzed diff pair (used by both `compare` and AI integration).
pub struct LoadedDiff {
    pub baseline: RunResultParsed,
    pub new: RunResultParsed,
    pub rows: Vec<DiffRow>,
    pub compat_warnings: Vec<String>,
}

/// Public loader для callers нужны both rows + parsed metadata (e.g. AI).
pub fn load_pair(baseline_path: &Path, new_path: &Path) -> Result<LoadedDiff> {
    let base_text = std::fs::read_to_string(baseline_path)
        .map_err(|e| anyhow!("read baseline {}: {}", baseline_path.display(), e))?;
    let new_text = std::fs::read_to_string(new_path)
        .map_err(|e| anyhow!("read new {}: {}", new_path.display(), e))?;
    let base_v: Value = serde_json::from_str(&base_text)
        .map_err(|e| anyhow!("parse baseline JSON: {}", e))?;
    let new_v: Value = serde_json::from_str(&new_text)
        .map_err(|e| anyhow!("parse new JSON: {}", e))?;
    let baseline = RunResultParsed::from_json(&base_v)
        .map_err(|e| anyhow!("baseline schema: {}", e))?;
    let new = RunResultParsed::from_json(&new_v)
        .map_err(|e| anyhow!("new schema: {}", e))?;
    let compat_warnings = baseline.metadata.compare_compatibility(&new.metadata);
    let rows = compute_diff(&baseline.benches, &new.benches);
    Ok(LoadedDiff { baseline, new, rows, compat_warnings })
}

pub fn compute_diff(baseline: &[AnalyzedBench], new: &[AnalyzedBench]) -> Vec<DiffRow> {
    use std::collections::HashMap;
    let base_map: HashMap<&str, &AnalyzedBench> = baseline.iter()
        .map(|b| (b.raw.name.as_str(), b))
        .collect();
    let new_map: HashMap<&str, &AnalyzedBench> = new.iter()
        .map(|b| (b.raw.name.as_str(), b))
        .collect();
    let mut names: Vec<&str> = base_map.keys().chain(new_map.keys()).cloned().collect();
    names.sort();
    names.dedup();

    let mut rows = Vec::with_capacity(names.len());
    for n in names {
        let b = base_map.get(n).copied();
        let nw = new_map.get(n).copied();
        let baseline_median_ns = b.map(|x| x.stats_ns.median);
        let new_median_ns = nw.map(|x| x.stats_ns.median);
        let delta_pct = match (baseline_median_ns, new_median_ns) {
            (Some(bm), Some(nm)) if bm > 0.0 => Some((nm - bm) / bm * 100.0),
            _ => None,
        };
        let p_value = match (b, nw) {
            (Some(b), Some(nw)) if b.raw.raw_ns.len() >= 2 && nw.raw.raw_ns.len() >= 2 => {
                let ba: Vec<f64> = b.raw.raw_ns.iter().map(|x| *x as f64).collect();
                let na: Vec<f64> = nw.raw.raw_ns.iter().map(|x| *x as f64).collect();
                let (_t, p, _df) = welch_t_test(&ba, &na);
                Some(p)
            }
            _ => None,
        };
        rows.push(DiffRow {
            name: n.to_string(),
            baseline_median_ns,
            new_median_ns,
            delta_pct, p_value,
            n_baseline: b.map(|x| x.stats_ns.n).unwrap_or(0),
            n_new: nw.map(|x| x.stats_ns.n).unwrap_or(0),
        });
    }
    rows
}

/// Plan 57.H.1 — derive group key from bench name (first slash-segment).
/// `hashmap/insert/n=10` → `hashmap`; `noop` → `noop` (own group).
/// `hashmap/insert/n=10/case=hot` → `hashmap`.
/// Returns None если ровно один name без slash (degenerate single-bench
/// suite — no useful per-group line).
pub fn group_key(name: &str) -> &str {
    name.split('/').next().unwrap_or(name)
}

/// Plan 57.H.1 — compute per-group geomean from rows.
/// Returns sorted Vec<(group_name, geomean_pct, sample_count)>; пустой
/// если только 1 group (degenerate — общий geomean уже отображается).
pub fn per_group_geomeans(rows: &[DiffRow]) -> Vec<(String, f64, usize)> {
    use std::collections::BTreeMap;
    let mut groups: BTreeMap<String, Vec<f64>> = BTreeMap::new();
    for r in rows {
        if let Some(d) = r.delta_pct {
            groups.entry(group_key(&r.name).to_string())
                .or_default()
                .push(1.0 + d / 100.0);
        }
    }
    if groups.len() <= 1 {
        return Vec::new();  // только один group → suite geomean достаточен
    }
    groups.into_iter()
        .map(|(g, ratios)| {
            let pct = (geomean(&ratios) - 1.0) * 100.0;
            (g, pct, ratios.len())
        })
        .collect()
}

pub fn terminal_format(rows: &[DiffRow], compat: &[String]) -> String {
    let mut out = String::new();
    let _ = writeln!(out, "name                       baseline        new             delta       p-value");
    let _ = writeln!(out, "─────────────────────────────────────────────────────────────────────────────");
    let mut ratios = Vec::new();
    for r in rows {
        let bn = r.baseline_median_ns.map(fmt_duration).unwrap_or_else(|| "—".to_string());
        let nn = r.new_median_ns.map(fmt_duration).unwrap_or_else(|| "—".to_string());
        let dp = r.delta_pct.map(|d| {
            let sign = if d >= 0.0 { "+" } else { "" };
            let stars = if let Some(p) = r.p_value {
                if p < 0.001 { " ***" } else if p < 0.01 { " **" } else if p < 0.05 { " *" } else { "" }
            } else { "" };
            format!("{}{:.1}%{}", sign, d, stars)
        }).unwrap_or_else(|| "—".to_string());
        let pv = r.p_value.map(|p| {
            if p < 0.001 { "<0.001".to_string() } else { format!("{:.3}", p) }
        }).unwrap_or_else(|| "—".to_string());
        let _ = writeln!(out, "{:<26} {:<15} {:<15} {:<12} {}",
            truncate(&r.name, 26), bn, nn, dp, pv);
        if let Some(d) = r.delta_pct {
            ratios.push(1.0 + d / 100.0);
        }
    }
    let _ = writeln!(out, "─────────────────────────────────────────────────────────────────────────────");
    if !ratios.is_empty() {
        let g = geomean(&ratios);
        let pct = (g - 1.0) * 100.0;
        let _ = writeln!(out, "                                          geomean delta: {}{:.1}%",
            if pct >= 0.0 { "+" } else { "" }, pct);
    }
    // Plan 57.H.1 — per-group geomean (benchstat-style).
    let groups = per_group_geomeans(rows);
    if !groups.is_empty() {
        let _ = writeln!(out, "");
        let _ = writeln!(out, "Per-group geomean (group = first '/'-segment of bench name):");
        for (g, pct, n) in &groups {
            let sign = if *pct >= 0.0 { "+" } else { "" };
            let _ = writeln!(out, "  {:<24} {:>6}{:.1}%  ({} benches)",
                truncate(g, 24), sign, pct, n);
        }
    }
    let _ = writeln!(out, "");
    let _ = writeln!(out, "Legend: *** p<0.001  ** p<0.01  * p<0.05");
    let _ = writeln!(out, "Tests with p>0.05 are within noise floor — not statistically significant.");

    if !compat.is_empty() {
        let _ = writeln!(out, "");
        let _ = writeln!(out, "Reproducibility check:");
        for w in compat {
            let _ = writeln!(out, "  ⚠ {}", w);
        }
    } else {
        let _ = writeln!(out, "");
        let _ = writeln!(out, "Reproducibility: ✓ baseline and new collected on compatible environment");
    }
    out
}

pub fn markdown_format(rows: &[DiffRow], compat: &[String]) -> String {
    let mut out = String::new();
    let _ = writeln!(out, "## Bench diff\n");
    let _ = writeln!(out, "| Bench | baseline | new | delta | p |");
    let _ = writeln!(out, "|---|---|---|---|---|");
    let mut ratios = Vec::new();
    for r in rows {
        let bn = r.baseline_median_ns.map(fmt_duration).unwrap_or_else(|| "—".to_string());
        let nn = r.new_median_ns.map(fmt_duration).unwrap_or_else(|| "—".to_string());
        let dp = r.delta_pct.map(|d| {
            let sign = if d >= 0.0 { "+" } else { "" };
            let badge = if let Some(p) = r.p_value {
                if p < 0.01 && d.abs() >= 5.0 { " ⚠️" } else { "" }
            } else { "" };
            format!("{}{:.1}%{}", sign, d, badge)
        }).unwrap_or_else(|| "—".to_string());
        let pv = r.p_value.map(|p| {
            if p < 0.001 { "<0.001".to_string() } else { format!("{:.3}", p) }
        }).unwrap_or_else(|| "—".to_string());
        let _ = writeln!(out, "| {} | {} | {} | {} | {} |", r.name, bn, nn, dp, pv);
        if let Some(d) = r.delta_pct {
            ratios.push(1.0 + d / 100.0);
        }
    }
    if !ratios.is_empty() {
        let g = geomean(&ratios);
        let pct = (g - 1.0) * 100.0;
        let _ = writeln!(out, "\n**Geomean delta: {}{:.1}%**",
            if pct >= 0.0 { "+" } else { "" }, pct);
    }
    // Plan 57.H.1 — per-group geomean table.
    let groups = per_group_geomeans(rows);
    if !groups.is_empty() {
        let _ = writeln!(out, "\n### Per-group geomean\n");
        let _ = writeln!(out, "| Group | Delta | N benches |");
        let _ = writeln!(out, "|---|---|---|");
        for (g, pct, n) in &groups {
            let sign = if *pct >= 0.0 { "+" } else { "" };
            let _ = writeln!(out, "| {} | {}{:.1}% | {} |", g, sign, pct, n);
        }
    }
    if !compat.is_empty() {
        let _ = writeln!(out, "\n### ⚠️ Compatibility warnings");
        for w in compat {
            let _ = writeln!(out, "- {}", w);
        }
    }
    out
}

pub fn json_format(rows: &[DiffRow], compat: &[String]) -> Result<String> {
    use serde_json::json;
    let rows_json: Vec<_> = rows.iter().map(|r| json!({
        "name": r.name,
        "baseline_median_ns": r.baseline_median_ns,
        "new_median_ns": r.new_median_ns,
        "delta_pct": r.delta_pct,
        "p_value": r.p_value,
        "n_baseline": r.n_baseline,
        "n_new": r.n_new,
    })).collect();
    let mut ratios = Vec::new();
    for r in rows {
        if let Some(d) = r.delta_pct { ratios.push(1.0 + d / 100.0); }
    }
    let g = if !ratios.is_empty() { geomean(&ratios) } else { 1.0 };
    // Plan 57.H.1 — per-group geomeans in JSON output.
    let groups = per_group_geomeans(rows);
    let groups_json: Vec<_> = groups.iter().map(|(name, pct, n)| json!({
        "group": name,
        "geomean_delta_pct": pct,
        "n_benches": n,
    })).collect();
    let out = json!({
        "rows": rows_json,
        "geomean_ratio": g,
        "geomean_delta_pct": (g - 1.0) * 100.0,
        "per_group_geomeans": groups_json,
        "compatibility_warnings": compat,
    });
    Ok(serde_json::to_string_pretty(&out)? + "\n")
}

fn truncate(s: &str, max: usize) -> &str {
    if s.len() <= max { s } else { &s[..max] }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::bench::schema::RawBenchResult;

    fn mk(name: &str, ns: Vec<u64>) -> AnalyzedBench {
        let raw = RawBenchResult {
            name: name.to_string(),
            iters_per_sample: 1,
            samples_count: ns.len() as u64,
            raw_ns: ns,
            throughput_bytes: None,
            throughput_elements: None,
            allocs_per_iter: None,
            allocs_total: None,
            cpu_instructions: Vec::new(),
            custom_metrics: Vec::new(),
        };
        AnalyzedBench::from_raw(raw).unwrap()
    }

    #[test]
    fn diff_no_change() {
        let b = vec![mk("foo", vec![100; 30])];
        let n = vec![mk("foo", vec![100; 30])];
        let rows = compute_diff(&b, &n);
        assert_eq!(rows.len(), 1);
        // 0% delta.
        assert!(rows[0].delta_pct.unwrap().abs() < 1e-9);
        // p-value high (no difference).
        assert!(rows[0].p_value.unwrap() > 0.9);
    }

    #[test]
    fn diff_regression() {
        let b = vec![mk("foo", vec![100, 102, 98, 101, 99, 100, 103, 97, 100, 100])];
        let n = vec![mk("foo", vec![200, 205, 195, 202, 198, 200, 203, 197, 200, 201])];
        let rows = compute_diff(&b, &n);
        assert_eq!(rows.len(), 1);
        let d = rows[0].delta_pct.unwrap();
        assert!(d > 90.0 && d < 110.0, "expected ~100% regression, got {}", d);
        // Very significant.
        assert!(rows[0].p_value.unwrap() < 0.001);
    }

    #[test]
    fn diff_missing_in_new() {
        let b = vec![mk("foo", vec![100; 5]), mk("bar", vec![200; 5])];
        let n = vec![mk("foo", vec![100; 5])];
        let rows = compute_diff(&b, &n);
        // bar is missing in new — new_median_ns = None.
        let bar = rows.iter().find(|r| r.name == "bar").unwrap();
        assert!(bar.new_median_ns.is_none());
        assert!(bar.delta_pct.is_none());
    }

    // ── Plan 57.H.1 — multi-group geomean tests ────────────────────

    #[test]
    fn group_key_first_segment() {
        assert_eq!(group_key("hashmap/insert/n=10"), "hashmap");
        assert_eq!(group_key("noop"),                  "noop");
        assert_eq!(group_key("a/b/c"),                 "a");
        assert_eq!(group_key(""),                      "");
    }

    #[test]
    fn per_group_geomeans_skips_single_group() {
        // Все benches в одной группе → empty Vec (suite geomean уже есть).
        let b = vec![mk("foo", vec![100; 5]), mk("bar", vec![200; 5])];
        let n = vec![mk("foo", vec![110; 5]), mk("bar", vec![220; 5])];
        let rows = compute_diff(&b, &n);
        let groups = per_group_geomeans(&rows);
        // "foo" + "bar" — different group_keys, so 2 groups.
        assert_eq!(groups.len(), 2);
    }

    #[test]
    fn per_group_geomeans_two_groups() {
        // hashmap/* и vec/* — два разных groups.
        let b = vec![
            mk("hashmap/insert", vec![100; 5]),
            mk("hashmap/lookup", vec![50; 5]),
            mk("vec/push",       vec![10; 5]),
            mk("vec/pop",        vec![10; 5]),
        ];
        let n = vec![
            mk("hashmap/insert", vec![110; 5]),  // +10%
            mk("hashmap/lookup", vec![55; 5]),   // +10%
            mk("vec/push",       vec![10; 5]),   //  0%
            mk("vec/pop",        vec![10; 5]),   //  0%
        ];
        let rows = compute_diff(&b, &n);
        let groups = per_group_geomeans(&rows);
        assert_eq!(groups.len(), 2);
        let hm = groups.iter().find(|(g, _, _)| g == "hashmap").unwrap();
        let v  = groups.iter().find(|(g, _, _)| g == "vec").unwrap();
        assert!((hm.1 - 10.0).abs() < 0.1, "hashmap group ≈ +10%, got {}", hm.1);
        assert!(v.1.abs() < 0.1, "vec group ≈ 0%, got {}", v.1);
        assert_eq!(hm.2, 2);
        assert_eq!(v.2, 2);
    }

    #[test]
    fn per_group_geomeans_sorted_alphabetically() {
        let b = vec![
            mk("zebra/x", vec![100; 5]),
            mk("apple/y", vec![100; 5]),
        ];
        let n = b.clone();
        let rows = compute_diff(&b, &n);
        let groups = per_group_geomeans(&rows);
        assert_eq!(groups.len(), 2);
        // BTreeMap iteration is sorted; apple должен быть first.
        assert_eq!(groups[0].0, "apple");
        assert_eq!(groups[1].0, "zebra");
    }

    #[test]
    fn per_group_geomeans_skips_unmatched_rows() {
        // bar отсутствует в new → delta_pct = None → не входит в geomean.
        let b = vec![mk("g1/foo", vec![100; 5]), mk("g1/bar", vec![100; 5])];
        let n = vec![mk("g1/foo", vec![110; 5])];
        let rows = compute_diff(&b, &n);
        let groups = per_group_geomeans(&rows);
        // только один group (g1), single-group → пустой результат.
        assert_eq!(groups.len(), 0);
    }
}