Skip to main content

aios_cli/benchmark_next_app/
metrics.rs

1//! Metric aggregation for the next-app benchmark.
2
3use super::types::{BackendMetrics, LatencySummary};
4
5#[derive(Debug, Clone, Default)]
6pub struct PredictionRecord {
7    pub rank: Option<usize>,
8    pub latency_us: u64,
9    pub noop: bool,
10    /// True when the underlying predictor produced at least one intent with
11    /// non-empty `rationale_tags`. False for statistical baselines.
12    pub rationale_present: bool,
13}
14
15pub fn compute_backend_metrics(records: &[PredictionRecord], eligible: usize) -> BackendMetrics {
16    let predicted = records.iter().filter(|r| r.rank.is_some()).count();
17    let top1 = records.iter().filter(|r| r.rank == Some(1)).count();
18    let top3 = records
19        .iter()
20        .filter(|r| r.rank.is_some_and(|rank| rank <= 3))
21        .count();
22    let rr_sum: f64 = records
23        .iter()
24        .map(|r| r.rank.map_or(0.0, |rank| 1.0 / rank as f64))
25        .sum();
26    let noop = records.iter().filter(|r| r.noop).count();
27    let rationale_windows = records.iter().filter(|r| r.rationale_present).count();
28
29    let latencies: Vec<u64> = records.iter().map(|r| r.latency_us).collect();
30
31    BackendMetrics {
32        eligible_windows: eligible,
33        predicted_windows: predicted,
34        top1_hits: top1,
35        top3_hits: top3,
36        top1_accuracy_pct: pct(top1, eligible),
37        top3_accuracy_pct: pct(top3, eligible),
38        prediction_coverage_pct: pct(predicted, eligible),
39        conditional_top1_accuracy_pct: pct(top1, predicted),
40        wrong_prediction_rate_pct: pct(predicted.saturating_sub(top1), predicted),
41        no_prediction_rate_pct: pct(eligible.saturating_sub(predicted), eligible),
42        noop_windows: noop,
43        noop_rate_pct: pct(noop, records.len()),
44        mean_reciprocal_rank: round3(rr_sum / eligible.max(1) as f64),
45        macro_top1_accuracy_pct: None,
46        macro_top3_accuracy_pct: None,
47        rationale_coverage_pct: pct(rationale_windows, records.len()),
48        latency_us: latency_summary(&latencies),
49    }
50}
51
52pub fn round3(v: f64) -> f64 {
53    (v * 1000.0).round() / 1000.0
54}
55
56fn pct(part: usize, whole: usize) -> f64 {
57    if whole == 0 {
58        0.0
59    } else {
60        round3((part as f64 / whole as f64) * 100.0)
61    }
62}
63
64pub fn latency_summary(latencies: &[u64]) -> LatencySummary {
65    if latencies.is_empty() {
66        return LatencySummary::default();
67    }
68    let mut sorted = latencies.to_vec();
69    sorted.sort_unstable();
70    let mean = sorted.iter().map(|&x| x as f64).sum::<f64>() / sorted.len() as f64;
71    LatencySummary {
72        mean: round3(mean),
73        p50: percentile(&sorted, 0.5),
74        p95: percentile(&sorted, 0.95),
75    }
76}
77
78fn percentile(sorted: &[u64], p: f64) -> f64 {
79    let idx = ((sorted.len() as f64 - 1.0) * p).round() as usize;
80    sorted[idx.min(sorted.len() - 1)] as f64
81}