aios_cli/benchmark_next_app/
metrics.rs1use super::types::{BackendMetrics, LatencySummary};
4
5#[derive(Debug, Clone, Default)]
6pub struct PredictionRecord {
7 pub rank: Option<usize>,
8 pub latency_us: u64,
9 pub noop: bool,
10 pub rationale_present: bool,
13}
14
15pub fn compute_backend_metrics(records: &[PredictionRecord], eligible: usize) -> BackendMetrics {
16 let predicted = records.iter().filter(|r| r.rank.is_some()).count();
17 let top1 = records.iter().filter(|r| r.rank == Some(1)).count();
18 let top3 = records
19 .iter()
20 .filter(|r| r.rank.is_some_and(|rank| rank <= 3))
21 .count();
22 let rr_sum: f64 = records
23 .iter()
24 .map(|r| r.rank.map_or(0.0, |rank| 1.0 / rank as f64))
25 .sum();
26 let noop = records.iter().filter(|r| r.noop).count();
27 let rationale_windows = records.iter().filter(|r| r.rationale_present).count();
28
29 let latencies: Vec<u64> = records.iter().map(|r| r.latency_us).collect();
30
31 BackendMetrics {
32 eligible_windows: eligible,
33 predicted_windows: predicted,
34 top1_hits: top1,
35 top3_hits: top3,
36 top1_accuracy_pct: pct(top1, eligible),
37 top3_accuracy_pct: pct(top3, eligible),
38 prediction_coverage_pct: pct(predicted, eligible),
39 conditional_top1_accuracy_pct: pct(top1, predicted),
40 wrong_prediction_rate_pct: pct(predicted.saturating_sub(top1), predicted),
41 no_prediction_rate_pct: pct(eligible.saturating_sub(predicted), eligible),
42 noop_windows: noop,
43 noop_rate_pct: pct(noop, records.len()),
44 mean_reciprocal_rank: round3(rr_sum / eligible.max(1) as f64),
45 macro_top1_accuracy_pct: None,
46 macro_top3_accuracy_pct: None,
47 rationale_coverage_pct: pct(rationale_windows, records.len()),
48 latency_us: latency_summary(&latencies),
49 }
50}
51
52pub fn round3(v: f64) -> f64 {
53 (v * 1000.0).round() / 1000.0
54}
55
56fn pct(part: usize, whole: usize) -> f64 {
57 if whole == 0 {
58 0.0
59 } else {
60 round3((part as f64 / whole as f64) * 100.0)
61 }
62}
63
64pub fn latency_summary(latencies: &[u64]) -> LatencySummary {
65 if latencies.is_empty() {
66 return LatencySummary::default();
67 }
68 let mut sorted = latencies.to_vec();
69 sorted.sort_unstable();
70 let mean = sorted.iter().map(|&x| x as f64).sum::<f64>() / sorted.len() as f64;
71 LatencySummary {
72 mean: round3(mean),
73 p50: percentile(&sorted, 0.5),
74 p95: percentile(&sorted, 0.95),
75 }
76}
77
78fn percentile(sorted: &[u64], p: f64) -> f64 {
79 let idx = ((sorted.len() as f64 - 1.0) * p).round() as usize;
80 sorted[idx.min(sorted.len() - 1)] as f64
81}