aios_cli/benchmark_next_app/
runner.rs1use std::collections::{BTreeMap, HashMap};
4use std::path::Path;
5
6use anyhow::{bail, Context, Result};
7use tracing::info;
8
9use super::baselines::{
10 AlwaysNoOpBackend, FirstCandidateBackend, GlobalMajorityBackend, LastAppPrewarmBackend,
11 LastForegroundBackend, MarkovBackend, NotificationPriorityBackend,
12 PerCurrentAppMajorityBackend, RandomCandidateBackend, RecentNotificationBackend,
13};
14use super::context_loader::{extract_observable_candidates, load_contexts_by_label, load_labels};
15use super::metrics::{compute_backend_metrics, round3, PredictionRecord};
16use super::predictor::{LocalEvaluatorNextAppBackend, RuleBasedNextAppBackend};
17use super::report::SCHEMA_VERSION;
18use super::types::{
19 BackendMetrics, BenchmarkConfig, BenchmarkReport, ExclusionCounts, NextAppLabel,
20 NextAppPredictor, ScenarioReport,
21};
22
23pub struct BenchmarkRunConfig {
24 pub inputs: Vec<std::path::PathBuf>,
25 pub labels: std::path::PathBuf,
26 pub train_split: f64,
27 pub window_secs: u64,
28}
29
30pub fn run_benchmark(config: &BenchmarkRunConfig) -> Result<BenchmarkReport> {
31 if !(0.0..1.0).contains(&config.train_split) {
32 bail!("train_split must be in (0, 1), got {}", config.train_split);
33 }
34
35 let labels = load_labels(&config.labels)
36 .with_context(|| format!("loading labels {}", config.labels.display()))?;
37
38 let mut by_scenario: HashMap<String, Vec<NextAppLabel>> = HashMap::new();
39 let mut exclusion_counts = ExclusionCounts::default();
40 for label in &labels {
41 by_scenario
42 .entry(label.scenario.clone())
43 .or_default()
44 .push(label.clone());
45 if label.eligible {
46 exclusion_counts.eligible += 1;
47 } else if let Some(reason) = &label.excluded_reason {
48 match reason.as_str() {
49 "label_not_observable" => exclusion_counts.label_not_observable += 1,
50 "no_future_switch" => exclusion_counts.no_future_switch += 1,
51 _ => {},
52 }
53 }
54 }
55
56 let horizon_secs = labels
57 .first()
58 .map(|l| l.prediction_horizon_ms / 1000)
59 .unwrap_or(30);
60
61 let input_pairs: Vec<(String, &Path)> = config
62 .inputs
63 .iter()
64 .map(|p| {
65 let scenario = p
66 .file_stem()
67 .and_then(|s| s.to_str())
68 .map(|s| s.to_string())
69 .with_context(|| format!("invalid input path {}", p.display()))?;
70 Ok((scenario, p.as_path()))
71 })
72 .collect::<Result<Vec<_>>>()?;
73
74 let contexts = load_contexts_by_label(&input_pairs, &labels, config.window_secs)
75 .context("loading contexts from traces")?;
76
77 let mut scenarios = Vec::new();
78 let mut aggregate_records: HashMap<String, Vec<PredictionRecord>> = HashMap::new();
79 let mut aggregate_latencies: HashMap<String, Vec<u64>> = HashMap::new();
80
81 for (scenario_name, trace_path) in &input_pairs {
82 let scenario_labels = by_scenario
83 .remove(scenario_name)
84 .with_context(|| format!("no labels for scenario '{}'", scenario_name))?;
85
86 let mut eligible: Vec<NextAppLabel> = scenario_labels
87 .iter()
88 .filter(|l| l.eligible)
89 .cloned()
90 .collect();
91 eligible.sort_by_key(|l| l.window_start_ms);
92
93 let n = eligible.len();
94 let split_idx = ((n as f64) * config.train_split)
95 .floor()
96 .max(1.0)
97 .min(n.saturating_sub(1) as f64) as usize;
98 let (train, test) = eligible.split_at(split_idx);
99
100 let mut predictors: Vec<Box<dyn NextAppPredictor>> = vec![
101 Box::new(RuleBasedNextAppBackend),
102 Box::new(LocalEvaluatorNextAppBackend),
103 Box::new(AlwaysNoOpBackend),
104 Box::new(RandomCandidateBackend::new(42)),
105 Box::new(FirstCandidateBackend),
106 Box::new(GlobalMajorityBackend::default()),
107 Box::new(PerCurrentAppMajorityBackend::default()),
108 Box::new(MarkovBackend::default()),
109 Box::new(RecentNotificationBackend),
110 Box::new(LastForegroundBackend),
111 Box::new(NotificationPriorityBackend),
112 Box::new(LastAppPrewarmBackend),
113 ];
114
115 for predictor in &mut predictors {
116 predictor.train(train);
117 }
118
119 let mut backend_metrics: BTreeMap<String, BackendMetrics> = BTreeMap::new();
120 for predictor in predictors.iter() {
121 let (records, latencies) =
122 evaluate_predictor(predictor.as_ref(), test, &contexts, scenario_name)?;
123 let metrics = compute_backend_metrics(&records, test.len());
124 backend_metrics.insert(predictor.name().to_string(), metrics);
125 aggregate_records
126 .entry(predictor.name().to_string())
127 .or_default()
128 .extend(records);
129 aggregate_latencies
130 .entry(predictor.name().to_string())
131 .or_default()
132 .extend(latencies);
133 }
134
135 let scenario_exclusions =
136 exclusion_counts_for_scenario(&scenario_labels.iter().collect::<Vec<_>>());
137
138 scenarios.push(ScenarioReport {
139 scenario: (*scenario_name).to_string(),
140 windows: scenario_labels.len(),
141 future_switch_windows: scenario_labels.len() - scenario_exclusions.no_future_switch,
142 eligible_windows: scenario_exclusions.eligible,
143 train_windows: train.len(),
144 test_windows: test.len(),
145 exclusions: scenario_exclusions,
146 backends: backend_metrics,
147 });
148
149 info!(
150 scenario = %scenario_name,
151 train = train.len(),
152 test = test.len(),
153 trace = %trace_path.display(),
154 "benchmarked scenario"
155 );
156 }
157
158 if !by_scenario.is_empty() {
159 bail!(
160 "missing input traces for scenarios: {:?}",
161 by_scenario.keys()
162 );
163 }
164
165 let mut aggregate: BTreeMap<String, BackendMetrics> = BTreeMap::new();
166 for (name, records) in &aggregate_records {
167 let mut metrics = compute_backend_metrics(records, records.len());
168 let scenario_top1: Vec<f64> = scenarios
169 .iter()
170 .filter(|s| s.test_windows > 0)
171 .filter_map(|s| s.backends.get(name).map(|m| m.top1_accuracy_pct))
172 .collect();
173 let scenario_top3: Vec<f64> = scenarios
174 .iter()
175 .filter(|s| s.test_windows > 0)
176 .filter_map(|s| s.backends.get(name).map(|m| m.top3_accuracy_pct))
177 .collect();
178 metrics.macro_top1_accuracy_pct = Some(round3(
179 scenario_top1.iter().sum::<f64>() / scenario_top1.len().max(1) as f64,
180 ));
181 metrics.macro_top3_accuracy_pct = Some(round3(
182 scenario_top3.iter().sum::<f64>() / scenario_top3.len().max(1) as f64,
183 ));
184 if let Some(latencies) = aggregate_latencies.get(name) {
185 metrics.latency_us = super::metrics::latency_summary(latencies);
186 }
187 aggregate.insert(name.clone(), metrics);
188 }
189
190 let future_switch_windows = exclusion_counts.eligible + exclusion_counts.label_not_observable;
191 let coverage_pct =
192 round3((exclusion_counts.eligible as f64 / future_switch_windows.max(1) as f64) * 100.0);
193
194 Ok(BenchmarkReport {
195 schema_version: SCHEMA_VERSION.into(),
196 dataset_id: labels
197 .first()
198 .map(|l| l.dataset_id.clone())
199 .unwrap_or_default(),
200 source: "synthetic".into(),
201 evaluation_source: "synthetic_trace".into(),
202 action_value_source: "synthetic_model_prediction_only".into(),
203 benefit_claim_policy: "no_measured_action_benefit_claim".into(),
204 config: BenchmarkConfig {
205 window_secs: config.window_secs,
206 horizon_secs,
207 label_definition: "first foreground transition within the horizon whose package differs from the app current at window end".into(),
208 eligibility_definition: "actual_next_app is present in the current window's foreground or notified app set after removing current_app".into(),
209 train_split: config.train_split,
210 },
211 inputs: config.inputs.clone(),
212 total_windows: labels.len(),
213 eligible_windows: exclusion_counts.eligible,
214 train_windows: scenarios.iter().map(|s| s.train_windows).sum(),
215 test_windows: scenarios.iter().map(|s| s.test_windows).sum(),
216 context_supported_switch_coverage_pct: coverage_pct,
217 exclusions: exclusion_counts,
218 aggregate,
219 scenarios,
220 limitations: vec![
221 "All labels are derived from deterministic synthetic traces, not real-user behavior.".into(),
222 "Accuracy is reported only for context-supported switches; coverage reports how selective that cohort is.".into(),
223 "Latency is measured during this run and is not byte-deterministic across machines.".into(),
224 "Statistical baselines are trained on the per-scenario train split and evaluated on the held-out test split.".into(),
225 "This synthetic benchmark validates prediction-to-action mapping only; it does not import measured action-value data and must not be cited as measured device benefit.".into(),
226 ],
227 })
228}
229
230fn evaluate_predictor(
231 predictor: &dyn NextAppPredictor,
232 test: &[NextAppLabel],
233 contexts: &BTreeMap<(String, i64, i64), aios_spec::StructuredContext>,
234 scenario: &str,
235) -> Result<(Vec<PredictionRecord>, Vec<u64>)> {
236 let mut records = Vec::with_capacity(test.len());
237 let mut latencies = Vec::with_capacity(test.len());
238
239 for label in test {
240 let ctx = contexts
241 .get(&(
242 label.scenario.clone(),
243 label.window_start_ms,
244 label.window_end_ms,
245 ))
246 .with_context(|| {
247 format!(
248 "missing context for scenario {} window {}-{}",
249 scenario, label.window_start_ms, label.window_end_ms
250 )
251 })?;
252 let current_app = &label.current_app;
253 let candidates = extract_observable_candidates(ctx, current_app);
254
255 let result = predictor.predict(ctx, current_app, &candidates);
256 let rank = if let Some(actual) = &label.actual_next_app {
257 result
258 .ranked
259 .iter()
260 .position(|p| &p.package == actual)
261 .map(|i| i + 1)
262 } else {
263 None
264 };
265
266 records.push(PredictionRecord {
267 rank,
268 latency_us: result.latency_us,
269 noop: result.ranked.is_empty(),
270 rationale_present: result.rationale_present,
271 });
272 latencies.push(result.latency_us);
273 }
274
275 Ok((records, latencies))
276}
277
278fn exclusion_counts_for_scenario(labels: &[&NextAppLabel]) -> ExclusionCounts {
279 let mut counts = ExclusionCounts::default();
280 for label in labels {
281 if label.eligible {
282 counts.eligible += 1;
283 } else if let Some(reason) = &label.excluded_reason {
284 match reason.as_str() {
285 "label_not_observable" => counts.label_not_observable += 1,
286 "no_future_switch" => counts.no_future_switch += 1,
287 _ => {},
288 }
289 }
290 }
291 counts
292}