Skip to main content

aios_cli/benchmark_next_app/
runner.rs

1//! Benchmark runner: train/test split, per-scenario evaluation, and report generation.
2
3use std::collections::{BTreeMap, HashMap};
4use std::path::Path;
5
6use anyhow::{bail, Context, Result};
7use tracing::info;
8
9use super::baselines::{
10    AlwaysNoOpBackend, FirstCandidateBackend, GlobalMajorityBackend, LastAppPrewarmBackend,
11    LastForegroundBackend, MarkovBackend, NotificationPriorityBackend,
12    PerCurrentAppMajorityBackend, RandomCandidateBackend, RecentNotificationBackend,
13};
14use super::context_loader::{extract_observable_candidates, load_contexts_by_label, load_labels};
15use super::metrics::{compute_backend_metrics, round3, PredictionRecord};
16use super::predictor::{LocalEvaluatorNextAppBackend, RuleBasedNextAppBackend};
17use super::report::SCHEMA_VERSION;
18use super::types::{
19    BackendMetrics, BenchmarkConfig, BenchmarkReport, ExclusionCounts, NextAppLabel,
20    NextAppPredictor, ScenarioReport,
21};
22
23pub struct BenchmarkRunConfig {
24    pub inputs: Vec<std::path::PathBuf>,
25    pub labels: std::path::PathBuf,
26    pub train_split: f64,
27    pub window_secs: u64,
28}
29
30pub fn run_benchmark(config: &BenchmarkRunConfig) -> Result<BenchmarkReport> {
31    if !(0.0..1.0).contains(&config.train_split) {
32        bail!("train_split must be in (0, 1), got {}", config.train_split);
33    }
34
35    let labels = load_labels(&config.labels)
36        .with_context(|| format!("loading labels {}", config.labels.display()))?;
37
38    let mut by_scenario: HashMap<String, Vec<NextAppLabel>> = HashMap::new();
39    let mut exclusion_counts = ExclusionCounts::default();
40    for label in &labels {
41        by_scenario
42            .entry(label.scenario.clone())
43            .or_default()
44            .push(label.clone());
45        if label.eligible {
46            exclusion_counts.eligible += 1;
47        } else if let Some(reason) = &label.excluded_reason {
48            match reason.as_str() {
49                "label_not_observable" => exclusion_counts.label_not_observable += 1,
50                "no_future_switch" => exclusion_counts.no_future_switch += 1,
51                _ => {},
52            }
53        }
54    }
55
56    let horizon_secs = labels
57        .first()
58        .map(|l| l.prediction_horizon_ms / 1000)
59        .unwrap_or(30);
60
61    let input_pairs: Vec<(String, &Path)> = config
62        .inputs
63        .iter()
64        .map(|p| {
65            let scenario = p
66                .file_stem()
67                .and_then(|s| s.to_str())
68                .map(|s| s.to_string())
69                .with_context(|| format!("invalid input path {}", p.display()))?;
70            Ok((scenario, p.as_path()))
71        })
72        .collect::<Result<Vec<_>>>()?;
73
74    let contexts = load_contexts_by_label(&input_pairs, &labels, config.window_secs)
75        .context("loading contexts from traces")?;
76
77    let mut scenarios = Vec::new();
78    let mut aggregate_records: HashMap<String, Vec<PredictionRecord>> = HashMap::new();
79    let mut aggregate_latencies: HashMap<String, Vec<u64>> = HashMap::new();
80
81    for (scenario_name, trace_path) in &input_pairs {
82        let scenario_labels = by_scenario
83            .remove(scenario_name)
84            .with_context(|| format!("no labels for scenario '{}'", scenario_name))?;
85
86        let mut eligible: Vec<NextAppLabel> = scenario_labels
87            .iter()
88            .filter(|l| l.eligible)
89            .cloned()
90            .collect();
91        eligible.sort_by_key(|l| l.window_start_ms);
92
93        let n = eligible.len();
94        let split_idx = ((n as f64) * config.train_split)
95            .floor()
96            .max(1.0)
97            .min(n.saturating_sub(1) as f64) as usize;
98        let (train, test) = eligible.split_at(split_idx);
99
100        let mut predictors: Vec<Box<dyn NextAppPredictor>> = vec![
101            Box::new(RuleBasedNextAppBackend),
102            Box::new(LocalEvaluatorNextAppBackend),
103            Box::new(AlwaysNoOpBackend),
104            Box::new(RandomCandidateBackend::new(42)),
105            Box::new(FirstCandidateBackend),
106            Box::new(GlobalMajorityBackend::default()),
107            Box::new(PerCurrentAppMajorityBackend::default()),
108            Box::new(MarkovBackend::default()),
109            Box::new(RecentNotificationBackend),
110            Box::new(LastForegroundBackend),
111            Box::new(NotificationPriorityBackend),
112            Box::new(LastAppPrewarmBackend),
113        ];
114
115        for predictor in &mut predictors {
116            predictor.train(train);
117        }
118
119        let mut backend_metrics: BTreeMap<String, BackendMetrics> = BTreeMap::new();
120        for predictor in predictors.iter() {
121            let (records, latencies) =
122                evaluate_predictor(predictor.as_ref(), test, &contexts, scenario_name)?;
123            let metrics = compute_backend_metrics(&records, test.len());
124            backend_metrics.insert(predictor.name().to_string(), metrics);
125            aggregate_records
126                .entry(predictor.name().to_string())
127                .or_default()
128                .extend(records);
129            aggregate_latencies
130                .entry(predictor.name().to_string())
131                .or_default()
132                .extend(latencies);
133        }
134
135        let scenario_exclusions =
136            exclusion_counts_for_scenario(&scenario_labels.iter().collect::<Vec<_>>());
137
138        scenarios.push(ScenarioReport {
139            scenario: (*scenario_name).to_string(),
140            windows: scenario_labels.len(),
141            future_switch_windows: scenario_labels.len() - scenario_exclusions.no_future_switch,
142            eligible_windows: scenario_exclusions.eligible,
143            train_windows: train.len(),
144            test_windows: test.len(),
145            exclusions: scenario_exclusions,
146            backends: backend_metrics,
147        });
148
149        info!(
150            scenario = %scenario_name,
151            train = train.len(),
152            test = test.len(),
153            trace = %trace_path.display(),
154            "benchmarked scenario"
155        );
156    }
157
158    if !by_scenario.is_empty() {
159        bail!(
160            "missing input traces for scenarios: {:?}",
161            by_scenario.keys()
162        );
163    }
164
165    let mut aggregate: BTreeMap<String, BackendMetrics> = BTreeMap::new();
166    for (name, records) in &aggregate_records {
167        let mut metrics = compute_backend_metrics(records, records.len());
168        let scenario_top1: Vec<f64> = scenarios
169            .iter()
170            .filter(|s| s.test_windows > 0)
171            .filter_map(|s| s.backends.get(name).map(|m| m.top1_accuracy_pct))
172            .collect();
173        let scenario_top3: Vec<f64> = scenarios
174            .iter()
175            .filter(|s| s.test_windows > 0)
176            .filter_map(|s| s.backends.get(name).map(|m| m.top3_accuracy_pct))
177            .collect();
178        metrics.macro_top1_accuracy_pct = Some(round3(
179            scenario_top1.iter().sum::<f64>() / scenario_top1.len().max(1) as f64,
180        ));
181        metrics.macro_top3_accuracy_pct = Some(round3(
182            scenario_top3.iter().sum::<f64>() / scenario_top3.len().max(1) as f64,
183        ));
184        if let Some(latencies) = aggregate_latencies.get(name) {
185            metrics.latency_us = super::metrics::latency_summary(latencies);
186        }
187        aggregate.insert(name.clone(), metrics);
188    }
189
190    let future_switch_windows = exclusion_counts.eligible + exclusion_counts.label_not_observable;
191    let coverage_pct =
192        round3((exclusion_counts.eligible as f64 / future_switch_windows.max(1) as f64) * 100.0);
193
194    Ok(BenchmarkReport {
195        schema_version: SCHEMA_VERSION.into(),
196        dataset_id: labels
197            .first()
198            .map(|l| l.dataset_id.clone())
199            .unwrap_or_default(),
200        source: "synthetic".into(),
201        evaluation_source: "synthetic_trace".into(),
202        action_value_source: "synthetic_model_prediction_only".into(),
203        benefit_claim_policy: "no_measured_action_benefit_claim".into(),
204        config: BenchmarkConfig {
205            window_secs: config.window_secs,
206            horizon_secs,
207            label_definition: "first foreground transition within the horizon whose package differs from the app current at window end".into(),
208            eligibility_definition: "actual_next_app is present in the current window's foreground or notified app set after removing current_app".into(),
209            train_split: config.train_split,
210        },
211        inputs: config.inputs.clone(),
212        total_windows: labels.len(),
213        eligible_windows: exclusion_counts.eligible,
214        train_windows: scenarios.iter().map(|s| s.train_windows).sum(),
215        test_windows: scenarios.iter().map(|s| s.test_windows).sum(),
216        context_supported_switch_coverage_pct: coverage_pct,
217        exclusions: exclusion_counts,
218        aggregate,
219        scenarios,
220        limitations: vec![
221            "All labels are derived from deterministic synthetic traces, not real-user behavior.".into(),
222            "Accuracy is reported only for context-supported switches; coverage reports how selective that cohort is.".into(),
223            "Latency is measured during this run and is not byte-deterministic across machines.".into(),
224            "Statistical baselines are trained on the per-scenario train split and evaluated on the held-out test split.".into(),
225            "This synthetic benchmark validates prediction-to-action mapping only; it does not import measured action-value data and must not be cited as measured device benefit.".into(),
226        ],
227    })
228}
229
230fn evaluate_predictor(
231    predictor: &dyn NextAppPredictor,
232    test: &[NextAppLabel],
233    contexts: &BTreeMap<(String, i64, i64), aios_spec::StructuredContext>,
234    scenario: &str,
235) -> Result<(Vec<PredictionRecord>, Vec<u64>)> {
236    let mut records = Vec::with_capacity(test.len());
237    let mut latencies = Vec::with_capacity(test.len());
238
239    for label in test {
240        let ctx = contexts
241            .get(&(
242                label.scenario.clone(),
243                label.window_start_ms,
244                label.window_end_ms,
245            ))
246            .with_context(|| {
247                format!(
248                    "missing context for scenario {} window {}-{}",
249                    scenario, label.window_start_ms, label.window_end_ms
250                )
251            })?;
252        let current_app = &label.current_app;
253        let candidates = extract_observable_candidates(ctx, current_app);
254
255        let result = predictor.predict(ctx, current_app, &candidates);
256        let rank = if let Some(actual) = &label.actual_next_app {
257            result
258                .ranked
259                .iter()
260                .position(|p| &p.package == actual)
261                .map(|i| i + 1)
262        } else {
263            None
264        };
265
266        records.push(PredictionRecord {
267            rank,
268            latency_us: result.latency_us,
269            noop: result.ranked.is_empty(),
270            rationale_present: result.rationale_present,
271        });
272        latencies.push(result.latency_us);
273    }
274
275    Ok((records, latencies))
276}
277
278fn exclusion_counts_for_scenario(labels: &[&NextAppLabel]) -> ExclusionCounts {
279    let mut counts = ExclusionCounts::default();
280    for label in labels {
281        if label.eligible {
282            counts.eligible += 1;
283        } else if let Some(reason) = &label.excluded_reason {
284            match reason.as_str() {
285                "label_not_observable" => counts.label_not_observable += 1,
286                "no_future_switch" => counts.no_future_switch += 1,
287                _ => {},
288            }
289        }
290    }
291    counts
292}