1use std::collections::BTreeMap;
4use std::fs::File;
5use std::io::{BufReader, BufWriter};
6use std::path::PathBuf;
7
8use aios_agent::{
9 prediction_features_for_example, train_next_app_artifact, NextAppAlgorithm,
10 NextAppModelArtifact, NextAppModelConfig, NextAppPredictor,
11};
12use anyhow::{bail, Context, Result};
13use clap::ValueEnum;
14use serde::Serialize;
15
16mod adaptive_baseline;
17mod baselines;
18mod loader;
19mod metrics;
20mod net_benefit;
21mod split;
22mod strong_baseline;
23
24pub use net_benefit::{
25 build_prewarm_net_benefit_fixture, compute_measured_net_benefit, compute_net_benefit,
26 MeasuredNetBenefitInputs, MeasuredValue, MeasurementSource, NetBenefitInputs, NetBenefitReport,
27 NetBenefitTrace, PrewarmFixtureBuildInputs, PrewarmNetBenefitFixture,
28};
29
30use adaptive_baseline::AdaptiveBaseline;
31use baselines::BaselineTables;
32use loader::load_examples;
33use metrics::evaluate_ranker;
34use split::split_examples;
35use strong_baseline::StrongPredictiveActionBaseline;
36
37#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)]
38pub enum NextAppDataset {
39 Lsapp,
40}
41
42#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)]
43pub enum NextAppSplit {
44 Standard,
45 ColdStart,
46}
47
48#[derive(Debug, Clone)]
49pub struct TrainOptions {
50 pub dataset: NextAppDataset,
51 pub input: PathBuf,
52 pub output: PathBuf,
53 pub horizon_secs: u64,
54 pub history_len: usize,
55 pub split: NextAppSplit,
56}
57
58#[derive(Debug, Clone)]
59pub struct EvalOptions {
60 pub dataset: NextAppDataset,
61 pub input: PathBuf,
62 pub artifact: PathBuf,
63 pub output: PathBuf,
64 pub horizon_secs: u64,
65 pub history_len: usize,
66 pub split: NextAppSplit,
67}
68
69#[derive(Debug, Serialize)]
70struct EvalReport {
71 schema_version: String,
72 dataset: String,
73 split: String,
74 artifact_model_id: String,
75 artifact_dataset_id: String,
76 total_examples: usize,
77 train_examples: usize,
78 test_examples: usize,
79 metrics: BTreeMap<String, MetricsReport>,
80}
81
82#[derive(Debug, Default, Clone, Serialize)]
83pub(crate) struct MetricsReport {
84 pub examples: usize,
85 pub predicted: usize,
86 pub hit_rate_at_1_pct: f32,
87 pub hit_rate_at_3_pct: f32,
88 pub hit_rate_at_5_pct: f32,
89 pub mean_reciprocal_rank_at_5: f32,
90 pub prediction_coverage_pct: f32,
91 pub macro_hit_rate_at_1_pct: f32,
92}
93
94#[derive(Debug, Default, Clone)]
95pub(crate) struct UserAccum {
96 pub examples: usize,
97 pub hit1: usize,
98}
99
100pub fn train(opts: TrainOptions) -> Result<()> {
101 let examples = load_examples(&opts.input, opts.horizon_secs, opts.history_len)
102 .with_context(|| format!("loading LSApp dataset from {}", opts.input.display()))?;
103 let (train_examples, _) = split_examples(&examples, opts.split);
104 let config = NextAppModelConfig {
105 horizon_secs: opts.horizon_secs,
106 history_len: opts.history_len,
107 };
108 let artifact =
109 train_next_app_artifact("lsapp", config, &train_examples).map_err(anyhow::Error::msg)?;
110
111 if let Some(parent) = opts.output.parent() {
112 if !parent.as_os_str().is_empty() {
113 std::fs::create_dir_all(parent)
114 .with_context(|| format!("creating output dir {}", parent.display()))?;
115 }
116 }
117 let file = File::create(&opts.output)
118 .with_context(|| format!("creating artifact {}", opts.output.display()))?;
119 serde_json::to_writer_pretty(BufWriter::new(file), &artifact)
120 .with_context(|| format!("writing artifact {}", opts.output.display()))?;
121 eprintln!(
122 "trained next-app artifact: {} examples, {} apps -> {}",
123 artifact.training_summary.examples,
124 artifact.training_summary.apps,
125 opts.output.display()
126 );
127 Ok(())
128}
129
130pub fn evaluate(opts: EvalOptions) -> Result<()> {
131 let artifact_file = File::open(&opts.artifact)
132 .with_context(|| format!("opening artifact {}", opts.artifact.display()))?;
133 let artifact: NextAppModelArtifact = serde_json::from_reader(BufReader::new(artifact_file))
134 .with_context(|| format!("parsing artifact {}", opts.artifact.display()))?;
135 let predictor = NextAppPredictor::new(artifact.clone()).map_err(anyhow::Error::msg)?;
136
137 let examples = load_examples(&opts.input, opts.horizon_secs, opts.history_len)
138 .with_context(|| format!("loading LSApp dataset from {}", opts.input.display()))?;
139 let (train_examples, test_examples) = split_examples(&examples, opts.split);
140 if test_examples.is_empty() {
141 bail!("split produced zero test examples");
142 }
143 let baseline = BaselineTables::from_training(&train_examples);
144 let strong_baseline = StrongPredictiveActionBaseline::from_training(&train_examples);
145
146 let mut metrics = BTreeMap::new();
147 metrics.insert(
148 "global_popularity".into(),
149 evaluate_ranker(&test_examples, |_example| {
150 baseline.global_popularity.iter().take(5).cloned().collect()
151 }),
152 );
153 metrics.insert(
154 "mfu".into(),
155 evaluate_ranker(&test_examples, |example| baseline.mfu(&example.user_id)),
156 );
157 metrics.insert(
158 "mru".into(),
159 evaluate_ranker(&test_examples, |example| {
160 example
161 .history
162 .last()
163 .cloned()
164 .map(|app| vec![app])
165 .unwrap_or_default()
166 }),
167 );
168 for (name, algorithm) in [
169 ("naive_bayes", NextAppAlgorithm::NaiveBayes),
170 ("markov", NextAppAlgorithm::Markov),
171 ("xgboost", NextAppAlgorithm::FeatureLift),
175 ("ensemble", NextAppAlgorithm::Ensemble),
176 ] {
177 metrics.insert(
178 name.into(),
179 evaluate_ranker(&test_examples, |example| {
180 predictor
181 .rank(&prediction_features_for_example(example), algorithm, 5)
182 .into_iter()
183 .map(|score| score.app)
184 .collect()
185 }),
186 );
187 }
188 metrics.insert(
189 "strong_predictive".into(),
190 evaluate_ranker(&test_examples, |example| {
191 strong_baseline.predict_for_example(example, 5)
192 }),
193 );
194 let adaptive_baseline = AdaptiveBaseline::from_training(&train_examples);
195 metrics.insert(
196 "adaptive_predictive".into(),
197 evaluate_ranker(&test_examples, |example| {
198 adaptive_baseline.predict_for_example(example, 5)
199 }),
200 );
201
202 let report = EvalReport {
203 schema_version: "dipecs.next_app_eval.v1".into(),
204 dataset: format!("{:?}", opts.dataset),
205 split: format!("{:?}", opts.split),
206 artifact_model_id: artifact.model_id,
207 artifact_dataset_id: artifact.dataset_id,
208 total_examples: examples.len(),
209 train_examples: train_examples.len(),
210 test_examples: test_examples.len(),
211 metrics,
212 };
213
214 if let Some(parent) = opts.output.parent() {
215 if !parent.as_os_str().is_empty() {
216 std::fs::create_dir_all(parent)
217 .with_context(|| format!("creating output dir {}", parent.display()))?;
218 }
219 }
220 let file = File::create(&opts.output)
221 .with_context(|| format!("creating report {}", opts.output.display()))?;
222 serde_json::to_writer_pretty(BufWriter::new(file), &report)
223 .with_context(|| format!("writing report {}", opts.output.display()))?;
224 eprintln!(
225 "evaluated next-app artifact on {} test examples -> {}",
226 report.test_examples,
227 opts.output.display()
228 );
229 Ok(())
230}
231
232#[cfg(test)]
233mod tests {
234 use std::fs;
235 use std::io::Write;
236 use std::path::PathBuf;
237 use std::time::{SystemTime, UNIX_EPOCH};
238
239 use aios_agent::NextAppTrainingExample;
240
241 use super::*;
242
243 #[test]
244 fn train_and_eval_lsapp_fixture() {
245 let dir = unique_tmp_dir();
246 fs::create_dir_all(&dir).expect("create tmp dir");
247 let input = dir.join("lsapp.csv");
248 let artifact = dir.join("artifact.json");
249 let report = dir.join("report.json");
250 write_fixture(&input);
251
252 train(TrainOptions {
253 dataset: NextAppDataset::Lsapp,
254 input: input.clone(),
255 output: artifact.clone(),
256 horizon_secs: 30,
257 history_len: 3,
258 split: NextAppSplit::Standard,
259 })
260 .expect("train should succeed");
261 evaluate(EvalOptions {
262 dataset: NextAppDataset::Lsapp,
263 input,
264 artifact,
265 output: report.clone(),
266 horizon_secs: 30,
267 history_len: 3,
268 split: NextAppSplit::Standard,
269 })
270 .expect("eval should succeed");
271
272 let value: serde_json::Value =
273 serde_json::from_reader(File::open(report).expect("report should exist"))
274 .expect("report should parse");
275 assert_eq!(value["schema_version"], "dipecs.next_app_eval.v1");
276 assert!(value["metrics"]["ensemble"]["examples"].as_u64().unwrap() > 0);
277 }
278
279 #[test]
280 fn lsapp_loader_orders_sessions_chronologically_not_lexically() {
281 let dir = unique_tmp_dir();
282 fs::create_dir_all(&dir).expect("create tmp dir");
283 let input = dir.join("lsapp.csv");
284 let mut file = fs::File::create(&input).unwrap();
285 writeln!(file, "user_id,session_id,timestamp_ms,app_name,event_type").unwrap();
286 writeln!(file, "u1,10,3000,com.late,foreground").unwrap();
287 writeln!(file, "u1,10,4000,com.final,foreground").unwrap();
288 writeln!(file, "u1,2,1000,com.early,foreground").unwrap();
289 writeln!(file, "u1,2,2000,com.shared,foreground").unwrap();
290
291 let examples = super::loader::load_examples(&input, 30, 4).expect("fixture should load");
292
293 let late = examples
294 .iter()
295 .find(|example| example.current_app == "com.late")
296 .expect("late session should produce an example");
297 assert_eq!(
298 late.history,
299 vec!["com.early".to_string(), "com.shared".to_string()],
300 "numeric-looking session ids must not reorder later sessions before earlier history"
301 );
302 }
303
304 #[test]
305 fn explicit_timestamp_ms_parse_errors_are_rejected() {
306 let dir = unique_tmp_dir();
307 fs::create_dir_all(&dir).expect("create tmp dir");
308 let input = dir.join("bad-timestamp.csv");
309 let mut file = fs::File::create(&input).unwrap();
310 writeln!(file, "user_id,session_id,timestamp_ms,app_name,event_type").unwrap();
311 writeln!(file, "u1,s1,not-a-time,com.chat,foreground").unwrap();
312 writeln!(file, "u1,s1,2000,com.mail,foreground").unwrap();
313
314 let err = super::loader::load_examples(&input, 30, 3)
315 .expect_err("malformed explicit timestamp_ms must not fall back to ordinal time");
316 assert!(
317 err.to_string().contains("timestamp_ms"),
318 "error should name the malformed timestamp_ms column: {err:#}"
319 );
320 }
321
322 #[test]
323 fn directory_loader_ignores_plain_json_files() {
324 let dir = unique_tmp_dir();
325 fs::create_dir_all(&dir).expect("create tmp dir");
326 let csv = dir.join("lsapp.csv");
327 write_fixture(&csv);
328 let mut json = fs::File::create(dir.join("notes.json")).unwrap();
329 writeln!(json, r#"{{"not":"a supported LSApp record"}}"#).unwrap();
330 writeln!(json, r#"{{"still":"not delimited"}}"#).unwrap();
331
332 let examples = super::loader::load_examples(&dir, 30, 3)
333 .expect("unsupported .json sidecars should be ignored");
334 assert!(!examples.is_empty());
335 }
336
337 #[test]
338 fn cold_start_split_uses_stable_hash_order_not_user_id_sort() {
339 let examples: Vec<NextAppTrainingExample> = ["u0", "u1", "u2", "u3", "u4"]
340 .into_iter()
341 .map(|user| NextAppTrainingExample {
342 user_id: user.into(),
343 current_app: "com.chat".into(),
344 history: vec![],
345 hour_bucket: 9,
346 weekday: 1,
347 event_type: "foreground".into(),
348 label_app: "com.mail".into(),
349 })
350 .collect();
351 let (train, test) = split_examples(&examples, NextAppSplit::ColdStart);
352 assert_eq!(train.len(), 4);
353 assert_eq!(test.len(), 1);
354 assert_eq!(test[0].user_id, "u3");
356 }
357
358 fn write_fixture(path: &PathBuf) {
359 let mut file = fs::File::create(path).unwrap();
360 writeln!(file, "user_id,session_id,timestamp,app_name,event_type").unwrap();
361 let mut ts = 1_700_000_000_000_i64;
362 for user in ["u1", "u2"] {
363 for session in ["s1", "s2"] {
364 for app in [
365 "com.chat",
366 "com.mail",
367 "com.chat",
368 "com.mail",
369 "com.browser",
370 ] {
371 writeln!(file, "{user},{session},{ts},{app},foreground").unwrap();
372 ts += 5_000;
373 }
374 ts += 60_000;
375 }
376 }
377 }
378
379 fn unique_tmp_dir() -> PathBuf {
380 let suffix = SystemTime::now()
381 .duration_since(UNIX_EPOCH)
382 .unwrap()
383 .as_nanos();
384 std::env::temp_dir().join(format!("dipecs-next-app-test-{suffix}"))
385 }
386
387 #[test]
388 fn evaluate_ranker_counts_non_empty_wrong_predictions_as_predicted() {
389 let examples = vec![
390 NextAppTrainingExample {
391 user_id: "u1".into(),
392 current_app: "com.chat".into(),
393 history: vec![],
394 hour_bucket: 9,
395 weekday: 1,
396 event_type: "foreground".into(),
397 label_app: "com.mail".into(),
398 },
399 NextAppTrainingExample {
400 user_id: "u1".into(),
401 current_app: "com.chat".into(),
402 history: vec![],
403 hour_bucket: 9,
404 weekday: 1,
405 event_type: "foreground".into(),
406 label_app: "com.browser".into(),
407 },
408 ];
409 let report = evaluate_ranker(&examples, |_example| {
410 vec!["com.other".into(), "com.yetanother".into()]
411 });
412 assert_eq!(report.examples, 2);
413 assert_eq!(
414 report.predicted, 2,
415 "non-empty predictions must count toward coverage"
416 );
417 assert_eq!(report.hit_rate_at_1_pct, 0.0);
418 assert_eq!(report.hit_rate_at_3_pct, 0.0);
419 assert_eq!(report.hit_rate_at_5_pct, 0.0);
420 assert_eq!(report.prediction_coverage_pct, 100.0);
421 }
422}