Skip to main content

aios_cli/next_app/
mod.rs

1//! Offline next-app training/evaluation helpers for LSApp-shaped datasets.
2
3use std::collections::BTreeMap;
4use std::fs::File;
5use std::io::{BufReader, BufWriter};
6use std::path::PathBuf;
7
8use aios_agent::{
9    prediction_features_for_example, train_next_app_artifact, NextAppAlgorithm,
10    NextAppModelArtifact, NextAppModelConfig, NextAppPredictor,
11};
12use anyhow::{bail, Context, Result};
13use clap::ValueEnum;
14use serde::Serialize;
15
16mod adaptive_baseline;
17mod baselines;
18mod loader;
19mod metrics;
20mod net_benefit;
21mod split;
22mod strong_baseline;
23
24pub use net_benefit::{
25    build_prewarm_net_benefit_fixture, compute_measured_net_benefit, compute_net_benefit,
26    MeasuredNetBenefitInputs, MeasuredValue, MeasurementSource, NetBenefitInputs, NetBenefitReport,
27    NetBenefitTrace, PrewarmFixtureBuildInputs, PrewarmNetBenefitFixture,
28};
29
30use adaptive_baseline::AdaptiveBaseline;
31use baselines::BaselineTables;
32use loader::load_examples;
33use metrics::evaluate_ranker;
34use split::split_examples;
35use strong_baseline::StrongPredictiveActionBaseline;
36
37#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)]
38pub enum NextAppDataset {
39    Lsapp,
40}
41
42#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)]
43pub enum NextAppSplit {
44    Standard,
45    ColdStart,
46}
47
48#[derive(Debug, Clone)]
49pub struct TrainOptions {
50    pub dataset: NextAppDataset,
51    pub input: PathBuf,
52    pub output: PathBuf,
53    pub horizon_secs: u64,
54    pub history_len: usize,
55    pub split: NextAppSplit,
56}
57
58#[derive(Debug, Clone)]
59pub struct EvalOptions {
60    pub dataset: NextAppDataset,
61    pub input: PathBuf,
62    pub artifact: PathBuf,
63    pub output: PathBuf,
64    pub horizon_secs: u64,
65    pub history_len: usize,
66    pub split: NextAppSplit,
67}
68
69#[derive(Debug, Serialize)]
70struct EvalReport {
71    schema_version: String,
72    dataset: String,
73    split: String,
74    artifact_model_id: String,
75    artifact_dataset_id: String,
76    total_examples: usize,
77    train_examples: usize,
78    test_examples: usize,
79    metrics: BTreeMap<String, MetricsReport>,
80}
81
82#[derive(Debug, Default, Clone, Serialize)]
83pub(crate) struct MetricsReport {
84    pub examples: usize,
85    pub predicted: usize,
86    pub hit_rate_at_1_pct: f32,
87    pub hit_rate_at_3_pct: f32,
88    pub hit_rate_at_5_pct: f32,
89    pub mean_reciprocal_rank_at_5: f32,
90    pub prediction_coverage_pct: f32,
91    pub macro_hit_rate_at_1_pct: f32,
92}
93
94#[derive(Debug, Default, Clone)]
95pub(crate) struct UserAccum {
96    pub examples: usize,
97    pub hit1: usize,
98}
99
100pub fn train(opts: TrainOptions) -> Result<()> {
101    let examples = load_examples(&opts.input, opts.horizon_secs, opts.history_len)
102        .with_context(|| format!("loading LSApp dataset from {}", opts.input.display()))?;
103    let (train_examples, _) = split_examples(&examples, opts.split);
104    let config = NextAppModelConfig {
105        horizon_secs: opts.horizon_secs,
106        history_len: opts.history_len,
107    };
108    let artifact =
109        train_next_app_artifact("lsapp", config, &train_examples).map_err(anyhow::Error::msg)?;
110
111    if let Some(parent) = opts.output.parent() {
112        if !parent.as_os_str().is_empty() {
113            std::fs::create_dir_all(parent)
114                .with_context(|| format!("creating output dir {}", parent.display()))?;
115        }
116    }
117    let file = File::create(&opts.output)
118        .with_context(|| format!("creating artifact {}", opts.output.display()))?;
119    serde_json::to_writer_pretty(BufWriter::new(file), &artifact)
120        .with_context(|| format!("writing artifact {}", opts.output.display()))?;
121    eprintln!(
122        "trained next-app artifact: {} examples, {} apps -> {}",
123        artifact.training_summary.examples,
124        artifact.training_summary.apps,
125        opts.output.display()
126    );
127    Ok(())
128}
129
130pub fn evaluate(opts: EvalOptions) -> Result<()> {
131    let artifact_file = File::open(&opts.artifact)
132        .with_context(|| format!("opening artifact {}", opts.artifact.display()))?;
133    let artifact: NextAppModelArtifact = serde_json::from_reader(BufReader::new(artifact_file))
134        .with_context(|| format!("parsing artifact {}", opts.artifact.display()))?;
135    let predictor = NextAppPredictor::new(artifact.clone()).map_err(anyhow::Error::msg)?;
136
137    let examples = load_examples(&opts.input, opts.horizon_secs, opts.history_len)
138        .with_context(|| format!("loading LSApp dataset from {}", opts.input.display()))?;
139    let (train_examples, test_examples) = split_examples(&examples, opts.split);
140    if test_examples.is_empty() {
141        bail!("split produced zero test examples");
142    }
143    let baseline = BaselineTables::from_training(&train_examples);
144    let strong_baseline = StrongPredictiveActionBaseline::from_training(&train_examples);
145
146    let mut metrics = BTreeMap::new();
147    metrics.insert(
148        "global_popularity".into(),
149        evaluate_ranker(&test_examples, |_example| {
150            baseline.global_popularity.iter().take(5).cloned().collect()
151        }),
152    );
153    metrics.insert(
154        "mfu".into(),
155        evaluate_ranker(&test_examples, |example| baseline.mfu(&example.user_id)),
156    );
157    metrics.insert(
158        "mru".into(),
159        evaluate_ranker(&test_examples, |example| {
160            example
161                .history
162                .last()
163                .cloned()
164                .map(|app| vec![app])
165                .unwrap_or_default()
166        }),
167    );
168    for (name, algorithm) in [
169        ("naive_bayes", NextAppAlgorithm::NaiveBayes),
170        ("markov", NextAppAlgorithm::Markov),
171        // The report key remains "xgboost" for backward compatibility with
172        // committed baseline reports; the underlying model is a log-lift
173        // feature ensemble, not XGBoost.
174        ("xgboost", NextAppAlgorithm::FeatureLift),
175        ("ensemble", NextAppAlgorithm::Ensemble),
176    ] {
177        metrics.insert(
178            name.into(),
179            evaluate_ranker(&test_examples, |example| {
180                predictor
181                    .rank(&prediction_features_for_example(example), algorithm, 5)
182                    .into_iter()
183                    .map(|score| score.app)
184                    .collect()
185            }),
186        );
187    }
188    metrics.insert(
189        "strong_predictive".into(),
190        evaluate_ranker(&test_examples, |example| {
191            strong_baseline.predict_for_example(example, 5)
192        }),
193    );
194    let adaptive_baseline = AdaptiveBaseline::from_training(&train_examples);
195    metrics.insert(
196        "adaptive_predictive".into(),
197        evaluate_ranker(&test_examples, |example| {
198            adaptive_baseline.predict_for_example(example, 5)
199        }),
200    );
201
202    let report = EvalReport {
203        schema_version: "dipecs.next_app_eval.v1".into(),
204        dataset: format!("{:?}", opts.dataset),
205        split: format!("{:?}", opts.split),
206        artifact_model_id: artifact.model_id,
207        artifact_dataset_id: artifact.dataset_id,
208        total_examples: examples.len(),
209        train_examples: train_examples.len(),
210        test_examples: test_examples.len(),
211        metrics,
212    };
213
214    if let Some(parent) = opts.output.parent() {
215        if !parent.as_os_str().is_empty() {
216            std::fs::create_dir_all(parent)
217                .with_context(|| format!("creating output dir {}", parent.display()))?;
218        }
219    }
220    let file = File::create(&opts.output)
221        .with_context(|| format!("creating report {}", opts.output.display()))?;
222    serde_json::to_writer_pretty(BufWriter::new(file), &report)
223        .with_context(|| format!("writing report {}", opts.output.display()))?;
224    eprintln!(
225        "evaluated next-app artifact on {} test examples -> {}",
226        report.test_examples,
227        opts.output.display()
228    );
229    Ok(())
230}
231
232#[cfg(test)]
233mod tests {
234    use std::fs;
235    use std::io::Write;
236    use std::path::PathBuf;
237    use std::time::{SystemTime, UNIX_EPOCH};
238
239    use aios_agent::NextAppTrainingExample;
240
241    use super::*;
242
243    #[test]
244    fn train_and_eval_lsapp_fixture() {
245        let dir = unique_tmp_dir();
246        fs::create_dir_all(&dir).expect("create tmp dir");
247        let input = dir.join("lsapp.csv");
248        let artifact = dir.join("artifact.json");
249        let report = dir.join("report.json");
250        write_fixture(&input);
251
252        train(TrainOptions {
253            dataset: NextAppDataset::Lsapp,
254            input: input.clone(),
255            output: artifact.clone(),
256            horizon_secs: 30,
257            history_len: 3,
258            split: NextAppSplit::Standard,
259        })
260        .expect("train should succeed");
261        evaluate(EvalOptions {
262            dataset: NextAppDataset::Lsapp,
263            input,
264            artifact,
265            output: report.clone(),
266            horizon_secs: 30,
267            history_len: 3,
268            split: NextAppSplit::Standard,
269        })
270        .expect("eval should succeed");
271
272        let value: serde_json::Value =
273            serde_json::from_reader(File::open(report).expect("report should exist"))
274                .expect("report should parse");
275        assert_eq!(value["schema_version"], "dipecs.next_app_eval.v1");
276        assert!(value["metrics"]["ensemble"]["examples"].as_u64().unwrap() > 0);
277    }
278
279    #[test]
280    fn lsapp_loader_orders_sessions_chronologically_not_lexically() {
281        let dir = unique_tmp_dir();
282        fs::create_dir_all(&dir).expect("create tmp dir");
283        let input = dir.join("lsapp.csv");
284        let mut file = fs::File::create(&input).unwrap();
285        writeln!(file, "user_id,session_id,timestamp_ms,app_name,event_type").unwrap();
286        writeln!(file, "u1,10,3000,com.late,foreground").unwrap();
287        writeln!(file, "u1,10,4000,com.final,foreground").unwrap();
288        writeln!(file, "u1,2,1000,com.early,foreground").unwrap();
289        writeln!(file, "u1,2,2000,com.shared,foreground").unwrap();
290
291        let examples = super::loader::load_examples(&input, 30, 4).expect("fixture should load");
292
293        let late = examples
294            .iter()
295            .find(|example| example.current_app == "com.late")
296            .expect("late session should produce an example");
297        assert_eq!(
298            late.history,
299            vec!["com.early".to_string(), "com.shared".to_string()],
300            "numeric-looking session ids must not reorder later sessions before earlier history"
301        );
302    }
303
304    #[test]
305    fn explicit_timestamp_ms_parse_errors_are_rejected() {
306        let dir = unique_tmp_dir();
307        fs::create_dir_all(&dir).expect("create tmp dir");
308        let input = dir.join("bad-timestamp.csv");
309        let mut file = fs::File::create(&input).unwrap();
310        writeln!(file, "user_id,session_id,timestamp_ms,app_name,event_type").unwrap();
311        writeln!(file, "u1,s1,not-a-time,com.chat,foreground").unwrap();
312        writeln!(file, "u1,s1,2000,com.mail,foreground").unwrap();
313
314        let err = super::loader::load_examples(&input, 30, 3)
315            .expect_err("malformed explicit timestamp_ms must not fall back to ordinal time");
316        assert!(
317            err.to_string().contains("timestamp_ms"),
318            "error should name the malformed timestamp_ms column: {err:#}"
319        );
320    }
321
322    #[test]
323    fn directory_loader_ignores_plain_json_files() {
324        let dir = unique_tmp_dir();
325        fs::create_dir_all(&dir).expect("create tmp dir");
326        let csv = dir.join("lsapp.csv");
327        write_fixture(&csv);
328        let mut json = fs::File::create(dir.join("notes.json")).unwrap();
329        writeln!(json, r#"{{"not":"a supported LSApp record"}}"#).unwrap();
330        writeln!(json, r#"{{"still":"not delimited"}}"#).unwrap();
331
332        let examples = super::loader::load_examples(&dir, 30, 3)
333            .expect("unsupported .json sidecars should be ignored");
334        assert!(!examples.is_empty());
335    }
336
337    #[test]
338    fn cold_start_split_uses_stable_hash_order_not_user_id_sort() {
339        let examples: Vec<NextAppTrainingExample> = ["u0", "u1", "u2", "u3", "u4"]
340            .into_iter()
341            .map(|user| NextAppTrainingExample {
342                user_id: user.into(),
343                current_app: "com.chat".into(),
344                history: vec![],
345                hour_bucket: 9,
346                weekday: 1,
347                event_type: "foreground".into(),
348                label_app: "com.mail".into(),
349            })
350            .collect();
351        let (train, test) = split_examples(&examples, NextAppSplit::ColdStart);
352        assert_eq!(train.len(), 4);
353        assert_eq!(test.len(), 1);
354        // Seeded FNV hash order for these user ids places u3 in test.
355        assert_eq!(test[0].user_id, "u3");
356    }
357
358    fn write_fixture(path: &PathBuf) {
359        let mut file = fs::File::create(path).unwrap();
360        writeln!(file, "user_id,session_id,timestamp,app_name,event_type").unwrap();
361        let mut ts = 1_700_000_000_000_i64;
362        for user in ["u1", "u2"] {
363            for session in ["s1", "s2"] {
364                for app in [
365                    "com.chat",
366                    "com.mail",
367                    "com.chat",
368                    "com.mail",
369                    "com.browser",
370                ] {
371                    writeln!(file, "{user},{session},{ts},{app},foreground").unwrap();
372                    ts += 5_000;
373                }
374                ts += 60_000;
375            }
376        }
377    }
378
379    fn unique_tmp_dir() -> PathBuf {
380        let suffix = SystemTime::now()
381            .duration_since(UNIX_EPOCH)
382            .unwrap()
383            .as_nanos();
384        std::env::temp_dir().join(format!("dipecs-next-app-test-{suffix}"))
385    }
386
387    #[test]
388    fn evaluate_ranker_counts_non_empty_wrong_predictions_as_predicted() {
389        let examples = vec![
390            NextAppTrainingExample {
391                user_id: "u1".into(),
392                current_app: "com.chat".into(),
393                history: vec![],
394                hour_bucket: 9,
395                weekday: 1,
396                event_type: "foreground".into(),
397                label_app: "com.mail".into(),
398            },
399            NextAppTrainingExample {
400                user_id: "u1".into(),
401                current_app: "com.chat".into(),
402                history: vec![],
403                hour_bucket: 9,
404                weekday: 1,
405                event_type: "foreground".into(),
406                label_app: "com.browser".into(),
407            },
408        ];
409        let report = evaluate_ranker(&examples, |_example| {
410            vec!["com.other".into(), "com.yetanother".into()]
411        });
412        assert_eq!(report.examples, 2);
413        assert_eq!(
414            report.predicted, 2,
415            "non-empty predictions must count toward coverage"
416        );
417        assert_eq!(report.hit_rate_at_1_pct, 0.0);
418        assert_eq!(report.hit_rate_at_3_pct, 0.0);
419        assert_eq!(report.hit_rate_at_5_pct, 0.0);
420        assert_eq!(report.prediction_coverage_pct, 100.0);
421    }
422}