Skip to main content

aios_collector/
android_jsonl.rs

1//! Android collector JSONL ingress.
2//!
3//! The Android app writes one `CollectorEvent` JSON object per line. When a
4//! row contains a Rust-compatible `rawEvent` field, this module wraps it in a
5//! `CollectorEnvelope` so the daemon can feed it into the normal
6//! `RustCollectorIngress -> PrivacyAirGap -> WindowAggregator` pipeline.
7
8use std::fs::File;
9use std::io::{Read, Seek, SeekFrom};
10use std::path::{Path, PathBuf};
11
12use aios_spec::{CollectorEnvelope, RawEvent, SourceTier};
13use serde_json::Value;
14use thiserror::Error;
15
16const SCHEMA_VERSION: &str = "dipecs.collector.v1";
17const DEFAULT_SOURCE: &str = "apps.android-collector";
18const FIELD_RAW_EVENT: &str = "rawEvent";
19const FIELD_SOURCE: &str = "source";
20const FIELD_EVENT_ID: &str = "eventId";
21const FIELD_TIMESTAMP_MS: &str = "timestampMs";
22
23/// Parsed outcome for one Android JSONL row.
24///
25/// `parse_line` keeps the historical `Option<CollectorEnvelope>` API for
26/// callers that only need ingestion. This shape is intentionally more verbose
27/// for code that wants to keep counters or diagnostics without re-parsing the
28/// original JSON row.
29#[derive(Debug, Clone, PartialEq, Eq)]
30pub enum AndroidJsonlRecord {
31    EmptyLine,
32    NoRawEvent,
33    Envelope(Box<CollectorEnvelope>),
34}
35
36impl AndroidJsonlRecord {
37    pub fn into_envelope(self) -> Option<CollectorEnvelope> {
38        match self {
39            Self::Envelope(envelope) => Some(*envelope),
40            Self::EmptyLine | Self::NoRawEvent => None,
41        }
42    }
43}
44
45#[derive(Debug, Clone)]
46pub struct AndroidJsonlIngress {
47    source_tier: SourceTier,
48}
49
50impl AndroidJsonlIngress {
51    pub fn new() -> Self {
52        Self {
53            source_tier: SourceTier::PublicApi,
54        }
55    }
56
57    pub fn parse_line(&self, line: &str) -> Result<Option<CollectorEnvelope>, AndroidJsonlError> {
58        let record = self.parse_record(line)?;
59        Ok(record.into_envelope())
60    }
61
62    pub fn parse_record(&self, line: &str) -> Result<AndroidJsonlRecord, AndroidJsonlError> {
63        if line.trim().is_empty() {
64            return Ok(AndroidJsonlRecord::EmptyLine);
65        }
66
67        let value: Value = serde_json::from_str(line)?;
68        let Some(raw_event_value) = raw_event_json(&value) else {
69            return Ok(AndroidJsonlRecord::NoRawEvent);
70        };
71
72        let raw_event = parse_raw_event(raw_event_value)?;
73        let source = collector_source(&value);
74        let device_trace_id = collector_event_id(&value);
75        let captured_at_ms = collector_timestamp_ms(&value, &raw_event);
76
77        let envelope = CollectorEnvelope {
78            schema_version: SCHEMA_VERSION.to_string(),
79            source,
80            source_tier: self.source_tier,
81            device_trace_id,
82            captured_at_ms,
83            received_at_ms: None,
84            raw_event,
85        };
86
87        Ok(AndroidJsonlRecord::Envelope(Box::new(envelope)))
88    }
89}
90
91impl Default for AndroidJsonlIngress {
92    fn default() -> Self {
93        Self::new()
94    }
95}
96
97/// Polls an append-only Android `actions.jsonl` file.
98///
99/// The tailer keeps byte offset state between polls. If the file is truncated
100/// or rotated, the offset is reset and the next poll starts from the beginning
101/// of the new file.
102#[derive(Debug)]
103pub struct AndroidJsonlTailer {
104    path: PathBuf,
105    offset: u64,
106    partial_line: String,
107    ingress: AndroidJsonlIngress,
108}
109
110impl AndroidJsonlTailer {
111    pub fn new(path: impl Into<PathBuf>) -> Self {
112        Self {
113            path: path.into(),
114            offset: 0,
115            partial_line: String::new(),
116            ingress: AndroidJsonlIngress::new(),
117        }
118    }
119
120    pub fn path(&self) -> &Path {
121        &self.path
122    }
123
124    pub fn poll(&mut self) -> Result<Vec<CollectorEnvelope>, AndroidJsonlError> {
125        let Some(chunk) = self.read_new_chunk()? else {
126            return Ok(Vec::new());
127        };
128
129        let complete_lines = self.take_complete_lines(chunk);
130        let envelopes = self.parse_complete_lines(complete_lines)?;
131
132        Ok(envelopes)
133    }
134
135    fn read_new_chunk(&mut self) -> Result<Option<String>, AndroidJsonlError> {
136        let mut file = match File::open(&self.path) {
137            Ok(file) => file,
138            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
139            Err(error) => return Err(AndroidJsonlError::Io(error)),
140        };
141
142        let file_len = file.metadata()?.len();
143        if self.offset_is_past_end(file_len) {
144            self.reset_for_recreated_file();
145        }
146
147        let old_offset = self.offset;
148        let mut chunk = self.read_chunk_at_offset(&mut file, old_offset)?;
149
150        if self.should_rewind_after_chunk_read(old_offset, &chunk) {
151            self.reset_for_recreated_file();
152            chunk = self.read_chunk_at_offset(&mut file, 0)?;
153        }
154
155        if chunk.is_empty() {
156            Ok(None)
157        } else {
158            Ok(Some(chunk))
159        }
160    }
161
162    fn read_chunk_at_offset(
163        &mut self,
164        file: &mut File,
165        offset: u64,
166    ) -> Result<String, AndroidJsonlError> {
167        file.seek(SeekFrom::Start(offset))?;
168
169        let mut chunk = String::new();
170        let bytes_read = file.read_to_string(&mut chunk)? as u64;
171        self.offset = offset + bytes_read;
172
173        Ok(chunk)
174    }
175
176    fn offset_is_past_end(&self, file_len: u64) -> bool {
177        file_len < self.offset
178    }
179
180    fn reset_for_recreated_file(&mut self) {
181        self.offset = 0;
182        self.partial_line.clear();
183    }
184
185    fn should_rewind_after_chunk_read(&self, old_offset: u64, chunk: &str) -> bool {
186        if old_offset == 0 || !self.partial_line.is_empty() {
187            return false;
188        }
189
190        let Some(first_char) = chunk.chars().find(|c| !c.is_whitespace()) else {
191            return false;
192        };
193
194        first_char != '{'
195    }
196
197    fn take_complete_lines(&mut self, chunk: String) -> Vec<String> {
198        let mut pending = self.pending_with_new_chunk(chunk);
199        let has_partial_tail = !pending.ends_with('\n');
200        let mut complete_lines = Vec::new();
201
202        if has_partial_tail {
203            let partial_start = pending.rfind('\n').map(|idx| idx + 1).unwrap_or(0);
204            self.partial_line = pending.split_off(partial_start);
205        }
206
207        for line in pending.split('\n') {
208            if line.is_empty() {
209                continue;
210            }
211            complete_lines.push(line.trim_end_matches('\r').to_string());
212        }
213
214        complete_lines
215    }
216
217    fn pending_with_new_chunk(&mut self, chunk: String) -> String {
218        if self.partial_line.is_empty() {
219            return chunk;
220        }
221
222        let mut pending = String::new();
223        std::mem::swap(&mut pending, &mut self.partial_line);
224        pending.push_str(&chunk);
225        pending
226    }
227
228    fn parse_complete_lines(
229        &self,
230        complete_lines: Vec<String>,
231    ) -> Result<Vec<CollectorEnvelope>, AndroidJsonlError> {
232        let mut envelopes = Vec::new();
233
234        for line in complete_lines {
235            let trimmed = line.trim_end_matches('\r');
236            if let Some(envelope) = self.ingress.parse_line(trimmed)? {
237                envelopes.push(envelope);
238            }
239        }
240
241        Ok(envelopes)
242    }
243}
244
245#[derive(Debug, Error)]
246pub enum AndroidJsonlError {
247    #[error("read Android JSONL trace failed: {0}")]
248    Io(#[from] std::io::Error),
249    #[error("parse Android CollectorEvent JSON failed: {0}")]
250    Json(#[from] serde_json::Error),
251}
252
253fn raw_event_json(value: &Value) -> Option<Value> {
254    match value.get(FIELD_RAW_EVENT) {
255        Some(raw_event) if !raw_event.is_null() => Some(raw_event.clone()),
256        Some(_) | None => None,
257    }
258}
259
260fn parse_raw_event(value: Value) -> Result<RawEvent, AndroidJsonlError> {
261    let raw_event = serde_json::from_value(value)?;
262    Ok(raw_event)
263}
264
265fn collector_source(value: &Value) -> String {
266    value
267        .get(FIELD_SOURCE)
268        .and_then(Value::as_str)
269        .filter(|source| !source.trim().is_empty())
270        .unwrap_or(DEFAULT_SOURCE)
271        .to_string()
272}
273
274fn collector_event_id(value: &Value) -> Option<String> {
275    value
276        .get(FIELD_EVENT_ID)
277        .and_then(Value::as_str)
278        .filter(|event_id| !event_id.trim().is_empty())
279        .map(String::from)
280}
281
282fn collector_timestamp_ms(value: &Value, raw_event: &RawEvent) -> i64 {
283    value
284        .get(FIELD_TIMESTAMP_MS)
285        .and_then(Value::as_i64)
286        .or_else(|| event_timestamp_ms(raw_event))
287        .unwrap_or(0)
288}
289
290fn event_timestamp_ms(raw_event: &RawEvent) -> Option<i64> {
291    Some(match raw_event {
292        RawEvent::AppTransition(event) => event.timestamp_ms,
293        RawEvent::BinderTransaction(event) => event.timestamp_ms,
294        RawEvent::ProcStateChange(event) => event.timestamp_ms,
295        RawEvent::FileSystemAccess(event) => event.timestamp_ms,
296        RawEvent::NotificationPosted(event) => event.timestamp_ms,
297        RawEvent::NotificationInteraction(event) => event.timestamp_ms,
298        RawEvent::ScreenState(event) => event.timestamp_ms,
299        RawEvent::SystemState(event) => event.timestamp_ms,
300    })
301}
302
303#[cfg(test)]
304mod tests {
305    use super::{AndroidJsonlIngress, AndroidJsonlRecord, AndroidJsonlTailer, DEFAULT_SOURCE};
306    use aios_spec::{AppTransition, NetworkType, RawEvent, SourceTier};
307    use std::fs::OpenOptions;
308    use std::io::Write;
309    use std::time::{SystemTime, UNIX_EPOCH};
310
311    const APP_TRANSITION_LINE: &str = r#"{"eventId":"evt-1","timestampMs":1000,"source":"UsageCollector","eventType":"app_transition","rawEvent":{"AppTransition":{"timestamp_ms":1000,"package_name":"com.android.chrome","activity_class":"MainActivity","transition":"Foreground"}},"rawPayload":{}}"#;
312    const NOTIFICATION_POSTED_LINE: &str = r#"{"eventId":"evt-2","timestampMs":2000,"source":"NotificationCollectorService","eventType":"notification_posted","rawEvent":{"NotificationPosted":{"timestamp_ms":2000,"package_name":"com.chat","category":"msg","channel_id":"messages","raw_title":"Alice","raw_text":"sent a file","is_ongoing":false,"group_key":"conversation","has_picture":false}},"rawPayload":{}}"#;
313    const SYSTEM_STATE_LINE: &str = r#"{"eventId":"evt-3","timestampMs":3000,"source":"CollectorForegroundService","eventType":"system_state","rawEvent":{"SystemState":{"timestamp_ms":3000,"battery_pct":88,"is_charging":true,"network":"Wifi","ringer_mode":"Normal","location_type":"Unknown","headphone_connected":false,"bluetooth_connected":false}},"rawPayload":{}}"#;
314
315    #[test]
316    fn parse_line_wraps_raw_event_in_collector_envelope() {
317        let envelope = AndroidJsonlIngress::new()
318            .parse_line(APP_TRANSITION_LINE)
319            .unwrap()
320            .unwrap();
321
322        assert_eq!(envelope.schema_version, "dipecs.collector.v1");
323        assert_eq!(envelope.source, "UsageCollector");
324        assert_eq!(envelope.source_tier, SourceTier::PublicApi);
325        assert_eq!(envelope.device_trace_id.as_deref(), Some("evt-1"));
326        assert_eq!(envelope.captured_at_ms, 1000);
327        match envelope.raw_event {
328            RawEvent::AppTransition(event) => {
329                assert_eq!(event.package_name, "com.android.chrome");
330                assert_eq!(event.transition, AppTransition::Foreground);
331            },
332            other => panic!("unexpected raw event: {other:?}"),
333        }
334    }
335
336    #[test]
337    fn parse_line_skips_rows_without_raw_event() {
338        let line = r#"{"eventId":"evt-2","timestampMs":1000,"source":"AccessibilityCollectorService","rawEvent":null}"#;
339        let parsed = AndroidJsonlIngress::new().parse_line(line).unwrap();
340        assert!(parsed.is_none());
341    }
342
343    #[test]
344    fn parse_record_distinguishes_empty_and_no_raw_event_rows() {
345        let ingress = AndroidJsonlIngress::new();
346
347        let empty = ingress.parse_record("  \r\n").unwrap();
348        assert_eq!(empty, AndroidJsonlRecord::EmptyLine);
349
350        let no_raw_event = ingress
351            .parse_record(
352                r#"{"eventId":"evt-screen","source":"AccessibilityCollectorService","rawEvent":null}"#,
353            )
354            .unwrap();
355        assert_eq!(no_raw_event, AndroidJsonlRecord::NoRawEvent);
356    }
357
358    #[test]
359    fn parse_line_uses_defaults_for_missing_outer_metadata() {
360        let line = r#"{"rawEvent":{"ScreenState":{"timestamp_ms":1234,"state":"Interactive"}}}"#;
361
362        let envelope = AndroidJsonlIngress::new()
363            .parse_line(line)
364            .unwrap()
365            .unwrap();
366
367        assert_eq!(envelope.source, DEFAULT_SOURCE);
368        assert_eq!(envelope.device_trace_id, None);
369        assert_eq!(envelope.captured_at_ms, 1234);
370        assert!(matches!(envelope.raw_event, RawEvent::ScreenState(_)));
371    }
372
373    #[test]
374    fn parse_line_prefers_outer_timestamp_over_raw_event_timestamp() {
375        let line = r#"{"eventId":"evt-screen","timestampMs":9999,"source":"CollectorForegroundService","rawEvent":{"ScreenState":{"timestamp_ms":1234,"state":"Interactive"}}}"#;
376
377        let envelope = AndroidJsonlIngress::new()
378            .parse_line(line)
379            .unwrap()
380            .unwrap();
381
382        assert_eq!(envelope.device_trace_id.as_deref(), Some("evt-screen"));
383        assert_eq!(envelope.captured_at_ms, 9999);
384    }
385
386    #[test]
387    fn parse_line_ignores_blank_source_and_event_id() {
388        let line = r#"{"eventId":"   ","timestampMs":1000,"source":"  ","rawEvent":{"ScreenState":{"timestamp_ms":1000,"state":"Interactive"}}}"#;
389
390        let envelope = AndroidJsonlIngress::new()
391            .parse_line(line)
392            .unwrap()
393            .unwrap();
394
395        assert_eq!(envelope.source, DEFAULT_SOURCE);
396        assert_eq!(envelope.device_trace_id, None);
397    }
398
399    #[test]
400    fn promoted_android_sources_parse_as_public_api_ingress() {
401        let ingress = AndroidJsonlIngress::new();
402        let rows = [
403            (APP_TRANSITION_LINE, "UsageCollector"),
404            (NOTIFICATION_POSTED_LINE, "NotificationCollectorService"),
405            (SYSTEM_STATE_LINE, "CollectorForegroundService"),
406        ];
407
408        let envelopes = rows
409            .into_iter()
410            .map(|(line, source)| {
411                let envelope = ingress.parse_line(line).unwrap().unwrap();
412                assert_eq!(envelope.source, source);
413                assert_eq!(envelope.source_tier, SourceTier::PublicApi);
414                envelope
415            })
416            .collect::<Vec<_>>();
417
418        assert!(matches!(envelopes[0].raw_event, RawEvent::AppTransition(_)));
419        match &envelopes[1].raw_event {
420            RawEvent::NotificationPosted(event) => {
421                assert_eq!(event.package_name, "com.chat");
422                assert_eq!(event.raw_text, "sent a file");
423            },
424            other => panic!("unexpected raw event: {other:?}"),
425        }
426        match &envelopes[2].raw_event {
427            RawEvent::SystemState(event) => {
428                assert_eq!(event.battery_pct, Some(88));
429                assert_eq!(event.network, NetworkType::Wifi);
430            },
431            other => panic!("unexpected raw event: {other:?}"),
432        }
433    }
434
435    #[test]
436    fn tailer_reads_only_new_complete_lines() {
437        let path = temp_trace_path();
438        {
439            let mut file = OpenOptions::new()
440                .create(true)
441                .append(true)
442                .open(&path)
443                .unwrap();
444            writeln!(file, "{APP_TRANSITION_LINE}").unwrap();
445        }
446
447        let mut tailer = AndroidJsonlTailer::new(&path);
448        assert_eq!(tailer.poll().unwrap().len(), 1);
449        assert!(tailer.poll().unwrap().is_empty());
450
451        {
452            let mut file = OpenOptions::new().append(true).open(&path).unwrap();
453            write!(file, "{APP_TRANSITION_LINE}").unwrap();
454        }
455        assert!(tailer.poll().unwrap().is_empty());
456
457        {
458            let mut file = OpenOptions::new().append(true).open(&path).unwrap();
459            writeln!(file).unwrap();
460        }
461        assert_eq!(tailer.poll().unwrap().len(), 1);
462
463        let _ = std::fs::remove_file(path);
464    }
465
466    #[test]
467    fn tailer_resets_when_file_is_truncated_or_recreated() {
468        let path = temp_trace_path();
469        {
470            let mut file = OpenOptions::new()
471                .create(true)
472                .append(true)
473                .open(&path)
474                .unwrap();
475            writeln!(file, "{APP_TRANSITION_LINE}").unwrap();
476        }
477
478        let mut tailer = AndroidJsonlTailer::new(&path);
479        assert_eq!(tailer.poll().unwrap().len(), 1);
480
481        {
482            let mut file = OpenOptions::new()
483                .write(true)
484                .truncate(true)
485                .open(&path)
486                .unwrap();
487            writeln!(file, "{SYSTEM_STATE_LINE}").unwrap();
488        }
489
490        let envelopes = tailer.poll().unwrap();
491        assert_eq!(envelopes.len(), 1);
492        assert!(matches!(envelopes[0].raw_event, RawEvent::SystemState(_)));
493
494        let _ = std::fs::remove_file(path);
495    }
496
497    #[test]
498    fn tailer_keeps_partial_line_across_multiple_polls() {
499        let path = temp_trace_path();
500        {
501            let mut file = OpenOptions::new()
502                .create(true)
503                .append(true)
504                .open(&path)
505                .unwrap();
506            write!(file, "{{\"rawEvent\":").unwrap();
507        }
508
509        let mut tailer = AndroidJsonlTailer::new(&path);
510        assert!(tailer.poll().unwrap().is_empty());
511
512        {
513            let mut file = OpenOptions::new().append(true).open(&path).unwrap();
514            writeln!(
515                file,
516                "{{\"ScreenState\":{{\"timestamp_ms\":5000,\"state\":\"Interactive\"}}}}}}"
517            )
518            .unwrap();
519        }
520
521        let envelopes = tailer.poll().unwrap();
522        assert_eq!(envelopes.len(), 1);
523        assert_eq!(envelopes[0].captured_at_ms, 5000);
524
525        let _ = std::fs::remove_file(path);
526    }
527
528    fn temp_trace_path() -> std::path::PathBuf {
529        let unique = SystemTime::now()
530            .duration_since(UNIX_EPOCH)
531            .unwrap()
532            .as_nanos();
533        std::env::temp_dir().join(format!("dipecs-android-jsonl-{unique}.jsonl"))
534    }
535}