Skip to main content

aios_cli/next_app/
net_benefit.rs

1//! Net-benefit arithmetic for next-app PreWarm predictions.
2//!
3//! All inputs are measured or assumed externally; this module only provides the
4//! deterministic cost/benefit combination used in evaluation reports.
5
6use serde::{Deserialize, Serialize};
7
8/// Inputs required to compute the net benefit of a ranker.
9#[derive(Debug, Clone, Copy, PartialEq)]
10pub struct NetBenefitInputs {
11    /// Top-1 hit rate expressed as a percentage (0.0-100.0).
12    pub hit_rate_at_1_pct: f32,
13    /// Measured startup time saved on a correct PreWarm (ms).
14    pub prewarm_saved_ms: f64,
15    /// Measured cost of a PreWarm that does not match the next app (ms).
16    pub prewarm_wasted_ms: f64,
17    /// DiPECS control-plane overhead per prediction (ms).
18    pub control_plane_ms: f64,
19}
20
21/// A measured scalar input loaded from an evaluation fixture.
22#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
23pub struct MeasuredValue {
24    pub mean_ms: f64,
25    #[serde(default)]
26    pub p95_ms: Option<f64>,
27    pub samples: usize,
28    pub source: MeasurementSource,
29}
30
31impl MeasuredValue {
32    pub fn validate(&self, name: &str) -> Result<(), String> {
33        if !self.mean_ms.is_finite() || self.mean_ms < 0.0 {
34            return Err(format!(
35                "{name}.mean_ms must be a finite non-negative value"
36            ));
37        }
38        if let Some(p95_ms) = self.p95_ms {
39            if !p95_ms.is_finite() || p95_ms < 0.0 {
40                return Err(format!("{name}.p95_ms must be a finite non-negative value"));
41            }
42        }
43        if self.samples == 0 {
44            return Err(format!("{name}.samples must be greater than zero"));
45        }
46        self.source.validate(name)
47    }
48}
49
50/// Provenance for a measured value. A value may be conservative or derived from
51/// measured counters, but it must not be an unlabelled placeholder.
52#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
53pub struct MeasurementSource {
54    pub kind: String,
55    pub path: String,
56    #[serde(default)]
57    pub field: Option<String>,
58    #[serde(default)]
59    pub note: Option<String>,
60}
61
62impl MeasurementSource {
63    pub fn validate(&self, name: &str) -> Result<(), String> {
64        if self.kind.trim().is_empty() {
65            return Err(format!("{name}.source.kind must be non-empty"));
66        }
67        if self.path.trim().is_empty() {
68            return Err(format!("{name}.source.path must be non-empty"));
69        }
70        let kind = self.kind.to_ascii_lowercase();
71        if kind.contains("placeholder") || kind.contains("synthetic_constant") {
72            return Err(format!("{name}.source.kind must not be a placeholder"));
73        }
74        Ok(())
75    }
76}
77
78/// Fully measured PreWarmProcess net-benefit fixture.
79#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
80pub struct PrewarmNetBenefitFixture {
81    pub schema_version: String,
82    pub dataset_id: String,
83    pub action: String,
84    pub status: String,
85    pub trace: NetBenefitTrace,
86    pub measurements: PrewarmMeasurements,
87}
88
89impl PrewarmNetBenefitFixture {
90    pub const SCHEMA_VERSION: &'static str = "dipecs.action_net_benefit.prewarm.v1";
91
92    pub fn validate(&self) -> Result<(), String> {
93        if self.schema_version != Self::SCHEMA_VERSION {
94            return Err(format!(
95                "schema_version must be {}, got {}",
96                Self::SCHEMA_VERSION,
97                self.schema_version
98            ));
99        }
100        if self.action != "PreWarmProcess" {
101            return Err(format!(
102                "action must be PreWarmProcess, got {}",
103                self.action
104            ));
105        }
106        if self.dataset_id.trim().is_empty() {
107            return Err("dataset_id must be non-empty".into());
108        }
109        if self.status.to_ascii_lowercase().contains("placeholder") {
110            return Err("status must not mark the fixture as placeholder".into());
111        }
112        if self.trace.source.trim().is_empty() {
113            return Err("trace.source must be non-empty".into());
114        }
115        if self.trace.split.trim().is_empty() {
116            return Err("trace.split must be non-empty".into());
117        }
118        if self.trace.examples == 0 {
119            return Err("trace.examples must be greater than zero".into());
120        }
121        self.measurements
122            .prewarm_saved
123            .validate("measurements.prewarm_saved")?;
124        self.measurements
125            .wasted_prewarm
126            .validate("measurements.wasted_prewarm")?;
127        self.measurements
128            .control_plane
129            .dipecs
130            .validate("measurements.control_plane.dipecs")?;
131        self.measurements
132            .control_plane
133            .strong_baseline
134            .validate("measurements.control_plane.strong_baseline")?;
135        Ok(())
136    }
137
138    pub fn dipecs_inputs(&self, hit_rate_at_1_pct: f32) -> MeasuredNetBenefitInputs {
139        MeasuredNetBenefitInputs {
140            hit_rate_at_1_pct,
141            prewarm_saved_ms: self.measurements.prewarm_saved.clone(),
142            prewarm_wasted_ms: self.measurements.wasted_prewarm.clone(),
143            control_plane_ms: self.measurements.control_plane.dipecs.clone(),
144        }
145    }
146
147    pub fn strong_baseline_inputs(&self, hit_rate_at_1_pct: f32) -> MeasuredNetBenefitInputs {
148        MeasuredNetBenefitInputs {
149            hit_rate_at_1_pct,
150            prewarm_saved_ms: self.measurements.prewarm_saved.clone(),
151            prewarm_wasted_ms: self.measurements.wasted_prewarm.clone(),
152            control_plane_ms: self.measurements.control_plane.strong_baseline.clone(),
153        }
154    }
155}
156
157#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
158pub struct NetBenefitTrace {
159    pub source: String,
160    pub split: String,
161    pub examples: usize,
162}
163
164#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
165pub struct PrewarmMeasurements {
166    pub prewarm_saved: MeasuredValue,
167    pub wasted_prewarm: MeasuredValue,
168    pub control_plane: ControlPlaneMeasurements,
169}
170
171#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
172pub struct ControlPlaneMeasurements {
173    pub dipecs: MeasuredValue,
174    pub strong_baseline: MeasuredValue,
175}
176
177/// Inputs required to compute measured net benefit of a ranker.
178#[derive(Debug, Clone, PartialEq)]
179pub struct MeasuredNetBenefitInputs {
180    pub hit_rate_at_1_pct: f32,
181    pub prewarm_saved_ms: MeasuredValue,
182    pub prewarm_wasted_ms: MeasuredValue,
183    pub control_plane_ms: MeasuredValue,
184}
185
186/// Inputs used to build a PreWarmProcess fixture from already-collected
187/// offline measurements.
188#[derive(Debug, Clone, PartialEq)]
189pub struct PrewarmFixtureBuildInputs {
190    pub dataset_id: String,
191    pub status: String,
192    pub trace: NetBenefitTrace,
193    pub prewarm_saved: MeasuredValue,
194    pub wasted_prewarm: MeasuredValue,
195    pub dipecs_control_plane: MeasuredValue,
196    pub strong_control_plane: MeasuredValue,
197}
198
199pub fn build_prewarm_net_benefit_fixture(
200    inputs: PrewarmFixtureBuildInputs,
201) -> Result<PrewarmNetBenefitFixture, String> {
202    let fixture = PrewarmNetBenefitFixture {
203        schema_version: PrewarmNetBenefitFixture::SCHEMA_VERSION.into(),
204        dataset_id: inputs.dataset_id,
205        action: "PreWarmProcess".into(),
206        status: inputs.status,
207        trace: inputs.trace,
208        measurements: PrewarmMeasurements {
209            prewarm_saved: inputs.prewarm_saved,
210            wasted_prewarm: inputs.wasted_prewarm,
211            control_plane: ControlPlaneMeasurements {
212                dipecs: inputs.dipecs_control_plane,
213                strong_baseline: inputs.strong_control_plane,
214            },
215        },
216    };
217    fixture.validate()?;
218    Ok(fixture)
219}
220
221impl From<&MeasuredNetBenefitInputs> for NetBenefitInputs {
222    fn from(inputs: &MeasuredNetBenefitInputs) -> Self {
223        Self {
224            hit_rate_at_1_pct: inputs.hit_rate_at_1_pct,
225            prewarm_saved_ms: inputs.prewarm_saved_ms.mean_ms,
226            prewarm_wasted_ms: inputs.prewarm_wasted_ms.mean_ms,
227            control_plane_ms: inputs.control_plane_ms.mean_ms,
228        }
229    }
230}
231
232impl NetBenefitInputs {
233    /// Build provisional inputs whose `prewarm_wasted_ms` and
234    /// `control_plane_ms` are **placeholders, not measurements**.
235    ///
236    /// Use this only for exploratory reporting. Measured gates must use
237    /// committed fixture inputs instead, via `next_app_net_benefit_test`.
238    pub fn placeholder_pending_measurement(hit_rate_at_1_pct: f32, prewarm_saved_ms: f64) -> Self {
239        Self {
240            hit_rate_at_1_pct,
241            prewarm_saved_ms,
242            // Placeholder. Not measured; do not use for gating.
243            prewarm_wasted_ms: 12.0,
244            // Placeholder. Not measured; do not use for gating.
245            control_plane_ms: 0.0,
246        }
247    }
248}
249
250/// Result of a net-benefit computation.
251#[derive(Debug, Clone, Copy, PartialEq)]
252pub struct NetBenefitReport {
253    pub net_benefit_ms: f64,
254    pub gross_saved_ms: f64,
255    pub gross_wasted_ms: f64,
256    pub control_plane_cost_ms: f64,
257}
258
259/// Compute net benefit for `examples` predictions.
260///
261/// - `gross_saved_ms = examples * hit_rate * saved_ms / 100.0`
262/// - `gross_wasted_ms = examples * (1 - hit_rate/100.0) * wasted_ms`
263/// - `control_plane_cost_ms = examples * control_plane_ms`
264/// - `net_benefit_ms = gross_saved - gross_wasted - control_plane_cost`
265pub fn compute_net_benefit(inputs: &NetBenefitInputs, examples: usize) -> NetBenefitReport {
266    let examples_f = examples as f64;
267    let hit = inputs.hit_rate_at_1_pct as f64;
268
269    let gross_saved_ms = examples_f * hit * inputs.prewarm_saved_ms / 100.0;
270    let gross_wasted_ms = examples_f * (1.0 - hit / 100.0) * inputs.prewarm_wasted_ms;
271    let control_plane_cost_ms = examples_f * inputs.control_plane_ms;
272    let net_benefit_ms = gross_saved_ms - gross_wasted_ms - control_plane_cost_ms;
273
274    NetBenefitReport {
275        net_benefit_ms,
276        gross_saved_ms,
277        gross_wasted_ms,
278        control_plane_cost_ms,
279    }
280}
281
282/// Compute net benefit from fixture-backed measured inputs.
283pub fn compute_measured_net_benefit(
284    inputs: &MeasuredNetBenefitInputs,
285    examples: usize,
286) -> Result<NetBenefitReport, String> {
287    inputs.prewarm_saved_ms.validate("prewarm_saved_ms")?;
288    inputs.prewarm_wasted_ms.validate("prewarm_wasted_ms")?;
289    inputs.control_plane_ms.validate("control_plane_ms")?;
290    Ok(compute_net_benefit(
291        &NetBenefitInputs::from(inputs),
292        examples,
293    ))
294}
295
296#[cfg(test)]
297mod tests {
298    use super::*;
299
300    fn source() -> MeasurementSource {
301        MeasurementSource {
302            kind: "measured_fixture".into(),
303            path: "data/evaluation/example.json".into(),
304            field: None,
305            note: None,
306        }
307    }
308
309    fn value(mean_ms: f64) -> MeasuredValue {
310        MeasuredValue {
311            mean_ms,
312            p95_ms: None,
313            samples: 10,
314            source: source(),
315        }
316    }
317
318    fn inputs(hit: f32, saved: f64, wasted: f64, control: f64) -> NetBenefitInputs {
319        NetBenefitInputs {
320            hit_rate_at_1_pct: hit,
321            prewarm_saved_ms: saved,
322            prewarm_wasted_ms: wasted,
323            control_plane_ms: control,
324        }
325    }
326
327    #[test]
328    fn zero_examples_yields_zero_benefit() {
329        let report = compute_net_benefit(&inputs(50.0, 100.0, 10.0, 1.0), 0);
330        assert_eq!(report.net_benefit_ms, 0.0);
331        assert_eq!(report.gross_saved_ms, 0.0);
332        assert_eq!(report.gross_wasted_ms, 0.0);
333        assert_eq!(report.control_plane_cost_ms, 0.0);
334    }
335
336    #[test]
337    fn perfect_hit_rate_avoids_waste() {
338        let report = compute_net_benefit(&inputs(100.0, 80.0, 20.0, 0.5), 10);
339        assert!((report.gross_saved_ms - 800.0).abs() < 1e-9);
340        assert!((report.gross_wasted_ms - 0.0).abs() < 1e-9);
341        assert!((report.control_plane_cost_ms - 5.0).abs() < 1e-9);
342        assert!((report.net_benefit_ms - 795.0).abs() < 1e-9);
343    }
344
345    #[test]
346    fn zero_hit_rate_is_pure_cost() {
347        let report = compute_net_benefit(&inputs(0.0, 80.0, 15.0, 0.5), 20);
348        assert!((report.gross_saved_ms - 0.0).abs() < 1e-9);
349        assert!((report.gross_wasted_ms - 300.0).abs() < 1e-9);
350        assert!((report.control_plane_cost_ms - 10.0).abs() < 1e-9);
351        assert!((report.net_benefit_ms - (-310.0)).abs() < 1e-9);
352    }
353
354    #[test]
355    fn mixed_hit_rate_matches_formula() {
356        // 100 examples, 40% hit, saved=200ms, wasted=20ms, control=2ms.
357        // saved = 100 * 0.4 * 200 = 8000
358        // wasted = 100 * 0.6 * 20 = 1200
359        // control = 100 * 2 = 200
360        // net = 6600
361        let report = compute_net_benefit(&inputs(40.0, 200.0, 20.0, 2.0), 100);
362        assert!((report.gross_saved_ms - 8000.0).abs() < 1e-9);
363        assert!((report.gross_wasted_ms - 1200.0).abs() < 1e-9);
364        assert!((report.control_plane_cost_ms - 200.0).abs() < 1e-9);
365        assert!((report.net_benefit_ms - 6600.0).abs() < 1e-9);
366    }
367
368    #[test]
369    fn measured_inputs_reject_placeholder_sources() {
370        let mut measured = MeasuredNetBenefitInputs {
371            hit_rate_at_1_pct: 50.0,
372            prewarm_saved_ms: value(100.0),
373            prewarm_wasted_ms: value(10.0),
374            control_plane_ms: value(1.0),
375        };
376        measured.control_plane_ms.source.kind = "placeholder".into();
377
378        let err = compute_measured_net_benefit(&measured, 10)
379            .expect_err("placeholder measured input must be rejected");
380        assert!(err.contains("placeholder"));
381    }
382
383    fn valid_build_inputs() -> PrewarmFixtureBuildInputs {
384        PrewarmFixtureBuildInputs {
385            dataset_id: "fixture-test".into(),
386            status: "measured".into(),
387            trace: NetBenefitTrace {
388                source: "data/evaluation/next-app/lsapp-standard.report.json".into(),
389                split: "Standard".into(),
390                examples: 100,
391            },
392            prewarm_saved: value(100.0),
393            wasted_prewarm: value(10.0),
394            dipecs_control_plane: value(1.0),
395            strong_control_plane: value(0.5),
396        }
397    }
398
399    #[test]
400    fn prewarm_fixture_builder_rejects_bad_action_shape() {
401        let mut fixture =
402            build_prewarm_net_benefit_fixture(valid_build_inputs()).expect("fixture should build");
403        fixture.action = "ReleaseMemory".into();
404
405        let err = fixture
406            .validate()
407            .expect_err("wrong action type must be rejected");
408        assert!(err.contains("PreWarmProcess"));
409    }
410
411    #[test]
412    fn prewarm_fixture_builder_rejects_zero_examples() {
413        let mut inputs = valid_build_inputs();
414        inputs.trace.examples = 0;
415
416        let err =
417            build_prewarm_net_benefit_fixture(inputs).expect_err("zero examples must be rejected");
418        assert!(err.contains("trace.examples"));
419    }
420
421    #[test]
422    fn prewarm_fixture_builder_rejects_zero_samples() {
423        let mut inputs = valid_build_inputs();
424        inputs.wasted_prewarm.samples = 0;
425
426        let err =
427            build_prewarm_net_benefit_fixture(inputs).expect_err("zero samples must be rejected");
428        assert!(err.contains("samples"));
429    }
430
431    #[test]
432    fn prewarm_fixture_builder_rejects_negative_measurements() {
433        let mut inputs = valid_build_inputs();
434        inputs.dipecs_control_plane.mean_ms = -0.1;
435
436        let err = build_prewarm_net_benefit_fixture(inputs)
437            .expect_err("negative measurements must be rejected");
438        assert!(err.contains("mean_ms"));
439    }
440
441    #[test]
442    fn prewarm_fixture_builder_rejects_placeholder_status() {
443        let mut inputs = valid_build_inputs();
444        inputs.status = "placeholder_pending_measurement".into();
445
446        let err = build_prewarm_net_benefit_fixture(inputs)
447            .expect_err("placeholder status must be rejected");
448        assert!(err.contains("placeholder"));
449    }
450
451    #[test]
452    fn measured_value_validation_rejects_many_bad_shapes() {
453        let cases = [
454            (
455                "negative_mean",
456                -1.0,
457                Some(1.0),
458                10,
459                "measured_fixture",
460                "x",
461            ),
462            ("negative_p95", 1.0, Some(-1.0), 10, "measured_fixture", "x"),
463            ("zero_samples", 1.0, Some(1.0), 0, "measured_fixture", "x"),
464            ("empty_kind", 1.0, Some(1.0), 10, "", "x"),
465            ("empty_path", 1.0, Some(1.0), 10, "measured_fixture", ""),
466            (
467                "placeholder_kind",
468                1.0,
469                Some(1.0),
470                10,
471                "placeholder_cost",
472                "x",
473            ),
474            (
475                "synthetic_constant_kind",
476                1.0,
477                Some(1.0),
478                10,
479                "synthetic_constant",
480                "x",
481            ),
482        ];
483
484        for (name, mean_ms, p95_ms, samples, kind, path) in cases {
485            let bad = MeasuredValue {
486                mean_ms,
487                p95_ms,
488                samples,
489                source: MeasurementSource {
490                    kind: kind.into(),
491                    path: path.into(),
492                    field: None,
493                    note: None,
494                },
495            };
496            assert!(
497                bad.validate(name).is_err(),
498                "{name} should fail measured value validation"
499            );
500        }
501    }
502
503    #[test]
504    fn prewarm_fixture_validation_rejects_many_corrupt_fixture_variants() {
505        type Mutator = fn(&mut PrewarmFixtureBuildInputs);
506        let cases: [(&str, Mutator); 16] = [
507            ("empty_dataset", |i| i.dataset_id.clear()),
508            ("placeholder_status", |i| i.status = "placeholder".into()),
509            ("empty_trace_source", |i| i.trace.source.clear()),
510            ("empty_trace_split", |i| i.trace.split.clear()),
511            ("zero_examples", |i| i.trace.examples = 0),
512            ("negative_saved", |i| i.prewarm_saved.mean_ms = -0.01),
513            ("negative_saved_p95", |i| {
514                i.prewarm_saved.p95_ms = Some(-0.01)
515            }),
516            ("zero_saved_samples", |i| i.prewarm_saved.samples = 0),
517            ("negative_wasted", |i| i.wasted_prewarm.mean_ms = -0.01),
518            ("zero_wasted_samples", |i| i.wasted_prewarm.samples = 0),
519            ("negative_dipecs_control", |i| {
520                i.dipecs_control_plane.mean_ms = -0.01
521            }),
522            ("zero_dipecs_control_samples", |i| {
523                i.dipecs_control_plane.samples = 0
524            }),
525            ("negative_strong_control", |i| {
526                i.strong_control_plane.mean_ms = -0.01
527            }),
528            ("zero_strong_control_samples", |i| {
529                i.strong_control_plane.samples = 0
530            }),
531            ("placeholder_source", |i| {
532                i.prewarm_saved.source.kind = "placeholder".into()
533            }),
534            ("empty_source_path", |i| i.prewarm_saved.source.path.clear()),
535        ];
536
537        for (name, mutate) in cases {
538            let mut inputs = valid_build_inputs();
539            mutate(&mut inputs);
540            assert!(
541                build_prewarm_net_benefit_fixture(inputs).is_err(),
542                "{name} should fail fixture validation"
543            );
544        }
545    }
546
547    #[test]
548    fn net_benefit_formula_matches_large_deterministic_case_matrix() {
549        let hit_rates = [0.0_f32, 1.0, 25.0, 50.0, 75.0, 99.0, 100.0];
550        let saved_values = [0.0_f64, 1.0, 31.231, 394.8, 1_000.0];
551        let wasted_values = [0.0_f64, 1.0, 12.0, 31.231, 250.0];
552        let control_values = [0.0_f64, 0.07848, 1.0, 10.0];
553        let example_counts = [0_usize, 1, 7, 100, 272_519];
554
555        let mut checked = 0usize;
556        for hit in hit_rates {
557            for saved in saved_values {
558                for wasted in wasted_values {
559                    for control in control_values {
560                        for examples in example_counts {
561                            let inputs = NetBenefitInputs {
562                                hit_rate_at_1_pct: hit,
563                                prewarm_saved_ms: saved,
564                                prewarm_wasted_ms: wasted,
565                                control_plane_ms: control,
566                            };
567                            let report = compute_net_benefit(&inputs, examples);
568                            let examples_f = examples as f64;
569                            let hit_f = hit as f64 / 100.0;
570                            let expected_saved = examples_f * hit_f * saved;
571                            let expected_wasted = examples_f * (1.0 - hit_f) * wasted;
572                            let expected_control = examples_f * control;
573                            let expected_net = expected_saved - expected_wasted - expected_control;
574
575                            assert!((report.gross_saved_ms - expected_saved).abs() < 1e-6);
576                            assert!((report.gross_wasted_ms - expected_wasted).abs() < 1e-6);
577                            assert!((report.control_plane_cost_ms - expected_control).abs() < 1e-6);
578                            assert!((report.net_benefit_ms - expected_net).abs() < 1e-6);
579                            checked += 1;
580                        }
581                    }
582                }
583            }
584        }
585
586        assert_eq!(checked, 7 * 5 * 5 * 4 * 5);
587    }
588
589    #[test]
590    fn measured_net_benefit_monotonicity_holds_over_many_cases() {
591        let examples = 1_000;
592        let source = source();
593        let mut checked = 0usize;
594
595        for hit in [0.0_f32, 10.0, 50.0, 90.0, 100.0] {
596            for saved in [1.0_f64, 100.0, 394.8] {
597                for wasted in [0.0_f64, 10.0, 31.231] {
598                    for control in [0.0_f64, 0.07848, 5.0] {
599                        let base = MeasuredNetBenefitInputs {
600                            hit_rate_at_1_pct: hit,
601                            prewarm_saved_ms: MeasuredValue {
602                                mean_ms: saved,
603                                p95_ms: None,
604                                samples: 10,
605                                source: source.clone(),
606                            },
607                            prewarm_wasted_ms: MeasuredValue {
608                                mean_ms: wasted,
609                                p95_ms: None,
610                                samples: 10,
611                                source: source.clone(),
612                            },
613                            control_plane_ms: MeasuredValue {
614                                mean_ms: control,
615                                p95_ms: None,
616                                samples: 10,
617                                source: source.clone(),
618                            },
619                        };
620                        let base_report =
621                            compute_measured_net_benefit(&base, examples).expect("valid base");
622
623                        let mut more_saved = base.clone();
624                        more_saved.prewarm_saved_ms.mean_ms += 1.0;
625                        let more_saved_report =
626                            compute_measured_net_benefit(&more_saved, examples).unwrap();
627                        assert!(more_saved_report.net_benefit_ms >= base_report.net_benefit_ms);
628
629                        let mut more_waste = base.clone();
630                        more_waste.prewarm_wasted_ms.mean_ms += 1.0;
631                        let more_waste_report =
632                            compute_measured_net_benefit(&more_waste, examples).unwrap();
633                        assert!(more_waste_report.net_benefit_ms <= base_report.net_benefit_ms);
634
635                        let mut more_control = base.clone();
636                        more_control.control_plane_ms.mean_ms += 1.0;
637                        let more_control_report =
638                            compute_measured_net_benefit(&more_control, examples).unwrap();
639                        assert!(more_control_report.net_benefit_ms <= base_report.net_benefit_ms);
640                        checked += 1;
641                    }
642                }
643            }
644        }
645
646        assert_eq!(checked, 5 * 3 * 3 * 3);
647    }
648
649    #[test]
650    fn measured_net_benefit_matches_raw_formula_over_large_fixture_like_grid() {
651        let source = source();
652        let hit_rates = [
653            0.0_f32, 0.1, 1.0, 5.0, 12.5, 25.0, 33.333, 50.0, 56.442, 75.0, 90.0, 99.9, 100.0,
654        ];
655        let saved_values = [0.001_f64, 1.0, 12.0, 31.231, 120.0, 394.8, 805.8, 1_500.0];
656        let wasted_values = [0.0_f64, 0.001, 1.0, 12.0, 31.231, 120.0, 394.8];
657        let control_values = [0.0_f64, 0.0001, 0.07848, 0.5, 1.0, 7.5, 31.231];
658        let example_counts = [1_usize, 2, 17, 999, 10_000, 272_519, 1_000_000];
659
660        let mut checked = 0usize;
661        for hit_rate_at_1_pct in hit_rates {
662            for saved in saved_values {
663                for wasted in wasted_values {
664                    for control in control_values {
665                        for examples in example_counts {
666                            let inputs = MeasuredNetBenefitInputs {
667                                hit_rate_at_1_pct,
668                                prewarm_saved_ms: MeasuredValue {
669                                    mean_ms: saved,
670                                    p95_ms: Some(saved),
671                                    samples: 30,
672                                    source: source.clone(),
673                                },
674                                prewarm_wasted_ms: MeasuredValue {
675                                    mean_ms: wasted,
676                                    p95_ms: Some(wasted),
677                                    samples: 30,
678                                    source: source.clone(),
679                                },
680                                control_plane_ms: MeasuredValue {
681                                    mean_ms: control,
682                                    p95_ms: Some(control),
683                                    samples: 30,
684                                    source: source.clone(),
685                                },
686                            };
687                            let report = compute_measured_net_benefit(&inputs, examples).unwrap();
688                            let examples_f = examples as f64;
689                            let hit = hit_rate_at_1_pct as f64 / 100.0;
690                            let expected = examples_f * hit * saved
691                                - examples_f * (1.0 - hit) * wasted
692                                - examples_f * control;
693                            assert!(
694                                (report.net_benefit_ms - expected).abs() < 1e-5,
695                                "case hit={hit_rate_at_1_pct} saved={saved} wasted={wasted} control={control} examples={examples}"
696                            );
697                            checked += 1;
698                        }
699                    }
700                }
701            }
702        }
703
704        assert_eq!(checked, 13 * 8 * 7 * 7 * 7);
705    }
706
707    #[test]
708    fn break_even_threshold_cases_stay_numerically_stable() {
709        let examples = 100_000;
710        let saved_values = [1.0_f64, 31.231, 394.8, 805.8, 1_500.0];
711        let wasted_values = [0.0_f64, 1.0, 12.0, 31.231, 394.8];
712        let control_values = [0.0_f64, 0.07848, 1.0, 12.0, 31.231];
713        let mut checked = 0usize;
714
715        for saved in saved_values {
716            for wasted in wasted_values {
717                for control in control_values {
718                    let denominator = saved + wasted;
719                    if denominator <= 0.0 {
720                        continue;
721                    }
722                    let break_even_hit =
723                        ((wasted + control) / denominator * 100.0).clamp(0.0, 100.0) as f32;
724                    for offset in [-0.01_f32, 0.0, 0.01] {
725                        let hit = (break_even_hit + offset).clamp(0.0, 100.0);
726                        let report = compute_net_benefit(
727                            &NetBenefitInputs {
728                                hit_rate_at_1_pct: hit,
729                                prewarm_saved_ms: saved,
730                                prewarm_wasted_ms: wasted,
731                                control_plane_ms: control,
732                            },
733                            examples,
734                        );
735                        assert!(report.net_benefit_ms.is_finite());
736                        if offset < 0.0 && break_even_hit > 0.01 && break_even_hit < 99.99 {
737                            assert!(
738                                report.net_benefit_ms <= 1e-3,
739                                "below break-even should not be positive: saved={saved} wasted={wasted} control={control} hit={hit}"
740                            );
741                        }
742                        if offset > 0.0 && break_even_hit > 0.01 && break_even_hit < 99.99 {
743                            assert!(
744                                report.net_benefit_ms >= -1e-3,
745                                "above break-even should not be negative: saved={saved} wasted={wasted} control={control} hit={hit}"
746                            );
747                        }
748                        checked += 1;
749                    }
750                }
751            }
752        }
753
754        assert_eq!(checked, 5 * 5 * 5 * 3);
755    }
756
757    #[test]
758    fn fixture_builder_accepts_many_valid_fixture_shapes() {
759        let source = source();
760        let mut checked = 0usize;
761
762        for examples in [1_usize, 10, 1_000, 272_519, 1_000_000] {
763            for saved in [0.001_f64, 31.231, 394.8, 1_500.0] {
764                for wasted in [0.0_f64, 0.001, 31.231, 394.8] {
765                    for control in [0.0_f64, 0.0001, 0.07848, 10.0] {
766                        let fixture =
767                            build_prewarm_net_benefit_fixture(PrewarmFixtureBuildInputs {
768                                dataset_id: format!(
769                                    "fixture-{examples}-{saved}-{wasted}-{control}"
770                                ),
771                                status: "generated_from_offline_measurements".into(),
772                                trace: NetBenefitTrace {
773                                    source: "data/evaluation/next-app/lsapp-standard.report.json"
774                                        .into(),
775                                    split: "Standard".into(),
776                                    examples,
777                                },
778                                prewarm_saved: MeasuredValue {
779                                    mean_ms: saved,
780                                    p95_ms: Some(saved),
781                                    samples: 20,
782                                    source: source.clone(),
783                                },
784                                wasted_prewarm: MeasuredValue {
785                                    mean_ms: wasted,
786                                    p95_ms: Some(wasted),
787                                    samples: 1,
788                                    source: source.clone(),
789                                },
790                                dipecs_control_plane: MeasuredValue {
791                                    mean_ms: control,
792                                    p95_ms: None,
793                                    samples: 1_631,
794                                    source: source.clone(),
795                                },
796                                strong_control_plane: MeasuredValue {
797                                    mean_ms: 0.0,
798                                    p95_ms: None,
799                                    samples: examples,
800                                    source: source.clone(),
801                                },
802                            })
803                            .expect("valid fixture shape should build");
804                        fixture.validate().expect("valid fixture should validate");
805                        checked += 1;
806                    }
807                }
808            }
809        }
810
811        assert_eq!(checked, 5 * 4 * 4 * 4);
812    }
813}