1use serde::{Deserialize, Serialize};
7
8#[derive(Debug, Clone, Copy, PartialEq)]
10pub struct NetBenefitInputs {
11 pub hit_rate_at_1_pct: f32,
13 pub prewarm_saved_ms: f64,
15 pub prewarm_wasted_ms: f64,
17 pub control_plane_ms: f64,
19}
20
21#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
23pub struct MeasuredValue {
24 pub mean_ms: f64,
25 #[serde(default)]
26 pub p95_ms: Option<f64>,
27 pub samples: usize,
28 pub source: MeasurementSource,
29}
30
31impl MeasuredValue {
32 pub fn validate(&self, name: &str) -> Result<(), String> {
33 if !self.mean_ms.is_finite() || self.mean_ms < 0.0 {
34 return Err(format!(
35 "{name}.mean_ms must be a finite non-negative value"
36 ));
37 }
38 if let Some(p95_ms) = self.p95_ms {
39 if !p95_ms.is_finite() || p95_ms < 0.0 {
40 return Err(format!("{name}.p95_ms must be a finite non-negative value"));
41 }
42 }
43 if self.samples == 0 {
44 return Err(format!("{name}.samples must be greater than zero"));
45 }
46 self.source.validate(name)
47 }
48}
49
50#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
53pub struct MeasurementSource {
54 pub kind: String,
55 pub path: String,
56 #[serde(default)]
57 pub field: Option<String>,
58 #[serde(default)]
59 pub note: Option<String>,
60}
61
62impl MeasurementSource {
63 pub fn validate(&self, name: &str) -> Result<(), String> {
64 if self.kind.trim().is_empty() {
65 return Err(format!("{name}.source.kind must be non-empty"));
66 }
67 if self.path.trim().is_empty() {
68 return Err(format!("{name}.source.path must be non-empty"));
69 }
70 let kind = self.kind.to_ascii_lowercase();
71 if kind.contains("placeholder") || kind.contains("synthetic_constant") {
72 return Err(format!("{name}.source.kind must not be a placeholder"));
73 }
74 Ok(())
75 }
76}
77
78#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
80pub struct PrewarmNetBenefitFixture {
81 pub schema_version: String,
82 pub dataset_id: String,
83 pub action: String,
84 pub status: String,
85 pub trace: NetBenefitTrace,
86 pub measurements: PrewarmMeasurements,
87}
88
89impl PrewarmNetBenefitFixture {
90 pub const SCHEMA_VERSION: &'static str = "dipecs.action_net_benefit.prewarm.v1";
91
92 pub fn validate(&self) -> Result<(), String> {
93 if self.schema_version != Self::SCHEMA_VERSION {
94 return Err(format!(
95 "schema_version must be {}, got {}",
96 Self::SCHEMA_VERSION,
97 self.schema_version
98 ));
99 }
100 if self.action != "PreWarmProcess" {
101 return Err(format!(
102 "action must be PreWarmProcess, got {}",
103 self.action
104 ));
105 }
106 if self.dataset_id.trim().is_empty() {
107 return Err("dataset_id must be non-empty".into());
108 }
109 if self.status.to_ascii_lowercase().contains("placeholder") {
110 return Err("status must not mark the fixture as placeholder".into());
111 }
112 if self.trace.source.trim().is_empty() {
113 return Err("trace.source must be non-empty".into());
114 }
115 if self.trace.split.trim().is_empty() {
116 return Err("trace.split must be non-empty".into());
117 }
118 if self.trace.examples == 0 {
119 return Err("trace.examples must be greater than zero".into());
120 }
121 self.measurements
122 .prewarm_saved
123 .validate("measurements.prewarm_saved")?;
124 self.measurements
125 .wasted_prewarm
126 .validate("measurements.wasted_prewarm")?;
127 self.measurements
128 .control_plane
129 .dipecs
130 .validate("measurements.control_plane.dipecs")?;
131 self.measurements
132 .control_plane
133 .strong_baseline
134 .validate("measurements.control_plane.strong_baseline")?;
135 Ok(())
136 }
137
138 pub fn dipecs_inputs(&self, hit_rate_at_1_pct: f32) -> MeasuredNetBenefitInputs {
139 MeasuredNetBenefitInputs {
140 hit_rate_at_1_pct,
141 prewarm_saved_ms: self.measurements.prewarm_saved.clone(),
142 prewarm_wasted_ms: self.measurements.wasted_prewarm.clone(),
143 control_plane_ms: self.measurements.control_plane.dipecs.clone(),
144 }
145 }
146
147 pub fn strong_baseline_inputs(&self, hit_rate_at_1_pct: f32) -> MeasuredNetBenefitInputs {
148 MeasuredNetBenefitInputs {
149 hit_rate_at_1_pct,
150 prewarm_saved_ms: self.measurements.prewarm_saved.clone(),
151 prewarm_wasted_ms: self.measurements.wasted_prewarm.clone(),
152 control_plane_ms: self.measurements.control_plane.strong_baseline.clone(),
153 }
154 }
155}
156
157#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
158pub struct NetBenefitTrace {
159 pub source: String,
160 pub split: String,
161 pub examples: usize,
162}
163
164#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
165pub struct PrewarmMeasurements {
166 pub prewarm_saved: MeasuredValue,
167 pub wasted_prewarm: MeasuredValue,
168 pub control_plane: ControlPlaneMeasurements,
169}
170
171#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
172pub struct ControlPlaneMeasurements {
173 pub dipecs: MeasuredValue,
174 pub strong_baseline: MeasuredValue,
175}
176
177#[derive(Debug, Clone, PartialEq)]
179pub struct MeasuredNetBenefitInputs {
180 pub hit_rate_at_1_pct: f32,
181 pub prewarm_saved_ms: MeasuredValue,
182 pub prewarm_wasted_ms: MeasuredValue,
183 pub control_plane_ms: MeasuredValue,
184}
185
186#[derive(Debug, Clone, PartialEq)]
189pub struct PrewarmFixtureBuildInputs {
190 pub dataset_id: String,
191 pub status: String,
192 pub trace: NetBenefitTrace,
193 pub prewarm_saved: MeasuredValue,
194 pub wasted_prewarm: MeasuredValue,
195 pub dipecs_control_plane: MeasuredValue,
196 pub strong_control_plane: MeasuredValue,
197}
198
199pub fn build_prewarm_net_benefit_fixture(
200 inputs: PrewarmFixtureBuildInputs,
201) -> Result<PrewarmNetBenefitFixture, String> {
202 let fixture = PrewarmNetBenefitFixture {
203 schema_version: PrewarmNetBenefitFixture::SCHEMA_VERSION.into(),
204 dataset_id: inputs.dataset_id,
205 action: "PreWarmProcess".into(),
206 status: inputs.status,
207 trace: inputs.trace,
208 measurements: PrewarmMeasurements {
209 prewarm_saved: inputs.prewarm_saved,
210 wasted_prewarm: inputs.wasted_prewarm,
211 control_plane: ControlPlaneMeasurements {
212 dipecs: inputs.dipecs_control_plane,
213 strong_baseline: inputs.strong_control_plane,
214 },
215 },
216 };
217 fixture.validate()?;
218 Ok(fixture)
219}
220
221impl From<&MeasuredNetBenefitInputs> for NetBenefitInputs {
222 fn from(inputs: &MeasuredNetBenefitInputs) -> Self {
223 Self {
224 hit_rate_at_1_pct: inputs.hit_rate_at_1_pct,
225 prewarm_saved_ms: inputs.prewarm_saved_ms.mean_ms,
226 prewarm_wasted_ms: inputs.prewarm_wasted_ms.mean_ms,
227 control_plane_ms: inputs.control_plane_ms.mean_ms,
228 }
229 }
230}
231
232impl NetBenefitInputs {
233 pub fn placeholder_pending_measurement(hit_rate_at_1_pct: f32, prewarm_saved_ms: f64) -> Self {
239 Self {
240 hit_rate_at_1_pct,
241 prewarm_saved_ms,
242 prewarm_wasted_ms: 12.0,
244 control_plane_ms: 0.0,
246 }
247 }
248}
249
250#[derive(Debug, Clone, Copy, PartialEq)]
252pub struct NetBenefitReport {
253 pub net_benefit_ms: f64,
254 pub gross_saved_ms: f64,
255 pub gross_wasted_ms: f64,
256 pub control_plane_cost_ms: f64,
257}
258
259pub fn compute_net_benefit(inputs: &NetBenefitInputs, examples: usize) -> NetBenefitReport {
266 let examples_f = examples as f64;
267 let hit = inputs.hit_rate_at_1_pct as f64;
268
269 let gross_saved_ms = examples_f * hit * inputs.prewarm_saved_ms / 100.0;
270 let gross_wasted_ms = examples_f * (1.0 - hit / 100.0) * inputs.prewarm_wasted_ms;
271 let control_plane_cost_ms = examples_f * inputs.control_plane_ms;
272 let net_benefit_ms = gross_saved_ms - gross_wasted_ms - control_plane_cost_ms;
273
274 NetBenefitReport {
275 net_benefit_ms,
276 gross_saved_ms,
277 gross_wasted_ms,
278 control_plane_cost_ms,
279 }
280}
281
282pub fn compute_measured_net_benefit(
284 inputs: &MeasuredNetBenefitInputs,
285 examples: usize,
286) -> Result<NetBenefitReport, String> {
287 inputs.prewarm_saved_ms.validate("prewarm_saved_ms")?;
288 inputs.prewarm_wasted_ms.validate("prewarm_wasted_ms")?;
289 inputs.control_plane_ms.validate("control_plane_ms")?;
290 Ok(compute_net_benefit(
291 &NetBenefitInputs::from(inputs),
292 examples,
293 ))
294}
295
296#[cfg(test)]
297mod tests {
298 use super::*;
299
300 fn source() -> MeasurementSource {
301 MeasurementSource {
302 kind: "measured_fixture".into(),
303 path: "data/evaluation/example.json".into(),
304 field: None,
305 note: None,
306 }
307 }
308
309 fn value(mean_ms: f64) -> MeasuredValue {
310 MeasuredValue {
311 mean_ms,
312 p95_ms: None,
313 samples: 10,
314 source: source(),
315 }
316 }
317
318 fn inputs(hit: f32, saved: f64, wasted: f64, control: f64) -> NetBenefitInputs {
319 NetBenefitInputs {
320 hit_rate_at_1_pct: hit,
321 prewarm_saved_ms: saved,
322 prewarm_wasted_ms: wasted,
323 control_plane_ms: control,
324 }
325 }
326
327 #[test]
328 fn zero_examples_yields_zero_benefit() {
329 let report = compute_net_benefit(&inputs(50.0, 100.0, 10.0, 1.0), 0);
330 assert_eq!(report.net_benefit_ms, 0.0);
331 assert_eq!(report.gross_saved_ms, 0.0);
332 assert_eq!(report.gross_wasted_ms, 0.0);
333 assert_eq!(report.control_plane_cost_ms, 0.0);
334 }
335
336 #[test]
337 fn perfect_hit_rate_avoids_waste() {
338 let report = compute_net_benefit(&inputs(100.0, 80.0, 20.0, 0.5), 10);
339 assert!((report.gross_saved_ms - 800.0).abs() < 1e-9);
340 assert!((report.gross_wasted_ms - 0.0).abs() < 1e-9);
341 assert!((report.control_plane_cost_ms - 5.0).abs() < 1e-9);
342 assert!((report.net_benefit_ms - 795.0).abs() < 1e-9);
343 }
344
345 #[test]
346 fn zero_hit_rate_is_pure_cost() {
347 let report = compute_net_benefit(&inputs(0.0, 80.0, 15.0, 0.5), 20);
348 assert!((report.gross_saved_ms - 0.0).abs() < 1e-9);
349 assert!((report.gross_wasted_ms - 300.0).abs() < 1e-9);
350 assert!((report.control_plane_cost_ms - 10.0).abs() < 1e-9);
351 assert!((report.net_benefit_ms - (-310.0)).abs() < 1e-9);
352 }
353
354 #[test]
355 fn mixed_hit_rate_matches_formula() {
356 let report = compute_net_benefit(&inputs(40.0, 200.0, 20.0, 2.0), 100);
362 assert!((report.gross_saved_ms - 8000.0).abs() < 1e-9);
363 assert!((report.gross_wasted_ms - 1200.0).abs() < 1e-9);
364 assert!((report.control_plane_cost_ms - 200.0).abs() < 1e-9);
365 assert!((report.net_benefit_ms - 6600.0).abs() < 1e-9);
366 }
367
368 #[test]
369 fn measured_inputs_reject_placeholder_sources() {
370 let mut measured = MeasuredNetBenefitInputs {
371 hit_rate_at_1_pct: 50.0,
372 prewarm_saved_ms: value(100.0),
373 prewarm_wasted_ms: value(10.0),
374 control_plane_ms: value(1.0),
375 };
376 measured.control_plane_ms.source.kind = "placeholder".into();
377
378 let err = compute_measured_net_benefit(&measured, 10)
379 .expect_err("placeholder measured input must be rejected");
380 assert!(err.contains("placeholder"));
381 }
382
383 fn valid_build_inputs() -> PrewarmFixtureBuildInputs {
384 PrewarmFixtureBuildInputs {
385 dataset_id: "fixture-test".into(),
386 status: "measured".into(),
387 trace: NetBenefitTrace {
388 source: "data/evaluation/next-app/lsapp-standard.report.json".into(),
389 split: "Standard".into(),
390 examples: 100,
391 },
392 prewarm_saved: value(100.0),
393 wasted_prewarm: value(10.0),
394 dipecs_control_plane: value(1.0),
395 strong_control_plane: value(0.5),
396 }
397 }
398
399 #[test]
400 fn prewarm_fixture_builder_rejects_bad_action_shape() {
401 let mut fixture =
402 build_prewarm_net_benefit_fixture(valid_build_inputs()).expect("fixture should build");
403 fixture.action = "ReleaseMemory".into();
404
405 let err = fixture
406 .validate()
407 .expect_err("wrong action type must be rejected");
408 assert!(err.contains("PreWarmProcess"));
409 }
410
411 #[test]
412 fn prewarm_fixture_builder_rejects_zero_examples() {
413 let mut inputs = valid_build_inputs();
414 inputs.trace.examples = 0;
415
416 let err =
417 build_prewarm_net_benefit_fixture(inputs).expect_err("zero examples must be rejected");
418 assert!(err.contains("trace.examples"));
419 }
420
421 #[test]
422 fn prewarm_fixture_builder_rejects_zero_samples() {
423 let mut inputs = valid_build_inputs();
424 inputs.wasted_prewarm.samples = 0;
425
426 let err =
427 build_prewarm_net_benefit_fixture(inputs).expect_err("zero samples must be rejected");
428 assert!(err.contains("samples"));
429 }
430
431 #[test]
432 fn prewarm_fixture_builder_rejects_negative_measurements() {
433 let mut inputs = valid_build_inputs();
434 inputs.dipecs_control_plane.mean_ms = -0.1;
435
436 let err = build_prewarm_net_benefit_fixture(inputs)
437 .expect_err("negative measurements must be rejected");
438 assert!(err.contains("mean_ms"));
439 }
440
441 #[test]
442 fn prewarm_fixture_builder_rejects_placeholder_status() {
443 let mut inputs = valid_build_inputs();
444 inputs.status = "placeholder_pending_measurement".into();
445
446 let err = build_prewarm_net_benefit_fixture(inputs)
447 .expect_err("placeholder status must be rejected");
448 assert!(err.contains("placeholder"));
449 }
450
451 #[test]
452 fn measured_value_validation_rejects_many_bad_shapes() {
453 let cases = [
454 (
455 "negative_mean",
456 -1.0,
457 Some(1.0),
458 10,
459 "measured_fixture",
460 "x",
461 ),
462 ("negative_p95", 1.0, Some(-1.0), 10, "measured_fixture", "x"),
463 ("zero_samples", 1.0, Some(1.0), 0, "measured_fixture", "x"),
464 ("empty_kind", 1.0, Some(1.0), 10, "", "x"),
465 ("empty_path", 1.0, Some(1.0), 10, "measured_fixture", ""),
466 (
467 "placeholder_kind",
468 1.0,
469 Some(1.0),
470 10,
471 "placeholder_cost",
472 "x",
473 ),
474 (
475 "synthetic_constant_kind",
476 1.0,
477 Some(1.0),
478 10,
479 "synthetic_constant",
480 "x",
481 ),
482 ];
483
484 for (name, mean_ms, p95_ms, samples, kind, path) in cases {
485 let bad = MeasuredValue {
486 mean_ms,
487 p95_ms,
488 samples,
489 source: MeasurementSource {
490 kind: kind.into(),
491 path: path.into(),
492 field: None,
493 note: None,
494 },
495 };
496 assert!(
497 bad.validate(name).is_err(),
498 "{name} should fail measured value validation"
499 );
500 }
501 }
502
503 #[test]
504 fn prewarm_fixture_validation_rejects_many_corrupt_fixture_variants() {
505 type Mutator = fn(&mut PrewarmFixtureBuildInputs);
506 let cases: [(&str, Mutator); 16] = [
507 ("empty_dataset", |i| i.dataset_id.clear()),
508 ("placeholder_status", |i| i.status = "placeholder".into()),
509 ("empty_trace_source", |i| i.trace.source.clear()),
510 ("empty_trace_split", |i| i.trace.split.clear()),
511 ("zero_examples", |i| i.trace.examples = 0),
512 ("negative_saved", |i| i.prewarm_saved.mean_ms = -0.01),
513 ("negative_saved_p95", |i| {
514 i.prewarm_saved.p95_ms = Some(-0.01)
515 }),
516 ("zero_saved_samples", |i| i.prewarm_saved.samples = 0),
517 ("negative_wasted", |i| i.wasted_prewarm.mean_ms = -0.01),
518 ("zero_wasted_samples", |i| i.wasted_prewarm.samples = 0),
519 ("negative_dipecs_control", |i| {
520 i.dipecs_control_plane.mean_ms = -0.01
521 }),
522 ("zero_dipecs_control_samples", |i| {
523 i.dipecs_control_plane.samples = 0
524 }),
525 ("negative_strong_control", |i| {
526 i.strong_control_plane.mean_ms = -0.01
527 }),
528 ("zero_strong_control_samples", |i| {
529 i.strong_control_plane.samples = 0
530 }),
531 ("placeholder_source", |i| {
532 i.prewarm_saved.source.kind = "placeholder".into()
533 }),
534 ("empty_source_path", |i| i.prewarm_saved.source.path.clear()),
535 ];
536
537 for (name, mutate) in cases {
538 let mut inputs = valid_build_inputs();
539 mutate(&mut inputs);
540 assert!(
541 build_prewarm_net_benefit_fixture(inputs).is_err(),
542 "{name} should fail fixture validation"
543 );
544 }
545 }
546
547 #[test]
548 fn net_benefit_formula_matches_large_deterministic_case_matrix() {
549 let hit_rates = [0.0_f32, 1.0, 25.0, 50.0, 75.0, 99.0, 100.0];
550 let saved_values = [0.0_f64, 1.0, 31.231, 394.8, 1_000.0];
551 let wasted_values = [0.0_f64, 1.0, 12.0, 31.231, 250.0];
552 let control_values = [0.0_f64, 0.07848, 1.0, 10.0];
553 let example_counts = [0_usize, 1, 7, 100, 272_519];
554
555 let mut checked = 0usize;
556 for hit in hit_rates {
557 for saved in saved_values {
558 for wasted in wasted_values {
559 for control in control_values {
560 for examples in example_counts {
561 let inputs = NetBenefitInputs {
562 hit_rate_at_1_pct: hit,
563 prewarm_saved_ms: saved,
564 prewarm_wasted_ms: wasted,
565 control_plane_ms: control,
566 };
567 let report = compute_net_benefit(&inputs, examples);
568 let examples_f = examples as f64;
569 let hit_f = hit as f64 / 100.0;
570 let expected_saved = examples_f * hit_f * saved;
571 let expected_wasted = examples_f * (1.0 - hit_f) * wasted;
572 let expected_control = examples_f * control;
573 let expected_net = expected_saved - expected_wasted - expected_control;
574
575 assert!((report.gross_saved_ms - expected_saved).abs() < 1e-6);
576 assert!((report.gross_wasted_ms - expected_wasted).abs() < 1e-6);
577 assert!((report.control_plane_cost_ms - expected_control).abs() < 1e-6);
578 assert!((report.net_benefit_ms - expected_net).abs() < 1e-6);
579 checked += 1;
580 }
581 }
582 }
583 }
584 }
585
586 assert_eq!(checked, 7 * 5 * 5 * 4 * 5);
587 }
588
589 #[test]
590 fn measured_net_benefit_monotonicity_holds_over_many_cases() {
591 let examples = 1_000;
592 let source = source();
593 let mut checked = 0usize;
594
595 for hit in [0.0_f32, 10.0, 50.0, 90.0, 100.0] {
596 for saved in [1.0_f64, 100.0, 394.8] {
597 for wasted in [0.0_f64, 10.0, 31.231] {
598 for control in [0.0_f64, 0.07848, 5.0] {
599 let base = MeasuredNetBenefitInputs {
600 hit_rate_at_1_pct: hit,
601 prewarm_saved_ms: MeasuredValue {
602 mean_ms: saved,
603 p95_ms: None,
604 samples: 10,
605 source: source.clone(),
606 },
607 prewarm_wasted_ms: MeasuredValue {
608 mean_ms: wasted,
609 p95_ms: None,
610 samples: 10,
611 source: source.clone(),
612 },
613 control_plane_ms: MeasuredValue {
614 mean_ms: control,
615 p95_ms: None,
616 samples: 10,
617 source: source.clone(),
618 },
619 };
620 let base_report =
621 compute_measured_net_benefit(&base, examples).expect("valid base");
622
623 let mut more_saved = base.clone();
624 more_saved.prewarm_saved_ms.mean_ms += 1.0;
625 let more_saved_report =
626 compute_measured_net_benefit(&more_saved, examples).unwrap();
627 assert!(more_saved_report.net_benefit_ms >= base_report.net_benefit_ms);
628
629 let mut more_waste = base.clone();
630 more_waste.prewarm_wasted_ms.mean_ms += 1.0;
631 let more_waste_report =
632 compute_measured_net_benefit(&more_waste, examples).unwrap();
633 assert!(more_waste_report.net_benefit_ms <= base_report.net_benefit_ms);
634
635 let mut more_control = base.clone();
636 more_control.control_plane_ms.mean_ms += 1.0;
637 let more_control_report =
638 compute_measured_net_benefit(&more_control, examples).unwrap();
639 assert!(more_control_report.net_benefit_ms <= base_report.net_benefit_ms);
640 checked += 1;
641 }
642 }
643 }
644 }
645
646 assert_eq!(checked, 5 * 3 * 3 * 3);
647 }
648
649 #[test]
650 fn measured_net_benefit_matches_raw_formula_over_large_fixture_like_grid() {
651 let source = source();
652 let hit_rates = [
653 0.0_f32, 0.1, 1.0, 5.0, 12.5, 25.0, 33.333, 50.0, 56.442, 75.0, 90.0, 99.9, 100.0,
654 ];
655 let saved_values = [0.001_f64, 1.0, 12.0, 31.231, 120.0, 394.8, 805.8, 1_500.0];
656 let wasted_values = [0.0_f64, 0.001, 1.0, 12.0, 31.231, 120.0, 394.8];
657 let control_values = [0.0_f64, 0.0001, 0.07848, 0.5, 1.0, 7.5, 31.231];
658 let example_counts = [1_usize, 2, 17, 999, 10_000, 272_519, 1_000_000];
659
660 let mut checked = 0usize;
661 for hit_rate_at_1_pct in hit_rates {
662 for saved in saved_values {
663 for wasted in wasted_values {
664 for control in control_values {
665 for examples in example_counts {
666 let inputs = MeasuredNetBenefitInputs {
667 hit_rate_at_1_pct,
668 prewarm_saved_ms: MeasuredValue {
669 mean_ms: saved,
670 p95_ms: Some(saved),
671 samples: 30,
672 source: source.clone(),
673 },
674 prewarm_wasted_ms: MeasuredValue {
675 mean_ms: wasted,
676 p95_ms: Some(wasted),
677 samples: 30,
678 source: source.clone(),
679 },
680 control_plane_ms: MeasuredValue {
681 mean_ms: control,
682 p95_ms: Some(control),
683 samples: 30,
684 source: source.clone(),
685 },
686 };
687 let report = compute_measured_net_benefit(&inputs, examples).unwrap();
688 let examples_f = examples as f64;
689 let hit = hit_rate_at_1_pct as f64 / 100.0;
690 let expected = examples_f * hit * saved
691 - examples_f * (1.0 - hit) * wasted
692 - examples_f * control;
693 assert!(
694 (report.net_benefit_ms - expected).abs() < 1e-5,
695 "case hit={hit_rate_at_1_pct} saved={saved} wasted={wasted} control={control} examples={examples}"
696 );
697 checked += 1;
698 }
699 }
700 }
701 }
702 }
703
704 assert_eq!(checked, 13 * 8 * 7 * 7 * 7);
705 }
706
707 #[test]
708 fn break_even_threshold_cases_stay_numerically_stable() {
709 let examples = 100_000;
710 let saved_values = [1.0_f64, 31.231, 394.8, 805.8, 1_500.0];
711 let wasted_values = [0.0_f64, 1.0, 12.0, 31.231, 394.8];
712 let control_values = [0.0_f64, 0.07848, 1.0, 12.0, 31.231];
713 let mut checked = 0usize;
714
715 for saved in saved_values {
716 for wasted in wasted_values {
717 for control in control_values {
718 let denominator = saved + wasted;
719 if denominator <= 0.0 {
720 continue;
721 }
722 let break_even_hit =
723 ((wasted + control) / denominator * 100.0).clamp(0.0, 100.0) as f32;
724 for offset in [-0.01_f32, 0.0, 0.01] {
725 let hit = (break_even_hit + offset).clamp(0.0, 100.0);
726 let report = compute_net_benefit(
727 &NetBenefitInputs {
728 hit_rate_at_1_pct: hit,
729 prewarm_saved_ms: saved,
730 prewarm_wasted_ms: wasted,
731 control_plane_ms: control,
732 },
733 examples,
734 );
735 assert!(report.net_benefit_ms.is_finite());
736 if offset < 0.0 && break_even_hit > 0.01 && break_even_hit < 99.99 {
737 assert!(
738 report.net_benefit_ms <= 1e-3,
739 "below break-even should not be positive: saved={saved} wasted={wasted} control={control} hit={hit}"
740 );
741 }
742 if offset > 0.0 && break_even_hit > 0.01 && break_even_hit < 99.99 {
743 assert!(
744 report.net_benefit_ms >= -1e-3,
745 "above break-even should not be negative: saved={saved} wasted={wasted} control={control} hit={hit}"
746 );
747 }
748 checked += 1;
749 }
750 }
751 }
752 }
753
754 assert_eq!(checked, 5 * 5 * 5 * 3);
755 }
756
757 #[test]
758 fn fixture_builder_accepts_many_valid_fixture_shapes() {
759 let source = source();
760 let mut checked = 0usize;
761
762 for examples in [1_usize, 10, 1_000, 272_519, 1_000_000] {
763 for saved in [0.001_f64, 31.231, 394.8, 1_500.0] {
764 for wasted in [0.0_f64, 0.001, 31.231, 394.8] {
765 for control in [0.0_f64, 0.0001, 0.07848, 10.0] {
766 let fixture =
767 build_prewarm_net_benefit_fixture(PrewarmFixtureBuildInputs {
768 dataset_id: format!(
769 "fixture-{examples}-{saved}-{wasted}-{control}"
770 ),
771 status: "generated_from_offline_measurements".into(),
772 trace: NetBenefitTrace {
773 source: "data/evaluation/next-app/lsapp-standard.report.json"
774 .into(),
775 split: "Standard".into(),
776 examples,
777 },
778 prewarm_saved: MeasuredValue {
779 mean_ms: saved,
780 p95_ms: Some(saved),
781 samples: 20,
782 source: source.clone(),
783 },
784 wasted_prewarm: MeasuredValue {
785 mean_ms: wasted,
786 p95_ms: Some(wasted),
787 samples: 1,
788 source: source.clone(),
789 },
790 dipecs_control_plane: MeasuredValue {
791 mean_ms: control,
792 p95_ms: None,
793 samples: 1_631,
794 source: source.clone(),
795 },
796 strong_control_plane: MeasuredValue {
797 mean_ms: 0.0,
798 p95_ms: None,
799 samples: examples,
800 source: source.clone(),
801 },
802 })
803 .expect("valid fixture shape should build");
804 fixture.validate().expect("valid fixture should validate");
805 checked += 1;
806 }
807 }
808 }
809 }
810
811 assert_eq!(checked, 5 * 4 * 4 * 4);
812 }
813}