honeybadger/test_curriculum_gate.py at main · kroq86/honeybadger · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import unittest

from curriculum_gate import evaluate_summary


class CurriculumGateTests(unittest.TestCase):
    def test_next_focus_is_first_missing_or_failed_stage(self) -> None:
        summary = {
            "model": "demo",
            "stages": {
                "single_step": {
                    "val_avg_field_accuracy": 0.90,
                    "test_avg_field_accuracy": 0.90,
                    "val_repaired_exact_match_rate": 0.20,
                    "test_repaired_exact_match_rate": 0.20,
                },
                "next_2_steps": {
                    "val_avg_field_accuracy": 0.70,
                    "test_avg_field_accuracy": 0.60,
                    "val_repaired_exact_match_rate": 0.10,
                    "test_repaired_exact_match_rate": 0.05,
                },
            },
        }
        result = evaluate_summary(summary)
        self.assertEqual(result["next_focus_stage"], "next_2_steps")
        self.assertEqual(result["active_curriculum"], ["single_step", "next_2_steps", "short_trace", "terminal_state"])
        self.assertTrue(result["stages"]["single_step"]["signal"])
        self.assertFalse(result["stages"]["next_2_steps"]["signal"])

    def test_pass_stage_requires_thresholds(self) -> None:
        summary = {
            "model": "demo",
            "stages": {
                "single_step": {
                    "val_avg_field_accuracy": 0.96,
                    "test_avg_field_accuracy": 0.95,
                    "val_repaired_exact_match_rate": 0.72,
                    "test_repaired_exact_match_rate": 0.71,
                },
                "next_2_steps": {
                    "val_avg_field_accuracy": 0.90,
                    "test_avg_field_accuracy": 0.88,
                    "val_repaired_exact_match_rate": 0.60,
                    "test_repaired_exact_match_rate": 0.55,
                },
                "short_trace": {
                    "val_repaired_exact_match_rate": 0.03,
                    "test_repaired_exact_match_rate": 0.02,
                },
            },
        }
        result = evaluate_summary(summary)
        self.assertTrue(result["stages"]["single_step"]["pass"])
        self.assertTrue(result["stages"]["next_2_steps"]["pass"])
        self.assertEqual(result["next_focus_stage"], "short_trace")


if __name__ == "__main__":
    unittest.main()