tenant.openagents/omega

1from __future__ import annotations
2
3import json
4import tempfile
5import unittest
6from pathlib import Path
7
8from zed_eval import config, rejudge
9
10
11class TaskResolutionTests(unittest.TestCase):
12    def test_task_tests_dir_layout(self) -> None:
13        root = Path("/cache")
14        self.assertEqual(
15            rejudge.task_tests_dir("scale-ai/task-abc", "sha256:deadbeef", root),
16            root / "scale-ai" / "task-abc" / "deadbeef" / "tests",
17        )
18
19    def test_task_tests_dir_without_hash_prefix(self) -> None:
20        root = Path("/cache")
21        self.assertEqual(
22            rejudge.task_tests_dir("org/task-1", "rawref", root),
23            root / "org" / "task-1" / "rawref" / "tests",
24        )
25
26    def test_detect_part(self) -> None:
27        with tempfile.TemporaryDirectory() as tmp:
28            tests = Path(tmp)
29            self.assertIsNone(rejudge.detect_part(tests))
30            (tests / rejudge.RF_VERIFIER).write_text("")
31            self.assertEqual(rejudge.detect_part(tests), rejudge.PART_RF)
32
33    def test_detect_part_qna(self) -> None:
34        with tempfile.TemporaryDirectory() as tmp:
35            tests = Path(tmp)
36            (tests / rejudge.QNA_VERIFIER).write_text("")
37            self.assertEqual(rejudge.detect_part(tests), rejudge.PART_QNA)
38
39
40class RewardRecombinationTests(unittest.TestCase):
41    def test_rf_pass_requires_rubric_and_tests(self) -> None:
42        # rubric passes, tests passed -> overall pass
43        rewards = rejudge.recompute_rf_rewards(
44            {"must_have_pass": True, "agg_score": 0.9},
45            {"tests_reward": 1.0},
46        )
47        self.assertEqual(rewards["reward"], 1.0)
48        self.assertEqual(rewards["overall_pass"], 1.0)
49        self.assertEqual(rewards["must_have_pass"], 1.0)
50        self.assertEqual(rewards["tests_reward"], 1.0)
51
52    def test_rf_rubric_pass_but_tests_fail_is_fail(self) -> None:
53        rewards = rejudge.recompute_rf_rewards(
54            {"must_have_pass": True, "agg_score": 0.9},
55            {"tests_reward": 0.0},
56        )
57        self.assertEqual(rewards["reward"], 0.0)
58
59    def test_rf_tests_pass_but_rubric_fail_is_fail(self) -> None:
60        rewards = rejudge.recompute_rf_rewards(
61            {"must_have_pass": False, "agg_score": 0.2},
62            {"tests_reward": 1.0},
63        )
64        self.assertEqual(rewards["reward"], 0.0)
65        self.assertEqual(rewards["rubrics_agg_score"], 0.2)
66
67    def test_rf_preserves_tests_reward_from_parent(self) -> None:
68        # tests_reward is invariant under rejudge (patch unchanged).
69        rewards = rejudge.recompute_rf_rewards(
70            {"must_have_pass": False, "agg_score": 0.0},
71            {"tests_reward": 1.0, "reward": 1.0},
72        )
73        self.assertEqual(rewards["tests_reward"], 1.0)
74
75    def test_qna_reward_is_pass_verdict(self) -> None:
76        self.assertEqual(rejudge.recompute_qna_rewards({"pass": True})["reward"], 1.0)
77        self.assertEqual(rejudge.recompute_qna_rewards({"pass": False})["reward"], 0.0)
78
79
80class ResultPatchingTests(unittest.TestCase):
81    def test_patch_preserves_agent_metrics(self) -> None:
82        with tempfile.TemporaryDirectory() as tmp:
83            result_path = Path(tmp) / "result.json"
84            result_path.write_text(
85                json.dumps(
86                    {
87                        "task_name": "scale-ai/task-1",
88                        "agent_result": {"tokens": 12345},
89                        "verifier_result": {"rewards": {"reward": 0.0}},
90                    }
91                )
92            )
93            rejudge.patch_result_rewards(result_path, {"reward": 1.0})
94            patched = json.loads(result_path.read_text())
95            # Verdict updated...
96            self.assertEqual(patched["verifier_result"]["rewards"], {"reward": 1.0})
97            # ...but agent metrics and other fields untouched.
98            self.assertEqual(patched["agent_result"], {"tokens": 12345})
99            self.assertEqual(patched["task_name"], "scale-ai/task-1")
100
101
102class JudgeEnvironmentTests(unittest.TestCase):
103    def test_proxy_environment_from_judge_config(self) -> None:
104        judge = config.get_judge("deepseek-v4-pro")
105        env = rejudge.proxy_environment(judge)
106        self.assertEqual(env["ZED_JUDGE_UPSTREAM"], judge.upstream)
107        self.assertEqual(env["ZED_JUDGE_AUTH_ENV"], judge.auth_env)
108        self.assertEqual(env["ZED_JUDGE_MAX_TOKENS"], str(judge.max_tokens))
109
110    def test_proxy_environment_omits_unset_max_tokens(self) -> None:
111        judge = config.get_judge("leaderboard")
112        env = rejudge.proxy_environment(judge)
113        self.assertNotIn("ZED_JUDGE_MAX_TOKENS", env)
114
115    def test_verifier_environment_points_at_proxy(self) -> None:
116        env = rejudge.verifier_environment("some/model", 8089)
117        self.assertEqual(env["EVAL_MODEL"], "some/model")
118        self.assertEqual(env["EVAL_BASE_URL"], "http://127.0.0.1:8089/v1")
119        self.assertTrue(env["EVAL_API_KEY"])
120
121
122if __name__ == "__main__":
123    unittest.main()
124