tenant.openagents/omega

1from __future__ import annotations
2
3import shlex
4from dataclasses import dataclass
5
6DEFAULT_APP_NAME = "agent-evals"
7DEFAULT_VOLUME_NAME = "agent-evals"
8# Modal secret holding the LLM provider keys (ANTHROPIC_API_KEY, BASETEN_API_KEY,
9# ...) mounted into trial sandboxes.
10DEFAULT_LLM_PROVIDERS_SECRET_NAME = "agent-evals-llm-providers"
11# Modal secret holding the controller's Modal token (MODAL_TOKEN_ID/SECRET).
12DEFAULT_MODAL_TOKEN_SECRET_NAME = "agent-evals-modal-token"
13HARBOR_VERSION = "0.15.0"
14MODAL_VERSION = "1.5.0"
15CONTROLLER_IMAGE_RECIPE_VERSION = "agent-evals-controller-image-v1"
16DEFAULT_MODEL = "anthropic/claude-sonnet-4-6"
17DEFAULT_JUDGE_PRESET = "auto"
18BASETEN_API_URL = "https://inference.baseten.co/v1"
19BASETEN_PROVIDER_ID = "baseten"
20BASETEN_DEFAULT_MAX_TOKENS = 262_144
21BASETEN_DEFAULT_MAX_OUTPUT_TOKENS = 65_536
22
23# Kept modest on purpose: the agent model's API key is the real bottleneck,
24# not Modal capacity. Snickerdoodle-EAP is capped at 2M ITPM / 400K OTPM, and
25# ~150 concurrent agent loops (3 jobs x 50) blew past that. One job at this
26# concurrency stays well under the ceiling; raise only if you've confirmed the
27# model's rate limits have headroom.
28DEFAULT_N_CONCURRENT = 25
29DEFAULT_SANDBOX_TIMEOUT_SECS = 14_400
30DEFAULT_SANDBOX_IDLE_TIMEOUT_SECS = 300
31# No per-task resource override by default. When these are None the
32# --override-cpus/--override-memory-mb flags are omitted entirely, so Harbor
33# applies each task's declared cpus/memory (SWE-Atlas tasks declare 16 CPU /
34# 16 GB). Overriding *below* the declared values OOM-kills memory-heavy tasks
35# (SIGKILL / exit 137) and, per the resource policy, changes the experimental
36# conditions. Pass --override-cpus/--override-memory-mb to opt into smaller
37# sandboxes deliberately.
38DEFAULT_OVERRIDE_CPUS = None
39DEFAULT_OVERRIDE_MEMORY_MB = None
40EVAL_BASE_URL_IN_SANDBOX = "http://127.0.0.1:8089/v1"
41JUDGE_PROXY_VERIFIER_IMPORT_PATH = "zed_eval.verifier:ZedJudgeProxyVerifier"
42
43
44@dataclass(frozen=True)
45class JudgeConfig:
46    model: str
47    upstream: str
48    auth_env: str
49    # Floor for the judge's max_completion_tokens, applied by the in-sandbox
50    # judge shim (ZED_JUDGE_MAX_TOKENS). Set for Baseten judges so the runtime
51    # judge matches the offline calibration (8192); left None for the
52    # opus/leaderboard path so it stays leaderboard-faithful at the verifier's
53    # hardcoded 2048.
54    max_tokens: int | None = None
55
56
57def judge_verifier_args(judge: JudgeConfig, judge_model: str) -> list[str]:
58    args = [
59        "--verifier-import-path",
60        JUDGE_PROXY_VERIFIER_IMPORT_PATH,
61        "--ve",
62        f"EVAL_MODEL={judge_model}",
63        "--ve",
64        f"EVAL_BASE_URL={EVAL_BASE_URL_IN_SANDBOX}",
65        "--ve",
66        "EVAL_API_KEY=unused-by-agent-evals-proxy",
67        "--ve",
68        f"ZED_JUDGE_UPSTREAM={judge.upstream}",
69        "--ve",
70        f"ZED_JUDGE_AUTH_ENV={judge.auth_env}",
71    ]
72    if judge.max_tokens is not None:
73        args.extend(["--ve", f"ZED_JUDGE_MAX_TOKENS={judge.max_tokens}"])
74    return args
75
76
77def redacted_command(command: list[str]) -> str:
78    return shlex.join(
79        "EVAL_API_KEY=<redacted>" if argument.startswith("EVAL_API_KEY=") else argument
80        for argument in command
81    )
82
83
84JUDGES: dict[str, JudgeConfig] = {
85    "leaderboard": JudgeConfig(
86        model="claude-opus-4-5-20251101",
87        upstream="https://api.anthropic.com/v1",
88        auth_env="ANTHROPIC_API_KEY",
89    ),
90    "opus-leaderboard": JudgeConfig(
91        model="claude-opus-4-5-20251101",
92        upstream="https://api.anthropic.com/v1",
93        auth_env="ANTHROPIC_API_KEY",
94    ),
95    "deepseek-v4-pro": JudgeConfig(
96        model="deepseek-ai/DeepSeek-V4-Pro",
97        upstream=BASETEN_API_URL,
98        auth_env="BASETEN_API_KEY",
99        max_tokens=8192,
100    ),
101    "deepseek": JudgeConfig(
102        model="deepseek-ai/DeepSeek-V4-Pro",
103        upstream=BASETEN_API_URL,
104        auth_env="BASETEN_API_KEY",
105        max_tokens=8192,
106    ),
107    "kimi-k2.7-code": JudgeConfig(
108        model="moonshotai/Kimi-K2.7-Code",
109        upstream=BASETEN_API_URL,
110        auth_env="BASETEN_API_KEY",
111        max_tokens=8192,
112    ),
113    "kimi": JudgeConfig(
114        model="moonshotai/Kimi-K2.7-Code",
115        upstream=BASETEN_API_URL,
116        auth_env="BASETEN_API_KEY",
117        max_tokens=8192,
118    ),
119    "gpt55": JudgeConfig(
120        model="gpt-5.5-2026-04-23",
121        upstream="https://api.openai.com/v1",
122        auth_env="OPENAI_API_KEY",
123    ),
124}
125
126
127MODEL_PRESETS: dict[str, str] = {
128    "sonnet-4.6": DEFAULT_MODEL,
129    "claude-sonnet-4.6": DEFAULT_MODEL,
130    "sonnet-4.6-latest": "anthropic/claude-sonnet-4-6-latest",
131    "opus-4.5": "anthropic/claude-opus-4-5",
132    "baseten:kimi-k2.7-code": "baseten/moonshotai/Kimi-K2.7-Code",
133    "baseten:kimi": "baseten/moonshotai/Kimi-K2.7-Code",
134    "baseten:deepseek-v4-pro": "baseten/deepseek-ai/DeepSeek-V4-Pro",
135    "baseten:deepseek": "baseten/deepseek-ai/DeepSeek-V4-Pro",
136}
137
138
139def orchestration_info() -> dict[str, str]:
140    return {
141        "controller_image_recipe_version": CONTROLLER_IMAGE_RECIPE_VERSION,
142        "harbor_version": HARBOR_VERSION,
143        "modal_client_version": MODAL_VERSION,
144    }
145
146
147def get_judge(name: str) -> JudgeConfig:
148    try:
149        return JUDGES[name]
150    except KeyError as error:
151        valid = ", ".join(sorted(JUDGES))
152        raise ValueError(f"unknown judge preset '{name}' (valid: {valid})") from error
153
154
155def canonical_part(part: str) -> str:
156    normalized = part.strip().lower()
157    if normalized in ("qna", "rf", "tw"):
158        return normalized
159    valid = ", ".join(("qna", "rf", "tw"))
160    raise ValueError(f"unknown SWE-Atlas part '{part}' (valid: {valid})")
161
162
163def resolve_model_preset(model: str) -> str:
164    return MODEL_PRESETS.get(model, model)
165