tenant.openagents/omega

1from __future__ import annotations
2
3import argparse
4import json
5import os
6import subprocess
7import sys
8from pathlib import Path
9
10from . import benchmarks, config, source
11from .baseline import (
12    command_baseline_list,
13    command_baseline_record,
14    command_baseline_show,
15)
16from .common import (
17    AppNotDeployedError,
18    command_exists,
19    default_namespace,
20    deploy_app,
21    deployed_function,
22    print_json,
23)
24from .launch import command_build, command_rejudge, command_run, command_swe_atlas
25from .report import command_report
26from .volume import (
27    command_builds,
28    command_fetch,
29    command_list,
30    command_logs,
31    command_runs,
32    command_status,
33    command_suite_fetch,
34    command_suite_logs,
35    command_suite_status,
36    ensure_volume_exists,
37)
38
39
40def modal_secret_names() -> set[str]:
41    result = subprocess.run(
42        ["modal", "secret", "list", "--json"],
43        check=True,
44        capture_output=True,
45        text=True,
46    )
47    data = json.loads(result.stdout)
48    if not isinstance(data, list):
49        return set()
50    return {
51        item["name"] for item in data if isinstance(item, dict) and item.get("name")
52    }
53
54
55def command_doctor(args: argparse.Namespace) -> int:
56    namespace = default_namespace(args)
57    print("zed-eval doctor")
58    print(f"  namespace:          {namespace}")
59    print(f"  app:                {args.app_name}")
60    print(f"  volume:             {args.volume}")
61    print(f"  api secret:         {args.api_secret}")
62    print(f"  modal token secret: {args.modal_token_secret}")
63    print(f"  repo root:          {source.repo_root()}")
64    print(f"  base sha:           {source.current_base_sha()}")
65    print(f"  default model:      {config.DEFAULT_MODEL}")
66    print("  default judges:     qna=deepseek-v4-pro, rf/tw=kimi-k2.7-code")
67
68    missing = []
69    for executable in ("git", "modal", "harbor"):
70        if command_exists(executable):
71            print(f"  {executable}:             found")
72        else:
73            print(f"  {executable}:             missing")
74            missing.append(executable)
75
76    if missing:
77        print("\nMissing required executables: " + ", ".join(missing), file=sys.stderr)
78        return 1
79
80    try:
81        secrets = modal_secret_names()
82    except (subprocess.CalledProcessError, json.JSONDecodeError) as error:
83        print(
84            f"  secrets:          could not list Modal secrets: {error}",
85            file=sys.stderr,
86        )
87    else:
88        missing_secrets = [
89            secret_name
90            for secret_name in (args.api_secret, args.modal_token_secret)
91            if secret_name not in secrets
92        ]
93        if missing_secrets:
94            print(
95                "\nMissing Modal secret(s): " + ", ".join(missing_secrets),
96                file=sys.stderr,
97            )
98            print(
99                "Use --api-secret/--modal-token-secret to point at existing secrets, "
100                "or create the missing ones with `modal secret create`.",
101                file=sys.stderr,
102            )
103            return 1
104        print("  secrets:          found")
105        print(
106            "  controller token: use a dedicated Modal service-user token for production"
107        )
108
109    if args.create_volume:
110        return ensure_volume_exists(args)
111    return 0
112
113
114def command_deploy(args: argparse.Namespace) -> int:
115    deploy_app(args)
116    print(f"Deployed Modal app '{args.app_name}'.")
117    return 0
118
119
120def command_cleanup(args: argparse.Namespace) -> int:
121    request: dict[str, object] = {"dry_run": bool(args.dry_run)}
122    if args.build_retention_days is not None:
123        request["build_retention_days"] = args.build_retention_days
124    result = deployed_function(args, "cleanup_artifacts").remote(request)
125    print_json(result)
126    return 0
127
128
129def env_default(
130    variable_name: str, default: object, *, include_defaults: bool = True
131) -> object:
132    if not include_defaults:
133        return argparse.SUPPRESS
134    return os.environ.get(variable_name, default)
135
136
137def add_common_options(
138    parser: argparse.ArgumentParser, *, include_defaults: bool = True
139) -> None:
140    default_volume = env_default(
141        "AGENT_EVALS_VOLUME",
142        config.DEFAULT_VOLUME_NAME,
143        include_defaults=include_defaults,
144    )
145    default_namespace = None if include_defaults else argparse.SUPPRESS
146    default_api_secret = env_default(
147        "AGENT_EVALS_LLM_PROVIDERS_SECRET",
148        config.DEFAULT_LLM_PROVIDERS_SECRET_NAME,
149        include_defaults=include_defaults,
150    )
151    default_modal_token_secret = env_default(
152        "AGENT_EVALS_MODAL_TOKEN_SECRET",
153        config.DEFAULT_MODAL_TOKEN_SECRET_NAME,
154        include_defaults=include_defaults,
155    )
156    default_app_name = env_default(
157        "AGENT_EVALS_APP_NAME",
158        config.DEFAULT_APP_NAME,
159        include_defaults=include_defaults,
160    )
161    infra = parser.add_argument_group(
162        "infra options", "Modal app/volume/namespace and secret names (rarely changed)"
163    )
164    infra.add_argument(
165        "--app-name",
166        default=default_app_name,
167        help=f"Deployed Modal app name (default: {config.DEFAULT_APP_NAME})",
168    )
169    infra.add_argument(
170        "--volume",
171        default=default_volume,
172        help=f"Modal volume for builds/runs (default: {config.DEFAULT_VOLUME_NAME})",
173    )
174    infra.add_argument(
175        "--namespace",
176        default=default_namespace,
177        help="Run namespace inside the shared volume (default: env/git user/local user)",
178    )
179    infra.add_argument(
180        "--api-secret",
181        default=default_api_secret,
182        help=(
183            "Modal secret with the LLM provider keys mounted into trial sandboxes "
184            f"(default: {config.DEFAULT_LLM_PROVIDERS_SECRET_NAME})"
185        ),
186    )
187    infra.add_argument(
188        "--modal-token-secret",
189        default=default_modal_token_secret,
190        help=(
191            "Modal secret mounted into the controller with MODAL_TOKEN_ID and MODAL_TOKEN_SECRET; "
192            f"use a dedicated service-user token for production (default: {config.DEFAULT_MODAL_TOKEN_SECRET_NAME})"
193        ),
194    )
195
196
197def add_build_source_options(parser: argparse.ArgumentParser) -> None:
198    parser.add_argument(
199        "--from",
200        dest="from_source",
201        default=os.environ.get("AGENT_EVALS_FROM"),
202        help=(
203            "Unified build source: 'local' for current HEAD + tracked changes, "
204            "'main' for origin/main's tip, or a git ref/tag/SHA (e.g. v0.210.0) "
205            "for a clean build resolved canonically against the remote so "
206            "teammates share one build"
207        ),
208    )
209    parser.add_argument("--base-sha")
210    parser.add_argument("--patch-path")
211    parser.add_argument(
212        "--repo-url",
213        default=os.environ.get("AGENT_EVALS_REPO_URL", source.DEFAULT_REPO_URL),
214        help="Git remote the Modal builder fetches the base SHA from",
215    )
216    parser.add_argument(
217        "--allow-untracked",
218        action="store_true",
219        help="Proceed even when untracked files exist; they are not included in the build patch",
220    )
221    parser.add_argument(
222        "--require-clean",
223        action="store_true",
224        help="Fail if tracked changes are present",
225    )
226    parser.add_argument(
227        "--clean-source",
228        action="store_true",
229        help="Build exactly --base-sha/--zed-version with no local patch",
230    )
231    parser.add_argument(
232        "--zed-version",
233        help="Git ref/tag/SHA of Zed to build as a clean source snapshot",
234    )
235
236
237def add_model_options(parser: argparse.ArgumentParser) -> None:
238    parser.add_argument(
239        "-m",
240        "--model",
241        default=os.environ.get("AGENT_EVALS_MODEL", config.DEFAULT_MODEL),
242        help="Base model as provider/model, sonnet-4.6, baseten:kimi-k2.7-code, or baseten:<model-id>",
243    )
244    parser.add_argument(
245        "--extra-api-secret",
246        action="append",
247        help="Additional Modal secret name to mount into the trial sandbox",
248    )
249    advanced = parser.add_argument_group(
250        "advanced model options",
251        "Lower-level model routing. For Baseten, prefer --model baseten:<model-id>.",
252    )
253    advanced.add_argument(
254        "--model-provider",
255        choices=("zed", "baseten"),
256        default="zed",
257        help="Use 'zed' for built-in provider/model ids, or 'baseten' for Baseten Model APIs",
258    )
259    advanced.add_argument(
260        "--baseten-model",
261        help="Baseten model id when --model-provider baseten, e.g. moonshotai/Kimi-K2.7-Code",
262    )
263    advanced.add_argument("--baseten-model-display-name")
264    advanced.add_argument(
265        "--baseten-api-url",
266        default=config.BASETEN_API_URL,
267        help=f"Baseten OpenAI-compatible API URL (default: {config.BASETEN_API_URL})",
268    )
269    advanced.add_argument(
270        "--baseten-model-max-tokens",
271        type=int,
272        default=config.BASETEN_DEFAULT_MAX_TOKENS,
273    )
274    advanced.add_argument(
275        "--baseten-model-max-output-tokens",
276        type=int,
277        default=config.BASETEN_DEFAULT_MAX_OUTPUT_TOKENS,
278    )
279    advanced.add_argument(
280        "--openai-compatible-provider-json",
281        help="JSON object merged into language_models.openai_compatible before eval-cli resolves --model",
282    )
283    advanced.add_argument(
284        "--anthropic-available-models-json",
285        help="JSON array merged into language_models.anthropic.available_models before eval-cli resolves --model",
286    )
287
288
289def add_launch_options(parser: argparse.ArgumentParser) -> None:
290    add_model_options(parser)
291    parser.add_argument(
292        "-j",
293        "--judge",
294        default=os.environ.get("AGENT_EVALS_JUDGE", config.DEFAULT_JUDGE_PRESET),
295        choices=["auto", *sorted(config.JUDGES)],
296        help="Judge preset; auto uses qna=deepseek-v4-pro and rf/tw=kimi-k2.7-code",
297    )
298    parser.add_argument("--judge-model")
299    parser.add_argument(
300        "--build",
301        metavar="ID",
302        help="Reuse this build if it exists, otherwise create it with this id",
303    )
304    add_build_source_options(parser)
305    parser.add_argument("--build-wait-timeout-secs", type=int, default=7200)
306    parser.add_argument("--tasks", help="File containing one full task name per line")
307    parser.add_argument("--include-task-name", action="append")
308    parser.add_argument("-n", "--n-tasks", type=int, help="Forward harness --n-tasks")
309    parser.add_argument("--n-concurrent", type=int, default=config.DEFAULT_N_CONCURRENT)
310    parser.add_argument(
311        "--override-cpus", type=int, default=config.DEFAULT_OVERRIDE_CPUS
312    )
313    parser.add_argument(
314        "--override-memory-mb", type=int, default=config.DEFAULT_OVERRIDE_MEMORY_MB
315    )
316    parser.add_argument(
317        "--sandbox-timeout-secs", type=int, default=config.DEFAULT_SANDBOX_TIMEOUT_SECS
318    )
319    parser.add_argument(
320        "--sandbox-idle-timeout-secs",
321        type=int,
322        default=config.DEFAULT_SANDBOX_IDLE_TIMEOUT_SECS,
323    )
324    parser.add_argument(
325        "--eval-cli-timeout",
326        type=int,
327        help="Override the per-task agent timeout (defaults to the benchmark's)",
328    )
329    parser.add_argument("--run-id")
330    parser.add_argument(
331        "--dry-run",
332        action="store_true",
333        help="Print full manifests and harness commands without launching",
334    )
335    parser.add_argument(
336        "--plan",
337        action="store_true",
338        help="Print a concise launch plan without launching",
339    )
340    parser.add_argument(
341        "--verbose",
342        action="store_true",
343        help="With --plan, include full manifests and harness commands",
344    )
345    parser.add_argument(
346        "--extra-harbor-arg",
347        action="append",
348        help="Append one raw argument to the harness run command",
349    )
350    parser.add_argument(
351        "--swe-atlas-repo-url",
352        default=os.environ.get(
353            "AGENT_EVALS_SWE_ATLAS_REPO_URL", benchmarks.SWE_ATLAS_REPO_URL
354        ),
355        help="SWE-Atlas repo URL used for path-backed datasets",
356    )
357    parser.add_argument(
358        "--swe-atlas-repo-ref",
359        default=os.environ.get(
360            "AGENT_EVALS_SWE_ATLAS_REPO_REF", benchmarks.SWE_ATLAS_REPO_REF
361        ),
362        help="SWE-Atlas repo ref used for path-backed datasets",
363    )
364
365
366def add_suite_options(parser: argparse.ArgumentParser) -> None:
367    add_launch_options(parser)
368    parser.add_argument(
369        "--parts",
370        help="Comma-separated SWE-Atlas parts to run: qna,rf,tw (or 'all')",
371    )
372    parser.add_argument("--experiment-prefix", default="swe-atlas")
373    parser.add_argument("--run-id-prefix")
374    parser.add_argument(
375        "--suite-id", help="Explicit suite id for grouping multi-part runs"
376    )
377    parser.add_argument("--interactive", action="store_true")
378    parser.add_argument(
379        "-y",
380        "--yes",
381        action="store_true",
382        help="Never prompt; require flags/defaults",
383    )
384
385
386def add_run_lookup_options(
387    parser: argparse.ArgumentParser,
388    *,
389    require_experiment: bool = False,
390    run_id_optional: bool = False,
391) -> None:
392    if run_id_optional:
393        parser.add_argument(
394            "run_id",
395            nargs="?",
396            help="Run id (defaults to the most recent run launched from this machine)",
397        )
398    else:
399        parser.add_argument("run_id")
400    parser.add_argument(
401        "-e",
402        "--experiment-name",
403        required=require_experiment,
404        help="Benchmark storage name (auto-resolved from the local run index when omitted)",
405    )
406
407
408def add_command_parser(
409    subparsers,
410    name: str,
411    *,
412    func: object | None = None,
413    **kwargs,
414) -> argparse.ArgumentParser:
415    parser = subparsers.add_parser(name, **kwargs)
416    add_common_options(parser, include_defaults=False)
417    if func is not None:
418        parser.set_defaults(func=func)
419    return parser
420
421
422def build_parser() -> argparse.ArgumentParser:
423    parser = argparse.ArgumentParser(
424        prog="zed-eval",
425        description="Launch, monitor, and fetch remote SWE-Atlas agent evals on Modal.",
426        formatter_class=argparse.RawDescriptionHelpFormatter,
427        epilog=(
428            "Common workflows:\n"
429            "  zed-eval run swe-atlas -m sonnet-4.6       # launch the SWE-Atlas suite\n"
430            "  zed-eval run rf -m sonnet-4.6              # launch one benchmark\n"
431            "  zed-eval runs                              # what did I launch recently?\n"
432            "  zed-eval status                            # check my most recent run\n"
433            "  zed-eval status <run-id>                   # check a specific run\n"
434            "  zed-eval logs <run-id>                     # print controller logs\n"
435            "  zed-eval report <run-id> --fetch           # fetch + score a run\n"
436            "\n"
437            "After launching, a run id alone locates the run — namespace and\n"
438            "benchmark storage name are resolved from this machine's local run index."
439        ),
440    )
441    add_common_options(parser)
442    subparsers = parser.add_subparsers(dest="command", required=True)
443
444    doctor = add_command_parser(
445        subparsers,
446        "doctor",
447        func=command_doctor,
448        help="Check local prerequisites and defaults",
449    )
450    doctor.add_argument("--create-volume", action="store_true")
451
452    add_command_parser(
453        subparsers,
454        "deploy",
455        func=command_deploy,
456        help=(
457            "Publish harness code to Modal. This is the ONLY command that "
458            "deploys; deploying cancels in-flight runs. Run it once after "
459            "changing harness code — all other commands just invoke the "
460            "already-deployed functions and never deploy."
461        ),
462    )
463
464    build = add_command_parser(
465        subparsers,
466        "build",
467        func=command_build,
468        help="Build eval-cli on Modal into builds/<build-id>",
469    )
470    add_build_source_options(build)
471    build.add_argument(
472        "--build",
473        metavar="ID",
474        help="Reuse this build if it exists, otherwise create it with this id",
475    )
476    build.add_argument(
477        "--detach", action="store_true", help="Spawn the build and return immediately"
478    )
479
480    builds = add_command_parser(
481        subparsers,
482        "builds",
483        func=command_builds,
484        help="List content-addressed builds on the volume",
485    )
486    builds.add_argument("--details", action="store_true")
487    builds.add_argument("--json", action="store_true")
488    builds.add_argument("--limit", type=int, default=50)
489
490    run_cmd = add_command_parser(
491        subparsers,
492        "run",
493        func=command_run,
494        help="Run benchmarks — the everyday entry point",
495        description=(
496            "Launch runs by benchmark id, alias, or group. It picks the right "
497            "default judge for each benchmark and never prompts. The sibling "
498            "`swe-atlas` command launches SWE-Atlas parts (qna/rf/tw) with "
499            "interactive prompts."
500        ),
501    )
502    run_cmd.add_argument(
503        "benchmark",
504        nargs="+",
505        metavar="target",
506        help=(
507            "Benchmark id/alias/group (swe-atlas (= qna,rf,tw), swe-atlas-rf, "
508            "qna, rf, tw, terminal-bench-2.1 (tb21), deepswe). Comma-separated "
509            "and repeated values are combined."
510        ),
511    )
512    add_launch_options(run_cmd)
513
514    run_cmd.add_argument(
515        "--suite-id", help="Explicit suite id for multi-benchmark runs"
516    )
517    run_cmd.add_argument(
518        "--staff",
519        action="store_true",
520        help=(
521            "Run the agent with staff mode ON (default OFF). Staff mode enables "
522            "the sandboxed terminal, which hangs inside Modal sandboxes, so keep "
523            "it off for remote runs."
524        ),
525    )
526    run_cmd.add_argument(
527        "-y",
528        "--yes",
529        action="store_true",
530        help="Accepted for parity with swe-atlas; run never prompts",
531    )
532
533    report = add_command_parser(
534        subparsers,
535        "report",
536        func=command_report,
537        help="Success-conditioned metrics (rate, tokens, tool calls, steps)",
538    )
539    report.add_argument("run_id", nargs="?")
540    report.add_argument(
541        "-e",
542        "--experiment-name",
543        help="Benchmark storage name (auto-resolved from the local run index when omitted)",
544    )
545    report.add_argument("--job-dir", help="Analyze a local job directory directly")
546    report.add_argument("--jobs-dir", default=str(Path.home() / ".cache/harbor/jobs"))
547    report.add_argument(
548        "--fetch", action="store_true", help="Fetch the run archive before reporting"
549    )
550    report.add_argument(
551        "--json",
552        dest="as_json",
553        action="store_true",
554        help="Emit JSON instead of a table",
555    )
556    report.add_argument(
557        "--timeouts-as-failures",
558        dest="timeouts_as_failures",
559        action=argparse.BooleanOptionalAction,
560        default=None,
561        help=(
562            "Count agent timeouts as failures rather than excluded errors "
563            "(default: auto — on for test-scored benchmarks like terminal-bench "
564            "and deepswe)"
565        ),
566    )
567
568    cleanup = add_command_parser(
569        subparsers,
570        "cleanup",
571        func=command_cleanup,
572        help="Prune stale builds and cold build cache (never eval results)",
573    )
574    cleanup.add_argument(
575        "--dry-run",
576        action="store_true",
577        help="Report what would be removed without deleting anything",
578    )
579    cleanup.add_argument(
580        "--build-retention-days",
581        type=float,
582        help="Remove builds older than this many days (default 14)",
583    )
584
585    suite = add_command_parser(
586        subparsers,
587        "swe-atlas",
588        func=command_swe_atlas,
589        help="Launch an interactive benchmark suite (SWE-Atlas, Terminal-Bench, DeepSWE)",
590    )
591    add_suite_options(suite)
592
593    list_runs = add_command_parser(
594        subparsers,
595        "list",
596        func=command_list,
597        help="List benchmark runs on the volume, optionally with metadata",
598    )
599    list_runs.add_argument("-e", "--experiment-name")
600    list_runs.add_argument("--details", action="store_true")
601    list_runs.add_argument("--json", action="store_true")
602    list_runs.add_argument("--limit", type=int, default=50)
603    list_runs.add_argument("--all-namespaces", action="store_true")
604
605    runs = add_command_parser(
606        subparsers,
607        "runs",
608        func=command_runs,
609        help="List recent runs launched from this machine (local, fast)",
610        description=(
611            "Show runs recorded in the local run index when you launched them "
612            "from this machine — newest first, no network call. Use this for a "
613            "quick 'what did I launch lately?'; use `list --details` to query "
614            "run state on the volume, or `status <run-id>` for one run's state."
615        ),
616    )
617    runs.add_argument("--json", action="store_true")
618    runs.add_argument("--limit", type=int, default=20)
619
620    status = add_command_parser(
621        subparsers,
622        "status",
623        func=command_status,
624        help="Print a run's state.json once (no run id = most recent run)",
625    )
626    add_run_lookup_options(status, run_id_optional=True)
627
628    logs = add_command_parser(
629        subparsers,
630        "logs",
631        func=command_logs,
632        help="Print a run's controller.log once (no run id = most recent run)",
633    )
634    add_run_lookup_options(logs, run_id_optional=True)
635
636    fetch = add_command_parser(
637        subparsers,
638        "fetch",
639        func=command_fetch,
640        help="Fetch and extract a run's Harbor job archive",
641    )
642    add_run_lookup_options(fetch)
643    fetch.add_argument("--jobs-dir", default=str(Path.home() / ".cache/harbor/jobs"))
644
645    rejudge = add_command_parser(
646        subparsers,
647        "rejudge",
648        func=command_rejudge,
649        help="Re-grade a finished run with a different judge (new derived run, "
650        "reuses the agent work)",
651    )
652    add_run_lookup_options(rejudge)
653    rejudge.add_argument(
654        "-j",
655        "--judge",
656        required=True,
657        help="Judge preset to re-grade with (see config.JUDGES, e.g. "
658        "deepseek-v4-pro, kimi-k2.7-code, leaderboard)",
659    )
660    rejudge.add_argument(
661        "--judge-model",
662        dest="judge_model",
663        help="Override the judge preset's model id",
664    )
665    rejudge.add_argument(
666        "--parent-namespace",
667        dest="parent_namespace",
668        help="Namespace of the source run, if different from --namespace",
669    )
670    rejudge.add_argument(
671        "--new-run-id",
672        dest="new_run_id",
673        help="Explicit id for the derived run (default: <parent>-rejudge-<judge>-<rand>)",
674    )
675    rejudge.add_argument(
676        "--dry-run",
677        action="store_true",
678        help="Print the rejudge request without spawning the controller",
679    )
680
681    suite_group = add_command_parser(
682        subparsers,
683        "suite",
684        help="Inspect or fetch grouped benchmark suite runs",
685    )
686    suite_subparsers = suite_group.add_subparsers(dest="suite_command", required=True)
687    suite_status = add_command_parser(
688        suite_subparsers,
689        "status",
690        func=command_suite_status,
691        help="Show status for each run in a suite",
692    )
693    suite_status.add_argument("suite_id")
694    suite_status.add_argument("--json", action="store_true")
695    suite_logs = add_command_parser(
696        suite_subparsers,
697        "logs",
698        func=command_suite_logs,
699        help="Print logs for each run in a suite",
700    )
701    suite_logs.add_argument("suite_id")
702    suite_logs.add_argument("--follow", action="store_true")
703    suite_logs.add_argument("--interval", type=float, default=30.0)
704    suite_fetch = add_command_parser(
705        suite_subparsers,
706        "fetch",
707        func=command_suite_fetch,
708        help="Fetch all run archives in a suite",
709    )
710    suite_fetch.add_argument("suite_id")
711    suite_fetch.add_argument(
712        "--jobs-dir", default=str(Path.home() / ".cache/harbor/jobs")
713    )
714
715    baseline_group = add_command_parser(
716        subparsers,
717        "baseline",
718        help="Record and inspect baseline-of-record results (clean commits on main)",
719    )
720    baseline_subparsers = baseline_group.add_subparsers(
721        dest="baseline_command", required=True
722    )
723    baseline_record = add_command_parser(
724        baseline_subparsers,
725        "record",
726        func=command_baseline_record,
727        help="Promote completed run(s) to the baseline of record for their (benchmark, model)",
728    )
729    baseline_record.add_argument("run_id", nargs="+")
730    baseline_record.add_argument("--experiment-name", required=True)
731    baseline_record.add_argument(
732        "--allow-dirty",
733        action="store_true",
734        help="Record even though the build carries a local patch (not a clean commit)",
735    )
736    baseline_record.add_argument(
737        "--allow-off-main",
738        action="store_true",
739        help="Record even though base_sha can't be verified as reachable from origin/main",
740    )
741    baseline_record.add_argument(
742        "--repo-url",
743        default=os.environ.get("AGENT_EVALS_REPO_URL"),
744        help="Git remote to resolve origin/main against (default: AGENT_EVALS_REPO_URL or the canonical repo)",
745    )
746
747    baseline_list = add_command_parser(
748        baseline_subparsers,
749        "list",
750        func=command_baseline_list,
751        help="List the current baseline of record for every (benchmark, model)",
752    )
753    baseline_list.add_argument("--json", action="store_true")
754
755    baseline_show = add_command_parser(
756        baseline_subparsers,
757        "show",
758        func=command_baseline_show,
759        help="Show the baseline record for one (benchmark, model)",
760    )
761    baseline_show.add_argument("experiment_name")
762    baseline_show.add_argument("--model", required=True)
763    baseline_show.add_argument(
764        "--history", action="store_true", help="Include superseded baselines"
765    )
766
767    return parser
768
769
770def main(argv: list[str] | None = None) -> int:
771    parser = build_parser()
772    args = parser.parse_args(argv)
773    try:
774        return args.func(args)
775    except (ValueError, AppNotDeployedError) as error:
776        print(f"error: {error}", file=sys.stderr)
777        return 1
778    except subprocess.CalledProcessError as error:
779        if error.stdout:
780            print(error.stdout, end="")
781        if error.stderr:
782            print(error.stderr, end="", file=sys.stderr)
783        return error.returncode
784
785
786if __name__ == "__main__":
787    raise SystemExit(main())
788