tenant.openagents/omega

1"""Harbor installed-agent adapter for Zed's eval-cli binary."""
2
3import json
4import os
5import shlex
6import tomllib
7from pathlib import Path
8
9from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template
10from harbor.environments.base import BaseEnvironment
11from harbor.models.agent.context import AgentContext
12
13from .agent_common import (
14    add_anthropic_available_models_env,
15    add_openai_compatible_provider_env,
16    add_zed_eval_env,
17    detect_workdir,
18    eval_cli_with_log_command,
19    patch_command,
20    populate_context_from_result,
21    provider_api_env,
22)
23
24# Leave eval-cli time to exit cleanly and flush logs before Harbor's outer
25# asyncio.wait_for can kill the agent coroutine.
26EVAL_CLI_FINALIZE_BUFFER_SEC = 45
27EVAL_CLI_MIN_TIMEOUT_SEC = 60
28
29
30class ZedAgent(BaseInstalledAgent):
31    def __init__(
32        self,
33        logs_dir: Path,
34        binary_path: str | None = None,
35        download_url: str | None = None,
36        *args,
37        **kwargs,
38    ):
39        super().__init__(logs_dir, *args, **kwargs)
40        self._binary_path = binary_path
41        self._download_url = download_url or os.environ.get("EVAL_CLI_DOWNLOAD_URL")
42
43    @staticmethod
44    def name() -> str:
45        return "zed"
46
47    async def _detect_workdir(self, environment: BaseEnvironment) -> str:
48        return await detect_workdir(
49            environment,
50            self.exec_as_agent,
51            self._extra_env.get,
52            "Could not detect a working directory in the container. "
53            "Set EVAL_CLI_WORKDIR explicitly via --ae EVAL_CLI_WORKDIR=/path/to/repo",
54        )
55
56    async def install(self, environment: BaseEnvironment) -> None:
57        # Detect the package manager and install base dependencies.
58        # Supports Debian/Ubuntu (apt-get), Alpine (apk), and
59        # Fedora/RHEL/CentOS (dnf/yum).
60        await self.exec_as_root(
61            environment,
62            command=(
63                "if command -v apt-get >/dev/null 2>&1; then "
64                "  apt-get update && "
65                "  apt-get install -y --no-install-recommends ca-certificates curl git; "
66                "elif command -v apk >/dev/null 2>&1; then "
67                "  apk add --no-cache ca-certificates curl git bash coreutils gcompat libstdc++; "
68                "elif command -v dnf >/dev/null 2>&1; then "
69                "  dnf install -y ca-certificates curl git; "
70                "elif command -v yum >/dev/null 2>&1; then "
71                "  yum install -y ca-certificates curl git; "
72                "else "
73                "  echo 'WARNING: No supported package manager found (apt-get, apk, dnf, yum)' >&2; "
74                "fi"
75            ),
76            env={"DEBIAN_FRONTEND": "noninteractive"},
77        )
78
79        # Tooling setup is best-effort; benchmark images vary enough that one
80        # unsupported runtime should not fail the trial before eval-cli starts.
81        await self._install_node(environment)
82        await self._install_lsps(environment)
83        await self._install_uv_and_ruff(environment)
84
85        # Modal can mount a prebuilt binary inside each sandbox, avoiding uploads.
86        container_path = self._extra_env.get("EVAL_CLI_CONTAINER_PATH")
87        if container_path:
88            await self.exec_as_root(
89                environment,
90                command=(
91                    f"cp {shlex.quote(container_path)} /usr/local/bin/eval-cli && "
92                    "chmod +x /usr/local/bin/eval-cli && "
93                    "eval-cli --help"
94                ),
95            )
96            return
97
98        if self._binary_path:
99            binary = Path(self._binary_path)
100            if not binary.exists():
101                raise FileNotFoundError(
102                    f"eval-cli binary not found at {binary}. "
103                    "Build it with: cargo build --release -p eval_cli"
104                )
105            await environment.upload_file(
106                source_path=binary,
107                target_path="/usr/local/bin/eval-cli",
108            )
109            await self.exec_as_root(
110                environment,
111                command="chmod +x /usr/local/bin/eval-cli && eval-cli --help",
112            )
113            return
114
115        if self._download_url:
116            await self.exec_as_root(
117                environment,
118                command=(
119                    f"curl -fsSL {shlex.quote(self._download_url)} "
120                    "-o /usr/local/bin/eval-cli && "
121                    "chmod +x /usr/local/bin/eval-cli && "
122                    "eval-cli --help"
123                ),
124            )
125            return
126
127        raise ValueError(
128            "No eval-cli binary provided. "
129            "Either pass binary_path=/path/to/target/release/eval-cli, "
130            "set download_url=/EVAL_CLI_DOWNLOAD_URL, "
131            "or set --ae EVAL_CLI_CONTAINER_PATH=/path/inside/container."
132        )
133
134    async def _install_node(self, environment: BaseEnvironment) -> None:
135        """Install Node.js from official binary tarballs.
136
137        Uses the musl build on Alpine and the glibc build elsewhere.
138        Skips if node is already on PATH.
139        """
140        try:
141            await self.exec_as_root(
142                environment,
143                command=(
144                    "if command -v node >/dev/null 2>&1; then "
145                    '  echo "Node.js already available: $(node --version)"; '
146                    "else "
147                    "  NODE_VER=v22.14.0; "
148                    "  ARCH=$(uname -m); "
149                    '  case "$ARCH" in '
150                    "    x86_64)  NODE_ARCH=x64  ;; "
151                    "    aarch64) NODE_ARCH=arm64 ;; "
152                    '    *)       echo "WARNING: unsupported arch $ARCH for Node.js" >&2; exit 0 ;; '
153                    "  esac; "
154                    "  if ldd /bin/sh 2>&1 | grep -qi musl; then "
155                    '    NODE_URL="https://unofficial-builds.nodejs.org/download/release/${NODE_VER}/node-${NODE_VER}-linux-${NODE_ARCH}-musl.tar.gz"; '
156                    "  else "
157                    '    NODE_URL="https://nodejs.org/dist/${NODE_VER}/node-${NODE_VER}-linux-${NODE_ARCH}.tar.gz"; '
158                    "  fi; "
159                    '  echo "Downloading Node.js from $NODE_URL"; '
160                    '  curl -fsSL "$NODE_URL" | tar -xz -C /usr/local --strip-components=1; '
161                    '  echo "Installed Node.js $(node --version)"; '
162                    "fi"
163                ),
164            )
165        except Exception as exc:
166            self.logger.warning("Node.js installation failed (non-fatal): %s", exc)
167
168    async def _install_lsps(self, environment: BaseEnvironment) -> None:
169        """Pre-install language servers so Zed doesn't download them at runtime.
170
171        Each LSP is installed independently so one failure doesn't block the rest.
172        """
173        # npm-based LSPs — skip all if npm is not available.
174        try:
175            await self.exec_as_agent(
176                environment,
177                command="command -v npm >/dev/null 2>&1",
178            )
179        except Exception:
180            self.logger.warning("npm not available — skipping npm-based LSP installs")
181            return
182
183        lsp_installs = [
184            (
185                "basedpyright",
186                'DIR="$ZED_DATA_DIR/languages/basedpyright"; '
187                'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact basedpyright',
188            ),
189            (
190                "typescript-language-server",
191                'DIR="$ZED_DATA_DIR/languages/typescript-language-server"; '
192                'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact typescript typescript-language-server',
193            ),
194            (
195                "vtsls",
196                'DIR="$ZED_DATA_DIR/languages/vtsls"; '
197                'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact @vtsls/language-server typescript',
198            ),
199            (
200                "tailwindcss-language-server",
201                'DIR="$ZED_DATA_DIR/languages/tailwindcss-language-server"; '
202                'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact @tailwindcss/language-server',
203            ),
204        ]
205
206        for name, cmd in lsp_installs:
207            try:
208                await self.exec_as_agent(
209                    environment,
210                    command=(
211                        'ZED_DATA_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/zed"; '
212                        + cmd
213                    ),
214                )
215            except Exception as exc:
216                self.logger.warning(
217                    "LSP install '%s' failed (non-fatal): %s", name, exc
218                )
219
220        # eslint — downloaded from GitHub and compiled separately.
221        try:
222            await self.exec_as_agent(
223                environment,
224                command=(
225                    "set -euo pipefail; "
226                    'ZED_DATA_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/zed"; '
227                    'ESLINT_DIR="$ZED_DATA_DIR/languages/eslint/vscode-eslint-2.4.4"; '
228                    'mkdir -p "$ESLINT_DIR"; '
229                    'curl -fsSL "https://github.com/zed-industries/vscode-eslint/archive/refs/tags/release/2.4.4.tar.gz" '
230                    '| tar -xz -C "$ESLINT_DIR"; '
231                    'mv "$ESLINT_DIR"/vscode-eslint-release-2.4.4 "$ESLINT_DIR/vscode-eslint"; '
232                    'cd "$ESLINT_DIR/vscode-eslint" && npm install && npm run compile'
233                ),
234            )
235        except Exception as exc:
236            self.logger.warning("eslint LSP install failed (non-fatal): %s", exc)
237
238        # gopls — only when Go is present.  Guarded by a 120s timeout so slow
239        # compilation can never eat the full setup budget.
240        gopls_script = (
241            "if command -v go >/dev/null 2>&1; then "
242            "if go install golang.org/x/tools/gopls@latest 2>/dev/null; then "
243            "echo 'Installed gopls@latest'; "
244            "else "
245            '  MY_GO=$(go env GOVERSION | sed "s/^go//"); '
246            "  for v in $(curl -fsSL "
247            "https://proxy.golang.org/golang.org/x/tools/gopls/@v/list 2>/dev/null"
248            " | grep -E '^v[0-9]+\\.[0-9]+\\.[0-9]+$' | sort -rV | head -5); do "
249            "    NEED=$(curl -fsSL "
250            '"https://proxy.golang.org/golang.org/x/tools/gopls/@v/${v}.mod"'
251            " 2>/dev/null | awk '/^go /{print $2; exit}'); "
252            '    if [ -n "$NEED" ] '
253            '    && [ "$(printf \'%s\\n%s\\n\' "$NEED" "$MY_GO" '
254            '         | sort -V | head -1)" = "$NEED" ]; then '
255            '      echo "Installing gopls $v (compatible with Go $MY_GO)"; '
256            '      go install "golang.org/x/tools/gopls@$v" && break; '
257            "    fi; "
258            "  done; "
259            "fi; "
260            "fi"
261        )
262        try:
263            await self.exec_as_agent(
264                environment,
265                command=(
266                    "timeout 120 bash -c "
267                    + shlex.quote(gopls_script)
268                    + " || echo 'WARNING: gopls installation timed out or failed -- skipping'"
269                ),
270            )
271        except Exception as exc:
272            self.logger.warning("gopls install failed (non-fatal): %s", exc)
273
274    async def _install_uv_and_ruff(self, environment: BaseEnvironment) -> None:
275        """Install uv and ruff for Python tooling."""
276        try:
277            await self.exec_as_agent(
278                environment,
279                command=(
280                    "curl -LsSf https://astral.sh/uv/install.sh | sh && "
281                    '. "$HOME/.local/bin/env"'
282                ),
283            )
284
285            agent_home_result = await self.exec_as_agent(
286                environment,
287                command='printf %s "$HOME"',
288            )
289            agent_home = agent_home_result.stdout.strip()
290            if not agent_home:
291                self.logger.warning(
292                    "Could not determine agent home directory — skipping uv symlinks"
293                )
294                return
295
296            await self.exec_as_root(
297                environment,
298                command=(
299                    f"ln -sf {shlex.quote(agent_home + '/.local/bin/uv')} /usr/local/bin/uv && "
300                    f"ln -sf {shlex.quote(agent_home + '/.local/bin/uvx')} /usr/local/bin/uvx"
301                ),
302            )
303
304            await self.exec_as_agent(
305                environment,
306                command='export PATH="$HOME/.local/bin:$PATH" && uv tool install ruff',
307            )
308        except Exception as exc:
309            self.logger.warning("uv/ruff installation failed (non-fatal): %s", exc)
310
311    def populate_context_post_run(self, context: AgentContext) -> None:
312        populate_context_from_result(self.logs_dir, context, self.logger)
313
314    def _get_api_env(self) -> dict[str, str]:
315        env = provider_api_env(self.model_name)
316        add_openai_compatible_provider_env(
317            env, self._extra_env.get("ZED_OPENAI_COMPATIBLE_PROVIDERS")
318        )
319        add_anthropic_available_models_env(
320            env, self._extra_env.get("ZED_ANTHROPIC_AVAILABLE_MODELS")
321        )
322        return env
323
324    def _harbor_agent_budget_sec(self) -> float | None:
325        """Recover Harbor's hidden outer timeout so eval-cli can exit first.
326
327        Harbor 0.15.0 computes this on ``Trial`` but does not pass it to custom
328        installed agents. Replaying the calculation host-side lets us preserve
329        trajectories instead of losing them to Harbor's coroutine kill.
330        """
331        config_path = Path(self.logs_dir).parent / "config.json"
332        try:
333            trial_config = json.loads(config_path.read_text())
334        except (OSError, ValueError) as exc:
335            print(
336                f"[zed-eval] agent budget: could not read {config_path}: {exc!r}",
337                flush=True,
338            )
339            return None
340
341        agent_cfg = trial_config.get("agent") or {}
342        base_sec = agent_cfg.get("override_timeout_sec")
343        if base_sec is None:
344            base_sec = self._task_declared_agent_timeout_sec(
345                trial_config.get("task") or {}
346            )
347        if base_sec is None:
348            return None
349
350        max_sec = agent_cfg.get("max_timeout_sec")
351        if max_sec is not None:
352            base_sec = min(base_sec, max_sec)
353        multiplier = trial_config.get("agent_timeout_multiplier")
354        if multiplier is None:
355            multiplier = trial_config.get("timeout_multiplier", 1.0)
356        return float(base_sec) * float(multiplier)
357
358    def _task_declared_agent_timeout_sec(self, task_cfg: dict) -> float | None:
359        task_dir = self._resolve_task_dir(task_cfg)
360        if task_dir is None:
361            return None
362        task_toml = task_dir / "task.toml"
363        try:
364            data = tomllib.loads(task_toml.read_text())
365        except (OSError, ValueError) as exc:
366            print(
367                f"[zed-eval] agent budget: could not parse {task_toml}: {exc!r}",
368                flush=True,
369            )
370            return None
371        timeout_sec = (data.get("agent") or {}).get("timeout_sec")
372        return float(timeout_sec) if timeout_sec is not None else None
373
374    def _resolve_task_dir(self, task_cfg: dict) -> Path | None:
375        path = task_cfg.get("path")
376        if path:
377            return Path(path).expanduser().resolve()
378
379        # config.json omits Harbor's package content hash; use the newest cached
380        # digest that contains a task.toml.
381        name = task_cfg.get("name")
382        if name and "/" in name:
383            try:
384                from harbor.constants import PACKAGE_CACHE_DIR
385            except ImportError as exc:
386                print(
387                    f"[zed-eval] agent budget: harbor PACKAGE_CACHE_DIR "
388                    f"unavailable: {exc!r}",
389                    flush=True,
390                )
391                return None
392            download_dir = task_cfg.get("download_dir")
393            base_dir = Path(download_dir) if download_dir else PACKAGE_CACHE_DIR
394            org, package = name.split("/", 1)
395            package_root = base_dir / org / package
396            candidates = [
397                digest_dir
398                for digest_dir in package_root.glob("*")
399                if (digest_dir / "task.toml").is_file()
400            ]
401            if not candidates:
402                return None
403            return max(candidates, key=lambda p: p.stat().st_mtime)
404
405        return None
406
407    @with_prompt_template
408    async def run(
409        self, instruction: str, environment: BaseEnvironment, context: AgentContext
410    ) -> None:
411        escaped_instruction = shlex.quote(instruction)
412        env = self._get_api_env()
413
414        add_zed_eval_env(
415            env, self._extra_env, exclude={"ZED_EVAL_INSTRUCTION_SUFFIX_FILE"}
416        )
417
418        workdir = await self._detect_workdir(environment)
419
420        parts = [
421            "eval-cli",
422            f"--workdir {shlex.quote(workdir)}",
423            "--output-dir /logs/agent",
424        ]
425
426        if self.model_name:
427            parts.append(f"--model {shlex.quote(self.model_name)}")
428
429        instruction_suffix_file = self._extra_env.get(
430            "ZED_EVAL_INSTRUCTION_SUFFIX_FILE"
431        )
432        if instruction_suffix_file:
433            parts.append(
434                f"--instruction-suffix-file {shlex.quote(instruction_suffix_file)}"
435            )
436
437        # Prefer eval-cli's own timeout over Harbor's coroutine kill so logs and
438        # partial answers are still available for delivery.
439        configured_timeout_raw = self._extra_env.get("EVAL_CLI_TIMEOUT")
440        configured_timeout: int | None
441        if configured_timeout_raw:
442            try:
443                configured_timeout = int(configured_timeout_raw)
444            except ValueError:
445                print(
446                    "[zed-eval] ignoring non-integer EVAL_CLI_TIMEOUT="
447                    f"{configured_timeout_raw!r}",
448                    flush=True,
449                )
450                configured_timeout = None
451        else:
452            configured_timeout = None
453
454        budget_sec = self._harbor_agent_budget_sec()
455        timeout_arg: int | None = None
456        if budget_sec is not None:
457            ceiling = (
458                configured_timeout
459                if configured_timeout is not None
460                else int(budget_sec)
461            )
462            timeout_arg = min(ceiling, int(budget_sec - EVAL_CLI_FINALIZE_BUFFER_SEC))
463            timeout_arg = max(timeout_arg, EVAL_CLI_MIN_TIMEOUT_SEC)
464            print(
465                f"[zed-eval] eval-cli --timeout {timeout_arg}s (harbor agent "
466                f"budget {int(budget_sec)}s - {EVAL_CLI_FINALIZE_BUFFER_SEC}s "
467                f"finalize buffer; EVAL_CLI_TIMEOUT="
468                f"{configured_timeout if configured_timeout is not None else 'unset'})",
469                flush=True,
470            )
471        elif configured_timeout is not None:
472            timeout_arg = configured_timeout
473            print(
474                f"[zed-eval] eval-cli --timeout {timeout_arg}s (harbor agent "
475                "budget unavailable; using EVAL_CLI_TIMEOUT)",
476                flush=True,
477            )
478
479        if timeout_arg is not None:
480            parts.append(f"--timeout {timeout_arg}")
481
482        staff = self._extra_env.get("EVAL_CLI_STAFF")
483        if staff and staff.lower() == "false":
484            parts.append("--no-staff")
485
486        reasoning_effort = self._extra_env.get("EVAL_CLI_REASONING_EFFORT")
487        if reasoning_effort:
488            parts.append(f"--reasoning-effort {shlex.quote(reasoning_effort)}")
489
490        enable_thinking = self._extra_env.get("EVAL_CLI_ENABLE_THINKING")
491        if enable_thinking:
492            if enable_thinking.lower() == "true":
493                parts.append("--thinking true")
494            elif enable_thinking.lower() == "false":
495                parts.append("--thinking false")
496
497        parts.append(f"--instruction {escaped_instruction}")
498
499        # Exit 2 is eval-cli's timeout, not a crash; keep the trajectory judgeable.
500        await self.exec_as_agent(
501            environment,
502            command=eval_cli_with_log_command(
503                parts,
504                "/logs/agent/eval-cli.txt",
505                timeout_message=(
506                    "[zed-eval] eval-cli timed out (exit 2); continuing to "
507                    "delivery/verification"
508                ),
509                line_buffered=True,
510            ),
511            env=env,
512        )
513
514        # Modal-style remote runs can differ in agent-log collection behavior.
515        await self.exec_as_agent(
516            environment,
517            command=(
518                "mkdir -p /logs/artifacts && "
519                "for f in result.json thread.json thread.md eval-cli.txt; do "
520                '  cp "/logs/agent/$f" /logs/artifacts/ 2>/dev/null || true; '
521                "done; true"
522            ),
523        )
524
525        # Some harnesses mount an initialized repo before creating the first commit.
526        await self.exec_as_agent(
527            environment,
528            command=patch_command("/logs/agent"),
529            cwd=workdir,
530        )
531