*.tgz

__pycache__/

# Bench: the Gym's harness lane

`bench/adapters/openagents_coder.py` is the Harbor installed-agent adapter

for `openagents coder` (OpenAgentsInc/openagents#35). It runs the working

tree's CLI, not the published npm version, so pack a tarball first:

```sh

cd packages/openagents-cli

pnpm build

pnpm pack --pack-destination ../../bench

```

Use `pnpm pack`, not `npm pack`: the manifest carries pnpm `catalog:`

versions that only `pnpm pack` rewrites into versions npm can install.

Run against the dev forge on the host (`PHX_LISTEN_ALL=true mix

phx.server` in openagents.com, plus a `chat:account` token):

```sh

PYTHONPATH=bench OPENAGENTS_TOKEN=... harbor run \

  --dataset terminal-bench@2.0 \

  --agent-import-path adapters.openagents_coder:OpenAgentsCoder \

  -m openai/gpt-5.6-luna \

  -i fix-git --n-concurrent 1 --jobs-dir /tmp/gym-jobs

```

`bench/post_gym_run.py` posts a completed job's graded result to

`POST /api/v3/gym/runs`. Post only runs whose verifier actually ran: a

score is a claim, and a crashed grader is not a grade.

Known constraint on Apple Silicon: Terminal-Bench task images are amd64,

and under qemu emulation the verifier's `uv`/`pytest` segfaults after the

agent phase completes. Enable Docker Desktop's Rosetta emulation

(Settings → General → "Use Rosetta for x86_64/amd64 emulation"), use a

cloud environment (`--env daytona` and peers), or run on amd64 hardware

for scored runs. The agent phase itself runs fine under qemu.

"""Harbor installed-agent adapter for `openagents coder`.

OpenAgentsInc/openagents#35, first slice. Loads out-of-tree by import path:

    PYTHONPATH=bench harbor run \

      --dataset terminal-bench@2.0 \

      --agent adapters.openagents_coder:OpenAgentsCoder \

      --model openai/gpt-5.6-luna \

      --n-concurrent 1

The coder runs headless (`--plain`) on its thread lane against an

OpenAgents server: `OPENAGENTS_TOKEN` carries authority and

`OPENAGENTS_CODER_API_URL` names the server (default

`http://host.docker.internal:4000`, the dev forge on the container's host,

which needs `PHX_LISTEN_ALL=true`). The server holds the provider keys; the

container holds only the scoped OpenAgents token. Harbor's `--model`

provider prefix is dropped: `openai/gpt-5.6-luna` becomes catalog id

`gpt-5.6-luna` on `POST /api/v3/threads`.

The CLI installs from a tarball packed beside this file (`npm pack` in

`packages/openagents-cli`), so the run measures the working tree, not the

published npm version. The trajectory is the coder's own ATIF export: the

run pipes `/export` after the instruction and copies the newest export to

the trial's `trajectory.json`.

"""

import os

import shlex

from pathlib import Path

from typing import override

from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template

from harbor.environments.base import BaseEnvironment

from harbor.models.agent.context import AgentContext

_TARBALL = Path(__file__).resolve().parent.parent / "openagentsinc-cli-0.3.5.tgz"

_REMOTE_TARBALL = "/installed-agent/openagents-cli.tgz"

_DEFAULT_API_URL = "http://host.docker.internal:4000"

_EXPORT_DIR = "$HOME/.openagents/exports"

class OpenAgentsCoder(BaseInstalledAgent):

    SUPPORTS_ATIF = True

    @staticmethod

    @override

    def name() -> str:

        return "openagents-coder"

    @override

    def get_version_command(self) -> str | None:

        return "openagents --version"

    @property

    def _api_url(self) -> str:

        return os.environ.get("OPENAGENTS_CODER_API_URL", _DEFAULT_API_URL)

    @property

    def _token(self) -> str:

        token = os.environ.get("OPENAGENTS_TOKEN", "")

        if not token:

            raise ValueError(

                "OPENAGENTS_TOKEN is not set. The coder's thread lane needs a "

                "chat:account token for the server at OPENAGENTS_CODER_API_URL."

            )

        return token

    @property

    def _catalog_model(self) -> str | None:

        if not self.model_name:

            return None

        # Harbor spells models provider/name; the catalog id is the name.

        return self.model_name.split("/", 1)[-1]

    @override

    async def install(self, environment: BaseEnvironment) -> None:

        if not _TARBALL.exists():

            raise FileNotFoundError(

                f"CLI tarball missing at {_TARBALL}. Run `pnpm build && npm pack "

                "--pack-destination ../../bench` in packages/openagents-cli first."

            )

        await self.ensure_system_dependencies(environment, ("curl", "bash"))

        # Node >= 20 (the CLI's engines floor). Task images rarely carry it,

        # so provision Node 22 through NodeSource on apt images and fail

        # loudly elsewhere rather than running on a node that cannot.

        await self.exec_as_root(

            environment,

            command=(

                "set -euo pipefail; "

                "if command -v node >/dev/null 2>&1 && "

                '[ "$(node -e \'process.stdout.write(String(process.versions.node.split(".")[0]))\')" -ge 20 ]; '

                "then echo 'node present'; "

                "elif command -v apt-get >/dev/null 2>&1; then "

                "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && "

                "apt-get install -y nodejs; "

                "else echo 'no node >= 20 and no apt-get' >&2; exit 1; fi; "

                "node --version"

            ),

        )

        await environment.upload_file(_TARBALL, _REMOTE_TARBALL)

        await self.exec_as_root(

            environment,

            command=f"npm install -g {shlex.quote(_REMOTE_TARBALL)} && openagents --version",

        )

    @override

    @with_prompt_template

    async def run(

        self,

        instruction: str,

        environment: BaseEnvironment,

        context: AgentContext,

    ) -> None:

        model_flag = ""

        if self._catalog_model:

            model_flag = f"--model {shlex.quote(self._catalog_model)} "

        # The instruction goes down stdin, then /export so the session writes

        # its ATIF trajectory before exiting on end-of-file. stdin carries the

        # instruction verbatim; nothing here re-quotes its content.

        command = (

            "set -uo pipefail; "

            f"printf '%s\\n/export\\n' {shlex.quote(instruction)} | "

            "openagents coder --plain "

            f"--api-url {shlex.quote(self._api_url)} "

            f"{model_flag}"

            f"2>&1 | tee {shlex.quote(str(self.environment_logs_dir))}/coder.txt; "

            "status=$?; "

            f"latest=$(ls -t {_EXPORT_DIR}/*.json 2>/dev/null | head -1 || true); "

            'if [ -n "$latest" ]; then '

            f'cp "$latest" {shlex.quote(str(self.environment_logs_dir))}/trajectory.json; '

            "fi; "

            "exit $status"

        )

        await self.exec_as_agent(

            environment,

            command=command,

            env={"OPENAGENTS_TOKEN": self._token},

        )

"""Post one Harbor job's graded result to the Gym.

Usage:

    OPENAGENTS_TOKEN=... python3 bench/post_gym_run.py \

      <job-dir> [--api-url http://localhost:4000] [--lane proxy]

Reads the job's `result.json` and each trial's ATIF `trajectory.json` (for

token totals), derives the recipe digest from the job's `config.json` plus

the agent tarball digest, and POSTs to `/api/v3/gym/runs`. Idempotent: the

server replays a repeated digest rather than duplicating the row.

"""

import argparse

import hashlib

import json

import os

import sys

import urllib.request

from pathlib import Path

def sha256_file(path: Path) -> str:

    digest = hashlib.sha256()

    with open(path, "rb") as handle:

        for chunk in iter(lambda: handle.read(65536), b""):

            digest.update(chunk)

    return digest.hexdigest()

def main() -> int:

    parser = argparse.ArgumentParser()

    parser.add_argument("job_dir", type=Path)

    parser.add_argument("--api-url", default="http://localhost:4000")

    parser.add_argument("--lane", default="proxy")

    parser.add_argument("--suite", default="terminal-bench@2.0")

    arguments = parser.parse_args()

    token = os.environ.get("OPENAGENTS_TOKEN", "")

    if not token:

        print("OPENAGENTS_TOKEN is not set", file=sys.stderr)

        return 2

    job_dir = arguments.job_dir

    result = json.loads((job_dir / "result.json").read_text())

    config = json.loads((job_dir / "config.json").read_text())

    trials = []

    input_tokens = 0

    output_tokens = 0

    agent_version = None

    model = None

    for trial_dir in sorted(job_dir.iterdir()):

        trial_result = trial_dir / "result.json"

        if not trial_dir.is_dir() or not trial_result.exists():

            continue

        trial = json.loads(trial_result.read_text())

        trials.append(trial)

        trajectory_path = trial_dir / "agent" / "trajectory.json"

        if trajectory_path.exists():

            trajectory = json.loads(trajectory_path.read_text())

            metrics = trajectory.get("final_metrics") or {}

            input_tokens += metrics.get("total_prompt_tokens") or 0

            output_tokens += metrics.get("total_completion_tokens") or 0

            agent = trajectory.get("agent") or {}

            agent_version = agent.get("version") or agent_version

            model = agent.get("model_name") or model

    def passed(trial: dict) -> bool:

        verifier = trial.get("verifier_result") or {}

        rewards = verifier.get("rewards") or trial.get("rewards") or {}

        reward = rewards.get("reward")

        if reward is None and isinstance(rewards, dict) and rewards:

            reward = next(iter(rewards.values()))

        return bool(reward) and float(reward) > 0

    tasks_total = len(trials)

    tasks_passed = sum(1 for trial in trials if passed(trial))

    recipe_source = json.dumps(

        {

            "config": config,

            "suite": arguments.suite,

            "lane": arguments.lane,

            "job_id": result.get("id"),

        },

        sort_keys=True,

    ).encode()

    recipe_digest = "harbor:" + hashlib.sha256(recipe_source).hexdigest()

    payload = {

        "suite": arguments.suite,

        "agent": "openagents-coder",

        "agent_version": agent_version,

        "model": model or "unknown",

        "lane": arguments.lane,

        "tasks_total": tasks_total,

        "tasks_passed": tasks_passed,

        "input_tokens": input_tokens or None,

        "output_tokens": output_tokens or None,

        "recipe_digest": recipe_digest,

        "report": {

            "job_id": result.get("id"),

            "trials": [

                {

                    "task": (trial.get("task_name") or trial.get("trial_name") or "?"),

                    "passed": passed(trial),

                    "exception": (trial.get("exception_info") or {}).get("exception_type")

                    if trial.get("exception_info")

                    else None,

                }

                for trial in trials

            ],

        },

    }

    request = urllib.request.Request(

        f"{arguments.api_url}/api/v3/gym/runs",

        data=json.dumps(payload).encode(),

        headers={

            "Content-Type": "application/json",

            "Authorization": f"Bearer {token}",

        },

        method="POST",

    )

    with urllib.request.urlopen(request) as response:

        body = json.loads(response.read())

        run = body.get("run") or {}

        print(

            f"{response.status}: run {run.get('id')} score={run.get('score')} "

            f"replayed={body.get('replayed')}"

        )

    return 0

if __name__ == "__main__":

    raise SystemExit(main())

    accountToken: options.token,

    threadId: id,

    grantToken: Redacted.make(token),

    proxyUrl: url,

    proxyUrl: resolveProxyUrl(url, options.origin),

    model,

    budget: budgetOf(record(grant["limits"]), record(grant["limits"])),

  });

    accountToken: options.token,

    threadId: string(record(body["thread"])["id"]) ?? options.threadId,

    grantToken: Redacted.make(token),

    proxyUrl: url,

    proxyUrl: resolveProxyUrl(url, options.origin),

    model,

    budget: budgetOf(record(grant["remaining"]), record(grant["limits"])),

  });

  return `$${(microusd / 1_000_000).toFixed(2)}`;

}

/**

 * The grant names the proxy with the server's own idea of its host, and

 * inside a container the server's "localhost" is the wrong machine. The

 * path is the server's contract; the origin is the client's — the same one

 * it authenticated against — so the URL resolves against it.

 */

export const resolveProxyUrl = (grantUrl: string, origin: string): string => {

  try {

    const named = new URL(grantUrl);

    return new URL(named.pathname, origin).toString();

  } catch {

    return grantUrl;

  }

};

function record(value: unknown): Record<string, unknown> {

  return typeof value === "object" && value !== null && !Array.isArray(value)

    ? (value as Record<string, unknown>)

    : {};

import { afterEach, describe, expect, it, vi } from "vitest";

import type { ReplyChunk } from "../src/coder-session.js";

import { openThread, ThreadUnavailable, type ThreadReplySource } from "../src/coder-thread.js";

import { openThread, resolveProxyUrl, ThreadUnavailable, type ThreadReplySource } from "../src/coder-thread.js";

import { ThreadTranscriptWriter } from "../src/coder-transcript.js";

const ORIGIN = "https://openagents.test";

    expect(shown).not.toContain("composed by the server");

  });

});

describe("resolveProxyUrl", () => {

  it("resolves the grant's path against the client's origin", () => {

    expect(

      resolveProxyUrl("http://localhost:4000/api/inference/proxy", "http://host.docker.internal:4000"),

    ).toBe("http://host.docker.internal:4000/api/inference/proxy");

  });

  it("keeps a same-origin grant URL intact", () => {

    expect(

      resolveProxyUrl("https://openagents.com/api/inference/proxy", "https://openagents.com"),

    ).toBe("https://openagents.com/api/inference/proxy");

  });

});