Open the Gym: an operator scoreboard for graded agent runs

310dac16777d · AtlantisPleb · · parent 5d6f30c758dc

Open the Gym: an operator scoreboard for graded agent runs

The Gym framing (docs/2026-08-24-harbor-terminal-bench-plan.md): Harbor
and Terminal-Bench are the training ground, not only the measurement —
new coder capabilities, model-swapping policies, and plugins prove
themselves against graded suites, and this surface is the scoreboard
they are read against.

A gym_runs row records one graded run: suite, agent and version, model,
lane, task counts, tokens, cost, duration, and the digest of the exact
recipe that produced it — unique, so a retried upload replays rather
than double-counting a trend line. POST /api/v3/gym/runs ingests on an
ordinary forge:write bearer plus live operator standing rechecked per
request (the fleet-promotion shape without a privileged scope, because
recording a benchmark row moves no money and deploys nothing). /gym
renders the runs behind the operator pipeline with mount and per-event
rechecks, and the sidebar shows the Gym row under Projects to operators
only: the operator allowlist is the whitelist, deliberately, rather
than a second gating mechanism.

Route authority, API route authority, export inventory (:gym is
:not_user_data — aggregate measurement of the product, no record an
account authors), and the migration lineage all classified in the same
change. 15 new tests plus the guard suites, 99 passing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GoYpb8FEmdxVErsv7ABCYi
Co-Authored-By
Claude Fable 5 <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

pushed
by user · WAL seq 320 · 2026-08-24T23:15:33.803812Z

Changed files

  • modified lib/openagents/data_rights/export_inventory.ex
  • added lib/openagents/gym.ex
  • added lib/openagents/gym/run.ex
  • modified lib/openagents_web/api_route_authority.ex
  • modified lib/openagents_web/components/layouts.ex
  • added lib/openagents_web/controllers/gym_run_controller.ex
  • added lib/openagents_web/live/gym_live.ex
  • modified lib/openagents_web/route_authority.ex
  • modified lib/openagents_web/router.ex
  • modified priv/migration_lineages/prior-2026-08-19.json
  • added priv/repo/migrations/20260824230730_create_gym_runs.exs
  • added test/openagents/gym_test.exs
  • added test/openagents_web/controllers/gym_run_controller_test.exs
  • added test/openagents_web/live/gym_live_test.exs

Diff

14 files changed, +771 -1

lib/openagents/data_rights/export_inventory.ex modified +12

@@ -363,6 +363,18 @@ defmodule OpenAgents.DataRights.ExportInventory do

363 363
          "ceilings — and carries no record an account authors. What an " <>
364 364
          "account did with a model is its threads, which export whole."
365 365
    },
366
    %{
367
      family: :gym,
368
      api?: true,
369
      status: :not_user_data,
370
      mechanism: nil,
371
      proof: nil,
372
      issue: nil,
373
      note:
374
        "Operator benchmark rows: graded runs of our own agents against " <>
375
          "task suites, posted by the bench harness. Aggregate measurement " <>
376
          "of the product, carrying no record an account authors."
377
    },
366 378
    %{
367 379
      family: :capacity,
368 380
      api?: true,
lib/openagents/gym.ex added +88

@@ -0,0 +1,88 @@

1
defmodule OpenAgents.Gym do
2
  @moduledoc """
3
  The Gym: graded benchmark runs of our agents, recorded so capability work
4
  has a scoreboard.
5
6
  The framing (`docs/2026-08-24-harbor-terminal-bench-plan.md`): Harbor and
7
  Terminal-Bench are not only measurement, they are the training ground —
8
  new coder capabilities, model-swapping policies, and plugins prove
9
  themselves against graded task suites, and this context holds the results
10
  those proofs produce. The harness runs elsewhere (the monorepo's bench
11
  lane); this is the record and the surface.
12
13
  Operator-only on every path for now: the Gym is a workbench for the
14
  people building the agent, not a public leaderboard. Widening it later is
15
  a deliberate act, not a default.
16
  """
17
18
  import Ecto.Query
19
20
  alias OpenAgents.Gym.Run
21
  alias OpenAgents.Repo
22
23
  @maximum_listed 200
24
25
  @doc """
26
  Record one run, idempotently by recipe digest.
27
28
  A resubmitted digest returns the existing row as `{:ok, run, replayed?:
29
  true}` rather than duplicating or refusing: the harness retries uploads,
30
  and a retry is not a second run.
31
  """
32
  @spec record_run(map()) :: {:ok, Run.t(), boolean()} | {:error, Ecto.Changeset.t()}
33
  def record_run(attributes) when is_map(attributes) do
34
    changeset = Run.changeset(%Run{}, attributes)
35
36
    case Repo.insert(changeset) do
37
      {:ok, run} ->
38
        {:ok, run, false}
39
40
      {:error, %Ecto.Changeset{errors: errors} = failed} ->
41
        case Keyword.get(errors, :recipe_digest) do
42
          {_message, options} ->
43
            if options[:constraint] == :unique,
44
              do: replay(Ecto.Changeset.get_field(changeset, :recipe_digest), failed),
45
              else: {:error, failed}
46
47
          nil ->
48
            {:error, failed}
49
        end
50
    end
51
  end
52
53
  @doc "Runs, newest first, optionally filtered by suite. Bounded."
54
  @spec list_runs(keyword()) :: [Run.t()]
55
  def list_runs(options \\ []) do
56
    limit = options |> Keyword.get(:limit, 50) |> min(@maximum_listed) |> max(1)
57
58
    Run
59
    |> filter_suite(options[:suite])
60
    |> order_by(desc: :inserted_at)
61
    |> limit(^limit)
62
    |> Repo.all()
63
  end
64
65
  @doc "Distinct suites present, for the surface's filter row."
66
  @spec suites() :: [String.t()]
67
  def suites do
68
    Run
69
    |> distinct(true)
70
    |> select([r], r.suite)
71
    |> order_by(asc: :suite)
72
    |> Repo.all()
73
  end
74
75
  defp filter_suite(query, suite) when is_binary(suite) and suite != "",
76
    do: where(query, [r], r.suite == ^suite)
77
78
  defp filter_suite(query, _absent), do: query
79
80
  defp replay(digest, failed) when is_binary(digest) do
81
    case Repo.get_by(Run, recipe_digest: digest) do
82
      %Run{} = run -> {:ok, run, true}
83
      nil -> {:error, failed}
84
    end
85
  end
86
87
  defp replay(_digest, failed), do: {:error, failed}
88
end
lib/openagents/gym/run.ex added +108

@@ -0,0 +1,108 @@

1
defmodule OpenAgents.Gym.Run do
2
  @moduledoc """
3
  One graded benchmark run of an agent against a suite.
4
5
  A run is a record of measurement, never of execution: the Harbor harness
6
  (`docs/2026-08-24-harbor-terminal-bench-plan.md`) runs the trials and this
7
  row holds what came back — how many tasks the suite graded, how many
8
  passed, what it cost, and the digest of the exact recipe (CLI version,
9
  model catalog revision, plugin set, dataset version) that produced it.
10
11
  `recipe_digest` is unique: submitting the same run twice replays the first
12
  row rather than duplicating it, so a trend line never counts a run twice.
13
  The bounded `report` map carries per-task rows and anything else the
14
  harness wants to keep beside the headline numbers; it is data about the
15
  run, not a second transcript store.
16
  """
17
18
  use Ecto.Schema
19
20
  import Ecto.Changeset
21
22
  @primary_key {:id, :binary_id, autogenerate: true}
23
24
  @bounded_fields [:suite, :agent, :model, :recipe_digest]
25
  @maximum_report_bytes 262_144
26
27
  schema "gym_runs" do
28
    field :suite, :string
29
    field :agent, :string
30
    field :agent_version, :string
31
    field :model, :string
32
    field :lane, :string
33
    field :tasks_total, :integer
34
    field :tasks_passed, :integer
35
    field :input_tokens, :integer
36
    field :output_tokens, :integer
37
    field :cost_microusd, :integer
38
    field :duration_seconds, :integer
39
    field :recipe_digest, :string
40
    field :report, :map, default: %{}
41
42
    timestamps(type: :utc_datetime_usec, updated_at: false)
43
  end
44
45
  def changeset(run, attributes) do
46
    run
47
    |> cast(attributes, [
48
      :suite,
49
      :agent,
50
      :agent_version,
51
      :model,
52
      :lane,
53
      :tasks_total,
54
      :tasks_passed,
55
      :input_tokens,
56
      :output_tokens,
57
      :cost_microusd,
58
      :duration_seconds,
59
      :recipe_digest,
60
      :report
61
    ])
62
    |> validate_required([:suite, :agent, :model, :tasks_total, :tasks_passed, :recipe_digest])
63
    |> validate_bounded_fields()
64
    |> validate_number(:tasks_total, greater_than_or_equal_to: 0)
65
    |> validate_number(:tasks_passed, greater_than_or_equal_to: 0)
66
    |> validate_passed_within_total()
67
    |> validate_report_bound()
68
    |> check_constraint(:tasks_passed, name: :gym_runs_task_counts_check)
69
    |> unique_constraint(:recipe_digest)
70
  end
71
72
  @doc "Pass rate in [0.0, 1.0]; nil for an empty suite rather than a fake 1.0."
73
  def score(%__MODULE__{tasks_total: 0}), do: nil
74
  def score(%__MODULE__{tasks_total: total, tasks_passed: passed}), do: passed / total
75
76
  defp validate_bounded_fields(changeset) do
77
    Enum.reduce(@bounded_fields, changeset, fn field, acc ->
78
      validate_length(acc, field, min: 1, max: 200, count: :bytes)
79
    end)
80
  end
81
82
  defp validate_passed_within_total(changeset) do
83
    total = get_field(changeset, :tasks_total)
84
    passed = get_field(changeset, :tasks_passed)
85
86
    if is_integer(total) and is_integer(passed) and passed > total do
87
      add_error(changeset, :tasks_passed, "cannot exceed tasks_total")
88
    else
89
      changeset
90
    end
91
  end
92
93
  defp validate_report_bound(changeset) do
94
    report = get_field(changeset, :report)
95
96
    case report do
97
      map when is_map(map) ->
98
        if byte_size(Jason.encode!(map)) > @maximum_report_bytes do
99
          add_error(changeset, :report, "is larger than #{@maximum_report_bytes} bytes")
100
        else
101
          changeset
102
        end
103
104
      _other ->
105
        add_error(changeset, :report, "must be an object")
106
    end
107
  end
108
end
lib/openagents_web/api_route_authority.ex modified +2

@@ -337,6 +337,8 @@ defmodule OpenAgentsWeb.ApiRouteAuthority do

337 337
        {:required_bearer, :issue, :envelope},
338 338
      "get /api/v3/agent" => {:required_bearer, :agent, :legacy},
339 339
      "post /api/v3/agents/:handle/box-control" => {:required_bearer, :agent, :legacy},
340
      "post /api/v3/gym/runs" => {:required_bearer, :gym, :envelope},
341
      "get /api/v3/gym/runs" => {:required_bearer, :gym, :envelope},
340 342
      "delete /api/v3/agents/:handle/box-control" => {:required_bearer, :agent, :legacy},
341 343
      "post /api/v3/agents/:handle/computer-control" => {:required_bearer, :agent, :legacy},
342 344
      "delete /api/v3/agents/:handle/computer-control" => {:required_bearer, :agent, :legacy},
lib/openagents_web/components/layouts.ex modified +11

@@ -814,6 +814,17 @@ defmodule OpenAgentsWeb.Layouts do

814 814
          icon="folder"
815 815
          patchable={false}
816 816
        />
817
        <%!-- Operator-only: the Gym is a workbench for the people building
818
        the agent, not a public leaderboard yet. The operator allowlist IS
819
        the whitelist, deliberately, rather than a second gating
820
        mechanism. --%>
821
        <Layouts.sidebar_link
822
          :if={@operator?}
823
          path={~p"/gym"}
824
          label="Gym"
825
          icon="dumbbell"
826
          patchable={false}
827
        />
817 828
        <Layouts.sidebar_link
818 829
          path={~p"/artifact-catalog"}
819 830
          label="Artifact catalog"
lib/openagents_web/controllers/gym_run_controller.ex added +72

@@ -0,0 +1,72 @@

1
defmodule OpenAgentsWeb.GymRunController do
2
  @moduledoc """
3
  The door the bench harness posts graded runs through.
4
5
  Authority is the fleet-promotion shape without the privileged scope: an
6
  ordinary `forge:write` bearer plus live operator standing, rechecked on
7
  every request through `OpenAgents.Accounts.admin?/1`. Recording a
8
  benchmark row is operator work, but it moves no money and deploys
9
  nothing, so it does not need a scope of its own the way promotion does —
10
  the recheck, not the scope, is what keeps it operator-only.
11
12
  Idempotent by recipe digest: a retried upload answers `200` with the
13
  existing row where the first answered `201`, so harness retry policy
14
  needs no special casing.
15
  """
16
17
  use OpenAgentsWeb, :controller
18
19
  alias OpenAgents.Accounts
20
  alias OpenAgents.Gym
21
  alias OpenAgents.Gym.Run
22
  alias OpenAgentsWeb.ApiError
23
24
  def create(conn, params) do
25
    with :ok <- operator(conn) do
26
      case Gym.record_run(params) do
27
        {:ok, run, replayed?} ->
28
          conn
29
          |> put_status(if(replayed?, do: :ok, else: :created))
30
          |> json(%{"run" => run_view(run), "replayed" => replayed?})
31
32
        {:error, changeset} ->
33
          ApiError.changeset(conn, changeset)
34
      end
35
    end
36
  end
37
38
  def index(conn, params) do
39
    with :ok <- operator(conn) do
40
      runs = Gym.list_runs(suite: params["suite"])
41
      json(conn, %{"runs" => Enum.map(runs, &run_view/1)})
42
    end
43
  end
44
45
  defp operator(conn) do
46
    if Accounts.admin?(conn.assigns.current_user) do
47
      :ok
48
    else
49
      ApiError.refuse(conn, "not_operator")
50
    end
51
  end
52
53
  defp run_view(%Run{} = run) do
54
    %{
55
      "id" => run.id,
56
      "suite" => run.suite,
57
      "agent" => run.agent,
58
      "agent_version" => run.agent_version,
59
      "model" => run.model,
60
      "lane" => run.lane,
61
      "tasks_total" => run.tasks_total,
62
      "tasks_passed" => run.tasks_passed,
63
      "score" => Run.score(run),
64
      "input_tokens" => run.input_tokens,
65
      "output_tokens" => run.output_tokens,
66
      "cost_microusd" => run.cost_microusd,
67
      "duration_seconds" => run.duration_seconds,
68
      "recipe_digest" => run.recipe_digest,
69
      "recorded_at" => run.inserted_at
70
    }
71
  end
72
end
lib/openagents_web/live/gym_live.ex added +142

@@ -0,0 +1,142 @@

1
defmodule OpenAgentsWeb.GymLive do
2
  @moduledoc """
3
  The Gym: graded benchmark runs of our agents, operator-only.
4
5
  Read-only over `OpenAgents.Gym` — the harness runs elsewhere and posts
6
  results through `POST /api/v3/gym/runs`; this surface is the scoreboard
7
  that capability work (models, plugins, harness changes) is read against.
8
  Operator-gated the same way `/chat` is: the route sits behind the
9
  `:operator` pipeline, the mount re-checks, and every event re-checks,
10
  because a long-lived socket outlives the decision that opened it.
11
  """
12
13
  use OpenAgentsWeb, :live_view
14
15
  alias OpenAgents.Accounts
16
  alias OpenAgents.Gym
17
  alias OpenAgents.Gym.Run
18
19
  @impl true
20
  def mount(_params, _session, socket) do
21
    if Accounts.admin?(socket.assigns.current_user) do
22
      {:ok, load(socket, nil)}
23
    else
24
      {:ok, redirect(socket, to: ~p"/")}
25
    end
26
  end
27
28
  @impl true
29
  def handle_event("filter", %{"suite" => suite}, socket) do
30
    if Accounts.admin?(socket.assigns.current_user) do
31
      {:noreply, load(socket, presence(suite))}
32
    else
33
      {:noreply, redirect(socket, to: ~p"/")}
34
    end
35
  end
36
37
  defp presence(""), do: nil
38
  defp presence(suite) when is_binary(suite), do: suite
39
40
  defp load(socket, suite) do
41
    runs = Gym.list_runs(suite: suite)
42
43
    socket
44
    |> assign(:page_title, "Gym")
45
    |> assign(:suite, suite)
46
    |> assign(:suites, Gym.suites())
47
    |> assign(:runs_empty?, runs == [])
48
    |> stream(:runs, runs, reset: true)
49
  end
50
51
  defp percent(nil), do: "—"
52
  defp percent(score), do: "#{Float.round(score * 100, 1)}%"
53
54
  defp dollars(nil), do: "—"
55
  defp dollars(microusd), do: "$#{Float.round(microusd / 1_000_000, 4)}"
56
57
  defp tokens(nil, nil), do: "—"
58
  defp tokens(input, output), do: "#{format_count(input)} in / #{format_count(output)} out"
59
60
  defp format_count(nil), do: "—"
61
  defp format_count(count) when count >= 1_000_000, do: "#{Float.round(count / 1_000_000, 1)}M"
62
  defp format_count(count) when count >= 1_000, do: "#{Float.round(count / 1_000, 1)}k"
63
  defp format_count(count), do: Integer.to_string(count)
64
65
  @impl true
66
  def render(assigns) do
67
    ~H"""
68
    <Layouts.app
69
      flash={@flash}
70
      sidebar_sections={assigns[:sidebar_sections]}
71
      current_scope={@current_scope}
72
    >
73
      <div class="mx-auto max-w-6xl space-y-6">
74
        <.header>
75
          Gym
76
          <:subtitle>
77
            Graded benchmark runs, newest first. The harness posts results;
78
            this page reads them. Operator-only.
79
          </:subtitle>
80
        </.header>
81
82
        <form id="gym-suite-filter" phx-change="filter">
83
          <select name="suite" class="select" data-size="sm">
84
            <option value="" selected={@suite == nil}>All suites</option>
85
            <option :for={suite <- @suites} value={suite} selected={@suite == suite}>
86
              {suite}
87
            </option>
88
          </select>
89
        </form>
90
91
        <div :if={@runs_empty?}>
92
          <.empty title="No runs recorded yet">
93
            No graded runs have been posted. The harness records one with
94
            <code>POST /api/v3/gym/runs</code>
95
            — see <code>docs/2026-08-24-harbor-terminal-bench-plan.md</code>.
96
          </.empty>
97
        </div>
98
99
        <div :if={!@runs_empty?} class="overflow-x-auto">
100
          <table class="table">
101
            <thead>
102
              <tr>
103
                <th>Recorded</th>
104
                <th>Suite</th>
105
                <th>Agent</th>
106
                <th>Model</th>
107
                <th>Lane</th>
108
                <th>Score</th>
109
                <th>Tasks</th>
110
                <th>Tokens</th>
111
                <th>Cost</th>
112
              </tr>
113
            </thead>
114
            <tbody id="gym-runs" phx-update="stream">
115
              <tr :for={{id, run} <- @streams.runs} id={id}>
116
                <td class="whitespace-nowrap">
117
                  {Calendar.strftime(run.inserted_at, "%Y-%m-%d %H:%M")}
118
                </td>
119
                <td class="font-mono text-sm">{run.suite}</td>
120
                <td>
121
                  {run.agent}
122
                  <span :if={run.agent_version} class="text-muted-foreground">
123
                    @{run.agent_version}
124
                  </span>
125
                </td>
126
                <td class="font-mono text-sm">{run.model}</td>
127
                <td>{run.lane || "—"}</td>
128
                <td class="font-semibold">{percent(Run.score(run))}</td>
129
                <td>{run.tasks_passed}/{run.tasks_total}</td>
130
                <td class="whitespace-nowrap text-sm">
131
                  {tokens(run.input_tokens, run.output_tokens)}
132
                </td>
133
                <td>{dollars(run.cost_microusd)}</td>
134
              </tr>
135
            </tbody>
136
          </table>
137
        </div>
138
      </div>
139
    </Layouts.app>
140
    """
141
  end
142
end
lib/openagents_web/route_authority.ex modified +3

@@ -189,6 +189,9 @@ defmodule OpenAgentsWeb.RouteAuthority do

189 189
  defp policy(%{path: "/chat"}),
190 190
    do: declaration(:operator, "configured operator GitHub ID", "chat:preview", false)
191 191
192
  defp policy(%{path: "/gym"}),
193
    do: declaration(:operator, "configured operator GitHub ID", "gym:read", false)
194
192 195
  defp policy(%{path: "/admin/analytics"}),
193 196
    do: declaration(:operator, "configured operator GitHub ID", "analytics:read", false)
194 197
lib/openagents_web/router.ex modified +14

@@ -264,6 +264,13 @@ defmodule OpenAgentsWeb.Router do

264 264
        {OpenAgentsWeb.UserAuth, :ensure_admin}
265 265
      ] do
266 266
      live "/chat", ChatConsoleLive, :index
267
268
      # The Gym: graded benchmark runs of our agents
269
      # (docs/2026-08-24-harbor-terminal-bench-plan.md). Operator-only for
270
      # now — the whitelist is the operator allowlist, deliberately, rather
271
      # than a second gating mechanism. Widening it is a decision, not a
272
      # default.
273
      live "/gym", GymLive, :index
267 274
    end
268 275
  end
269 276

@@ -498,6 +505,13 @@ defmodule OpenAgentsWeb.Router do

498 505
    delete "/agents/:handle/box-control", AgentController, :revoke_box_control
499 506
    post "/agents/:handle/computer-control", AgentController, :grant_computer_control
500 507
    delete "/agents/:handle/computer-control", AgentController, :revoke_computer_control
508
509
    # The Gym's ingest and read: an ordinary forge:write bearer carries the
510
    # request, and the controller rechecks live operator standing on every
511
    # call — the fleet-promotion shape without a privileged scope, because
512
    # recording a benchmark row moves no money and deploys nothing.
513
    post "/gym/runs", GymRunController, :create
514
    get "/gym/runs", GymRunController, :index
501 515
  end
502 516
503 517
  scope "/api/v3", OpenAgentsWeb do
priv/migration_lineages/prior-2026-08-19.json modified +2 -1

@@ -294,7 +294,8 @@

294 294
    20260824203139,
295 295
    20260824204740,
296 296
    20260824210500,
297
    20260824230007
297
    20260824230007,
298
    20260824230730
298 299
  ],
299 300
  "required_tables": [
300 301
    "users",
priv/repo/migrations/20260824230730_create_gym_runs.exs added +50

@@ -0,0 +1,50 @@

1
defmodule OpenAgents.Repo.Migrations.CreateGymRuns do
2
  use Ecto.Migration
3
4
  def change do
5
    create table(:gym_runs, primary_key: false) do
6
      add :id, :binary_id, primary_key: true
7
      add :suite, :text, null: false
8
      add :agent, :text, null: false
9
      add :agent_version, :text
10
      add :model, :text, null: false
11
      add :lane, :text
12
      add :tasks_total, :integer, null: false
13
      add :tasks_passed, :integer, null: false
14
      add :input_tokens, :bigint
15
      add :output_tokens, :bigint
16
      add :cost_microusd, :bigint
17
      add :duration_seconds, :integer
18
      add :recipe_digest, :text, null: false
19
      add :report, :map, null: false, default: %{}
20
21
      timestamps(type: :utc_datetime_usec, updated_at: false)
22
    end
23
24
    create constraint(:gym_runs, :gym_runs_task_counts_check,
25
             check: "tasks_total >= 0 AND tasks_passed >= 0 AND tasks_passed <= tasks_total"
26
           )
27
28
    create constraint(:gym_runs, :gym_runs_suite_bound_check,
29
             check: "octet_length(suite) BETWEEN 1 AND 200"
30
           )
31
32
    create constraint(:gym_runs, :gym_runs_agent_bound_check,
33
             check: "octet_length(agent) BETWEEN 1 AND 200"
34
           )
35
36
    create constraint(:gym_runs, :gym_runs_model_bound_check,
37
             check: "octet_length(model) BETWEEN 1 AND 200"
38
           )
39
40
    create constraint(:gym_runs, :gym_runs_recipe_digest_bound_check,
41
             check: "octet_length(recipe_digest) BETWEEN 1 AND 200"
42
           )
43
44
    # One row per exact run of an exact recipe: a re-submitted result replays
45
    # rather than duplicating, so trend lines never double-count a run.
46
    create unique_index(:gym_runs, [:recipe_digest])
47
48
    create index(:gym_runs, [:suite, :inserted_at])
49
  end
50
end
test/openagents/gym_test.exs added +81

@@ -0,0 +1,81 @@

1
defmodule OpenAgents.GymTest do
2
  use OpenAgents.DataCase, async: true
3
4
  alias OpenAgents.Gym
5
  alias OpenAgents.Gym.Run
6
7
  defp attributes(overrides \\ %{}) do
8
    Map.merge(
9
      %{
10
        "suite" => "terminal-bench@2.0",
11
        "agent" => "openagents-coder",
12
        "agent_version" => "0.3.5",
13
        "model" => "ox-alpha",
14
        "lane" => "proxy",
15
        "tasks_total" => 20,
16
        "tasks_passed" => 13,
17
        "input_tokens" => 1_200_000,
18
        "output_tokens" => 240_000,
19
        "cost_microusd" => 1_850_000,
20
        "duration_seconds" => 3_600,
21
        "recipe_digest" => "sha256:" <> String.duplicate("a", 64)
22
      },
23
      overrides
24
    )
25
  end
26
27
  test "records a run and derives its score" do
28
    assert {:ok, %Run{} = run, false} = Gym.record_run(attributes())
29
    assert run.tasks_passed == 13
30
    assert Run.score(run) == 13 / 20
31
  end
32
33
  test "an empty suite has no score rather than a perfect one" do
34
    {:ok, run, false} =
35
      Gym.record_run(attributes(%{"tasks_total" => 0, "tasks_passed" => 0}))
36
37
    assert Run.score(run) == nil
38
  end
39
40
  test "a resubmitted recipe digest replays the first row" do
41
    {:ok, first, false} = Gym.record_run(attributes())
42
43
    {:ok, second, true} =
44
      Gym.record_run(attributes(%{"tasks_passed" => 1}))
45
46
    assert second.id == first.id
47
    assert second.tasks_passed == 13
48
    assert length(Gym.list_runs()) == 1
49
  end
50
51
  test "passed cannot exceed total" do
52
    assert {:error, changeset} =
53
             Gym.record_run(attributes(%{"tasks_passed" => 21}))
54
55
    assert %{tasks_passed: [_message]} = errors_on(changeset)
56
  end
57
58
  test "listing filters by suite and bounds the page" do
59
    {:ok, _one, false} = Gym.record_run(attributes())
60
61
    {:ok, _two, false} =
62
      Gym.record_run(
63
        attributes(%{
64
          "suite" => "swebench@lite",
65
          "recipe_digest" => "sha256:" <> String.duplicate("b", 64)
66
        })
67
      )
68
69
    assert [%Run{suite: "swebench@lite"}] = Gym.list_runs(suite: "swebench@lite")
70
    assert length(Gym.list_runs()) == 2
71
    assert Gym.suites() == ["swebench@lite", "terminal-bench@2.0"]
72
    assert [_only] = Gym.list_runs(limit: 1)
73
  end
74
75
  test "an oversized report is refused" do
76
    huge = %{"rows" => String.duplicate("x", 300_000)}
77
78
    assert {:error, changeset} = Gym.record_run(attributes(%{"report" => huge}))
79
    assert %{report: [_message]} = errors_on(changeset)
80
  end
81
end
test/openagents_web/controllers/gym_run_controller_test.exs added +101

@@ -0,0 +1,101 @@

1
defmodule OpenAgentsWeb.GymRunControllerTest do
2
  @moduledoc """
3
  The Gym's ingest door authenticates a `forge:write` bearer and then
4
  rechecks live operator standing on every request. An ordinary account with
5
  the same scope is refused with a typed `not_operator`, and nothing is
6
  recorded for it.
7
  """
8
9
  use OpenAgentsWeb.ConnCase, async: false
10
11
  alias OpenAgents.Gym
12
13
  @digest "sha256:" <> String.duplicate("c", 64)
14
15
  defp payload(overrides \\ %{}) do
16
    Map.merge(
17
      %{
18
        "suite" => "terminal-bench@2.0",
19
        "agent" => "openagents-coder",
20
        "model" => "ox-alpha",
21
        "tasks_total" => 10,
22
        "tasks_passed" => 7,
23
        "recipe_digest" => @digest
24
      },
25
      overrides
26
    )
27
  end
28
29
  defp operator_token(conn, key) do
30
    user = github_user("api-token-" <> key)
31
    grant_operator(user)
32
    put_forge_api_token(conn, key)
33
  end
34
35
  test "an operator records a run and a retry replays it", %{conn: conn} do
36
    authenticated = operator_token(conn, "gym-operator")
37
38
    created =
39
      authenticated
40
      |> post(~p"/api/v3/gym/runs", payload())
41
      |> json_response(201)
42
43
    assert created["run"]["score"] == 0.7
44
    assert created["replayed"] == false
45
46
    replayed =
47
      authenticated
48
      |> post(~p"/api/v3/gym/runs", payload(%{"tasks_passed" => 1}))
49
      |> json_response(200)
50
51
    assert replayed["replayed"] == true
52
    assert replayed["run"]["id"] == created["run"]["id"]
53
    assert replayed["run"]["tasks_passed"] == 7
54
  end
55
56
  test "an ordinary forge:write token is refused and records nothing", %{conn: conn} do
57
    refused =
58
      conn
59
      |> put_forge_api_token("gym-ordinary")
60
      |> post(~p"/api/v3/gym/runs", payload())
61
      |> json_response(403)
62
63
    assert refused["code"] == "not_operator"
64
    assert Gym.list_runs() == []
65
  end
66
67
  test "an invalid run refuses with field errors", %{conn: conn} do
68
    refused =
69
      conn
70
      |> operator_token("gym-invalid")
71
      |> post(~p"/api/v3/gym/runs", payload(%{"tasks_passed" => 99}))
72
      |> json_response(422)
73
74
    assert refused["errors"]["tasks_passed"]
75
  end
76
77
  test "listing is operator-only and filters by suite", %{conn: conn} do
78
    authenticated = operator_token(conn, "gym-lister")
79
80
    _created =
81
      authenticated |> post(~p"/api/v3/gym/runs", payload()) |> json_response(201)
82
83
    listed = authenticated |> get(~p"/api/v3/gym/runs") |> json_response(200)
84
    assert [%{"suite" => "terminal-bench@2.0"}] = listed["runs"]
85
86
    filtered =
87
      authenticated
88
      |> get(~p"/api/v3/gym/runs?suite=swebench@lite")
89
      |> json_response(200)
90
91
    assert filtered["runs"] == []
92
93
    refused =
94
      conn
95
      |> put_forge_api_token("gym-list-ordinary")
96
      |> get(~p"/api/v3/gym/runs")
97
      |> json_response(403)
98
99
    assert refused["code"] == "not_operator"
100
  end
101
end
test/openagents_web/live/gym_live_test.exs added +85

@@ -0,0 +1,85 @@

1
defmodule OpenAgentsWeb.GymLiveTest do
2
  @moduledoc """
3
  `/gym` gates like every operator surface: the operator sees the
4
  scoreboard, an ordinary account is redirected and told nothing, and the
5
  sidebar shows the Gym row only to operators.
6
  """
7
8
  use OpenAgentsWeb.ConnCase, async: false
9
10
  import Phoenix.LiveViewTest
11
12
  alias OpenAgents.Gym
13
14
  defp record_run(suite, digest_letter) do
15
    {:ok, run, false} =
16
      Gym.record_run(%{
17
        "suite" => suite,
18
        "agent" => "openagents-coder",
19
        "agent_version" => "0.3.5",
20
        "model" => "ox-alpha",
21
        "lane" => "proxy",
22
        "tasks_total" => 10,
23
        "tasks_passed" => 8,
24
        "recipe_digest" => "sha256:" <> String.duplicate(digest_letter, 64)
25
      })
26
27
    run
28
  end
29
30
  describe "access" do
31
    test "the operator reaches the surface", %{conn: conn} do
32
      conn = log_in_admin_user(conn, "gym-operator")
33
34
      {:ok, _view, html} = live(conn, ~p"/gym")
35
36
      assert html =~ "Gym"
37
      assert html =~ "No runs recorded yet"
38
    end
39
40
    test "an ordinary authenticated account is redirected", %{conn: conn} do
41
      conn = log_in_github_user(conn, "gym-ordinary")
42
43
      assert {:error, {:redirect, %{to: "/"}}} = live(conn, ~p"/gym")
44
    end
45
46
    test "an unauthenticated visitor is redirected", %{conn: conn} do
47
      assert {:error, {:redirect, %{to: "/"}}} = live(conn, ~p"/gym")
48
    end
49
  end
50
51
  describe "sidebar" do
52
    test "the Gym row shows for the operator and not for an ordinary account", %{conn: conn} do
53
      operator = log_in_admin_user(conn, "gym-nav-operator")
54
      operator_home = operator |> get(~p"/repositories") |> html_response(200)
55
      assert operator_home =~ ~p"/gym"
56
57
      ordinary = log_in_github_user(conn, "gym-nav-ordinary")
58
      ordinary_home = ordinary |> get(~p"/repositories") |> html_response(200)
59
      refute ordinary_home =~ ~p"/gym"
60
    end
61
  end
62
63
  describe "runs" do
64
    test "recorded runs render with score, and the suite filter narrows", %{conn: conn} do
65
      _bench = record_run("terminal-bench@2.0", "d")
66
      _swe = record_run("swebench@lite", "e")
67
68
      conn = log_in_admin_user(conn, "gym-runs-operator")
69
      {:ok, view, html} = live(conn, ~p"/gym")
70
71
      assert html =~ "terminal-bench@2.0"
72
      assert html =~ "swebench@lite"
73
      assert html =~ "80.0%"
74
      assert html =~ "8/10"
75
76
      filtered =
77
        view
78
        |> element("#gym-suite-filter")
79
        |> render_change(%{"suite" => "swebench@lite"})
80
81
      assert filtered =~ "swebench@lite"
82
      refute filtered =~ "terminal-bench@2.0</td>"
83
    end
84
  end
85
end

This page updates live while a promote is in flight · changelog