Report what the agent said, not everything it typed

01e0ec033ab2 · AtlantisPleb · · parent 273cbd5c5841

Report what the agent said, not everything it typed

When a delegation ended, its report posted the whole decoded ACP transcript
into the conversation: hundreds of `Terminal: …` lines, each command repeated
as title and detail, ending in `[transcript truncated]`. It landed the moment
the live panel went terminal, so it read as though dismissing the panel had
dumped the log into the chat.

`AcpTranscript.summarize/1` now splits the stream: the agent's prose, its
notes, and the tool calls that failed on one side, the tool-by-tool log on the
other. The delegation report keeps the first, names the second as a count, and
bounds the body to a chat-sized 2,000 characters. The rolling log stays where
it belongs, in the live delegation rail. Controller-reported detail is bounded
too, so no machine-supplied text reaches a message unbounded.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016o8HwTaqLKEWCHTjsjFtrB
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified lib/openagents/computer/acp_transcript.ex
  • modified lib/openagents/work/delegation_server.ex
  • modified test/openagents/computer/acp_transcript_test.exs
  • added test/openagents/work/delegation_report_test.exs

Diff

4 files changed, +308 -25

lib/openagents/computer/acp_transcript.ex modified +50 -14

@@ -7,6 +7,10 @@ defmodule OpenAgents.Computer.AcpTranscript do

7 7
  separator `0x1F`. Durable job reports used to post the raw bytes. This
8 8
  module is the server-side decoder so a timeout or completion never writes
9 9
  `Ttoolu_…0executeVGVybWluYWw=` into the conversation.
10
11
  `decode/1` renders the whole stream, prose and tool lines alike. A chat
12
  message wants less than that, so `summarize/1` splits the two: the agent's
13
  own words stay, and the tool-by-tool log becomes a count.
10 14
  """
11 15
12 16
  @record_separator <<30>>

@@ -33,12 +37,35 @@ defmodule OpenAgents.Computer.AcpTranscript do

33 37
  @spec decode(term()) :: String.t()
34 38
  def decode(binary) when is_binary(binary) do
35 39
    binary
36
    |> walk([], %{})
37
    |> finalize()
40
    |> entries()
41
    |> render()
38 42
  end
39 43
40 44
  def decode(_other), do: ""
41 45
46
  @doc """
47
  Splits the stream into what a conversation can carry and what it cannot.
48
49
  Returns the agent's own prose, its notes, and the tool calls that failed —
50
  the parts that explain an outcome — plus the total number of tool calls.
51
  The tool-by-tool log belongs to the live delegation rail: pasted into a
52
  message it buries the report under hundreds of `Terminal: …` lines, so the
53
  caller names the count instead.
54
  """
55
  @spec summarize(term()) :: {String.t(), non_neg_integer()}
56
  def summarize(binary) when is_binary(binary) do
57
    entries = entries(binary)
58
59
    text =
60
      entries
61
      |> Enum.reject(&(&1.kind == :tool and &1.status != :failed))
62
      |> render()
63
64
    {text, Enum.count(entries, &(&1.kind == :tool))}
65
  end
66
67
  def summarize(_other), do: {"", 0}
68
42 69
  defp walk(<<@record_separator, rest::binary>>, acc, tools) do
43 70
    case take_line(rest) do
44 71
      {line, remaining} ->

@@ -80,7 +107,7 @@ defmodule OpenAgents.Computer.AcpTranscript do

80 107
81 108
      ["N", encoded | rest] ->
82 109
        tone = Enum.at(rest, 0) || "info"
83
        {append_block(acc, note_line(decode64(encoded), tone)), tools}
110
        {append_block(:note, acc, note_line(decode64(encoded), tone), nil), tools}
84 111
85 112
      _other ->
86 113
        {acc, tools}

@@ -110,34 +137,43 @@ defmodule OpenAgents.Computer.AcpTranscript do

110 137
    if tool.emitted? do
111 138
      {acc, tools}
112 139
    else
113
      {append_block(acc, tool_line(tool, if(phase == "2", do: :failed, else: :done))),
140
      status = if phase == "2", do: :failed, else: :done
141
142
      {append_block(:tool, acc, tool_line(tool, status), status),
114 143
       Map.update!(tools, id, &Map.put(&1, :emitted?, true))}
115 144
    end
116 145
  end
117 146
118 147
  defp maybe_emit_tool(acc, _id, _phase, tools), do: {acc, tools}
119 148
120
  defp finalize({acc, tools}) do
149
  # The stream in reading order, each block tagged with what it is, so a caller
150
  # can keep the prose without the tool log.
151
  defp entries(binary) do
152
    {acc, tools} = walk(binary, [], %{})
153
121 154
    unfinished =
122 155
      tools
123 156
      |> Enum.reject(fn {_id, tool} -> tool.emitted? or tool.phase in ["1", "2"] end)
124
      |> Enum.map(fn {_id, tool} -> tool_line(tool, :running) end)
157
      |> Enum.map(fn {_id, tool} -> entry(:tool, tool_line(tool, :running), :running) end)
125 158
126
    [Enum.reverse(acc), unfinished]
127
    |> List.flatten()
128
    |> Enum.reject(&(&1 in [nil, ""]))
129
    |> Enum.join("\n\n")
159
    Enum.reverse(acc) ++ unfinished
160
  end
161
162
  defp render(entries) do
163
    entries
164
    |> Enum.map_join("\n\n", & &1.text)
130 165
    |> String.trim()
131 166
  end
132 167
168
  defp entry(kind, text, status), do: %{kind: kind, text: text, status: status}
169
133 170
  defp append_prose(acc, text) do
134 171
    trimmed = String.trim(text)
135
    if trimmed == "", do: acc, else: [trimmed | acc]
172
    if trimmed == "", do: acc, else: [entry(:prose, trimmed, nil) | acc]
136 173
  end
137 174
138
  defp append_block(acc, block) do
139
    if block in [nil, ""], do: acc, else: [block | acc]
140
  end
175
  defp append_block(_kind, acc, block, _status) when block in [nil, ""], do: acc
176
  defp append_block(kind, acc, block, status), do: [entry(kind, block, status) | acc]
141 177
142 178
  defp tool_line(tool, status) do
143 179
    label = Map.get(@kind_labels, tool.kind, "")
lib/openagents/work/delegation_server.ex modified +26 -11

@@ -23,7 +23,11 @@ defmodule OpenAgents.Work.DelegationServer do

23 23
  alias OpenAgents.Cluster.Sessions
24 24
  alias OpenAgents.Computer.AcpTranscript
25 25
26
  @maximum_report_output 6_000
26
  # A report is a chat message, not a terminal window. It carries what the
27
  # agent said and any tool call that failed; the tool-by-tool log stays in the
28
  # live delegation rail. The bound is characters of that composed body.
29
  @maximum_report_output 2_000
30
  @maximum_detail 500
27 31
28 32
  def start_link(job_id) do
29 33
    GenServer.start_link(__MODULE__, job_id, name: via(job_id))

@@ -261,26 +265,37 @@ defmodule OpenAgents.Work.DelegationServer do

261 265
  defp human_status("refused"), do: "refused"
262 266
  defp human_status(other), do: "ended (#{other})"
263 267
264
  defp detail_line(detail) when is_binary(detail) and detail != "", do: detail
268
  # The detail is controller-reported text: bound it before it reaches a message.
269
  defp detail_line(detail) when is_binary(detail) and detail != "",
270
    do: String.slice(detail, 0, @maximum_detail)
271
265 272
  defp detail_line(_detail), do: nil
266 273
274
  # The conversation gets the agent's answer, not its keystrokes. Posting the
275
  # whole decoded transcript dumped hundreds of `Terminal: …` lines into the
276
  # chat the moment a long delegation ended. What survives here is the prose,
277
  # the tool calls that failed, and a line counting the rest, so the work is
278
  # named without being replayed.
267 279
  defp output_block(output) when is_binary(output) and output != "" do
268
    transcript = AcpTranscript.decode(output)
280
    {summary, tool_count} = AcpTranscript.summarize(output)
269 281
270
    if transcript == "" do
271
      nil
272
    else
273
      bound_transcript(transcript)
282
    case Enum.reject([bound_report(summary), tool_count_line(tool_count)], &(&1 in [nil, ""])) do
283
      [] -> nil
284
      parts -> Enum.join(parts, "\n\n")
274 285
    end
275 286
  end
276 287
277 288
  defp output_block(_output), do: nil
278 289
279
  defp bound_transcript(transcript) do
280
    if String.length(transcript) <= @maximum_report_output do
281
      transcript
290
  defp tool_count_line(0), do: nil
291
  defp tool_count_line(1), do: "The agent ran 1 tool call on the machine."
292
  defp tool_count_line(count), do: "The agent ran #{count} tool calls on the machine."
293
294
  defp bound_report(summary) do
295
    if String.length(summary) <= @maximum_report_output do
296
      summary
282 297
    else
283
      String.slice(transcript, 0, @maximum_report_output) <> "\n\n[transcript truncated]"
298
      String.slice(summary, 0, @maximum_report_output) <> "\n\n[report truncated]"
284 299
    end
285 300
  end
286 301
test/openagents/computer/acp_transcript_test.exs modified +29

@@ -87,6 +87,35 @@ defmodule OpenAgents.Computer.AcpTranscriptTest do

87 87
    assert report =~ "3 hits"
88 88
  end
89 89
90
  test "summarize keeps the prose and failures, and counts the rest" do
91
    stream =
92
      "I read the failing test and fixed the selector.\n" <>
93
        frame("T", ["toolu_01run", "1", "execute", b64("git status"), b64("On branch main")]) <>
94
        frame("T", ["toolu_02read", "1", "read", b64("chat_live.ex"), b64("defmodule")]) <>
95
        frame("T", ["toolu_03edit", "2", "edit", b64("chat_live.ex"), b64("User refused")]) <>
96
        frame("N", [b64("Permission denied: Edit"), "warn"])
97
98
    assert {text, 3} = AcpTranscript.summarize(stream)
99
100
    # What explains the outcome stays: the agent's words, its notes, the failure.
101
    assert text =~ "I read the failing test and fixed the selector."
102
    assert text =~ "Edit: chat_live.ex (failed)"
103
    assert text =~ "Warning: Permission denied: Edit"
104
105
    # The tool-by-tool log does not.
106
    refute text =~ "git status"
107
    refute text =~ "On branch main"
108
    refute text =~ "Read: chat_live.ex"
109
  end
110
111
  test "summarize counts a tool that never finished, and empties safely" do
112
    stream = frame("T", ["toolu_01open", "0", "read", b64("chat_live.ex"), ""])
113
114
    assert {"", 1} = AcpTranscript.summarize(stream)
115
    assert AcpTranscript.summarize("") == {"", 0}
116
    assert AcpTranscript.summarize(nil) == {"", 0}
117
  end
118
90 119
  defp frame(kind, fields) do
91 120
    @rs <> Enum.join([kind | fields], @us) <> "\n"
92 121
  end
test/openagents/work/delegation_report_test.exs added +203

@@ -0,0 +1,203 @@

1
defmodule OpenAgents.Work.DelegationReportTest do
2
  @moduledoc """
3
  What a finished delegation is allowed to post into the conversation.
4
5
  A delegation's report becomes an assistant message, so it is chat, not a
6
  terminal window. The regression this pins: a long delegation used to end by
7
  dumping its whole decoded ACP transcript — hundreds of `Terminal: …` lines,
8
  each command repeated as title and detail — into the transcript, ending in
9
  `[transcript truncated]`. The tool-by-tool log belongs to the live
10
  delegation rail; the message carries what the agent said.
11
  """
12
  use OpenAgents.DataCase
13
14
  alias OpenAgents.Conversations.Message
15
  alias OpenAgents.Support.FakeController
16
  alias OpenAgents.{Accounts, Conversations, Machines, Work}
17
18
  @record_separator <<30>>
19
  @unit_separator <<31>>
20
21
  test "a completed delegation reports the agent's prose, never its tool log" do
22
    %{conversation: conversation, machine: machine} = delegation_owner("deleg-report-prose")
23
24
    connect(machine.id, fn {:agent, request_id, _payload, caller} ->
25
      FakeController.chunk(caller, request_id, noisy_transcript())
26
27
      FakeController.exit(caller, request_id, %{
28
        "status" => "completed",
29
        "stop_reason" => "end_turn",
30
        "session_id" => "acp-report-1",
31
        "duration_ms" => 118_000,
32
        "truncated" => false,
33
        "detail" => ""
34
      })
35
    end)
36
37
    job = run_delegation(conversation, machine)
38
    assert job.status == "completed"
39
    content = report_content(job)
40
41
    # The agent's own words, the session line the resume path parses, and the
42
    # one tool call that failed — the parts that explain the outcome.
43
    assert content =~ "Delegation to claude"
44
    assert content =~ "Session: acp-report-1"
45
    assert content =~ "I read the failing test and fixed the selector."
46
    assert content =~ "Edit: chat_live.ex (failed)"
47
48
    # Never the log of everything it typed.
49
    refute content =~ "Terminal:"
50
    refute content =~ "/usr/bin/bash -lc"
51
    refute content =~ "[transcript truncated]"
52
53
    # The work is still named, and the whole message stays chat-sized.
54
    assert content =~ "The agent ran 61 tool calls on the machine."
55
    assert String.length(content) < 1_000
56
  end
57
58
  test "cancelling a delegation reports the cancellation, never a partial transcript" do
59
    %{conversation: conversation, machine: machine} = delegation_owner("deleg-report-cancel")
60
61
    # A delegation that streams and never returns: the owner stops it from the
62
    # live panel while the tool log is still growing.
63
    connect(machine.id, fn {:agent, request_id, _payload, caller} ->
64
      FakeController.chunk(caller, request_id, noisy_transcript())
65
    end)
66
67
    {:ok, started} = Work.start_delegation(delegation_attributes(conversation, machine))
68
    assert eventually(fn -> Work.get_job!(started.id).status == "running" end)
69
70
    :ok = Work.cancel_active_delegations(conversation.id)
71
    assert eventually(fn -> Work.get_job!(started.id).status == "cancelled" end)
72
73
    content = started.id |> Work.get_job!() |> report_content()
74
75
    assert content =~ "Delegation cancelled by the owner."
76
    refute content =~ "Terminal:"
77
    refute content =~ "/usr/bin/bash -lc"
78
    refute content =~ "[transcript truncated]"
79
  end
80
81
  # ── helpers ──────────────────────────────────────────────────────────────
82
83
  # One realistic delegation stream: a sentence of prose, 60 successful
84
  # terminal calls whose title and detail both carry the command, one failed
85
  # edit, and a permission note.
86
  defp noisy_transcript do
87
    command = ~s(/usr/bin/bash -lc "sed -n 1,200p lib/openagents_web/live/chat_live.ex")
88
89
    terminals =
90
      Enum.map_join(1..60, "", fn index ->
91
        id = "toolu_0#{index}"
92
93
        frame("T", [id, "0", "execute", encode("Terminal"), ""]) <>
94
          frame("T", [id, "1", "execute", encode(command), encode(command)])
95
      end)
96
97
    "I read the failing test and fixed the selector.\n" <>
98
      terminals <>
99
      frame("T", [
100
        "toolu_0edit",
101
        "2",
102
        "edit",
103
        encode("chat_live.ex"),
104
        encode("User refused permission")
105
      ]) <> frame("N", [encode("Permission denied: Edit"), "warn"])
106
  end
107
108
  defp frame(kind, fields) do
109
    @record_separator <> Enum.join([kind | fields], @unit_separator) <> "\n"
110
  end
111
112
  defp encode(text), do: Base.encode64(text)
113
114
  defp run_delegation(conversation, machine) do
115
    {:ok, job} = Work.start_delegation(delegation_attributes(conversation, machine))
116
117
    assert eventually(fn -> Work.get_job!(job.id).status in Work.Job.terminal_statuses() end)
118
119
    Work.get_job!(job.id)
120
  end
121
122
  # The assistant message the delegation posted into the conversation — the row
123
  # the owner actually reads.
124
  defp report_content(job) do
125
    Repo.get_by!(Message, work_job_id: job.id, role: "assistant").content
126
  end
127
128
  defp delegation_owner(login) do
129
    {:ok, user} =
130
      Accounts.upsert_github_user(%{
131
        github_id: System.unique_integer([:positive]),
132
        github_login: login,
133
        github_avatar_url: "https://avatars.githubusercontent.com/u/1?v=4"
134
      })
135
136
    {:ok, conversation} = Conversations.ensure_conversation(user)
137
138
    {:ok, %{code: code}} =
139
      Machines.start_pairing(%{
140
        "name" => "report-box-#{login}",
141
        "tier" => "curated",
142
        "platform" => "linux-x64",
143
        "agent_version" => "0.1.0",
144
        "roots" => ["/tmp/openagents-work"]
145
      })
146
147
    {:ok, machine} = Machines.approve_pairing(user, code)
148
    %{user: user, conversation: conversation, machine: machine}
149
  end
150
151
  defp connect(machine_id, script) do
152
    start_supervised!({FakeController, machine_id: machine_id, script: script})
153
  end
154
155
  defp delegation_attributes(conversation, machine) do
156
    owner = Conversations.get_conversation_owner!(conversation)
157
158
    %{
159
      conversation_id: conversation.id,
160
      owner_visitor_id: owner.id,
161
      machine_id: machine.id,
162
      surface: "text",
163
      goal: "Delegate to claude on #{machine.name}: fix the failing test",
164
      kind: "delegation",
165
      delegation: %{
166
        "agent_id" => "claude",
167
        "machine_id" => machine.id,
168
        "machine_name" => machine.name,
169
        "prompt" => "fix the failing test",
170
        "cwd" => "/tmp/openagents-work",
171
        "timeout_ms" => 3_600_000
172
      },
173
      authority_snapshot: %{
174
        "machine_tier" => machine.tier,
175
        "roots" => machine.roots,
176
        "cwd" => "/tmp/openagents-work",
177
        "agent_id" => "claude",
178
        "machine_name" => machine.name
179
      },
180
      budget_snapshot: %{
181
        "wall_clock_ms" => 3_600_000,
182
        "maximum_prompt_bytes" => 8_000,
183
        "maximum_report_bytes" => 8_000
184
      }
185
    }
186
  end
187
188
  # The delegation runs in a supervised background process; the shared sandbox
189
  # makes its writes visible here.
190
  defp eventually(fun, attempts \\ 40) do
191
    cond do
192
      fun.() ->
193
        true
194
195
      attempts <= 0 ->
196
        false
197
198
      true ->
199
        Process.sleep(50)
200
        eventually(fun, attempts - 1)
201
    end
202
  end
203
end

This page updates live while a promote is in flight · changelog