Record what a turn cost, and name the model by its identifier

fd352755294b · AtlantisPleb · · parent f296672580f7

Record what a turn cost, and name the model by its identifier

Reading back an export turned up two things it was not saying.

`agent.model_name` carried "Ollama qwen3.8:27b-mtp-q8_0", which is the label
written for a narrow status bar. A record has to name something a reader could
run again, so `ReplySource` now reports an identifier alongside the label and
the export prefers it: `qwen3.8:27b-mtp-q8_0`.

`final_metrics` carried only a step count, though Ollama reports
`prompt_eval_count` and `eval_count` on the final chunk of every round and this
was reading past them. A turn may take several rounds -- a model that asks for
tools and then answers -- so the counts are summed across the turn and reported
once at the end of it with the number of calls they aggregate, rather than the
last round's figures presented as the turn's. The export puts them on the step
the turn ended on, with `llm_call_count`, which is what the account-side
exporter does for the same reason.

The same export now reads: model `qwen3.8:27b-mtp-q8_0`, 2334 prompt and 114
completion tokens, and `llm_call_count: 2` on the answer that followed a skill
call. Totals are omitted rather than zeroed when no source measured anything: a
`total_prompt_tokens` of 0 on a session that never measured would itself be a
measurement.

328 tests pass.

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified packages/openagents-cli/src/coder-export.ts
  • modified packages/openagents-cli/src/coder-ollama.ts
  • modified packages/openagents-cli/src/coder-session.ts
  • modified packages/openagents-cli/test/coder-export.test.ts
  • modified packages/openagents-cli/test/coder-ollama.test.ts
  • modified packages/openagents-cli/test/coder-session.test.ts

Diff

6 files changed, +208 -10

packages/openagents-cli/src/coder-export.ts modified +47 -2

@@ -44,6 +44,8 @@ interface AtifStep {

44 44
  message: string;
45 45
  model_name?: string;
46 46
  reasoning_content?: string;
47
  metrics?: { prompt_tokens?: number; completion_tokens?: number };
48
  llm_call_count?: number;
47 49
  tool_calls?: ReadonlyArray<{
48 50
    tool_call_id: string;
49 51
    function_name: string;

@@ -52,6 +54,27 @@ interface AtifStep {

52 54
  observation?: { results: ReadonlyArray<{ source_call_id: string; content: string }> };
53 55
}
54 56
57
/**
58
 * The trajectory's totals.
59
 *
60
 * Summed from what the sources reported and omitted when nothing did: a
61
 * `total_prompt_tokens` of 0 on a session that never measured any would be a
62
 * measurement, and this has none to give.
63
 */
64
const sumMetrics = (
65
  entries: ReadonlyArray<CoderEntry>,
66
): { total_prompt_tokens?: number; total_completion_tokens?: number } => {
67
  const measured = entries.filter((entry) => entry.metrics !== undefined);
68
  if (measured.length === 0) return {};
69
  return {
70
    total_prompt_tokens: measured.reduce((sum, entry) => sum + (entry.metrics?.promptTokens ?? 0), 0),
71
    total_completion_tokens: measured.reduce(
72
      (sum, entry) => sum + (entry.metrics?.completionTokens ?? 0),
73
      0,
74
    ),
75
  };
76
};
77
55 78
export interface ExportedTrajectory {
56 79
  /** Where the file was written. */
57 80
  readonly path: string;

@@ -87,6 +110,22 @@ const argumentsOf = (source: string): Record<string, unknown> => {

87 110
  }
88 111
};
89 112
113
/** A step's metrics, present only when the source reported any. */
114
const metricsOf = (entry: CoderEntry): Partial<AtifStep> => {
115
  const metrics = entry.metrics;
116
  if (metrics === undefined) return {};
117
  const figures = {
118
    ...(metrics.promptTokens === undefined ? {} : { prompt_tokens: metrics.promptTokens }),
119
    ...(metrics.completionTokens === undefined
120
      ? {}
121
      : { completion_tokens: metrics.completionTokens }),
122
  };
123
  return {
124
    ...(Object.keys(figures).length === 0 ? {} : { metrics: figures }),
125
    ...(metrics.calls === undefined ? {} : { llm_call_count: metrics.calls }),
126
  };
127
};
128
90 129
/** Fold the transcript into ATIF steps. */
91 130
const stepsOf = (entries: ReadonlyArray<CoderEntry>, model: string): ReadonlyArray<AtifStep> => {
92 131
  const steps: AtifStep[] = [];

@@ -99,7 +138,8 @@ const stepsOf = (entries: ReadonlyArray<CoderEntry>, model: string): ReadonlyArr

99 138
    if (entry.role === "notice") continue;
100 139
101 140
    if (entry.role === "reasoning") {
102
      pendingReasoning = pendingReasoning === undefined ? entry.text : `${pendingReasoning}\n${entry.text}`;
141
      pendingReasoning =
142
        pendingReasoning === undefined ? entry.text : `${pendingReasoning}\n${entry.text}`;
103 143
      continue;
104 144
    }
105 145

@@ -124,6 +164,7 @@ const stepsOf = (entries: ReadonlyArray<CoderEntry>, model: string): ReadonlyArr

124 164
        observation: {
125 165
          results: [{ source_call_id: callId, content: error ?? output ?? "" }],
126 166
        },
167
        ...metricsOf(entry),
127 168
      });
128 169
      pendingReasoning = undefined;
129 170
      continue;

@@ -140,6 +181,7 @@ const stepsOf = (entries: ReadonlyArray<CoderEntry>, model: string): ReadonlyArr

140 181
        message: entry.text,
141 182
        model_name: model,
142 183
        ...(pendingReasoning === undefined ? {} : { reasoning_content: pendingReasoning }),
184
        ...metricsOf(entry),
143 185
      });
144 186
      pendingReasoning = undefined;
145 187
    }

@@ -212,7 +254,10 @@ export function exportTrajectory(

212 254
        : { tool_definitions: options.toolDefinitions }),
213 255
    },
214 256
    steps,
215
    final_metrics: { total_steps: steps.length },
257
    final_metrics: {
258
      ...sumMetrics(snapshot.entries),
259
      total_steps: steps.length,
260
    },
216 261
    extra: {
217 262
      exporter: "openagents.coder.atif_export.v1",
218 263
      exported_at: at.toISOString(),
packages/openagents-cli/src/coder-ollama.ts modified +28 -2

@@ -143,6 +143,16 @@ export class OllamaReplySource implements ReplySource {

143 143
    return `Ollama ${this.modelName}`;
144 144
  }
145 145
146
  /**
147
   * The identifier, as against `model`, which is the label a status line shows.
148
   *
149
   * A record has to name the model a reader could run again. "Ollama qwen3.8"
150
   * is for a narrow bar; `qwen3.8:27b-mtp-q8_0` is the thing itself.
151
   */
152
  get modelId(): string {
153
    return this.modelName;
154
  }
155
146 156
  constructor(options: OllamaOptions) {
147 157
    this.client = new Ollama({ host: options.host ?? DEFAULT_HOST });
148 158
    this.modelName = options.model;

@@ -200,6 +210,10 @@ export class OllamaReplySource implements ReplySource {

200 210
    // A turn is a loop, not a single call: the model may answer, or it may ask
201 211
    // for tools and then answer once it has seen what they returned. The
202 212
    // ceiling is what stops a model that only ever delegates.
213
    let promptTokens = 0;
214
    let completionTokens = 0;
215
    let llmCalls = 0;
216
203 217
    for (let step = 0; step < MAX_TOOL_STEPS; step += 1) {
204 218
      if (signal.aborted) return;
205 219

@@ -252,7 +266,15 @@ export class OllamaReplySource implements ReplySource {

252 266
          const toolCalls = chunk.message.tool_calls;
253 267
          if (Array.isArray(toolCalls)) calls.push(...toolCalls);
254 268
255
          if (chunk.done) break;
269
          if (chunk.done) {
270
            // The counts ride on the final chunk of each round, so they are
271
            // summed across the rounds a turn took rather than reported from
272
            // the last one.
273
            promptTokens += chunk.prompt_eval_count ?? 0;
274
            completionTokens += chunk.eval_count ?? 0;
275
            llmCalls += 1;
276
            break;
277
          }
256 278
        }
257 279
      } finally {
258 280
        signal.removeEventListener("abort", onAbort);

@@ -268,7 +290,10 @@ export class OllamaReplySource implements ReplySource {

268 290
        ...(calls.length === 0 ? {} : { tool_calls: calls }),
269 291
      });
270 292
271
      if (calls.length === 0) return;
293
      if (calls.length === 0) {
294
        yield { type: "usage", promptTokens, completionTokens, calls: llmCalls };
295
        return;
296
      }
272 297
273 298
      for (const call of calls) {
274 299
        if (signal.aborted) return;

@@ -282,6 +307,7 @@ export class OllamaReplySource implements ReplySource {

282 307
      type: "text",
283 308
      value: `\n\nStopped after ${String(MAX_TOOL_STEPS)} rounds of tool calls without an answer.`,
284 309
    };
310
    yield { type: "usage", promptTokens, completionTokens, calls: llmCalls };
285 311
  }
286 312
287 313
  /**
packages/openagents-cli/src/coder-session.ts modified +44 -1

@@ -37,6 +37,19 @@ export type ReplyChunk =

37 37
      /** Arguments as JSON source, pretty-printed when the server printed it. */
38 38
      readonly arguments: string;
39 39
    }
40
  | {
41
      /**
42
       * What the turn cost, reported once at the end of it.
43
       *
44
       * A turn may take several LLM calls -- a model that asks for tools and
45
       * then answers -- so this is their total, with the count, rather than the
46
       * last call's figures presented as the turn's.
47
       */
48
      readonly type: "usage";
49
      readonly promptTokens?: number | undefined;
50
      readonly completionTokens?: number | undefined;
51
      readonly calls?: number | undefined;
52
    }
40 53
  | {
41 54
      readonly type: "tool_result";
42 55
      readonly callId: string;

@@ -54,6 +67,14 @@ export interface CoderToolCall {

54 67
  status: "running" | "succeeded" | "failed";
55 68
}
56 69
70
/** What a turn cost, on the entry that closed it. */
71
export interface CoderMetrics {
72
  readonly promptTokens?: number | undefined;
73
  readonly completionTokens?: number | undefined;
74
  /** How many LLM calls the figures aggregate. */
75
  readonly calls?: number | undefined;
76
}
77
57 78
/** One entry in the transcript. */
58 79
export interface CoderEntry {
59 80
  readonly role: "you" | "assistant" | "notice" | "tool" | "reasoning";

@@ -71,6 +92,8 @@ export interface CoderEntry {

71 92
  settled: boolean;
72 93
  /** Present on a `tool` entry only. */
73 94
  readonly tool?: CoderToolCall;
95
  /** Set on the entry a turn ended on, when the source reported the cost. */
96
  metrics?: CoderMetrics;
74 97
}
75 98
76 99
/** Everything a renderer needs. No renderer reads anything else. */

@@ -118,6 +141,14 @@ export interface CoderDelegation {

118 141
export interface ReplySource {
119 142
  /** The label the status line shows for the reply source. */
120 143
  readonly model: string;
144
  /**
145
   * The model's identifier, when it differs from the label above.
146
   *
147
   * `model` is written for a narrow status bar. A record has to name something
148
   * a reader could run again, so an export prefers this and falls back to the
149
   * label when a source has only one name for itself.
150
   */
151
  readonly modelId?: string | undefined;
121 152
  /**
122 153
   * What this source may still spend, already formatted for the status line,
123 154
   * or undefined for a source that meters nothing. Read on every snapshot, so

@@ -405,7 +436,7 @@ export class CoderSession {

405 436
      this.entries.push({ role: "you", text: prompt, settled: true, at: Date.now() });
406 437
      try {
407 438
        const written = exportTrajectory(this.snapshot(), {
408
          model: this.source.model,
439
          model: this.source.modelId ?? this.source.model,
409 440
          toolDefinitions: this.source.toolDefinitions?.(),
410 441
          version: VERSION,
411 442
        });

@@ -500,6 +531,18 @@ export class CoderSession {

500 531
              status: "running",
501 532
            },
502 533
          });
534
        } else if (chunk.type === "usage") {
535
          // Onto the entry the turn ended on, which is the step a reader of the
536
          // trajectory would attribute the cost to. `calls` says how many LLM
537
          // calls it aggregates, so a turn that used tools is not read as one.
538
          const closing = text ?? reasoning ?? this.entries.at(-1);
539
          if (closing !== undefined) {
540
            closing.metrics = {
541
              promptTokens: chunk.promptTokens,
542
              completionTokens: chunk.completionTokens,
543
              calls: chunk.calls,
544
            };
545
          }
503 546
        } else {
504 547
          this.applyToolResult(chunk);
505 548
        }
packages/openagents-cli/test/coder-export.test.ts modified +31

@@ -183,6 +183,37 @@ describe("exporting a conversation as ATIF", () => {

183 183
    expect(document["steps"]).toHaveLength(1);
184 184
  });
185 185
186
187
  it("records what a turn cost, and how many calls that was", () => {
188
    const { document } = write([
189
      entry({ role: "you", text: "ask" }),
190
      entry({
191
        role: "assistant",
192
        text: "answer",
193
        metrics: { promptTokens: 2334, completionTokens: 114, calls: 2 },
194
      }),
195
    ]);
196
197
    const steps = document["steps"] as ReadonlyArray<Record<string, unknown>>;
198
    expect(steps[1]).toMatchObject({
199
      metrics: { prompt_tokens: 2334, completion_tokens: 114 },
200
      // A turn that asked for a tool and then answered is two calls, not one.
201
      llm_call_count: 2,
202
    });
203
    expect(document["final_metrics"]).toMatchObject({
204
      total_prompt_tokens: 2334,
205
      total_completion_tokens: 114,
206
      total_steps: 2,
207
    });
208
  });
209
210
  it("reports no totals rather than zero when nothing measured any", () => {
211
    const { document } = write([entry({ role: "assistant", text: "answer" })]);
212
213
    // A total of 0 on a session that never measured would be a measurement.
214
    expect(document["final_metrics"]).toEqual({ total_steps: 1 });
215
  });
216
186 217
  it("writes one file per export, named so they sort by time", () => {
187 218
    const { directory } = write([entry({ role: "you", text: "one" })]);
188 219
packages/openagents-cli/test/coder-ollama.test.ts modified +40 -5

@@ -70,10 +70,13 @@ describe("an ollama reply", () => {

70 70
71 71
    const chunks = await collect(source, "hi");
72 72
73
    expect(chunks).toEqual([
73
    // Every turn closes with what it cost, so the text chunks are read apart
74
    // from the usage chunk that trails them.
75
    expect(chunks.filter((piece) => piece.type !== "usage")).toEqual([
74 76
      { type: "reasoning", value: "weighing it" },
75 77
      { type: "text", value: "Hello" },
76 78
    ]);
79
    expect(chunks.at(-1)).toMatchObject({ type: "usage" });
77 80
    expect(stub.requests[0]).not.toHaveProperty("tools");
78 81
  });
79 82
});

@@ -115,7 +118,8 @@ describe("an ollama turn that calls a tool", () => {

115 118
    // The call and its result share an id, which is how a renderer pairs them.
116 119
    expect(result).toMatchObject({ callId: (call as { callId: string }).callId });
117 120
    // The turn continued rather than ending on the tool result.
118
    expect(chunks.at(-1)).toEqual({ type: "text", value: "They said PONG." });
121
    const spoken = chunks.filter((piece) => piece.type !== "usage");
122
    expect(spoken.at(-1)).toEqual({ type: "text", value: "They said PONG." });
119 123
120 124
    // The tool was declared, and the second round carried the exchange back.
121 125
    expect(stub.requests[0]).toHaveProperty("tools");

@@ -129,6 +133,33 @@ describe("an ollama turn that calls a tool", () => {

129 133
    expect(messages.at(-1)).toMatchObject({ content: "child 1 said PONG", tool_name: "delegate" });
130 134
  });
131 135
136
137
  it("reports the turn's cost, summed over the rounds it took", async () => {
138
    const calls: Record<string, unknown>[] = [];
139
    const { source } = sourceWith([
140
      [
141
        chunk({
142
          content: "",
143
          tool_calls: [{ function: { name: "delegate", arguments: { prompt: "go" } } }],
144
        }),
145
        { message: {}, done: true, prompt_eval_count: 100, eval_count: 10 },
146
      ],
147
      [{ message: { content: "done" }, done: true, prompt_eval_count: 200, eval_count: 20 }],
148
    ] as never);
149
    source.useTools([delegate(calls)]);
150
151
    const chunks = await collect(source, "delegate this");
152
153
    // Two LLM calls in one turn: the counts are their total, not the last
154
    // round's presented as the turn's.
155
    expect(chunks.at(-1)).toEqual({
156
      type: "usage",
157
      promptTokens: 300,
158
      completionTokens: 30,
159
      calls: 2,
160
    });
161
  });
162
132 163
  it("reports a tool that throws instead of ending the turn", async () => {
133 164
    const { source } = sourceWith([CALLING, [chunk({ content: "It failed." }, true)]]);
134 165
    source.useTools([

@@ -146,7 +177,10 @@ describe("an ollama turn that calls a tool", () => {

146 177
      error: "the fleet is full",
147 178
      output: "the fleet is full",
148 179
    });
149
    expect(chunks.at(-1)).toEqual({ type: "text", value: "It failed." });
180
    expect(chunks.filter((piece) => piece.type !== "usage").at(-1)).toEqual({
181
      type: "text",
182
      value: "It failed.",
183
    });
150 184
  });
151 185
152 186
  it("tells the model when it asks for a tool the session does not have", async () => {

@@ -175,8 +209,9 @@ describe("an ollama turn that calls a tool", () => {

175 209
176 210
    // Six rounds, then a sentence saying why it stopped.
177 211
    expect(stub.chat).toHaveBeenCalledTimes(6);
178
    expect(chunks.at(-1)).toMatchObject({ type: "text" });
179
    expect((chunks.at(-1) as { value: string }).value).toContain("Stopped after 6 rounds");
212
    const spoken = chunks.filter((piece) => piece.type !== "usage");
213
    expect(spoken.at(-1)).toMatchObject({ type: "text" });
214
    expect((spoken.at(-1) as { value: string }).value).toContain("Stopped after 6 rounds");
180 215
  });
181 216
});
182 217
packages/openagents-cli/test/coder-session.test.ts modified +18

@@ -392,3 +392,21 @@ describe("the /export command", () => {

392 392
    if (written !== undefined) rmSync(written, { force: true });
393 393
  });
394 394
});
395
396
describe("a turn's cost", () => {
397
  it("lands on the entry the turn ended on", async () => {
398
    const session = new CoderSession(
399
      source([
400
        { type: "text", value: "answer" },
401
        { type: "usage", promptTokens: 12, completionTokens: 34, calls: 2 },
402
      ]),
403
      "repo",
404
      "main",
405
    );
406
407
    await session.submit("go");
408
409
    const assistant = session.snapshot().entries.find((entry) => entry.role === "assistant");
410
    expect(assistant?.metrics).toEqual({ promptTokens: 12, completionTokens: 34, calls: 2 });
411
  });
412
});

This page updates live while a promote is in flight · changelog