OpenAgents

Commit

Carry the cached-token split out to the trace

aa7a61de96df · AtlantisPleb · · parent c2b0889cdb2d 

Carry the cached-token split out to the trace

The server records the provider's cached-versus-fresh split; nothing
downstream carried it, so the CLI reported a total and the ATIF
document exported one. On agentic workloads that is the number that
matters — a real session read back 7,329 of 7,422 input tokens as
cache reads — and a corpus of traces that omits it prices every one of
them as fresh.

The split now travels the rest of the way: the thread lane's usage
chunk and the transcript carry it, and the ATIF export puts it in step
`metrics.extra`, which is the spec's designated place for
provider-specific token metrics.

Absent stays absent, end to end. A provider that reported no split
produces no split — no `extra` key at all rather than one filled with
zeros — because a zero there reads as "measured, and it was none",
which is a different claim from "nobody measured".

Built by a Devin child through the openagents coder's delegate tool;
835 CLI tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GoYpb8FEmdxVErsv7ABCYi
Co-Authored-By
Claude Fable 5 <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified packages/atif/src/trace-schema.ts
  • modified packages/openagents-cli/src/coder-export.ts
  • modified packages/openagents-cli/src/coder-session.ts
  • modified packages/openagents-cli/src/coder-thread.ts
  • modified packages/openagents-cli/test/coder-export.test.ts
  • modified packages/openagents-cli/test/coder-session.test.ts
  • modified packages/openagents-cli/test/coder-thread.test.ts

Diff

7 files changed, +123 -2

packages/atif/src/trace-schema.ts modified +1

@@ -51,6 +51,7 @@ export class AtifStepMetrics extends S.Class<AtifStepMetrics>('AtifStepMetrics')

51 51 
    prompt_tokens: S.optionalKey(S.Number),
52 52 
    completion_tokens: S.optionalKey(S.Number),
53 53 
    cost_usd: S.optionalKey(S.Number),
54 
    extra: S.optionalKey(S.Record(S.String, S.Number)),
54 55 
  },
55 56 
) {}
56 57
packages/openagents-cli/src/coder-export.ts modified +4 -1

@@ -49,7 +49,7 @@ interface AtifStep {

49 49 
  message: string;
50 50 
  model_name?: string;
51 51 
  reasoning_content?: string;
52 
  metrics?: { prompt_tokens?: number; completion_tokens?: number };
52 
  metrics?: { prompt_tokens?: number; completion_tokens?: number; extra?: Record<string, number> };
53 53 
  llm_call_count?: number;
54 54 
  tool_calls?: ReadonlyArray<{
55 55 
    tool_call_id: string;

@@ -137,6 +137,9 @@ const metricsOf = (entry: CoderEntry): Partial<AtifStep> => {

137 137 
    ...(metrics.completionTokens === undefined
138 138 
      ? {}
139 139 
      : { completion_tokens: metrics.completionTokens }),
140 
    ...(metrics.cacheReadInputTokens === undefined
141 
      ? {}
142 
      : { extra: { cache_read_input_tokens: metrics.cacheReadInputTokens } }),
140 143 
  };
141 144 
  return {
142 145 
    ...(Object.keys(figures).length === 0 ? {} : { metrics: figures }),
packages/openagents-cli/src/coder-session.ts modified +5

@@ -48,6 +48,7 @@ export type ReplyChunk =

48 48 
      readonly type: "usage";
49 49 
      readonly promptTokens?: number | undefined;
50 50 
      readonly completionTokens?: number | undefined;
51 
      readonly cacheReadInputTokens?: number | undefined;
51 52 
      readonly calls?: number | undefined;
52 53 
    }
53 54 
  | {

@@ -162,6 +163,7 @@ const describeFailure = (cause: unknown): string => {

162 163 
export interface CoderMetrics {
163 164 
  readonly promptTokens?: number | undefined;
164 165 
  readonly completionTokens?: number | undefined;
166 
  readonly cacheReadInputTokens?: number | undefined;
165 167 
  /** How many LLM calls the figures aggregate. */
166 168 
  readonly calls?: number | undefined;
167 169 
}

@@ -1022,6 +1024,9 @@ export class CoderSession {

1022 1024 
              promptTokens: chunk.promptTokens,
1023 1025 
              completionTokens: chunk.completionTokens,
1024 1026 
              calls: chunk.calls,
1027 
              ...(chunk.cacheReadInputTokens === undefined
1028 
                ? {}
1029 
                : { cacheReadInputTokens: chunk.cacheReadInputTokens }),
1025 1030 
            };
1026 1031 
          }
1027 1032 
        } else {
packages/openagents-cli/src/coder-thread.ts modified +17 -1

@@ -393,7 +393,13 @@ export class ThreadReplySource implements ReplySource {

393 393 
   */
394 394 
  private sink: TranscriptSink | undefined;
395 395 
  /** The running turn's token usage, accumulated across its model calls. */
396 
  private turnUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0, calls: 0 };
396 
  private turnUsage: {
397 
    promptTokens: number;
398 
    completionTokens: number;
399 
    totalTokens: number;
400 
    calls: number;
401 
    cacheReadInputTokens?: number;
402 
  } = { promptTokens: 0, completionTokens: 0, totalTokens: 0, calls: 0 };
397 403 
  /** Set for the one round that must answer rather than call another tool. */
398 404 
  private mustAnswer = false;
399 405 
  /**

@@ -603,6 +609,9 @@ export class ThreadReplySource implements ReplySource {

603 609 
              promptTokens: this.turnUsage.promptTokens,
604 610 
              completionTokens: this.turnUsage.completionTokens,
605 611 
              calls: this.turnUsage.calls,
612 
              ...(this.turnUsage.cacheReadInputTokens === undefined
613 
                ? {}
614 
                : { cacheReadInputTokens: this.turnUsage.cacheReadInputTokens }),
606 615 
            };
607 616 
          }
608 617 
          this.recordAnswer(turnText, turnToolCalls, signal.aborted);

@@ -682,6 +691,9 @@ export class ThreadReplySource implements ReplySource {

682 691 
        completion_tokens: this.turnUsage.completionTokens,
683 692 
        total_tokens: this.turnUsage.totalTokens,
684 693 
        calls: this.turnUsage.calls,
694 
        ...(this.turnUsage.cacheReadInputTokens === undefined
695 
          ? {}
696 
          : { cache_read_input_tokens: this.turnUsage.cacheReadInputTokens }),
685 697 
      },
686 698 
      tool_calls: toolCalls,
687 699 
      ...(interrupted ? { interrupted: true } : {}),

@@ -914,11 +926,15 @@ export class ThreadReplySource implements ReplySource {

914 926 
    // The same report feeds the turn's own tally, which `turn.assistant`
915 927 
    // carries: a turn is several calls, and the record holds their sum with
916 928 
    // the count rather than the last call's figures presented as the turn's.
929 
    const cache = optional(usage["cache_read_input_tokens"]);
917 930 
    this.turnUsage = {
918 931 
      promptTokens: this.turnUsage.promptTokens + number(usage["prompt_tokens"]),
919 932 
      completionTokens: this.turnUsage.completionTokens + number(usage["completion_tokens"]),
920 933 
      totalTokens: this.turnUsage.totalTokens + total,
921 934 
      calls: this.turnUsage.calls + 1,
935 
      ...(cache === undefined
936 
        ? {}
937 
        : { cacheReadInputTokens: (this.turnUsage.cacheReadInputTokens ?? 0) + cache }),
922 938 
    };
923 939 
  }
924 940
packages/openagents-cli/test/coder-export.test.ts modified +38

@@ -216,6 +216,44 @@ describe("exporting a conversation as ATIF", () => {

216 216 
    });
217 217 
  });
218 218
219 
  it("carries the cache read token split in step metrics extra", () => {
220 
    const { document } = write([
221 
      entry({
222 
        role: "assistant",
223 
        text: "answer",
224 
        metrics: {
225 
          promptTokens: 100,
226 
          completionTokens: 10,
227 
          cacheReadInputTokens: 25,
228 
          calls: 1,
229 
        },
230 
      }),
231 
    ]);
232
233 
    const steps = document["steps"] as ReadonlyArray<Record<string, unknown>>;
234 
    expect(steps[0]).toMatchObject({
235 
      metrics: {
236 
        prompt_tokens: 100,
237 
        completion_tokens: 10,
238 
        extra: { cache_read_input_tokens: 25 },
239 
      },
240 
      llm_call_count: 1,
241 
    });
242 
  });
243
244 
  it("omits the extra metrics block when no cache split was reported", () => {
245 
    const { document } = write([
246 
      entry({
247 
        role: "assistant",
248 
        text: "answer",
249 
        metrics: { promptTokens: 100, completionTokens: 10, calls: 1 },
250 
      }),
251 
    ]);
252
253 
    const steps = document["steps"] as ReadonlyArray<Record<string, unknown>>;
254 
    expect(steps[0]?.["metrics"]).toEqual({ prompt_tokens: 100, completion_tokens: 10 });
255 
  });
256
219 257 
  it("reports no totals rather than zero when nothing measured any", () => {
220 258 
    const { document } = write([entry({ role: "assistant", text: "answer" })]);
221 259
packages/openagents-cli/test/coder-session.test.ts modified +27

@@ -447,6 +447,33 @@ describe("a turn's cost", () => {

447 447 
    const assistant = session.snapshot().entries.find((entry) => entry.role === "assistant");
448 448 
    expect(assistant?.metrics).toEqual({ promptTokens: 12, completionTokens: 34, calls: 2 });
449 449 
  });
450
451 
  it("lands with a cache read split when the source reports one", async () => {
452 
    const session = new CoderSession(
453 
      source([
454 
        { type: "text", value: "answer" },
455 
        {
456 
          type: "usage",
457 
          promptTokens: 12,
458 
          completionTokens: 34,
459 
          calls: 2,
460 
          cacheReadInputTokens: 5,
461 
        },
462 
      ]),
463 
      "repo",
464 
      "main",
465 
    );
466
467 
    await session.submit("go");
468
469 
    const assistant = session.snapshot().entries.find((entry) => entry.role === "assistant");
470 
    expect(assistant?.metrics).toEqual({
471 
      promptTokens: 12,
472 
      completionTokens: 34,
473 
      calls: 2,
474 
      cacheReadInputTokens: 5,
475 
    });
476 
  });
450 477 
});
451 478
452 479 
describe("notices that replace one another", () => {
packages/openagents-cli/test/coder-thread.test.ts modified +31

@@ -747,6 +747,37 @@ describe("the thread's durable transcript", () => {

747 747 
    });
748 748 
  });
749 749
750 
  it("records the cache read split when the server reports one", async () => {
751 
    const CACHE_SSE = [
752 
      `data: {"choices":[{"delta":{"content":"Hello"},"index":0}]}`,
753 
      `data: {"choices":[],"usage":{"completion_tokens":11,"prompt_tokens":12,"total_tokens":23,"cache_read_input_tokens":5}}`,
754 
      `data: [DONE]`,
755 
      "",
756 
    ].join("\n\n");
757 
    stub({ proxy: [sse([CACHE_SSE])] });
758 
    const source = await open();
759 
    const sink = recorder();
760 
    source.useTranscript(sink);
761
762 
    const received = await chunks(source, "hello");
763
764 
    const answer = sink.events.find((event) => event.eventType === "turn.assistant");
765 
    expect(answer?.payload).toEqual({
766 
      text: "Hello",
767 
      usage: {
768 
        prompt_tokens: 12,
769 
        completion_tokens: 11,
770 
        total_tokens: 23,
771 
        calls: 1,
772 
        cache_read_input_tokens: 5,
773 
      },
774 
      tool_calls: 0,
775 
    });
776 
    expect(received.filter((chunk) => chunk.type === "usage")).toEqual([
777 
      { type: "usage", promptTokens: 12, completionTokens: 11, calls: 1, cacheReadInputTokens: 5 },
778 
    ]);
779 
  });
780
750 781 
  it("yields the turn's usage so the session's export can carry it", async () => {
751 782 
    stub({ proxy: [sse([TOOL_ROUND]), sse([ANSWER_ROUND])] });
752 783 
    const source = await open();

This page updates live while a promote is in flight · changelog