Retrieve capabilities for the model instead of asking it to look

858c9a68fdf2 · AtlantisPleb · · parent ac1c7221713f

Retrieve capabilities for the model instead of asking it to look

Models are not trained to consult registries, and the measured record
agrees: enumerate the catalog in the standing prompt and it bloats
with every install; hide it behind a search tool and the model shells
instead. So retrieval moves to the harness (#42). The capability
tool's description is constant now — no names, no growth — and its
query is a real bounded search. On every submitted prompt the session
scores the catalog, auto-loads what clearly matches (digest-verified,
recorded in plugin events), and attaches one line saying which tools
are now available; a weaker match rides as a suggestion, and no match
costs the turn nothing. Proven live: a fresh dev-lane session asked to
read a conversation calls read_conversation directly, with no
capability call in the trace. The system prompt also now says to
answer very concisely unless the reader asks for more — and gains
nothing else.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01E1mRkPGYmTVvMKqAzmQvy5
Co-Authored-By
Claude Fable 5 <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified packages/openagents-cli/src/cli.ts
  • modified packages/openagents-cli/src/coder-capability.ts
  • modified packages/openagents-cli/src/coder-session.ts
  • modified packages/openagents-cli/src/coder-system.ts
  • modified packages/openagents-cli/test/coder-capability.test.ts
  • modified packages/openagents-cli/test/coder-tiers.test.ts

Diff

6 files changed, +250 -53

packages/openagents-cli/src/cli.ts modified +42

@@ -100,6 +100,7 @@ import {

100 100
  capabilityTool,
101 101
  defaultCapabilityGapRecorder,
102 102
  discoverPluginCatalog,
103
  matchCapabilities,
103 104
} from "./coder-capability.js";
104 105
import { runForeignResume } from "./coder-foreign-resume.js";
105 106
import { existsSync } from "node:fs";

@@ -2457,6 +2458,7 @@ const coderCommand = Command.make(

2457 2458
          (buildTier !== undefined && initialTier !== undefined
2458 2459
            ? { initial: initialTier, build: buildTier }
2459 2460
            : undefined),
2461
        (prompt) => capabilityRetrieval(prompt),
2460 2462
      );
2461 2463
2462 2464
      // The resumed thread's history goes on the session before anything new,

@@ -2586,6 +2588,46 @@ const coderCommand = Command.make(

2586 2588
        return described;
2587 2589
      };
2588 2590
2591
      // Retrieval is the harness's job (OpenAgentsInc/openagents#42): score
2592
      // the catalog against each incoming message, auto-load what clearly
2593
      // matches so the model simply sees the right tool, and note a weaker
2594
      // match for the model to load itself. Nothing matched costs nothing.
2595
      const capabilityRetrieval = async (prompt: string): Promise<string | undefined> => {
2596
        const matches = matchCapabilities(catalog, prompt);
2597
        const held = new Set(plugins.map((plugin) => plugin.manifest.name));
2598
2599
        const materialized: string[] = [];
2600
        for (const match of matches.filter((candidate) => candidate.hits >= 2).slice(0, 2)) {
2601
          if (held.has(match.entry.name)) {
2602
            materialized.push(match.entry.name);
2603
            continue;
2604
          }
2605
          const described = loadPlugin(match.entry.manifestPath);
2606
          if (described.startsWith("Loaded plugin")) materialized.push(match.entry.name);
2607
        }
2608
2609
        if (materialized.length > 0) {
2610
          const names = materialized.map((name) => `\`${name}\``).join(", ");
2611
          return (
2612
            `[Attached by the harness: installed capabilities matched this request, and ` +
2613
            `${names} ${materialized.length === 1 ? "is" : "are"} loaded and available as ` +
2614
            `${materialized.length === 1 ? "a tool" : "tools"} right now. Prefer ` +
2615
            `${materialized.length === 1 ? "it" : "them"} over scripting the same thing.]`
2616
          );
2617
        }
2618
2619
        const near = matches.slice(0, 2);
2620
        if (near.length > 0) {
2621
          const names = near.map((candidate) => `\`${candidate.entry.name}\``).join(", ");
2622
          return (
2623
            `[Attached by the harness: ${names} in the installed capability catalog may ` +
2624
            `cover this. Load one with the capability tool, name set exactly, if it fits.]`
2625
          );
2626
        }
2627
2628
        return undefined;
2629
      };
2630
2589 2631
      const locateForeignSessionsManifest = (): string | undefined => {
2590 2632
        let here = dirname(fileURLToPath(import.meta.url));
2591 2633
        while (true) {
packages/openagents-cli/src/coder-capability.ts modified +80 -30

@@ -144,43 +144,83 @@ function catalogDescription(catalog: ReadonlyArray<PluginCatalogEntry>): string

144 144
 * Those appear only after a search with `query` returns the catalog and an
145 145
 * exact-name call with `name` loads the chosen plugin.
146 146
 */
147
/** Most capabilities the standing summary names; the rest ride behind `query`. */
148
const SUMMARY_CAP = 12;
147
/** Most candidates one search returns; the rest are counted, not listed. */
148
const SEARCH_LIMIT = 5;
149 149
150
/** A description's first sentence, for the one-line standing summary. */
150
/** A description's first sentence, for a one-line candidate row. */
151 151
const firstSentence = (text: string): string => {
152 152
  const at = text.indexOf(". ");
153 153
  return at > 0 ? text.slice(0, at + 1) : text;
154 154
};
155 155
156
/** "; and K more" when the catalog holds more than one answer showed. */
157
const remainder = (beyond: number): string =>
158
  beyond > 0 ? `\n…and ${String(beyond)} more; search again with other words.` : "";
159
160
/**
161
 * Lexical retrieval over the catalog: token overlap between the query and
162
 * each manifest's name and description, best first, at most [`SEARCH_LIMIT`].
163
 *
164
 * Retrieval narrows candidates; it never routes. The model still invokes by
165
 * the exact returned name and the load still verifies the digest, so the
166
 * no-keyword-routing law holds. An embedding index can replace this scoring
167
 * without changing the surface (OpenAgentsInc/openagents#42).
168
 */
169
const searchCatalog = (
170
  catalog: ReadonlyArray<PluginCatalogEntry>,
171
  query: string,
172
): PluginCatalogEntry[] =>
173
  matchCapabilities(catalog, query)
174
    .slice(0, SEARCH_LIMIT)
175
    .map((candidate) => candidate.entry);
176
177
/**
178
 * Score the catalog against free text, best first.
179
 *
180
 * Exported because retrieval is the harness's job, not only the model's
181
 * (OpenAgentsInc/openagents#42): the session runs this over each incoming
182
 * message and materializes what matches, so the model sees the right tool
183
 * without having consulted anything.
184
 */
185
export const matchCapabilities = (
186
  catalog: ReadonlyArray<PluginCatalogEntry>,
187
  text: string,
188
): Array<{ readonly entry: PluginCatalogEntry; readonly hits: number }> => {
189
  const terms = tokens(text);
190
  if (terms.length === 0) return [];
191
  const scored = catalog
192
    .map((entry) => {
193
      const haystack = new Set(tokens(`${entry.name} ${entry.description}`));
194
      const hits = terms.filter((term) => haystack.has(term)).length;
195
      return { entry, hits };
196
    })
197
    .filter((candidate) => candidate.hits > 0);
198
  scored.sort((left, right) => right.hits - left.hits);
199
  return scored;
200
};
201
202
/** Lowercased word stems of three letters or more; the rest is noise. */
203
const tokens = (text: string): string[] =>
204
  text
205
    .toLowerCase()
206
    .split(/[^a-z0-9]+/)
207
    .filter((word) => word.length >= 3);
208
156 209
export function capabilityTool(options: CapabilityOptions): CoderTool {
157 210
  const { catalog, approval, recordGap, onSelect, load = loadPluginFromManifest } = options;
158
  // The catalog's names and first sentences ride in the standing
159
  // description: a model that has never heard what is installed answers
160
  // "read that conversation back" with an improvised shell script, and the
161
  // sandboxed, bounded capability sits unused. One tool, but an honest one.
162
  const shown = catalog.slice(0, SUMMARY_CAP);
163
  const beyond = catalog.length - shown.length;
164
  const summary =
165
    catalog.length === 0
166
      ? ""
167
      : "Installed: " +
168
        shown
169
          .map((entry) => `\`${entry.name}\` (${firstSentence(entry.description)})`)
170
          .join("; ") +
171
        (beyond > 0 ? `; and ${String(beyond)} more via \`query\`` : "") +
172
        ". When one of these covers the work, load and call it instead of improvising a script: it is sandboxed, bounded, and returns structured output. ";
173 211
  return {
174 212
    name: "capability",
213
    // Constant-size on purpose (OpenAgentsInc/openagents#42): the catalog is
214
    // searched, never enumerated here, so the standing prompt does not grow
215
    // as capabilities are installed.
175 216
    description:
176
      "Discover and load a local plugin capability from the installed catalog. " +
177
      summary +
178
      "No semantic embedding is available in this package, so `query` returns " +
179
      "the full catalog of installed capabilities and their descriptions for you " +
180
      "to choose from. Do not try to guess a name by substring or keyword. " +
181
      "Once you see the exact catalog name, call `capability` again with `name` " +
182
      "set to that exact name to load it and make its dedicated tool available. " +
183
      "Every later call to the loaded capability uses that exact catalog name as the tool name.",
217
      "Discover and load installed plugin capabilities: sandboxed, sealed programs this " +
218
      "machine already holds for common agent work. Before writing a script for a task, " +
219
      "search here first — a capability that covers it is bounded, reviewable, and returns " +
220
      "structured output. Call with `query` describing what you need to get the best " +
221
      "matches; then call again with `name` set to the exact returned name to load it and " +
222
      "make its dedicated tool available. Every later call to the loaded capability uses " +
223
      "that exact name as the tool name.",
184 224
    parameters: {
185 225
      type: "object",
186 226
      properties: {

@@ -234,13 +274,23 @@ export function capabilityTool(options: CapabilityOptions): CoderTool {

234 274
      }
235 275
236 276
      if (query !== undefined && query.length > 0) {
237
        if (catalog.length === 0) {
277
        const ranked = searchCatalog(catalog, query);
278
        if (ranked.length === 0) {
238 279
          await recordGap({ requestedAt: Date.now(), query });
280
          return catalog.length === 0
281
            ? "No capabilities are installed on this machine."
282
            : "Nothing installed matches that. The full catalog:\n\n" +
283
                `${catalogDescription(catalog.slice(0, SEARCH_LIMIT))}` +
284
                remainder(catalog.length - SEARCH_LIMIT) +
285
                "\n\nCall `capability` with `name` set to an exact name to load it.";
239 286
        }
240 287
        return (
241
          "No semantic embedding is available, so the full catalog is shown for you to choose.\n\n" +
242
          `${catalogDescription(catalog)}\n\n` +
243
          "Call `capability` with `name` set to the exact catalog name you want to load."
288
          "Best matches, most relevant first:\n\n" +
289
          ranked
290
            .map((entry) => `- \`${entry.name}\` — ${firstSentence(entry.description)}`)
291
            .join("\n") +
292
          remainder(catalog.length - ranked.length) +
293
          "\n\nCall `capability` with `name` set to the exact name you want to load."
244 294
        );
245 295
      }
246 296
packages/openagents-cli/src/coder-session.ts modified +19 -1

@@ -552,6 +552,15 @@ export class CoderSession {

552 552
        history: ReadonlyArray<unknown>,
553 553
      ): Promise<ReplySource>;
554 554
    },
555
    /**
556
     * Harness-side capability retrieval, run over each submitted prompt.
557
     *
558
     * Returns a short note to attach to the outgoing turn — after loading
559
     * whatever matched, so the model sees the right tool rather than being
560
     * asked to go looking — or undefined when nothing matched, which costs
561
     * the turn nothing. The model is never expected to consult a registry.
562
     */
563
    private readonly retrieve?: (prompt: string) => Promise<string | undefined>,
555 564
  ) {
556 565
    this.tier = tiers?.initial;
557 566
    // A child reporting progress has to reach the renderer, and the renderer

@@ -1043,6 +1052,14 @@ export class CoderSession {

1043 1052
    try {
1044 1053
      await this.applyPendingTier();
1045 1054
1055
      // Retrieval before the turn goes out: what matched is loaded and its
1056
      // note rides the prompt. A failed retrieval is a turn without a note,
1057
      // never a failed turn.
1058
      const attached =
1059
        this.retrieve === undefined
1060
          ? undefined
1061
          : await this.retrieve(prompt).catch(() => undefined);
1062
1046 1063
      // The reader's entry above keeps what they typed; the model receives the
1047 1064
      // standing context ahead of it on the first turn only.
1048 1065
      const sent =

@@ -1055,8 +1072,9 @@ export class CoderSession {

1055 1072
        this.source.useContext !== undefined
1056 1073
          ? prompt
1057 1074
          : `${this.standing}\n\n---\n\n${prompt}`;
1075
      const outgoing = attached === undefined ? sent : `${sent}\n\n${attached}`;
1058 1076
1059
      for await (const chunk of this.source.reply(sent, controller.signal)) {
1077
      for await (const chunk of this.source.reply(outgoing, controller.signal)) {
1060 1078
        if (controller.signal.aborted) break;
1061 1079
1062 1080
        if (chunk.type === "text") {
packages/openagents-cli/src/coder-system.ts modified +6 -1

@@ -23,7 +23,12 @@ export const systemPrompt = (

23 23
  lane: string,
24 24
  standing?: string,
25 25
): string => {
26
  const lines = [`You are \`openagents coder\`, a coding assistant in a terminal. ${lane}`, ""];
26
  const lines = [
27
    `You are \`openagents coder\`, a coding assistant in a terminal. ${lane}`,
28
    "",
29
    "Answer very concisely unless the reader asks for a longer response.",
30
    "",
31
  ];
27 32
28 33
  if (tools.length === 0) {
29 34
    lines.push(
packages/openagents-cli/test/coder-capability.test.ts modified +67 -21

@@ -8,6 +8,7 @@ import {

8 8
  discoverPluginCatalog,
9 9
  type CapabilityGap,
10 10
  type PluginCatalogEntry,
11
  matchCapabilities,
11 12
} from "../src/coder-capability.js";
12 13
import {
13 14
  isRefusal,

@@ -39,35 +40,38 @@ const baseCatalog: ReadonlyArray<PluginCatalogEntry> = [

39 40
];
40 41
41 42
describe("capabilityTool", () => {
42
  it("declares one standing tool that names the catalog, capped rather than unbounded", () => {
43
    // The description deliberately carries the installed names and first
44
    // sentences: a model that has never heard what is installed answered
45
    // "read that conversation back" with an improvised shell script while
46
    // the capability sat unused. The growth is capped, not zero — past the
47
    // cap, the rest ride behind `query`.
48
    const bigCatalog = Array.from({ length: 20 }, (_unused, index) => ({
43
  it("declares one standing tool whose description is constant, however big the catalog", () => {
44
    // The constant-size contract (OpenAgentsInc/openagents#42): the catalog
45
    // is searched, never enumerated in the standing prompt, so installing a
46
    // capability costs zero standing bytes. The behavioral sentence — search
47
    // before scripting — is what keeps the tool reachable without the list.
48
    const bigCatalog = Array.from({ length: 500 }, (_unused, index) => ({
49 49
      ...baseCatalog[0],
50 50
      name: `demo_${String(index)}`,
51 51
    }));
52
    const tool = capabilityTool({
52
    const small = capabilityTool({
53
      catalog: [],
54
      approval: new PluginApproval(),
55
      recordGap: () => {},
56
      onSelect: () => {},
57
    });
58
    const big = capabilityTool({
53 59
      catalog: bigCatalog,
54 60
      approval: new PluginApproval(),
55 61
      recordGap: () => {},
56 62
      onSelect: () => {},
57 63
    });
58
    expect(tool.name).toBe("capability");
59
    const properties = tool.parameters["properties"] as Record<string, unknown>;
64
    expect(big.name).toBe("capability");
65
    const properties = big.parameters["properties"] as Record<string, unknown>;
60 66
    expect(Object.keys(properties)).toContain("query");
61 67
    expect(Object.keys(properties)).toContain("name");
62
    expect(tool.description).toMatch(/No semantic embedding is available/);
63
    expect(tool.description).toContain("demo_0");
64
    expect(tool.description).toContain("demo_11");
65
    expect(tool.description).not.toContain("demo_12");
66
    expect(tool.description).toContain("and 8 more");
67
    expect(tool.parameters["additionalProperties"]).toBe(false);
68
    expect(big.description).toBe(small.description);
69
    expect(big.description).not.toContain("demo_");
70
    expect(big.description).toContain("search here first");
71
    expect(big.parameters["additionalProperties"]).toBe(false);
68 72
  });
69 73
70
  it("returns the full catalog on a query and does not use keyword selection", async () => {
74
  it("searches the catalog on a query: candidates narrow, invocation stays exact-name", async () => {
71 75
    const catalog: ReadonlyArray<PluginCatalogEntry> = [
72 76
      baseCatalog[0],
73 77
      { ...baseCatalog[0], name: "git_lost_work", description: "Scan a git repository." },

@@ -78,10 +82,20 @@ describe("capabilityTool", () => {

78 82
      recordGap: () => {},
79 83
      onSelect: () => {},
80 84
    });
81
    const output = await tool.run({ query: "word" }, new AbortController().signal);
85
    const output = await tool.run(
86
      { query: "statistics about words" },
87
      new AbortController().signal,
88
    );
89
    // The match leads; the miss is counted, not listed.
82 90
    expect(output).toContain("word_stats");
83
    expect(output).toContain("git_lost_work");
84
    expect(output).toMatch(/No semantic embedding is available/);
91
    expect(output).not.toContain("git_lost_work");
92
    expect(output).toContain("and 1 more");
93
    expect(output).toContain("exact name");
94
95
    // A query nothing matches still shows a bounded catalog to choose from.
96
    const missed = await tool.run({ query: "zzzz" }, new AbortController().signal);
97
    expect(missed).toContain("Nothing installed matches");
98
    expect(missed).toContain("word_stats");
85 99
  });
86 100
87 101
  it("loads and selects a capability by exact catalog name", async () => {

@@ -130,7 +144,7 @@ describe("capabilityTool", () => {

130 144
    const output = await tool.run({ query: "something" }, new AbortController().signal);
131 145
    expect(gaps).toHaveLength(1);
132 146
    expect(gaps[0].query).toBe("something");
133
    expect(output).toMatch(/The local catalog is empty/);
147
    expect(output).toContain("No capabilities are installed");
134 148
  });
135 149
});
136 150

@@ -236,3 +250,35 @@ describe("defaultCapabilityGapRecorder", () => {

236 250
    expect(typeof recorder).toBe("function");
237 251
  });
238 252
});
253
254
describe("matchCapabilities", () => {
255
  const catalog: ReadonlyArray<PluginCatalogEntry> = [
256
    {
257
      name: "read_conversation",
258
      version: "0.1.0",
259
      description: "Read a conversation back from this machine: a Claude Code or Codex session.",
260
      manifestPath: "/tmp/a.json",
261
      artifact: { path: "a.wasm", digest: "sha256:aa" },
262
      capabilities: { mounts: [], hosts: [], timeout_ms: 1000, memory_max_mib: 64 },
263
    },
264
    {
265
      name: "word_stats",
266
      version: "0.1.0",
267
      description: "Compute statistics for a piece of text.",
268
      manifestPath: "/tmp/b.json",
269
      artifact: { path: "b.wasm", digest: "sha256:bb" },
270
      capabilities: { mounts: [], hosts: [], timeout_ms: 1000, memory_max_mib: 64 },
271
    },
272
  ];
273
274
  it("scores the catalog against free text, best first, misses dropped", () => {
275
    const matches = matchCapabilities(catalog, "read back the conversation on this machine");
276
    expect(matches[0]?.entry.name).toBe("read_conversation");
277
    expect(matches[0]?.hits).toBeGreaterThanOrEqual(2);
278
    expect(matches.some((match) => match.entry.name === "word_stats")).toBe(false);
279
  });
280
281
  it("returns nothing for text nothing matches, so a turn costs nothing", () => {
282
    expect(matchCapabilities(catalog, "deploy the kubernetes cluster")).toEqual([]);
283
  });
284
});
packages/openagents-cli/test/coder-tiers.test.ts modified +36

@@ -203,3 +203,39 @@ describe("session tier cycling", () => {

203 203
    expect(notices).not.toContain(TIER_MODELS.flash);
204 204
  });
205 205
});
206
207
// Retrieval is the harness's job: the hook runs over the submitted prompt
208
// and its note rides the outgoing turn, so the model sees the match without
209
// having consulted anything (OpenAgentsInc/openagents#42).
210
describe("session capability retrieval", () => {
211
  it("attaches the retrieval note to what the source receives", async () => {
212
    const seen: string[] = [];
213
    const source: ReplySource = {
214
      model: "Coder Auto",
215
      async *reply(prompt: string) {
216
        seen.push(prompt);
217
        yield { type: "text", value: "ok" } as const;
218
      },
219
    };
220
    const session = new CoderSession(
221
      source,
222
      "repo",
223
      "main",
224
      undefined,
225
      undefined,
226
      undefined,
227
      undefined,
228
      (prompt) =>
229
        Promise.resolve(
230
          prompt.includes("conversation") ? "[Attached by the harness: loaded.]" : undefined,
231
        ),
232
    );
233
234
    await session.submit("read the conversation");
235
    await session.submit("hello there");
236
237
    expect(seen[0]).toContain("read the conversation");
238
    expect(seen[0]).toContain("[Attached by the harness: loaded.]");
239
    expect(seen[1]).toBe("hello there");
240
  });
241
});

This page updates live while a promote is in flight · changelog