Project the engram log into something queryable

d1931a576ed2 · AtlantisPleb · · parent 82edb78eb8cf

Project the engram log into something queryable

Issue #223's projection layer. The log is the authority; this is a
view of it, derived and never stored: project() resolves each slug's
supersession chain, names tombstones rather than dropping them
silently, refuses a forked or broken chain whole rather than applying
half a correction, gathers slugs under the entity that names them and
the edges they were distilled from, and counts every event it refused
so a gap is countable rather than invisible.

Three properties the tests hold it to: derived only (a pure function,
no clock and no ambient state), idempotent (the same log gives the
same digest, so a cold rebuild and a warm one are one value), and
order-independent (a relay replaying out of sequence projects the
same thing, because order comes from the supersession links rather
than from arrival).

CoderMemory now reads through it, so the projection is the live path
rather than a parallel one, and it re-derives the moment it disagrees
with the log — a projection that cannot be trusted is discarded, never
repaired in place. Embeddings and semantic search stay ahead of this;
the relational half is what a cold start can rebuild deterministically.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GoYpb8FEmdxVErsv7ABCYi
Co-Authored-By
Claude Fable 5 <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified docs/assure-repo/false-green-candidates.v1.json
  • modified docs/assure-repo/surface-inventory.v1.json
  • modified packages/agent-experience-memory/src/index.ts
  • added packages/agent-experience-memory/src/projection.test.ts
  • added packages/agent-experience-memory/src/projection.ts
  • modified packages/openagents-cli/scripts/vendor-memory.mjs
  • modified packages/openagents-cli/src/coder-memory.ts
  • modified packages/openagents-cli/src/memory/index.ts
  • added packages/openagents-cli/src/memory/projection.ts

Diff

9 files changed, +726 -18

docs/assure-repo/false-green-candidates.v1.json modified +1 -1

@@ -4,7 +4,7 @@

4 4
  "note": "Heuristic false-green LEADS, not findings. A finding requires a demonstrated reproduction (surviving mutation via mutation-runner). Do not treat a candidate as a confirmed false green. Coverage-theater leads may include tests that delegate their assertion to a custom helper the classifier does not recognise; verify before acting.",
5 5
  "sourceDigest": "sha256:dd810dd48c5bdbc9becd7fcc01dd41a4ca2abf0b2d6f6a545907247f6e3e8361",
6 6
  "summary": {
7
    "filesScanned": 2464,
7
    "filesScanned": 2465,
8 8
    "candidateCount": 16,
9 9
    "byMode": {
10 10
      "false_green_coverage_theater": 15,
docs/assure-repo/surface-inventory.v1.json modified +2 -2

@@ -1,7 +1,7 @@

1 1
{
2 2
  "schemaVersion": "1",
3 3
  "repository": "OpenAgentsInc/openagents",
4
  "sourceDigest": "sha256:432d55f76eea481e2c0451fb1f2511fa7e93ec7ac0f709ccb0aaed7ed1fa912c",
4
  "sourceDigest": "sha256:d54f284f2e5b93477ae762b1e4ca044f5be9959a7496e1e3fb4933e29b74c68b",
5 5
  "surfaces": [
6 6
    {
7 7
      "id": "app:@openagentsinc/acceptance-runner",

@@ -1533,7 +1533,7 @@

1533 1533
      "oracles": [
1534 1534
        {
1535 1535
          "type": "test",
1536
          "ref": "packages/agent-experience-memory (9 tracked test files)"
1536
          "ref": "packages/agent-experience-memory (10 tracked test files)"
1537 1537
        }
1538 1538
      ],
1539 1539
      "obligation": {
packages/agent-experience-memory/src/index.ts modified +1

@@ -39,3 +39,4 @@ export * from "./graph-memory-store.js";

39 39
export * from "./engram.js";
40 40
export * from "./consolidation.js";
41 41
export * from "./subagent-memory.js";
42
export * from "./projection.js";
packages/agent-experience-memory/src/projection.test.ts added +136

@@ -0,0 +1,136 @@

1
import { describe, expect, test } from "vite-plus/test";
2
3
import {
4
  buildEngramBody,
5
  buildEngramEvent,
6
  engramContentDigest,
7
  signSupersedingEngram,
8
  type EngramEvent,
9
} from "./engram.js";
10
import { project, projectedValue, projectionMatches } from "./projection.js";
11
12
const PUBKEY = "a".repeat(64);
13
const sign = (eventId: string): string => `sig-${eventId.slice(0, 16)}`;
14
15
const engram = (
16
  slug: string,
17
  value: string | null,
18
  createdAt: number,
19
  entityId = "entity-1",
20
  derivedFromSlugs: ReadonlyArray<string> = [],
21
): EngramEvent => {
22
  const body = buildEngramBody(slug, value, {
23
    admission: "admitted",
24
    entityId,
25
    contentDigest: engramContentDigest(value),
26
    sourceEventRefs: [],
27
    relations: [],
28
    derivedFromSlugs: [...derivedFromSlugs],
29
  });
30
  return buildEngramEvent(PUBKEY, createdAt, slug, JSON.stringify(body), sign);
31
};
32
33
describe("projecting an engram log", () => {
34
  test("derives the live value per slug", () => {
35
    const projection = project([
36
      engram("note/one", "first", 1_000),
37
      engram("note/two", "second", 1_001),
38
    ]);
39
    expect(projectedValue(projection, "note/one")).toBe("first");
40
    expect(projectedValue(projection, "note/two")).toBe("second");
41
    expect(projection.entries).toHaveLength(2);
42
  });
43
44
  test("resolves a supersession chain to its newest value", () => {
45
    const first = engram("note/one", "first", 1_000);
46
    const second = signSupersedingEngram(first, "corrected", 1_001, PUBKEY, sign);
47
    const third = signSupersedingEngram(second, "corrected twice", 1_002, PUBKEY, sign);
48
49
    const projection = project([first, second, third]);
50
    expect(projectedValue(projection, "note/one")).toBe("corrected twice");
51
    expect(projection.entries[0]?.revisions).toBe(3);
52
    expect(projection.entries[0]?.head).toBe(third.id);
53
  });
54
55
  test("names a tombstoned slug rather than dropping it silently", () => {
56
    const first = engram("note/one", "first", 1_000);
57
    const tombstone = signSupersedingEngram(first, null, 1_001, PUBKEY, sign);
58
59
    const projection = project([first, tombstone]);
60
    expect(projection.tombstoned).toEqual(["note/one"]);
61
    expect(projectedValue(projection, "note/one")).toBeUndefined();
62
  });
63
64
  test("is order-independent: shuffled arrival projects identically", () => {
65
    const first = engram("note/one", "first", 1_000);
66
    const second = signSupersedingEngram(first, "corrected", 1_001, PUBKEY, sign);
67
    const other = engram("note/two", "second", 1_002);
68
69
    const forwards = project([first, second, other]);
70
    const backwards = project([other, second, first]);
71
    expect(backwards.digest).toBe(forwards.digest);
72
  });
73
74
  test("is idempotent: a cold rebuild equals the warm one", () => {
75
    const events = [
76
      engram("note/one", "first", 1_000),
77
      engram("note/two", "second", 1_001, "entity-2"),
78
    ];
79
    expect(project(events).digest).toBe(project(events).digest);
80
    expect(projectionMatches(project(events), events)).toBe(true);
81
  });
82
83
  test("refuses a tampered event and counts it", () => {
84
    const authentic = engram("note/one", "authentic", 1_000);
85
    const tampered: EngramEvent = {
86
      ...authentic,
87
      content: authentic.content.replace("authentic", "forged"),
88
    };
89
90
    const projection = project([tampered]);
91
    expect(projection.entries).toHaveLength(0);
92
    expect(projection.rejected.unverified).toBe(1);
93
  });
94
95
  test("drops a forked chain whole rather than applying half a correction", () => {
96
    const first = engram("note/one", "first", 1_000);
97
    const branchA = signSupersedingEngram(first, "branch a", 1_001, PUBKEY, sign);
98
    const branchB = signSupersedingEngram(first, "branch b", 1_002, PUBKEY, sign);
99
100
    const projection = project([first, branchA, branchB]);
101
    expect(projectedValue(projection, "note/one")).toBeUndefined();
102
    expect(projection.rejected.unresolvedChain).toBe(3);
103
  });
104
105
  test("drops a chain whose prior link is missing", () => {
106
    const first = engram("note/one", "first", 1_000);
107
    const second = signSupersedingEngram(first, "corrected", 1_001, PUBKEY, sign);
108
109
    // The root never arrived.
110
    const projection = project([second]);
111
    expect(projectedValue(projection, "note/one")).toBeUndefined();
112
    expect(projection.rejected.unresolvedChain).toBe(1);
113
  });
114
115
  test("gathers slugs under the entity that names them, and derivation edges", () => {
116
    const projection = project([
117
      engram("fact/a", "a", 1_000, "entity-1"),
118
      engram("fact/b", "b", 1_001, "entity-1"),
119
      engram("heuristic/x", "distilled", 1_002, "entity-2", ["fact/a", "fact/b"]),
120
    ]);
121
122
    const entity = projection.entities.find((candidate) => candidate.entityId === "entity-1");
123
    expect(entity?.slugs).toEqual(["fact/a", "fact/b"]);
124
    expect(projection.relations).toEqual([
125
      { from: "heuristic/x", to: "fact/a", type: "derived_from" },
126
      { from: "heuristic/x", to: "fact/b", type: "derived_from" },
127
    ]);
128
  });
129
130
  test("notices when the log has moved past the projection", () => {
131
    const events = [engram("note/one", "first", 1_000)];
132
    const projection = project(events);
133
    const grown = [...events, engram("note/two", "second", 1_001)];
134
    expect(projectionMatches(projection, grown)).toBe(false);
135
  });
136
});
packages/agent-experience-memory/src/projection.ts added +256

@@ -0,0 +1,256 @@

1
import { Schema as S } from "effect";
2
3
import { canonicalStringify } from "./internal/canonical.js";
4
import { sha256Hex } from "./internal/sha256.js";
5
import { computeEngramEventId, EngramBody, type EngramEvent } from "./engram.js";
6
7
/**
8
 * The derived projection over an engram log (issue #223).
9
 *
10
 * The engram stream is the authority; this is a view of it. Nothing here is
11
 * stored anywhere that survives a process, and nothing here can be repaired in
12
 * place — a projection that disagrees with the log is discarded and rebuilt.
13
 * That is the whole design: an append-only signed log is the thing you can
14
 * trust, and a queryable index over it is a convenience that must never become
15
 * a second source of truth.
16
 *
17
 * Three properties the tests hold this to:
18
 *
19
 * 1. **Derived only.** `project` is a pure function of the events it is given.
20
 *    No clock, no randomness, no ambient state.
21
 * 2. **Idempotent.** Projecting the same log twice produces an identical
22
 *    value, digest included, so a cold-start rebuild is indistinguishable from
23
 *    a warm one.
24
 * 3. **Order-independent.** Events arriving in any order — a relay replaying
25
 *    out of sequence, two clients merging — project to the same value, because
26
 *    order is taken from `created_at` and the supersession chain rather than
27
 *    from arrival.
28
 *
29
 * An event that fails verification is not projected. A chain whose
30
 * supersession does not resolve is dropped whole rather than partly applied:
31
 * half a correction is worse than none, because it reads as fact.
32
 */
33
34
export const PROJECTION_SCHEMA_ID = "openagents.engram_projection.v1" as const;
35
36
/** One live slug: the surviving value and where it came from. */
37
export const ProjectedEntry = S.Struct({
38
  slug: S.String,
39
  value: S.String,
40
  entityId: S.String,
41
  /** How many events stand behind this value, corrections included. */
42
  revisions: S.Number,
43
  /** The event id of the newest engram in the chain. */
44
  head: S.String.check(S.isPattern(/^[0-9a-f]{64}$/)),
45
  /** Seconds since the epoch, from the surviving engram. */
46
  updatedAt: S.Number,
47
  derivedFromSlugs: S.Array(S.String),
48
});
49
export type ProjectedEntry = typeof ProjectedEntry.Type;
50
51
/** One entity gathered from the slugs that name it. */
52
export const ProjectedEntity = S.Struct({
53
  entityId: S.String,
54
  slugs: S.Array(S.String),
55
});
56
export type ProjectedEntity = typeof ProjectedEntity.Type;
57
58
/** One derivation edge: a slug and something it was distilled from. */
59
export const ProjectedRelation = S.Struct({
60
  from: S.String,
61
  to: S.String,
62
  type: S.String,
63
});
64
export type ProjectedRelation = typeof ProjectedRelation.Type;
65
66
export const Projection = S.Struct({
67
  schema: S.Literal(PROJECTION_SCHEMA_ID),
68
  entries: S.Array(ProjectedEntry),
69
  entities: S.Array(ProjectedEntity),
70
  relations: S.Array(ProjectedRelation),
71
  /** Slugs whose surviving engram is a tombstone. Named, not silently gone. */
72
  tombstoned: S.Array(S.String),
73
  /** Events the projection refused, by reason, so a gap is countable. */
74
  rejected: S.Struct({
75
    unverified: S.Number,
76
    unresolvedChain: S.Number,
77
    malformed: S.Number,
78
  }),
79
  /** The digest of everything above: two equal logs give one digest. */
80
  digest: S.String.check(S.isPattern(/^sha256:[0-9a-f]{64}$/)),
81
});
82
export type Projection = typeof Projection.Type;
83
84
const dTagOf = (event: EngramEvent): string | undefined =>
85
  event.tags.find((tag) => tag[0] === "d")?.[1];
86
87
const bodyOf = (event: EngramEvent): EngramBody | undefined => {
88
  try {
89
    return JSON.parse(event.content) as EngramBody;
90
  } catch {
91
    return undefined;
92
  }
93
};
94
95
/**
96
 * Order one slug's events into a chain, oldest first.
97
 *
98
 * The `supersedes` links are the authority — `created_at` only breaks ties
99
 * among roots, because a clock is a claim and a reference is a fact. A chain
100
 * that does not resolve into a single line (a fork, a cycle, a missing link)
101
 * is refused whole.
102
 */
103
const resolveChain = (
104
  events: ReadonlyArray<EngramEvent>,
105
): ReadonlyArray<EngramEvent> | undefined => {
106
  if (events.length === 0) return undefined;
107
  const byId = new Map(events.map((event) => [event.id, event]));
108
  const supersededBy = new Map<string, EngramEvent>();
109
  const roots: Array<EngramEvent> = [];
110
111
  for (const event of events) {
112
    const body = bodyOf(event);
113
    if (body === undefined) return undefined;
114
    const prior = body.openagents.supersedes;
115
    if (prior === undefined) {
116
      roots.push(event);
117
      continue;
118
    }
119
    if (!byId.has(prior)) return undefined;
120
    // Two events superseding the same prior is a fork, not a chain.
121
    if (supersededBy.has(prior)) return undefined;
122
    supersededBy.set(prior, event);
123
  }
124
125
  if (roots.length !== 1) return undefined;
126
  const chain: Array<EngramEvent> = [];
127
  const seen = new Set<string>();
128
  let current: EngramEvent | undefined = roots[0];
129
  while (current !== undefined) {
130
    if (seen.has(current.id)) return undefined;
131
    seen.add(current.id);
132
    chain.push(current);
133
    current = supersededBy.get(current.id);
134
  }
135
  // Every event must be on the one line; a stray is an unresolved chain.
136
  return chain.length === events.length ? chain : undefined;
137
};
138
139
/**
140
 * Build the projection for a log.
141
 *
142
 * Pass every engram known for the scope. Order does not matter.
143
 */
144
export const project = (events: ReadonlyArray<EngramEvent>): Projection => {
145
  let unverified = 0;
146
  let malformed = 0;
147
  let unresolvedChain = 0;
148
149
  const bySlug = new Map<string, Array<EngramEvent>>();
150
  for (const event of events) {
151
    if (event.id !== computeEngramEventId(event)) {
152
      unverified += 1;
153
      continue;
154
    }
155
    const slug = dTagOf(event);
156
    if (slug === undefined || bodyOf(event) === undefined) {
157
      malformed += 1;
158
      continue;
159
    }
160
    const chain = bySlug.get(slug) ?? [];
161
    chain.push(event);
162
    bySlug.set(slug, chain);
163
  }
164
165
  const entries: Array<ProjectedEntry> = [];
166
  const tombstoned: Array<string> = [];
167
  const relations: Array<ProjectedRelation> = [];
168
  const entityToSlugs = new Map<string, Set<string>>();
169
170
  for (const [slug, slugEvents] of bySlug) {
171
    const ordered = [...slugEvents].sort((left, right) =>
172
      left.created_at !== right.created_at
173
        ? left.created_at - right.created_at
174
        : left.id.localeCompare(right.id),
175
    );
176
    const chain = resolveChain(ordered);
177
    if (chain === undefined) {
178
      unresolvedChain += slugEvents.length;
179
      continue;
180
    }
181
    const head = chain[chain.length - 1];
182
    if (head === undefined) continue;
183
    const body = bodyOf(head);
184
    if (body === undefined) {
185
      malformed += 1;
186
      continue;
187
    }
188
    if (body.value === null) {
189
      tombstoned.push(slug);
190
      continue;
191
    }
192
193
    entries.push({
194
      slug,
195
      value: body.value,
196
      entityId: body.openagents.entityId,
197
      revisions: chain.length,
198
      head: head.id,
199
      updatedAt: head.created_at,
200
      derivedFromSlugs: [...body.openagents.derivedFromSlugs].sort(),
201
    });
202
203
    const slugs = entityToSlugs.get(body.openagents.entityId) ?? new Set<string>();
204
    slugs.add(slug);
205
    entityToSlugs.set(body.openagents.entityId, slugs);
206
207
    for (const source of body.openagents.derivedFromSlugs) {
208
      relations.push({ from: slug, to: source, type: "derived_from" });
209
    }
210
    for (const relation of body.openagents.relations) {
211
      relations.push({ from: slug, to: relation.targetSlug, type: relation.type });
212
    }
213
  }
214
215
  entries.sort((left, right) => left.slug.localeCompare(right.slug));
216
  tombstoned.sort();
217
  relations.sort(
218
    (left, right) =>
219
      left.from.localeCompare(right.from) ||
220
      left.to.localeCompare(right.to) ||
221
      left.type.localeCompare(right.type),
222
  );
223
  const entities = [...entityToSlugs.entries()]
224
    .map(([entityId, slugs]) => ({ entityId, slugs: [...slugs].sort() }))
225
    .sort((left, right) => left.entityId.localeCompare(right.entityId));
226
227
  const withoutDigest = {
228
    schema: PROJECTION_SCHEMA_ID,
229
    entries,
230
    entities,
231
    relations,
232
    tombstoned,
233
    rejected: { unverified, unresolvedChain, malformed },
234
  } as const;
235
236
  return {
237
    ...withoutDigest,
238
    digest: `sha256:${sha256Hex(canonicalStringify(withoutDigest))}`,
239
  };
240
};
241
242
/**
243
 * Whether a projection still describes a log.
244
 *
245
 * The check is a rebuild: re-project and compare digests. There is no cheaper
246
 * honest answer, because a projection carries no authority of its own — if it
247
 * disagrees with the log, the log is right and the projection is discarded.
248
 */
249
export const projectionMatches = (
250
  projection: Projection,
251
  events: ReadonlyArray<EngramEvent>,
252
): boolean => project(events).digest === projection.digest;
253
254
/** The live value for one slug, or undefined when absent or tombstoned. */
255
export const projectedValue = (projection: Projection, slug: string): string | undefined =>
256
  projection.entries.find((entry) => entry.slug === slug)?.value;
packages/openagents-cli/scripts/vendor-memory.mjs modified +9

@@ -38,6 +38,14 @@ export const VENDORED = [

38 38
      ["./internal/sha256.js", "./sha256.js"],
39 39
    ],
40 40
  ],
41
  [
42
    "agent-experience-memory/src/projection.ts",
43
    "projection.ts",
44
    [
45
      ["./internal/canonical.js", "./canonical.js"],
46
      ["./internal/sha256.js", "./sha256.js"],
47
    ],
48
  ],
41 49
  [
42 50
    "agent-experience-memory/src/consolidation.ts",
43 51
    "consolidation.ts",

@@ -78,6 +86,7 @@ const BARREL = `// The barrel coder-memory.ts consumes; regenerated by scripts/v

78 86
export * from "./engram.js";
79 87
export * from "./consolidation.js";
80 88
export * from "./subagent-memory.js";
89
export * from "./projection.js";
81 90
`;
82 91
83 92
export const writeAll = () => {
packages/openagents-cli/src/coder-memory.ts modified +61 -15

@@ -21,12 +21,16 @@ import {

21 21
  ledgerEntriesAsHeuristics,
22 22
  promoteHeuristicToPattern,
23 23
  signSupersedingEngram,
24
  project,
25
  projectedValue,
26
  projectionMatches,
24 27
  verifyEngramEventId,
25
  verifySupersessionChain,
28
  COMPANION_SCHEMA_ID,
26 29
  HarvestedLedgerEntry,
27 30
  type EngramEvent,
28 31
  type EngramBody,
29 32
  type ParentHeuristic,
33
  type Projection,
30 34
} from "./memory/index.js";
31 35
import { Schema as S } from "effect";
32 36

@@ -112,6 +116,7 @@ export class CoderMemory implements CoderDelegationMemory {

112 116
  private readonly now: () => number;
113 117
  private readonly dreamThreshold: number;
114 118
  private key: Buffer | undefined;
119
  private cachedProjection: Projection | undefined;
115 120
116 121
  constructor(options: CoderMemoryOptions = {}) {
117 122
    this.directory = options.directory ?? join(homedir(), ".openagents", "memory");

@@ -174,6 +179,7 @@ export class CoderMemory implements CoderDelegationMemory {

174 179
      sign,
175 180
    );
176 181
    appendFileSync(this.ledgerPath, `${JSON.stringify(event)}\n`, { mode: 0o600 });
182
    this.cachedProjection = undefined;
177 183
    return event;
178 184
  }
179 185

@@ -200,6 +206,21 @@ export class CoderMemory implements CoderDelegationMemory {

200 206
    return event;
201 207
  }
202 208
209
  /** Every engram on disk, malformed lines dropped. The projection judges them. */
210
  private events(): ReadonlyArray<EngramEvent> {
211
    if (!existsSync(this.ledgerPath)) return [];
212
    const events: Array<EngramEvent> = [];
213
    for (const line of readFileSync(this.ledgerPath, "utf8").split("\n")) {
214
      if (line.trim().length === 0) continue;
215
      try {
216
        events.push(JSON.parse(line) as EngramEvent);
217
      } catch {
218
        continue;
219
      }
220
    }
221
    return events;
222
  }
223
203 224
  /** All ledger events grouped per slug in append order, invalid lines dropped. */
204 225
  private chains(): Map<string, Array<EngramEvent>> {
205 226
    const chains = new Map<string, Array<EngramEvent>>();

@@ -232,21 +253,46 @@ export class CoderMemory implements CoderDelegationMemory {

232 253
  }
233 254
234 255
  private living(): ReadonlyArray<{ body: EngramBody; createdAtMs: number }> {
235
    const out: Array<{ body: EngramBody; createdAtMs: number }> = [];
236
    for (const chain of this.chains().values()) {
237
      if (!verifySupersessionChain(chain)) continue;
238
      const last = chain[chain.length - 1];
239
      if (last === undefined) continue;
240
      let body: EngramBody;
241
      try {
242
        body = JSON.parse(last.content) as EngramBody;
243
      } catch {
244
        continue;
245
      }
246
      if (body.value === null) continue;
247
      out.push({ body, createdAtMs: last.created_at * 1000 });
256
    const projection = this.projection();
257
    return projection.entries.map((entry) => ({
258
      body: {
259
        slug: entry.slug,
260
        value: entry.value,
261
        openagents: {
262
          schema: COMPANION_SCHEMA_ID,
263
          admission: "admitted",
264
          entityId: entry.entityId,
265
          contentDigest: engramContentDigest(entry.value),
266
          sourceEventRefs: [],
267
          relations: [],
268
          derivedFromSlugs: [...entry.derivedFromSlugs],
269
        },
270
      },
271
      createdAtMs: entry.updatedAt * 1000,
272
    }));
273
  }
274
275
  /**
276
   * The queryable view over the ledger (issue #223).
277
   *
278
   * Derived, never stored: the projection resolves each slug's supersession
279
   * chain, names tombstones rather than dropping them, refuses a forked or
280
   * broken chain whole, and counts what it refused. It is cached against the
281
   * log it was built from and rebuilt the moment they disagree, so a cold
282
   * start and a warm one are the same value.
283
   */
284
  projection(): Projection {
285
    const events = this.events();
286
    if (this.cachedProjection !== undefined && projectionMatches(this.cachedProjection, events)) {
287
      return this.cachedProjection;
248 288
    }
249
    return out;
289
    this.cachedProjection = project(events);
290
    return this.cachedProjection;
291
  }
292
293
  /** The live value for one slug, or undefined when absent or tombstoned. */
294
  recall(slug: string): string | undefined {
295
    return projectedValue(this.projection(), slug);
250 296
  }
251 297
252 298
  /**
packages/openagents-cli/src/memory/index.ts modified +1

@@ -2,3 +2,4 @@

2 2
export * from "./engram.js";
3 3
export * from "./consolidation.js";
4 4
export * from "./subagent-memory.js";
5
export * from "./projection.js";
packages/openagents-cli/src/memory/projection.ts added +259

@@ -0,0 +1,259 @@

1
// Vendored from packages/agent-experience-memory/src/projection.ts by scripts/vendor-memory.mjs — do not edit here.
2
// The drift guard (test/vendored-memory-drift.test.ts) fails when this copy
3
// no longer matches the canonical source.
4
import { Schema as S } from "effect";
5
6
import { canonicalStringify } from "./canonical.js";
7
import { sha256Hex } from "./sha256.js";
8
import { computeEngramEventId, EngramBody, type EngramEvent } from "./engram.js";
9
10
/**
11
 * The derived projection over an engram log (issue #223).
12
 *
13
 * The engram stream is the authority; this is a view of it. Nothing here is
14
 * stored anywhere that survives a process, and nothing here can be repaired in
15
 * place — a projection that disagrees with the log is discarded and rebuilt.
16
 * That is the whole design: an append-only signed log is the thing you can
17
 * trust, and a queryable index over it is a convenience that must never become
18
 * a second source of truth.
19
 *
20
 * Three properties the tests hold this to:
21
 *
22
 * 1. **Derived only.** `project` is a pure function of the events it is given.
23
 *    No clock, no randomness, no ambient state.
24
 * 2. **Idempotent.** Projecting the same log twice produces an identical
25
 *    value, digest included, so a cold-start rebuild is indistinguishable from
26
 *    a warm one.
27
 * 3. **Order-independent.** Events arriving in any order — a relay replaying
28
 *    out of sequence, two clients merging — project to the same value, because
29
 *    order is taken from `created_at` and the supersession chain rather than
30
 *    from arrival.
31
 *
32
 * An event that fails verification is not projected. A chain whose
33
 * supersession does not resolve is dropped whole rather than partly applied:
34
 * half a correction is worse than none, because it reads as fact.
35
 */
36
37
export const PROJECTION_SCHEMA_ID = "openagents.engram_projection.v1" as const;
38
39
/** One live slug: the surviving value and where it came from. */
40
export const ProjectedEntry = S.Struct({
41
  slug: S.String,
42
  value: S.String,
43
  entityId: S.String,
44
  /** How many events stand behind this value, corrections included. */
45
  revisions: S.Number,
46
  /** The event id of the newest engram in the chain. */
47
  head: S.String.check(S.isPattern(/^[0-9a-f]{64}$/)),
48
  /** Seconds since the epoch, from the surviving engram. */
49
  updatedAt: S.Number,
50
  derivedFromSlugs: S.Array(S.String),
51
});
52
export type ProjectedEntry = typeof ProjectedEntry.Type;
53
54
/** One entity gathered from the slugs that name it. */
55
export const ProjectedEntity = S.Struct({
56
  entityId: S.String,
57
  slugs: S.Array(S.String),
58
});
59
export type ProjectedEntity = typeof ProjectedEntity.Type;
60
61
/** One derivation edge: a slug and something it was distilled from. */
62
export const ProjectedRelation = S.Struct({
63
  from: S.String,
64
  to: S.String,
65
  type: S.String,
66
});
67
export type ProjectedRelation = typeof ProjectedRelation.Type;
68
69
export const Projection = S.Struct({
70
  schema: S.Literal(PROJECTION_SCHEMA_ID),
71
  entries: S.Array(ProjectedEntry),
72
  entities: S.Array(ProjectedEntity),
73
  relations: S.Array(ProjectedRelation),
74
  /** Slugs whose surviving engram is a tombstone. Named, not silently gone. */
75
  tombstoned: S.Array(S.String),
76
  /** Events the projection refused, by reason, so a gap is countable. */
77
  rejected: S.Struct({
78
    unverified: S.Number,
79
    unresolvedChain: S.Number,
80
    malformed: S.Number,
81
  }),
82
  /** The digest of everything above: two equal logs give one digest. */
83
  digest: S.String.check(S.isPattern(/^sha256:[0-9a-f]{64}$/)),
84
});
85
export type Projection = typeof Projection.Type;
86
87
const dTagOf = (event: EngramEvent): string | undefined =>
88
  event.tags.find((tag) => tag[0] === "d")?.[1];
89
90
const bodyOf = (event: EngramEvent): EngramBody | undefined => {
91
  try {
92
    return JSON.parse(event.content) as EngramBody;
93
  } catch {
94
    return undefined;
95
  }
96
};
97
98
/**
99
 * Order one slug's events into a chain, oldest first.
100
 *
101
 * The `supersedes` links are the authority — `created_at` only breaks ties
102
 * among roots, because a clock is a claim and a reference is a fact. A chain
103
 * that does not resolve into a single line (a fork, a cycle, a missing link)
104
 * is refused whole.
105
 */
106
const resolveChain = (
107
  events: ReadonlyArray<EngramEvent>,
108
): ReadonlyArray<EngramEvent> | undefined => {
109
  if (events.length === 0) return undefined;
110
  const byId = new Map(events.map((event) => [event.id, event]));
111
  const supersededBy = new Map<string, EngramEvent>();
112
  const roots: Array<EngramEvent> = [];
113
114
  for (const event of events) {
115
    const body = bodyOf(event);
116
    if (body === undefined) return undefined;
117
    const prior = body.openagents.supersedes;
118
    if (prior === undefined) {
119
      roots.push(event);
120
      continue;
121
    }
122
    if (!byId.has(prior)) return undefined;
123
    // Two events superseding the same prior is a fork, not a chain.
124
    if (supersededBy.has(prior)) return undefined;
125
    supersededBy.set(prior, event);
126
  }
127
128
  if (roots.length !== 1) return undefined;
129
  const chain: Array<EngramEvent> = [];
130
  const seen = new Set<string>();
131
  let current: EngramEvent | undefined = roots[0];
132
  while (current !== undefined) {
133
    if (seen.has(current.id)) return undefined;
134
    seen.add(current.id);
135
    chain.push(current);
136
    current = supersededBy.get(current.id);
137
  }
138
  // Every event must be on the one line; a stray is an unresolved chain.
139
  return chain.length === events.length ? chain : undefined;
140
};
141
142
/**
143
 * Build the projection for a log.
144
 *
145
 * Pass every engram known for the scope. Order does not matter.
146
 */
147
export const project = (events: ReadonlyArray<EngramEvent>): Projection => {
148
  let unverified = 0;
149
  let malformed = 0;
150
  let unresolvedChain = 0;
151
152
  const bySlug = new Map<string, Array<EngramEvent>>();
153
  for (const event of events) {
154
    if (event.id !== computeEngramEventId(event)) {
155
      unverified += 1;
156
      continue;
157
    }
158
    const slug = dTagOf(event);
159
    if (slug === undefined || bodyOf(event) === undefined) {
160
      malformed += 1;
161
      continue;
162
    }
163
    const chain = bySlug.get(slug) ?? [];
164
    chain.push(event);
165
    bySlug.set(slug, chain);
166
  }
167
168
  const entries: Array<ProjectedEntry> = [];
169
  const tombstoned: Array<string> = [];
170
  const relations: Array<ProjectedRelation> = [];
171
  const entityToSlugs = new Map<string, Set<string>>();
172
173
  for (const [slug, slugEvents] of bySlug) {
174
    const ordered = [...slugEvents].sort((left, right) =>
175
      left.created_at !== right.created_at
176
        ? left.created_at - right.created_at
177
        : left.id.localeCompare(right.id),
178
    );
179
    const chain = resolveChain(ordered);
180
    if (chain === undefined) {
181
      unresolvedChain += slugEvents.length;
182
      continue;
183
    }
184
    const head = chain[chain.length - 1];
185
    if (head === undefined) continue;
186
    const body = bodyOf(head);
187
    if (body === undefined) {
188
      malformed += 1;
189
      continue;
190
    }
191
    if (body.value === null) {
192
      tombstoned.push(slug);
193
      continue;
194
    }
195
196
    entries.push({
197
      slug,
198
      value: body.value,
199
      entityId: body.openagents.entityId,
200
      revisions: chain.length,
201
      head: head.id,
202
      updatedAt: head.created_at,
203
      derivedFromSlugs: [...body.openagents.derivedFromSlugs].sort(),
204
    });
205
206
    const slugs = entityToSlugs.get(body.openagents.entityId) ?? new Set<string>();
207
    slugs.add(slug);
208
    entityToSlugs.set(body.openagents.entityId, slugs);
209
210
    for (const source of body.openagents.derivedFromSlugs) {
211
      relations.push({ from: slug, to: source, type: "derived_from" });
212
    }
213
    for (const relation of body.openagents.relations) {
214
      relations.push({ from: slug, to: relation.targetSlug, type: relation.type });
215
    }
216
  }
217
218
  entries.sort((left, right) => left.slug.localeCompare(right.slug));
219
  tombstoned.sort();
220
  relations.sort(
221
    (left, right) =>
222
      left.from.localeCompare(right.from) ||
223
      left.to.localeCompare(right.to) ||
224
      left.type.localeCompare(right.type),
225
  );
226
  const entities = [...entityToSlugs.entries()]
227
    .map(([entityId, slugs]) => ({ entityId, slugs: [...slugs].sort() }))
228
    .sort((left, right) => left.entityId.localeCompare(right.entityId));
229
230
  const withoutDigest = {
231
    schema: PROJECTION_SCHEMA_ID,
232
    entries,
233
    entities,
234
    relations,
235
    tombstoned,
236
    rejected: { unverified, unresolvedChain, malformed },
237
  } as const;
238
239
  return {
240
    ...withoutDigest,
241
    digest: `sha256:${sha256Hex(canonicalStringify(withoutDigest))}`,
242
  };
243
};
244
245
/**
246
 * Whether a projection still describes a log.
247
 *
248
 * The check is a rebuild: re-project and compare digests. There is no cheaper
249
 * honest answer, because a projection carries no authority of its own — if it
250
 * disagrees with the log, the log is right and the projection is discarded.
251
 */
252
export const projectionMatches = (
253
  projection: Projection,
254
  events: ReadonlyArray<EngramEvent>,
255
): boolean => project(events).digest === projection.digest;
256
257
/** The live value for one slug, or undefined when absent or tombstoned. */
258
export const projectedValue = (projection: Projection, slug: string): string | undefined =>
259
  projection.entries.find((entry) => entry.slug === slug)?.value;

This page updates live while a promote is in flight · changelog