|
1
|
+ |
// Vendored from packages/agent-experience-memory/src/sync.ts by scripts/vendor-memory.mjs — do not edit here.
|
|
2
|
+ |
// The drift guard (test/vendored-memory-drift.test.ts) fails when this copy
|
|
3
|
+ |
// no longer matches the canonical source.
|
|
4
|
+ |
import { Schema as S } from "effect";
|
|
5
|
+ |
|
|
6
|
+ |
import { computeEngramEventId, type EngramEvent } from "./engram.js";
|
|
7
|
+ |
|
|
8
|
+ |
/**
|
|
9
|
+ |
* The engram sync seam (issue #222).
|
|
10
|
+ |
*
|
|
11
|
+ |
* The ledger is local and authoritative. Sync is how a copy of it reaches
|
|
12
|
+ |
* somewhere else, and the whole design follows from one rule: **a turn never
|
|
13
|
+ |
* waits for it, and never fails because of it.** A relay that is slow, down,
|
|
14
|
+ |
* or gone must be indistinguishable from one that is fine, as far as the
|
|
15
|
+ |
* caller is concerned — because memory that can break a conversation is worse
|
|
16
|
+ |
* than memory that is briefly out of date.
|
|
17
|
+ |
*
|
|
18
|
+ |
* That gives the three properties the tests hold this to:
|
|
19
|
+ |
*
|
|
20
|
+ |
* 1. **Local-first.** `publish` records the engram as pending and returns
|
|
21
|
+ |
* immediately. The engram is already in the local ledger by then; sync is
|
|
22
|
+ |
* catching up, not gatekeeping.
|
|
23
|
+ |
* 2. **Nothing is lost.** A failed publish stays queued and is retried. The
|
|
24
|
+ |
* queue only forgets an engram once a transport has acknowledged it, so a
|
|
25
|
+ |
* transport that is down for an hour costs an hour of latency and no
|
|
26
|
+ |
* engrams.
|
|
27
|
+ |
* 3. **Degraded is a state, not an error.** The queue reports what it is
|
|
28
|
+ |
* holding and what failed last, so a caller can *say* it is behind rather
|
|
29
|
+ |
* than discovering it by silence.
|
|
30
|
+ |
*
|
|
31
|
+ |
* What is deliberately not here: relays, sockets, keys, encryption. A
|
|
32
|
+ |
* transport is anything that satisfies `EngramTransport`, and the real Nostr
|
|
33
|
+ |
* one lands behind this interface without any caller changing. The workspace
|
|
34
|
+ |
* has a shared Nostr implementation (`nostr-effect`) that a real transport
|
|
35
|
+ |
* should build on rather than reimplement.
|
|
36
|
+ |
*/
|
|
37
|
+ |
|
|
38
|
+ |
export const SYNC_SCHEMA_ID = "openagents.engram_sync.v1" as const;
|
|
39
|
+ |
|
|
40
|
+ |
/** Why a publish did not land. Distinct because they need distinct responses. */
|
|
41
|
+ |
export const SyncFailureReason = S.Literals([
|
|
42
|
+ |
/** The transport could not be reached at all. Retry later, unchanged. */
|
|
43
|
+ |
"unreachable",
|
|
44
|
+ |
/** The transport reached and refused this engram. Retrying will not help. */
|
|
45
|
+ |
"refused",
|
|
46
|
+ |
/** The transport accepted the call and failed inside it. Retry later. */
|
|
47
|
+ |
"failed",
|
|
48
|
+ |
]);
|
|
49
|
+ |
export type SyncFailureReason = typeof SyncFailureReason.Type;
|
|
50
|
+ |
|
|
51
|
+ |
export type PublishResult =
|
|
52
|
+ |
| { readonly ok: true; readonly eventId: string }
|
|
53
|
+ |
| { readonly ok: false; readonly reason: SyncFailureReason; readonly detail?: string };
|
|
54
|
+ |
|
|
55
|
+ |
/** A filter for what to fetch back. Absent fields mean "no constraint". */
|
|
56
|
+ |
export interface EngramFilter {
|
|
57
|
+ |
readonly authors?: ReadonlyArray<string>;
|
|
58
|
+ |
/** `d` tag values — engram slugs. */
|
|
59
|
+ |
readonly slugs?: ReadonlyArray<string>;
|
|
60
|
+ |
/** Unix seconds; inclusive. */
|
|
61
|
+ |
readonly since?: number;
|
|
62
|
+ |
readonly until?: number;
|
|
63
|
+ |
readonly limit?: number;
|
|
64
|
+ |
}
|
|
65
|
+ |
|
|
66
|
+ |
/**
|
|
67
|
+ |
* Whatever carries engrams somewhere else.
|
|
68
|
+ |
*
|
|
69
|
+ |
* Implementations must not throw: a transport that cannot answer returns a
|
|
70
|
+ |
* failure, because a throw from here would reach the caller's turn and the
|
|
71
|
+ |
* whole point is that it cannot.
|
|
72
|
+ |
*/
|
|
73
|
+ |
export interface EngramTransport {
|
|
74
|
+ |
publish(event: EngramEvent): Promise<PublishResult>;
|
|
75
|
+ |
fetch(filter: EngramFilter): Promise<ReadonlyArray<EngramEvent>>;
|
|
76
|
+ |
}
|
|
77
|
+ |
|
|
78
|
+ |
/** What the queue is holding, for a caller that wants to say so. */
|
|
79
|
+ |
export interface SyncStatus {
|
|
80
|
+ |
readonly pending: number;
|
|
81
|
+ |
readonly delivered: number;
|
|
82
|
+ |
/** Engrams the transport refused outright. They are not retried. */
|
|
83
|
+ |
readonly refused: number;
|
|
84
|
+ |
readonly lastFailure?: { readonly reason: SyncFailureReason; readonly detail?: string };
|
|
85
|
+ |
}
|
|
86
|
+ |
|
|
87
|
+ |
const matches = (event: EngramEvent, filter: EngramFilter): boolean => {
|
|
88
|
+ |
if (filter.authors !== undefined && !filter.authors.includes(event.pubkey)) return false;
|
|
89
|
+ |
if (filter.since !== undefined && event.created_at < filter.since) return false;
|
|
90
|
+ |
if (filter.until !== undefined && event.created_at > filter.until) return false;
|
|
91
|
+ |
if (filter.slugs !== undefined) {
|
|
92
|
+ |
const slug = event.tags.find((tag) => tag[0] === "d")?.[1];
|
|
93
|
+ |
if (slug === undefined || !filter.slugs.includes(slug)) return false;
|
|
94
|
+ |
}
|
|
95
|
+ |
return true;
|
|
96
|
+ |
};
|
|
97
|
+ |
|
|
98
|
+ |
/**
|
|
99
|
+ |
* A transport that keeps engrams in memory.
|
|
100
|
+ |
*
|
|
101
|
+ |
* The reference implementation and what the tests run against. It is also a
|
|
102
|
+ |
* useful real transport for a single process that wants sync's shape without
|
|
103
|
+ |
* a relay.
|
|
104
|
+ |
*/
|
|
105
|
+ |
export class MemoryTransport implements EngramTransport {
|
|
106
|
+ |
private readonly events = new Map<string, EngramEvent>();
|
|
107
|
+ |
/** Set to fail every publish, for degraded-mode tests. */
|
|
108
|
+ |
reachable = true;
|
|
109
|
+ |
|
|
110
|
+ |
publish(event: EngramEvent): Promise<PublishResult> {
|
|
111
|
+ |
if (!this.reachable) {
|
|
112
|
+ |
return Promise.resolve({ ok: false, reason: "unreachable", detail: "transport is down" });
|
|
113
|
+ |
}
|
|
114
|
+ |
if (event.id !== computeEngramEventId(event)) {
|
|
115
|
+ |
return Promise.resolve({ ok: false, reason: "refused", detail: "event id does not verify" });
|
|
116
|
+ |
}
|
|
117
|
+ |
this.events.set(event.id, event);
|
|
118
|
+ |
return Promise.resolve({ ok: true, eventId: event.id });
|
|
119
|
+ |
}
|
|
120
|
+ |
|
|
121
|
+ |
fetch(filter: EngramFilter): Promise<ReadonlyArray<EngramEvent>> {
|
|
122
|
+ |
if (!this.reachable) return Promise.resolve([]);
|
|
123
|
+ |
const found = [...this.events.values()]
|
|
124
|
+ |
.filter((event) => matches(event, filter))
|
|
125
|
+ |
.sort((left, right) =>
|
|
126
|
+ |
left.created_at !== right.created_at
|
|
127
|
+ |
? left.created_at - right.created_at
|
|
128
|
+ |
: left.id.localeCompare(right.id),
|
|
129
|
+ |
);
|
|
130
|
+ |
return Promise.resolve(filter.limit === undefined ? found : found.slice(0, filter.limit));
|
|
131
|
+ |
}
|
|
132
|
+ |
|
|
133
|
+ |
/** Everything the transport holds, for assertions. */
|
|
134
|
+ |
stored(): ReadonlyArray<EngramEvent> {
|
|
135
|
+ |
return [...this.events.values()];
|
|
136
|
+ |
}
|
|
137
|
+ |
}
|
|
138
|
+ |
|
|
139
|
+ |
/**
|
|
140
|
+ |
* The queue between the ledger and a transport.
|
|
141
|
+ |
*
|
|
142
|
+ |
* `publish` is synchronous from the caller's side: it enqueues and returns.
|
|
143
|
+ |
* `drain` is what actually talks to the transport, and a host calls it
|
|
144
|
+ |
* whenever it likes — after a turn, on a timer, at exit. Nothing about the
|
|
145
|
+ |
* caller's turn depends on when that happens.
|
|
146
|
+ |
*/
|
|
147
|
+ |
export class EngramSyncQueue {
|
|
148
|
+ |
private readonly queue: Array<EngramEvent> = [];
|
|
149
|
+ |
private readonly deliveredIds = new Set<string>();
|
|
150
|
+ |
private readonly refusedIds = new Set<string>();
|
|
151
|
+ |
private lastFailure: SyncStatus["lastFailure"];
|
|
152
|
+ |
private draining = false;
|
|
153
|
+ |
|
|
154
|
+ |
constructor(private readonly transport: EngramTransport) {}
|
|
155
|
+ |
|
|
156
|
+ |
/**
|
|
157
|
+ |
* Enqueue an engram for delivery. Returns nothing to wait on.
|
|
158
|
+ |
*
|
|
159
|
+ |
* An engram already delivered or already refused is not enqueued twice, so
|
|
160
|
+ |
* a caller that republishes its whole ledger costs one pass, not one
|
|
161
|
+ |
* delivery per pass.
|
|
162
|
+ |
*/
|
|
163
|
+ |
publish(event: EngramEvent): void {
|
|
164
|
+ |
if (this.deliveredIds.has(event.id) || this.refusedIds.has(event.id)) return;
|
|
165
|
+ |
if (this.queue.some((queued) => queued.id === event.id)) return;
|
|
166
|
+ |
this.queue.push(event);
|
|
167
|
+ |
}
|
|
168
|
+ |
|
|
169
|
+ |
/**
|
|
170
|
+ |
* Try to deliver everything queued.
|
|
171
|
+ |
*
|
|
172
|
+ |
* Returns how many landed. An unreachable or failing transport leaves the
|
|
173
|
+ |
* engram queued for the next drain; a refusal is terminal, because retrying
|
|
174
|
+ |
* something the transport has judged invalid only repeats the judgement.
|
|
175
|
+ |
* Never throws — a transport that throws anyway is treated as failing.
|
|
176
|
+ |
*/
|
|
177
|
+ |
async drain(): Promise<number> {
|
|
178
|
+ |
if (this.draining) return 0;
|
|
179
|
+ |
this.draining = true;
|
|
180
|
+ |
try {
|
|
181
|
+ |
let delivered = 0;
|
|
182
|
+ |
// Copy: a publish during a drain lands in the next one rather than
|
|
183
|
+ |
// mutating the array being walked.
|
|
184
|
+ |
const attempting = [...this.queue];
|
|
185
|
+ |
for (const event of attempting) {
|
|
186
|
+ |
let result: PublishResult;
|
|
187
|
+ |
try {
|
|
188
|
+ |
result = await this.transport.publish(event);
|
|
189
|
+ |
} catch (cause) {
|
|
190
|
+ |
result = {
|
|
191
|
+ |
ok: false,
|
|
192
|
+ |
reason: "failed",
|
|
193
|
+ |
detail: cause instanceof Error ? cause.message : String(cause),
|
|
194
|
+ |
};
|
|
195
|
+ |
}
|
|
196
|
+ |
if (result.ok) {
|
|
197
|
+ |
this.deliveredIds.add(event.id);
|
|
198
|
+ |
this.removeFromQueue(event.id);
|
|
199
|
+ |
delivered += 1;
|
|
200
|
+ |
continue;
|
|
201
|
+ |
}
|
|
202
|
+ |
this.lastFailure = {
|
|
203
|
+ |
reason: result.reason,
|
|
204
|
+ |
...(result.detail === undefined ? {} : { detail: result.detail }),
|
|
205
|
+ |
};
|
|
206
|
+ |
if (result.reason === "refused") {
|
|
207
|
+ |
this.refusedIds.add(event.id);
|
|
208
|
+ |
this.removeFromQueue(event.id);
|
|
209
|
+ |
}
|
|
210
|
+ |
// "unreachable" and "failed" stay queued for the next drain.
|
|
211
|
+ |
}
|
|
212
|
+ |
return delivered;
|
|
213
|
+ |
} finally {
|
|
214
|
+ |
this.draining = false;
|
|
215
|
+ |
}
|
|
216
|
+ |
}
|
|
217
|
+ |
|
|
218
|
+ |
private removeFromQueue(eventId: string): void {
|
|
219
|
+ |
const at = this.queue.findIndex((queued) => queued.id === eventId);
|
|
220
|
+ |
if (at >= 0) this.queue.splice(at, 1);
|
|
221
|
+ |
}
|
|
222
|
+ |
|
|
223
|
+ |
/** Fetch from the transport. An unreachable transport yields nothing. */
|
|
224
|
+ |
async fetch(filter: EngramFilter): Promise<ReadonlyArray<EngramEvent>> {
|
|
225
|
+ |
try {
|
|
226
|
+ |
return await this.transport.fetch(filter);
|
|
227
|
+ |
} catch {
|
|
228
|
+ |
return [];
|
|
229
|
+ |
}
|
|
230
|
+ |
}
|
|
231
|
+ |
|
|
232
|
+ |
status(): SyncStatus {
|
|
233
|
+ |
return {
|
|
234
|
+ |
pending: this.queue.length,
|
|
235
|
+ |
delivered: this.deliveredIds.size,
|
|
236
|
+ |
refused: this.refusedIds.size,
|
|
237
|
+ |
...(this.lastFailure === undefined ? {} : { lastFailure: this.lastFailure }),
|
|
238
|
+ |
};
|
|
239
|
+ |
}
|
|
240
|
+ |
|
|
241
|
+ |
/** Whether the queue is holding anything the transport has not taken. */
|
|
242
|
+ |
behind(): boolean {
|
|
243
|
+ |
return this.queue.length > 0;
|
|
244
|
+ |
}
|
|
245
|
+ |
}
|