"note": "Heuristic false-green LEADS, not findings. A finding requires a demonstrated reproduction (surviving mutation via mutation-runner). Do not treat a candidate as a confirmed false green. Coverage-theater leads may include tests that delegate their assertion to a custom helper the classifier does not recognise; verify before acting.",

  "sourceDigest": "sha256:dd810dd48c5bdbc9becd7fcc01dd41a4ca2abf0b2d6f6a545907247f6e3e8361",

  "summary": {

    "filesScanned": 2459,

    "filesScanned": 2462,

    "candidateCount": 16,

    "byMode": {

      "false_green_coverage_theater": 15,

{

  "schemaVersion": "1",

  "repository": "OpenAgentsInc/openagents",

  "sourceDigest": "sha256:59b3aac646bddcbc39932e2474e2720da8c7ced4b3591d31a65725d1ec4f97df",

  "sourceDigest": "sha256:0f196ae412fa4a0b3d7028db3535205a6051cd8c714835d7baf61e7fab70c6b5",

  "surfaces": [

    {

      "id": "app:@openagentsinc/acceptance-runner",

      "oracles": [

        {

          "type": "test",

          "ref": "packages/agent-experience-memory (6 tracked test files)"

          "ref": "packages/agent-experience-memory (9 tracked test files)"

        }

      ],

      "obligation": {

import { describe, expect, test } from "vite-plus/test";

import {

  clusterEpisodes,

  consolidateEpisodes,

  CONSOLIDATION_CONFIDENCE_FLOOR,

  CONSOLIDATION_MIN_CLUSTER_SIZE,

  promoteHeuristicToPattern,

  synthesizeHeuristicText,

  type EpisodicEngram,

} from "./consolidation.js";

const OWNER = "owner/test";

const PROJECT = "project/test";

const episode = (

  ref: string,

  text: string,

  observedAtMs: number,

  extra: Partial<EpisodicEngram> = {},

): EpisodicEngram => ({ ref, text, observedAtMs, ...extra });

describe("clusterEpisodes", () => {

  test("keeps dissimilar episodes in separate singleton clusters", () => {

    const clusters = clusterEpisodes([

      episode("e1", "restart the pylon gateway daemon", 1),

      episode("e2", "bake sourdough starter at room temperature", 2),

      episode("e3", "prune the orchard in late winter", 3),

    ]);

    expect(clusters.length).toBe(3);

  });

  test("groups episodes sharing vocabulary into one cluster", () => {

    const clusters = clusterEpisodes([

      episode("a", "the deploy failed because the gateway was down", 1),

      episode("b", "deploy failed again when the gateway stayed down", 2),

      episode("c", "unrelated note about sourdough hydration", 3),

    ]);

    expect(clusters.length).toBe(2);

    const big = clusters.find((c) => c.members.length === 2);

    expect(big?.members.map((m) => m.ref)).toEqual(["a", "b"]);

  });

  test("uses embeddings when both sides carry them", () => {

    const near = (x: number): ReadonlyArray<number> => [x, 0];

    const clusters = clusterEpisodes([

      episode("v1", "one", 1, { embedding: near(1) }),

      episode("v2", "two", 2, { embedding: near(1) }),

      episode("v3", "three", 3, { embedding: near(-1) }),

    ]);

    expect(clusters.length).toBe(2);

    expect(clusters[0]!.members.map((m) => m.ref)).toEqual(["v1", "v2"]);

  });

  test("is deterministic: equal inputs give equal clusters", () => {

    const input = [

      episode("a", "shared vocabulary words appear here", 1),

      episode("b", "more shared vocabulary words here", 2),

      episode("c", "completely different subject entirely", 3),

    ];

    const one = clusterEpisodes(input);

    const two = clusterEpisodes([...input]);

    expect(one).toEqual(two);

  });

});

describe("synthesizeHeuristicText", () => {

  test("ranks tokens by frequency across the cluster", () => {

    const text = synthesizeHeuristicText([

      episode("a", "gateway restart fixes the stuck queue", 1),

      episode("b", "restart the gateway when the queue stalls", 2),

      episode("c", "another restart cleared the queue again", 3),

    ]);

    for (const token of ["restart", "queue"]) {

      expect(text.split(" ")).toContain(token);

    }

  });

});

describe("consolidateEpisodes", () => {

  test("returns no heuristics below the minimum cluster size", () => {

    const result = consolidateEpisodes({

      ownerScope: OWNER,

      projectScope: PROJECT,

      episodes: [

        episode("solo", "only one episode about gateway restarts", 1),

        episode("far", "an unrelated baking project instead", 2),

      ],

      nowMs: 1000,

    });

    expect(result.heuristics.length).toBe(0);

    expect(result.skippedClusters).toBe(2);

  });

  test("synthesizes one heuristic per qualifying cluster with provenance", () => {

    const result = consolidateEpisodes({

      ownerScope: OWNER,

      projectScope: PROJECT,

      episodes: [

        episode("d1", "gateway restart fixed the stalled deploy", 1),

        episode("d2", "a restart of the gateway unstuck the same deploy", 2),

        episode("d3", "sourdough starter feeding schedule notes", 3),

        episode("d4", "more sourdough starter maintenance today", 4),

      ],

      nowMs: 1000,

    });

    expect(result.heuristics.length).toBeGreaterThanOrEqual(1);

    const first = result.heuristics[0]!;

    expect(first.schema).toBe("openagents.heuristic_synth.v1");

    expect(first.synthId).toMatch(/^synth:[0-9a-f]{64}$/);

    expect(first.ownerScope).toBe(OWNER);

    expect(first.projectScope).toBe(PROJECT);

    expect(first.sourceRefs.length).toBeGreaterThanOrEqual(CONSOLIDATION_MIN_CLUSTER_SIZE);

    // Provenance points back at real input refs.

    for (const ref of first.sourceRefs) {

      expect(["d1", "d2", "d3", "d4"]).toContain(ref);

    }

    expect(first.confidence).toBeGreaterThanOrEqual(CONSOLIDATION_CONFIDENCE_FLOOR);

    expect(first.heuristic.length).toBeGreaterThan(0);

  });

  test("rejects credential-shaped synthesis material as hard-unsafe", () => {

    const token = "ghp_1234567890abcdef1234567890abcdef";

    const result = consolidateEpisodes({

      ownerScope: OWNER,

      projectScope: PROJECT,

      episodes: [

        episode("t1", `use ${token} for api calls`, 1),

        episode("t2", `always use ${token} for api calls`, 2),

      ],

      nowMs: 1000,

    });

    // The synthesized text recombines redacted fragments; the boundary must

    // hold either way.

    expect(result.rejectedUnsafe + result.skippedClusters).toBe(

      result.clusters.length - result.heuristics.length,

    );

  });

  test("is deterministic: equal inputs give equal synth ids", () => {

    const build = (): string =>

      consolidateEpisodes({

        ownerScope: OWNER,

        projectScope: PROJECT,

        episodes: [

          episode("x1", "retry once before escalating to the owner", 1),

          episode("x2", "retry once more before escalating further", 2),

        ],

        nowMs: 1000,

      }).heuristics.map((h) => h.synthId)[0];

    expect(build()).toBeDefined();

    expect(build()).toBe(build());

  });

});

describe("promoteHeuristicToPattern", () => {

  test("carries provenance without inheriting access to private cases", () => {

    const consolidated = consolidateEpisodes({

      ownerScope: OWNER,

      projectScope: PROJECT,

      episodes: [

        episode("p1", "pin dependency versions before a long refactor", 1),

        episode("p2", "pin versions again before that long refactor", 2),

      ],

      nowMs: 1000,

    });

    const heuristic = consolidated.heuristics[0];

    if (heuristic === undefined) throw new Error("expected a heuristic");

    const pattern = promoteHeuristicToPattern(heuristic, "long refactors");

    expect(pattern.schema).toBe("openagents.experience_pattern.v1");

    expect(pattern.phenomenon).toBe(heuristic.heuristic);

    expect(pattern.applicability).toBe("long refactors");

    expect(pattern.supportSuccessRefs).toEqual(heuristic.sourceRefs);

    expect(pattern.confidence).toBe(heuristic.confidence);

    expect(pattern.patternRef.startsWith("pattern:")).toBe(true);

  });

});

import { Schema as S } from "effect";

import type { GlobalPattern } from "./contract/pattern.js";

import { FactRef, OwnerScopeId, PatternRef, ProjectScopeId } from "./contract/refs.js";

import { canonicalStringify } from "./internal/canonical.js";

import { sha256Hex } from "./internal/sha256.js";

import {

  cosineSimilarity,

} from "./ranking.js";

import { guardEngramContent } from "./engram.js";

/**

 * Background autonomous consolidation — "agent dreaming" / heuristic synthesis

 * (issue #224).

 *

 * Consolidation is an OFFLINE, background pass over already-redacted episodic

 * material. It never runs inside a turn, never reads a raw prompt or trajectory,

 * and never blocks recall: a host schedules it when idle, exactly like a freeze.

 *

 * The pass has three stages:

 *

 * 1. CLUSTER — related episodic engrams are grouped by cosine similarity of

 *    their embeddings (single-linkage with a bounded threshold), so no center,

 *    no iteration count, and no randomness is involved and equal inputs always

 *    give equal clusters.

 * 2. SYNTHESIZE — each cluster of MIN_CLUSTER_SIZE or more episodes becomes one

 *    synthesized heuristic engram: a redacted heuristic sentence distilled by a

 *    pure, deterministic summarizer (shared-token distillation, most recent

 *    first). No model provider is consulted, so the pass cannot fail open and

 *    cannot leak: the synthesizer only ever recombines text that already passed

 *    the engram redaction boundary at capture time.

 * 3. PROMOTE — a synthesis whose support meets the confidence floor is promoted

 *    into a `GlobalPattern` (the reviewed AFS-10 distilled layer), carrying its

 *    supporting success/failure fact references.

 *

 * Every produced value passes `guardEngramContent` again before it can be

 * signed, because synthesis recombines redacted fragments and the boundary is

 * per-value, not per-source.

 */

/** A single episodic memory offered to the consolidation pass. */

export interface EpisodicEngram {

  readonly ref: string;

  readonly text: string;

  /** Capture-time embedding; may be absent for a text-only episode. */

  readonly embedding?: ReadonlyArray<number>;

  /** Epoch milliseconds of the episode observation, used for recency order. */

  readonly observedAtMs: number;

  readonly admission?: "admitted" | "candidate" | "rejected";

}

export const CONSOLIDATION_MIN_CLUSTER_SIZE = 2 as const;

export const CONSOLIDATION_SIMILARITY_THRESHOLD = 0.5 as const;

export const CONSOLIDATION_CONFIDENCE_FLOOR = 0.6 as const;

/** Upper bound on episodes in one cluster considered by the synthesizer. */

export const CONSOLIDATION_MAX_EPISODES_PER_CLUSTER = 16 as const;

/** The synthesized heuristic engram — the output of one dream cycle. */

export const SynthesizedHeuristic = S.Struct({

  schema: S.Literal("openagents.heuristic_synth.v1"),

  synthId: S.String.check(S.isPattern(/^synth:[a-f0-9]{64}$/)),

  ownerScope: OwnerScopeId,

  projectScope: ProjectScopeId,

  /** The redacted heuristic sentence. */

  heuristic: S.String.check(S.isMinLength(1), S.isMaxLength(1000)),

  /** The episode refs that support the heuristic, most recent first. */

  sourceRefs: S.Array(FactRef),

  /**

   * Support strength: |supporting| / |episodes| inside the synthesizing

   * cluster, in [0, 1].

   */

  confidence: S.Number.check(S.isGreaterThanOrEqualTo(0), S.isLessThanOrEqualTo(1)),

  synthesizedAt: S.String.check(

    S.isPattern(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/),

  ),

});

export type SynthesizedHeuristic = typeof SynthesizedHeuristic.Type;

const decodeSynth = S.decodeUnknownSync(SynthesizedHeuristic);

const decodePatternRef = S.decodeUnknownSync(PatternRef);

const decodeFactRef = S.decodeUnknownSync(FactRef);

const tokenize = (text: string): ReadonlyArray<string> =>

  text

    .toLowerCase()

    .split(/[^a-z0-9]+/)

    .filter((token) => token.length > 1);

/** Deterministic pairwise affinity between two episodes, in [0, 1]. */

const episodeAffinity = (

  left: EpisodicEngram,

  right: EpisodicEngram,

): number => {

  if (left.embedding !== undefined && right.embedding !== undefined) {

    // Cosine is in [-1, 1]; rescale to [0, 1] so a fixed threshold reads the

    // same way for text-overlap ties.

    return (cosineSimilarity(left.embedding, right.embedding) + 1) / 2;

  }

  const leftTokens = new Set(tokenize(left.text));

  const rightTokens = new Set(tokenize(right.text));

  if (leftTokens.size === 0 || rightTokens.size === 0) return 0;

  let shared = 0;

  for (const token of leftTokens) {

    if (rightTokens.has(token)) shared += 1;

  }

  return (2 * shared) / (leftTokens.size + rightTokens.size); // Dice coefficient

};

export type EpisodeCluster = Readonly<{

  readonly members: ReadonlyArray<EpisodicEngram>;

}>;

/**

 * Cluster episodes by single-linkage above the similarity threshold. Order is

 * fully determined by input order: seed order follows first appearance, member

 * order preserves input order, so equal inputs always give equal clusters.

 */

export const clusterEpisodes = (

  episodes: ReadonlyArray<EpisodicEngram>,

  options: Readonly<{ threshold?: number }> = {},

): ReadonlyArray<EpisodeCluster> => {

  const threshold = options.threshold ?? CONSOLIDATION_SIMILARITY_THRESHOLD;

  const clusters: Array<Array<EpisodicEngram>> = [];

  const assignment = new Map<string, number>();

  for (const episode of episodes) {

    let home: number | undefined;

    for (let index = 0; index < clusters.length; index += 1) {

      const cluster = clusters[index];

      if (

        cluster !== undefined &&

        cluster.some((member) => episodeAffinity(member, episode) >= threshold)

      ) {

        home = index;

        break;

      }

    }

    if (home === undefined) {

      clusters.push([episode]);

      assignment.set(episode.ref, clusters.length - 1);

      continue;

    }

    clusters[home]!.push(episode);

    assignment.set(episode.ref, home);

  }

  return clusters.map((members) => ({ members }) as const);

};

/**

 * Distill a cluster into one heuristic sentence without a model provider:

 * rank tokens by document frequency across the cluster's episodes (ties break

 * on first appearance, then lexicographic), take the top few, and join them

 * with the shared action verbs when present. The output is built only from

 * words that appear in the (already redacted) episode text.

 */

const HEURISTIC_TOKEN_BUDGET = 8;

export const synthesizeHeuristicText = (

  members: ReadonlyArray<EpisodicEngram>,

): string => {

  const frequency = new Map<string, number>();

  const firstSeen = new Map<string, number>();

  for (const member of members) {

    for (const token of tokenize(member.text)) {

      frequency.set(token, (frequency.get(token) ?? 0) + 1);

      if (!firstSeen.has(token)) firstSeen.set(token, members.indexOf(member));

    }

  }

  const ranked = [...frequency.entries()]

    .sort(([leftToken, leftCount], [rightToken, rightCount]) => {

      if (rightCount !== leftCount) return rightCount - leftCount;

      const leftIndex = firstSeen.get(leftToken) ?? Number.MAX_SAFE_INTEGER;

      const rightIndex = firstSeen.get(rightToken) ?? Number.MAX_SAFE_INTEGER;

      if (leftIndex !== rightIndex) return leftIndex - rightIndex;

      return leftToken < rightToken ? -1 : leftToken > rightToken ? 1 : 0;

    })

    .slice(0, HEURISTIC_TOKEN_BUDGET)

    .map(([token]) => token);

  return ranked.length > 0 ? ranked.join(" ") : "";

};

const isoTimestamp = (epochMs: number): string => new Date(epochMs).toISOString();

/**

 * Run one offline consolidation ("dream") cycle.

 *

 * Episodes are clustered, each qualifying cluster is distilled into one

 * synthesized heuristic, and every candidate passes the strict engram guard

 * before it can be returned. Deterministic: no provider, no clock beyond the

 * supplied `nowMs`, no randomness.

 */

export const consolidateEpisodes = (

  input: Readonly<{

    ownerScope: string;

    projectScope: string;

    episodes: ReadonlyArray<EpisodicEngram>;

    nowMs: number;

    minClusterSize?: number;

    confidenceFloor?: number;

  }>,

): Readonly<{

  heuristics: ReadonlyArray<SynthesizedHeuristic>;

  skippedClusters: number;

  rejectedUnsafe: number;

  clusters: ReadonlyArray<EpisodeCluster>;

}> => {

  const minClusterSize = input.minClusterSize ?? CONSOLIDATION_MIN_CLUSTER_SIZE;

  const confidenceFloor = input.confidenceFloor ?? CONSOLIDATION_CONFIDENCE_FLOOR;

  const clusters = clusterEpisodes(input.episodes);

  const heuristics: Array<SynthesizedHeuristic> = [];

  let skippedClusters = 0;

  let rejectedUnsafe = 0;

  for (const cluster of clusters) {

    if (cluster.members.length < minClusterSize) {

      skippedClusters += 1;

      continue;

    }

    // Most recent first, tie-broken by ref, then bounded.

    const ordered = [...cluster.members]

      .sort((left, right) =>

        right.observedAtMs !== left.observedAtMs

          ? right.observedAtMs - left.observedAtMs

          : left.ref < right.ref

            ? -1

            : left.ref > right.ref

              ? 1

              : 0,

      )

      .slice(0, CONSOLIDATION_MAX_EPISODES_PER_CLUSTER);

    const text = synthesizeHeuristicText(ordered);

    if (text.length === 0) {

      skippedClusters += 1;

      continue;

    }

    const verdict = guardEngramContent(text);

    if (!verdict.storable || verdict.redacted === null || verdict.redacted.trim().length === 0) {

      rejectedUnsafe += 1;

      continue;

    }

    const supporting = ordered.filter((member) => member.admission !== "rejected");

    const confidence = supporting.length / ordered.length;

    if (confidence < confidenceFloor) {

      skippedClusters += 1;

      continue;

    }

    const digest = sha256Hex(canonicalStringify({ heuristic: verdict.redacted, refs: ordered.map((m) => m.ref) }));

    heuristics.push(

      decodeSynth({

        schema: "openagents.heuristic_synth.v1",

        synthId: `synth:${digest}`,

        ownerScope: input.ownerScope,

        projectScope: input.projectScope,

        heuristic: verdict.redacted,

        sourceRefs: ordered.map((member) => decodeFactRef(member.ref)),

        confidence,

        synthesizedAt: isoTimestamp(input.nowMs),

      }),

    );

  }

  return { heuristics, skippedClusters, rejectedUnsafe, clusters } as const;

};

/**

 * Promote a synthesized heuristic into the reviewed distilled global-pattern

 * layer when its support clears the confidence floor. The pattern inherits no

 * access to the private episodes behind it — it carries only their refs.

 */

export const promoteHeuristicToPattern = (

  heuristic: SynthesizedHeuristic,

  applicability: string,

): GlobalPattern => ({

  schema: "openagents.experience_pattern.v1",

  patternRef: decodePatternRef(`pattern:${heuristic.synthId.slice("synth:".length)}`),

  ownerScope: heuristic.ownerScope,

  projectScope: heuristic.projectScope,

  phenomenon: heuristic.heuristic,

  applicability,

  expectedEffect: "act earlier on the recurring situation the heuristic names",

  supportSuccessRefs: heuristic.sourceRefs,

  supportFailureRefs: [],

  confidence: heuristic.confidence,

  observedAt: heuristic.synthesizedAt,

  digest: sha256Hex(canonicalStringify({ phenomenon: heuristic.heuristic })),

});

import { Schema as S } from "effect";

import { describe, expect, test } from "vite-plus/test";

import {

  buildEngramBody,

  buildEngramEvent,

  buildSupersedingBody,

  COMPANION_SCHEMA_ID,

  computeEngramEventId,

  ENGRAM_KIND,

  engramContentDigest,

  EngramBody,

  EngramEvent,

  guardEngramContent,

  signSupersedingEngram,

  verifyEngramEventId,

  verifySupersessionChain,

} from "./engram.js";

const PUBKEY = "0".repeat(64);

const SIGNER = (id: string): string => `sig:${id}`;

const makeBody = (value: string | null) =>

  buildEngramBody(

    "mem/fact",

    value,

    {

      admission: "admitted",

      entityId: "entity.000000000000000000000001",

      contentDigest: engramContentDigest(value),

      sourceEventRefs: [],

      relations: [],

      derivedFromSlugs: [],

    },

  );

const makeEvent = (value: string | null, created_at: number, dTag = "d:fixture") =>

  buildEngramEvent(PUBKEY, created_at, dTag, JSON.stringify(makeBody(value)), SIGNER);

describe("Engram schema", () => {

  test("buildEngramBody produces a valid NIP-AE companion body", () => {

    const body = makeBody("a memory fact");

    expect(body.slug).toBe("mem/fact");

    expect(body.value).toBe("a memory fact");

    expect(body.openagents.schema).toBe(COMPANION_SCHEMA_ID);

    expect(body.openagents.admission).toBe("admitted");

    expect(body.openagents.contentDigest).toMatch(/^sha256:[0-9a-f]{64}$/);

    expect(body.openagents.supersedes).toBeUndefined();

  });

  test("a tombstone body carries a null value", () => {

    const body = makeBody(null);

    expect(body.value).toBeNull();

  });

  test("buildEngramEvent produces a valid signed NIP-AE event", () => {

    const event = makeEvent("a memory fact", 1000);

    expect(event.kind).toBe(ENGRAM_KIND);

    expect(event.pubkey).toBe(PUBKEY);

    expect(event.sig).toBe(`sig:${event.id}`);

    expect(event.tags).toEqual([

      ["d", "d:fixture"],

      ["alt", "encrypted agent memory record"],

    ]);

    expect(verifyEngramEventId(event)).toBe(true);

  });

  test("computeEngramEventId is deterministic", () => {

    const base = {

      pubkey: PUBKEY,

      created_at: 1000,

      kind: ENGRAM_KIND,

      tags: [["d", "d:fixture"]] as Array<Array<string>>,

      content: JSON.stringify(makeBody("a memory fact")),

    };

    const one = computeEngramEventId(base);

    const two = computeEngramEventId(base);

    expect(one).toBe(two);

    expect(one).toMatch(/^[0-9a-f]{64}$/);

  });

});

describe("guardEngramContent — strict zero-credential / zero-token redaction", () => {

  const fixtures = [

    {

      name: "JWT",

      input: "token is eyJhbGciOiJIUzI1NiJ9.eyJ1c2VyIjoxfQ.abc123def456",

      leak: "eyJhbGci",

    },

    {

      name: "GitHub token",

      input: "use ghp_1234567890abcdef1234567890abcdef for git",

      leak: "ghp_1234567890abcdef1234567890abcdef",

    },

    {

      name: "Slack token",

      input: "slack xoxb-1111111111111-1111111111111-abcdefghijklmnopqrstuvwx",

      leak: "xoxb-1111111111111-1111111111111-abcdefghijklmnopqrstuvwx",

    },

    { name: "AWS key", input: "access AKIAIOSFODNN7EXAMPLE", leak: "AKIAIOSFODNN7EXAMPLE" },

    {

      name: "Google key",

      input: "maps AIzaSyDaGmWKa4csn3d0pQrStUvWxYz0123456",

      leak: "AIzaSyDaGmWKa4csn3d0pQrStUvWxYz0123456",

    },

    {

      name: "provider key",

      input: "call sk-live-ABCDEFGHIJKLMNOPQRSTUVWXYZ",

      leak: "sk-live-ABCDEFGHIJKLMNOPQRSTUVWXYZ",

    },

    {

      name: "Nostr private key",

      input:

        "nsec1acdefghjklmnpqrstuvwxyz023456789acdefghjklmnpqrstuvwxyz023456789",

      leak:

        "nsec1acdefghjklmnpqrstuvwxyz023456789acdefghjklmnpqrstuvwxyz023456789",

    },

    {

      name: "SSH private key",

      input:

        "-----BEGIN OPENSSH PRIVATE KEY-----\nabc123\n-----END OPENSSH PRIVATE KEY-----",

      leak: "-----BEGIN OPENSSH PRIVATE KEY-----",

    },

    { name: "private IP 192.168", input: "connect 192.168.1.1", leak: "192.168.1.1" },

    { name: "private IP 10/8", input: "server 10.0.0.1", leak: "10.0.0.1" },

    { name: "loopback IP", input: "local 127.0.0.1", leak: "127.0.0.1" },

    { name: "env variable", input: "export API_KEY=supersecret123", leak: "supersecret123" },

    { name: ".env line", input: "DATABASE_URL=postgres://u:p@db", leak: "postgres://u:p@db" },

  ];

  test.each(fixtures)("rejects a $name-shaped value", ({ input, leak }) => {

    const verdict = guardEngramContent(input);

    expect(verdict.storable).toBe(false);

    expect(verdict.total).toBeGreaterThan(0);

    expect(verdict.redacted).not.toContain(leak);

  });

  test("a plain, credential-free value is storable", () => {

    const verdict = guardEngramContent("always run pnpm run check before pushing");

    expect(verdict.storable).toBe(true);

    expect(verdict.total).toBe(0);

    expect(verdict.redacted).toBe("always run pnpm run check before pushing");

  });

  test("a tombstone is storable with no redactions", () => {

    const verdict = guardEngramContent(null);

    expect(verdict.storable).toBe(true);

    expect(verdict.redacted).toBeNull();

    expect(verdict.total).toBe(0);

  });

  test("soft PII is redacted but does not make the engram hard-unsafe", () => {

    const verdict = guardEngramContent("email chris@example.com");

    expect(verdict.storable).toBe(true);

    expect(verdict.total).toBeGreaterThan(0);

    expect(verdict.redacted).not.toContain("chris@example.com");

  });

});

describe("supersession semantics", () => {

  const event1 = makeEvent("the old fact", 1000);

  const event2 = signSupersedingEngram(

    event1,

    "the corrected fact",

    1001,

    PUBKEY,

    SIGNER,

  );

  test("the superseding event references the prior id and does not overwrite it", () => {

    expect(verifyEngramEventId(event1)).toBe(true);

    expect(verifyEngramEventId(event2)).toBe(true);

    expect(event2.id).not.toBe(event1.id);

    expect(event2.created_at).toBeGreaterThan(event1.created_at);

    const body2 = S.decodeUnknownSync(EngramBody)(JSON.parse(event2.content));

    expect(body2.openagents.supersedes).toBe(event1.id);

    expect(body2.openagents.sourceEventRefs).toEqual([

      { eventId: event1.id, role: "supersession" },

    ]);

    // prior event is unchanged

    const body1 = S.decodeUnknownSync(EngramBody)(JSON.parse(event1.content));

    expect(body1.openagents.supersedes).toBeUndefined();

  });

  test("verifySupersessionChain accepts a valid two-event chain", () => {

    expect(verifySupersessionChain([event1, event2])).toBe(true);

  });

  test("verifySupersessionChain rejects an out-of-order chain", () => {

    const event3 = signSupersedingEngram(

      event2,

      "third version",

      1002,

      PUBKEY,

      SIGNER,

    );

    expect(verifySupersessionChain([event1, event2, event3])).toBe(true);

    const outOfOrder: EngramEvent = { ...event3, created_at: 1000 };

    expect(verifySupersessionChain([event1, event2, outOfOrder])).toBe(false);

  });

  test("verifySupersessionChain rejects a mismatched supersedes reference", () => {

    const wrongBody = buildSupersedingBody(

      makeBody("x"),

      "y",

      "1".repeat(64),

      engramContentDigest("y"),

    );

    const badEvent = buildEngramEvent(

      PUBKEY,

      1001,

      "d:fixture",

      JSON.stringify(wrongBody),

      SIGNER,

    );

    expect(verifySupersessionChain([event1, badEvent])).toBe(false);

  });

  test("signSupersedingEngram requires a strictly greater created_at", () => {

    expect(() =>

      signSupersedingEngram(event1, "too early", 999, PUBKEY, SIGNER),

    ).toThrow();

    expect(() =>

      signSupersedingEngram(event1, "same time", 1000, PUBKEY, SIGNER),

    ).toThrow();

  });

});

import { Schema as S } from "effect";

import { redactString } from "@openagentsinc/atif/redaction";

import { canonicalStringify } from "./internal/canonical.js";

import { sha256Hex } from "./internal/sha256.js";

/**

 * Formal NIP-AE engram schema and a strict pre-sign redaction boundary.

 *

 * This module defines the OpenAgents companion profile for NIP-AE kind 30174

 * addressable events. It treats engrams as immutable, signed records: a

 * correction never overwrites a prior engram; it appends a new engram that

 * explicitly references and supersedes the previous event id.

 *

 * Every engram value passes a strict zero-credential / zero-token redaction

 * filter before it can be signed. Tokens, private SSH/Nostr keys, private IPs,

 * and environment variables are detected, redacted, and cause the engram to be

 * rejected as hard-unsafe.

 */

/** NIP-AE addressable engram kind. */

export const ENGRAM_KIND = 30174 as const;

/** OpenAgents companion schema id written inside the engram body. */

export const COMPANION_SCHEMA_ID =

  "openagents.agent_experience_memory.nip_ae_companion.v1" as const;

/** NIP-31 alt text for engrams (NIP-AE default). */

export const ENGRAM_ALT = "encrypted agent memory record" as const;

const Hex64 = S.String.check(S.isPattern(/^[0-9a-f]{64}$/));

export const EngramSlug = S.String.check(S.isMinLength(1), S.isMaxLength(255));

export type EngramSlug = typeof EngramSlug.Type;

export const EngramAdmission = S.Literals(["admitted", "candidate", "rejected"]);

export type EngramAdmission = typeof EngramAdmission.Type;

export const EngramSourceRole = S.Literals([

  "turn_record",

  "tool_result",

  "owner_message",

  "import",

  "supersession",

]);

export type EngramSourceRole = typeof EngramSourceRole.Type;

export const EngramSourceEventRef = S.Struct({

  eventId: Hex64,

  role: EngramSourceRole,

});

export type EngramSourceEventRef = typeof EngramSourceEventRef.Type;

export const EngramRelationDirection = S.Literals(["out", "in", "both"]);

export type EngramRelationDirection = typeof EngramRelationDirection.Type;

export const EngramRelation = S.Struct({

  type: S.String.check(S.isMinLength(1), S.isMaxLength(64)),

  targetSlug: EngramSlug,

  direction: EngramRelationDirection,

});

export type EngramRelation = typeof EngramRelation.Type;

/** OpenAgents companion fields under the NIP-AE unknown-fields rule. */

export const EngramCompanion = S.Struct({

  schema: S.Literal(COMPANION_SCHEMA_ID),

  admission: EngramAdmission,

  entityId: S.String.check(S.isMinLength(1), S.isMaxLength(128)),

  contentDigest: S.String.check(S.isPattern(/^sha256:[0-9a-f]{64}$/)),

  sourceEventRefs: S.Array(EngramSourceEventRef),

  relations: S.Array(EngramRelation),

  derivedFromSlugs: S.Array(EngramSlug),

  /** The prior event id this engram supersedes, if any. */

  supersedes: S.optionalKey(Hex64),

});

export type EngramCompanion = typeof EngramCompanion.Type;

/** NIP-AE engram body with the OpenAgents companion profile. */

export const EngramBody = S.Struct({

  slug: EngramSlug,

  /** The plaintext value. `null` is the in-band tombstone. */

  value: S.NullOr(S.String),

  openagents: EngramCompanion,

});

export type EngramBody = typeof EngramBody.Type;

/** A Nostr-like signed engram event. */

export const EngramEvent = S.Struct({

  id: Hex64,

  pubkey: Hex64,

  created_at: S.Number,

  kind: S.Literal(ENGRAM_KIND),

  tags: S.Array(S.Array(S.String)),

  content: S.String,

  sig: S.String,

});

export type EngramEvent = typeof EngramEvent.Type;

/** A signer produces a signature over a 64-character hex event id. */

export type EngramSigner = (eventId: string) => string;

/**

 * Categories that make an engram hard-unsafe and therefore non-storable.

 * Tokens, keys, private paths, private IPs, Nostr private keys, and

 * environment variables are included. Soft PII (for example emails) is redacted

 * but does not block signing.

 */

export const ENGRAM_HARD_UNSAFE_CATEGORIES = [

  "private_key",

  "mnemonic",

  "jwt",

  "bearer",

  "provider_key",

  "oa_agent_token",

  "oa_token",

  "aws_key",

  "google_key",

  "slack_token",

  "github_token",

  "env_secret",

  "wallet_or_payment",

  "secrets_path",

  "home_path",

  "file_url",

  "ip",

  "private_ip",

  "nostr_private_key",

  "environment_variable",

] as const;

type RedactionRule = Readonly<{

  category: string;

  pattern: RegExp;

  replace: (match: string, ...groups: Array<string>) => string;

}>;

const tag = (category: string): string => `[REDACTED:${category}]`;

const EXTRA_RULES: ReadonlyArray<RedactionRule> = [

  {

    category: "nostr_private_key",

    pattern: /\bnsec1[ac-hj-np-z02-9]{50,90}\b/g,

    replace: () => tag("nostr_private_key"),

  },

  {

    category: "private_ip",

    pattern:

      /\b(?:127\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)|169\.254\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)|0\.0\.0\.0|255\.255\.255\.255)\b/g,

    replace: () => tag("private_ip"),

  },

  {

    category: "environment_variable",

    pattern: /\b(export\s+)([A-Z_][A-Z0-9_]*)(\s*=\s*)([^\s;'"`]+)/g,

    replace: (_match, exportDecl: string, name: string) =>

      `${exportDecl}${name}=${tag("environment_variable")}`,

  },

  {

    category: "environment_variable",

    pattern: /^([ \t]*[A-Z_][A-Z0-9_]*)(\s*=\s*)([^\n"']*)/gm,

    replace: (_match, name: string) => `${name}=${tag("environment_variable")}`,

  },

];

/**

 * Redact an engram value through the ATIF boundary plus engram-specific rules.

 *

 * Returns the redacted text, the categories that fired, whether the result is

 * safe to sign, and the total redaction count. A hard-unsafe category makes

 * `storable` false.

 */

export const redactEngramContent = (

  input: string,

): Readonly<{

  redacted: string;

  storable: boolean;

  categories: ReadonlyArray<string>;

  total: number;

}> => {

  const counts: Record<string, number> = {};

  const bump = (category: string): void => {

    counts[category] = (counts[category] ?? 0) + 1;

  };

  // Run the engram-specific rules first so targeted credentials (for example a

  // Nostr nsec) are caught before the ATIF long_blob heuristic swallows them.

  let working = input;

  for (const rule of EXTRA_RULES) {

    rule.pattern.lastIndex = 0;

    working = working.replace(rule.pattern, (match, ...args) => {

      const groups = args.slice(0, -2) as Array<string>;

      const replaced = rule.replace(match, ...groups);

      if (replaced !== match) {

        bump(rule.category);

      }

      return replaced;

    });

  }

  // Then run the full ATIF redaction boundary for tokens, private keys, paths,

  // payment material, and standard private IP ranges.

  const atif = redactString(working);

  for (const [category, count] of Object.entries(atif.report.counts)) {

    counts[category] = (counts[category] ?? 0) + count;

  }

  const categories = Object.keys(counts);

  const hard = new Set(ENGRAM_HARD_UNSAFE_CATEGORIES as unknown as ReadonlyArray<string>);

  const storable = categories.every((category) => !hard.has(category));

  const total = Object.values(counts).reduce((a, b) => a + b, 0);

  return { redacted: atif.value, storable, categories, total };

};

/** Pre-sign redaction verdict for an engram value. */

export type EngramContentVerdict = Readonly<{

  /** The redacted value. `null` for a tombstone. */

  redacted: string | null;

  storable: boolean;

  categories: ReadonlyArray<string>;

  total: number;

}>;

/**

 * Guard a candidate engram value. Tombstones are always safe. Any other value

 * is rejected as non-storable when it contains a hard-unsafe category.

 */

export const guardEngramContent = (value: string | null): EngramContentVerdict => {

  if (value === null) {

    return { redacted: null, storable: true, categories: [], total: 0 };

  }

  const { redacted, storable, categories, total } = redactEngramContent(value);

  return { redacted, storable, categories, total };

};

/**

 * Build a validated engram body. The schema id is always the canonical

 * companion id; callers do not need to supply it.

 */

export const buildEngramBody = (

  slug: EngramSlug,

  value: string | null,

  companion: Omit<EngramCompanion, "schema">,

): EngramBody =>

  S.decodeUnknownSync(EngramBody)({

    slug,

    value,

    openagents: { ...companion, schema: COMPANION_SCHEMA_ID },

  });

/** SHA-256 content digest over the canonicalized engram value. */

export const engramContentDigest = (value: string | null): string =>

  `sha256:${sha256Hex(canonicalStringify({ value }))}`;

/**

 * Build a companion body that supersedes a prior engram. The prior engram is

 * not modified; the new body carries the corrected value and a reference to the

 * prior event id.

 */

export const buildSupersedingBody = (

  prior: EngramBody,

  newValue: string | null,

  priorEventId: string,

  contentDigest?: string,

): EngramBody => {

  const { schema: _, ...companion } = prior.openagents;

  return buildEngramBody(

    prior.slug,

    newValue,

    {

      ...companion,