Measure the prefix reuse a transcript actually has

416a305bb202 · AtlantisPleb · · parent da9837e4ebc1

Measure the prefix reuse a transcript actually has

The replay tool read every turn's block list as the whole context sent
for that request. A weka-trace-v1 export is not that shape: each event
carries only its own new content, and the re-sent prefix is the
accumulation of everything before it. Read the wrong way round, a
session that is almost entirely prefix reported zero reuse — the tool
was measuring the one corpus shape our own exporter does not produce.

The context model is explicit now rather than assumed. `accumulate` is
the default and what weka-trace-v1 means: at turn N the request
carries every earlier block plus this turn's own. `explicit` is the
other shape, for corpora that record whole requests. A fixture pins
each, and the transcript fixture asserts both readings of the same
document — 5 of 9 blocks are prefix under the right model, zero under
the wrong one, which is the defect stated as a test.

Also fixes the Harbor adapter, which pinned the CLI tarball to version
0.3.5. The 0.4.0 bump silently broke every graded run: each trial
errored in install against a tarball nobody packs any more. It finds
the packed tarball instead of predicting its name, and the hint it
prints now says `pnpm pack`, which is what the workspace requires —
`npm pack` leaves catalog: protocol versions unresolved.

Found by a delegated channel that reported the acceptance could not
pass rather than making a test agree with the tool, which is what let
the real defect surface.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GoYpb8FEmdxVErsv7ABCYi
Co-Authored-By
Claude Fable 5 <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified bench/adapters/openagents_coder.py
  • added bench/fixtures/weka-trace-v1-transcript.json
  • modified bench/replay_weka.py
  • modified bench/test_replay_weka.py

Diff

4 files changed, +107 -12

bench/adapters/openagents_coder.py modified +19 -1

@@ -33,7 +33,25 @@ from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_templat

33 33
from harbor.environments.base import BaseEnvironment
34 34
from harbor.models.agent.context import AgentContext
35 35
36
_TARBALL = Path(__file__).resolve().parent.parent / "openagentsinc-cli-0.3.5.tgz"
36
def _find_tarball() -> Path:
37
    """The packed CLI, whatever version it is.
38
39
    Pinning the version here meant a version bump silently broke every graded
40
    run: the adapter looked for a tarball nobody packs any more and every trial
41
    errored in install with a message about the old number. The pack step
42
    produces exactly one tarball, so find it rather than predict its name; if
43
    there are several, take the newest, because that is the one just built.
44
    """
45
    bench = Path(__file__).resolve().parent.parent
46
    candidates = sorted(
47
        bench.glob("openagentsinc-cli-*.tgz"),
48
        key=lambda path: path.stat().st_mtime,
49
        reverse=True,
50
    )
51
    return candidates[0] if candidates else bench / "openagentsinc-cli-<version>.tgz"
52
53
54
_TARBALL = _find_tarball()
37 55
_REMOTE_TARBALL = "/installed-agent/openagents-cli.tgz"
38 56
_DEFAULT_API_URL = "http://host.docker.internal:4000"
39 57
_EXPORT_DIR = "$HOME/.openagents/exports"
bench/fixtures/weka-trace-v1-transcript.json added +10

@@ -0,0 +1,10 @@

1
{
2
  "format": "weka-trace-v1",
3
  "thread_id": "transcript-001",
4
  "event_count": 3,
5
  "events": [
6
    { "id": 1, "event_type": "turn.user",      "emitted_at": "2026-08-25T00:00:00Z", "role": "user",      "block_count": 2, "blocks": ["a1", "a2"] },
7
    { "id": 2, "event_type": "turn.assistant", "emitted_at": "2026-08-25T00:00:01Z", "role": "assistant", "block_count": 1, "blocks": ["b1"] },
8
    { "id": 3, "event_type": "turn.user",      "emitted_at": "2026-08-25T00:00:02Z", "role": "user",      "block_count": 1, "blocks": ["c1"] }
9
  ]
10
}
bench/replay_weka.py modified +46 -8

@@ -132,9 +132,25 @@ def _blocks_for_event(ev: Dict[str, Any]) -> List[Any]:

132 132
    return _get(ev, _HASH_KEYS) or []
133 133
134 134
135
def _analyze_session(events: List[Dict[str, Any]]) -> Dict[str, Any]:
135
def _analyze_session(events: List[Dict[str, Any]], context: str = "accumulate") -> Dict[str, Any]:
136
    """Per-turn prefix reuse for one session.
137
138
    Two corpora shapes exist and they mean different things by a turn's block
139
    list, so the model is explicit rather than guessed:
140
141
    - `accumulate` (the default, and what `weka-trace-v1` exports): each event
142
      carries only its OWN new content. The request context at turn N is
143
      everything the session has said up to and including N, so the re-sent
144
      prefix is the accumulation of turns 1..N-1. This is where the reuse
145
      lives, and reading each event's list as if it were the whole context
146
      reports zero reuse for a session that is almost entirely prefix.
147
    - `explicit`: each turn's list already IS the full context sent for that
148
      request, repeats included. Used by corpora that record requests rather
149
      than transcripts.
150
    """
136 151
    indexed = sorted(enumerate(events), key=lambda x: (_sortable_time(_get(x[1], _TIME_KEYS)), x[0]))
137 152
    seen: set = set()
153
    carried: List[Any] = []
138 154
    turns: List[Dict[str, Any]] = []
139 155
    total, repeated, new = 0, 0, 0
140 156
    first_total, last_total = 0, 0

@@ -142,10 +158,20 @@ def _analyze_session(events: List[Dict[str, Any]]) -> Dict[str, Any]:

142 158
        raw = _blocks_for_event(ev)
143 159
        if not isinstance(raw, list):
144 160
            raise ValueError("turn block list is not an array")
145
        keys = [_canon(b) for b in raw if _canon(b) is not None]
146
        n_total = len(keys)
147
        n_repeated = sum(1 for k in keys if k in seen)
148
        n_new = n_total - n_repeated
161
        own = [_canon(b) for b in raw if _canon(b) is not None]
162
        if context == "accumulate":
163
            # The request carries everything said so far, then this turn's own
164
            # blocks. The prefix is exactly what was carried in.
165
            n_repeated = len(carried)
166
            keys = carried + own
167
            n_total = len(keys)
168
            n_new = len(own)
169
            carried = keys
170
        else:
171
            keys = own
172
            n_total = len(keys)
173
            n_repeated = sum(1 for k in keys if k in seen)
174
            n_new = n_total - n_repeated
149 175
        seen.update(keys)
150 176
        total += n_total
151 177
        repeated += n_repeated

@@ -272,7 +298,7 @@ def _build_output(results: List[Dict[str, Any]], corpus: Dict[str, Any]) -> Dict

272 298
    return {"files": results, "corpus": corpus}
273 299
274 300
275
def process_paths(paths: List[Path]) -> Tuple[List[Dict[str, Any]], List[str]]:
301
def process_paths(paths: List[Path], context: str = "accumulate") -> Tuple[List[Dict[str, Any]], List[str]]:
276 302
    results: List[Dict[str, Any]] = []
277 303
    errors: List[str] = []
278 304
    for p in paths:

@@ -289,7 +315,7 @@ def process_paths(paths: List[Path]) -> Tuple[List[Dict[str, Any]], List[str]]:

289 315
                        "turns": a["turns"],
290 316
                        "summary": a["summary"],
291 317
                    }
292
                    for sid, a in ((sid, _analyze_session(ev)) for sid, ev in sessions)
318
                    for sid, a in ((sid, _analyze_session(ev, context)) for sid, ev in sessions)
293 319
                ]
294 320
                file_docs.append({"document_index": i, "sessions": analyzed})
295 321
            results.append({"path": str(p), "documents": file_docs})

@@ -306,9 +332,21 @@ def main(argv: Optional[List[str]] = None) -> int:

306 332
    )
307 333
    parser.add_argument("traces", nargs="+", type=Path, help="weka-trace-v1 JSON files")
308 334
    parser.add_argument("--json", action="store_true", help="emit machine-readable JSON output")
335
    parser.add_argument(
336
        "--context",
337
        choices=("accumulate", "explicit"),
338
        default="accumulate",
339
        help=(
340
            "how a turn's block list relates to the request context. "
341
            "accumulate (default, and what weka-trace-v1 exports): each event "
342
            "carries its own new content and the request re-sends everything "
343
            "before it. explicit: each turn's list is already the whole "
344
            "context, repeats included."
345
        ),
346
    )
309 347
    arguments = parser.parse_args(argv)
310 348
311
    results, errors = process_paths(arguments.traces)
349
    results, errors = process_paths(arguments.traces, arguments.context)
312 350
    all_sessions = [
313 351
        s for r in results for d in r.get("documents", []) for s in d["sessions"]
314 352
    ]
bench/test_replay_weka.py modified +32 -3

@@ -28,7 +28,9 @@ def _run(args: List[str]) -> subprocess.CompletedProcess:

28 28
29 29
30 30
def test_sample_reuse_ratio() -> None:
31
    proc = _run([str(_FIXTURE)])
31
    # The fixture records whole request contexts, so it is an `explicit`
32
    # corpus. `weka-trace-v1` is the other shape and is covered below.
33
    proc = _run([str(_FIXTURE), "--context", "explicit"])
32 34
    if proc.returncode != 0:
33 35
        raise AssertionError(f"tool failed: {proc.stderr}")
34 36
    data = json.loads(proc.stdout)

@@ -40,13 +42,36 @@ def test_sample_reuse_ratio() -> None:

40 42
    assert corpus["context_growth"] == 6
41 43
42 44
45
def test_transcript_corpus_reuse_ratio() -> None:
46
    """A weka-trace-v1 transcript: each event carries only its own blocks.
47
48
    Hand-computed: turn 1 sends 2 blocks and repeats none; turn 2 re-sends
49
    those 2 and adds 1; turn 3 re-sends 3 and adds 1. So 9 blocks travel, 5 of
50
    them are prefix, and the reuse is 5/9. Read under the `explicit` model
51
    this same document reports zero reuse, which is the bug this pins.
52
    """
53
    fixture = _FIXTURE.parent / "weka-trace-v1-transcript.json"
54
    proc = _run([str(fixture)])
55
    if proc.returncode != 0:
56
        raise AssertionError(f"tool failed: {proc.stderr}")
57
    corpus = json.loads(proc.stdout)["corpus"]
58
    assert corpus["turns"] == 3
59
    assert corpus["total_blocks"] == 9
60
    assert corpus["repeated_blocks"] == 5
61
    assert abs(corpus["prefix_reuse_ratio"] - 5 / 9) < 1e-9
62
    assert corpus["context_growth"] == 4
63
64
    explicit = _run([str(fixture), "--context", "explicit"])
65
    assert json.loads(explicit.stdout)["corpus"]["repeated_blocks"] == 0
66
67
43 68
def test_malformed_and_empty_do_not_block_valid() -> None:
44 69
    with tempfile.TemporaryDirectory() as d:
45 70
        empty = Path(d) / "empty.json"
46 71
        empty.write_text("")
47 72
        bad = Path(d) / "bad.json"
48 73
        bad.write_text("not json")
49
        proc = _run([str(empty), str(bad), str(_FIXTURE)])
74
        proc = _run([str(empty), str(bad), str(_FIXTURE), "--context", "explicit"])
50 75
        assert proc.returncode == 1, f"expected exit 1, got {proc.returncode}"
51 76
        data = json.loads(proc.stdout)
52 77
        errors = [f for f in data["files"] if "error" in f]

@@ -59,7 +84,11 @@ def test_malformed_and_empty_do_not_block_valid() -> None:

59 84
60 85
if __name__ == "__main__":
61 86
    failures = 0
62
    for test in (test_sample_reuse_ratio, test_malformed_and_empty_do_not_block_valid):
87
    for test in (
88
        test_sample_reuse_ratio,
89
        test_transcript_corpus_reuse_ratio,
90
        test_malformed_and_empty_do_not_block_valid,
91
    ):
63 92
        name = test.__name__
64 93
        try:
65 94
            test()

This page updates live while a promote is in flight · changelog