Stop calling a group isolated when it is the trend printed above it

e21ff2c175ea · Claude Fable 5 · 1d ago · parent 96c921c07c9c

Stop calling a group isolated when it is the trend printed above it

The first real two-run store ended its report with "1 run shape(s) in this
store had no comparable second row and were left out", directly under the
trend over exactly those rows. Two runs on one lane produce no lane comparison,
which is correct and expected, and the group was counted as isolated for it.

Isolated now means the group produced nothing: no lane comparison and no trend.
A footnote that contradicts the table above it costs more than it explains —
the reader has to decide which half of their own report to believe.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SoZMfWRSGnf6FZX2Ar9rQ2

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SoZMfWRSGnf6FZX2Ar9rQ2

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

modified packages/coder-effectiveness/src/compare.test.ts
modified packages/coder-effectiveness/src/compare.ts

Diff

2 files changed, +26 -3

    expect(comparison.isolatedGroups).toBe(1);
  });

  test("does not call a group isolated when it is the trend printed above", () => {
    // Two runs, one lane: no lane comparison is possible and none is wanted.
    // Counting the group as isolated told a reader that the trend they had
    // just read had been left out of the report they were reading.
    const first = row("priced-lane", "local", "2026-08-25T10:00:00.000Z");
    const second = row("regressed-lane", "local", "2026-08-25T11:00:00.000Z");

    const comparison = compareRuns([first, second]);

    expect(comparison.trends).toHaveLength(1);
    expect(comparison.laneComparisons).toEqual([]);
    expect(comparison.isolatedGroups).toBe(0);
    expect(renderComparison(comparison)).not.toContain("left out");
  });

  test("flags a lane comparison whose CLI version also varies", () => {
    const proxy = row("priced-lane", "proxy", "2026-08-25T10:00:00.000Z");
    const local: BenchResultRow = {


  for (const group of groups) {
    const latestByLane = latestPerLane(group);
    if (latestByLane.length < 2) {
      isolatedGroups += 1;
    } else {
    if (latestByLane.length >= 2) {
      laneComparisons.push(laneComparisonOf(latestByLane, options.baselineLane));
    }

    let trendsFromGroup = 0;
    for (const laneRows of groupBy(group, (row) => row.lane)) {
      if (laneRows.length < 2) continue;
      trends.push(trendOf(laneRows));
      trendsFromGroup += 1;
    }

    // Isolated means this group produced nothing — no lane comparison AND no
    // trend. A group with several runs on a single lane is not isolated: it is
    // the trend printed above, and counting it here made the report tell a
    // reader that the thing they had just read had been left out.
    if (latestByLane.length < 2 && trendsFromGroup === 0) {
      isolatedGroups += 1;
    }
  }


191	191		`expect(comparison.isolatedGroups).toBe(1);`
192	192		`});`
193	193
	194	+	`test("does not call a group isolated when it is the trend printed above", () => {`
	195	+	`// Two runs, one lane: no lane comparison is possible and none is wanted.`
	196	+	`// Counting the group as isolated told a reader that the trend they had`
	197	+	`// just read had been left out of the report they were reading.`
	198	+	`const first = row("priced-lane", "local", "2026-08-25T10:00:00.000Z");`
	199	+	`const second = row("regressed-lane", "local", "2026-08-25T11:00:00.000Z");`
	200	+
	201	+	`const comparison = compareRuns([first, second]);`
	202	+
	203	+	`expect(comparison.trends).toHaveLength(1);`
	204	+	`expect(comparison.laneComparisons).toEqual([]);`
	205	+	`expect(comparison.isolatedGroups).toBe(0);`
	206	+	`expect(renderComparison(comparison)).not.toContain("left out");`
	207	+	`});`
	208	+
194	209		`test("flags a lane comparison whose CLI version also varies", () => {`
195	210		`const proxy = row("priced-lane", "proxy", "2026-08-25T10:00:00.000Z");`
196	211		`const local: BenchResultRow = {`

107	107
108	108		`for (const group of groups) {`
109	109		`const latestByLane = latestPerLane(group);`
110		-	`if (latestByLane.length < 2) {`
111		-	`isolatedGroups += 1;`
112		-	`} else {`
	110	+	`if (latestByLane.length >= 2) {`
113	111		`laneComparisons.push(laneComparisonOf(latestByLane, options.baselineLane));`
114	112		`}`
115	113
	114	+	`let trendsFromGroup = 0;`
116	115		`for (const laneRows of groupBy(group, (row) => row.lane)) {`
117	116		`if (laneRows.length < 2) continue;`
118	117		`trends.push(trendOf(laneRows));`
	118	+	`trendsFromGroup += 1;`
	119	+	`}`
	120	+
	121	+	`// Isolated means this group produced nothing — no lane comparison AND no`
	122	+	`// trend. A group with several runs on a single lane is not isolated: it is`
	123	+	`// the trend printed above, and counting it here made the report tell a`
	124	+	`// reader that the thing they had just read had been left out.`
	125	+	`if (latestByLane.length < 2 && trendsFromGroup === 0) {`
	126	+	`isolatedGroups += 1;`
119	127		`}`
120	128		`}`
121	129