Rename environmentProven, which claimed more than it knew

  return names[0];
};

const registryTask = (datasets, dataset, name, rationale, environmentProven) => ({
const registryTask = (datasets, dataset, name, rationale, environmentAvailable) => ({
  id: name,
  pin: pinFor(datasets, dataset, name),
  environmentProven,
  environmentAvailable,
  ...(rationale === undefined ? {} : { rationale }),
});


 * be perfectly well closed, but without a commit there is nothing to grade
 * against.
 *
 * Every one of these is `environmentProven: false` today. The pin is real — the
 * Every one of these is `environmentAvailable: false` today. The pin is real — the
 * issue, its instruction, and the commit that satisfied it — and no container
 * has been built that can grade it, so `parseSuiteManifest` will refuse to let
 * them into a score-tier suite until one has. See `bench/tasks/owned/README.md`.

        issue: issue.number,
        acceptedCommit,
      },
      environmentProven: false,
      environmentAvailable: false,
      rationale: issue.title,
    });
  }

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "git-leak-recovery"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "sanitize-git-repo",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "sanitize-git-repo"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "merge-diff-arc-agi-task",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "merge-diff-arc-agi-task"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "build-cython-ext",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "build-cython-ext"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "sqlite-with-gcov",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "sqlite-with-gcov"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "fix-code-vulnerability",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "fix-code-vulnerability"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "regex-log",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "regex-log"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "count-dataset-tokens",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "count-dataset-tokens"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "password-recovery",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "password-recovery"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "openssl-selfsigned-cert",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "openssl-selfsigned-cert"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "nginx-request-logging",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "nginx-request-logging"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "schemelike-metacircular-eval",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "schemelike-metacircular-eval"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "astropy__astropy-12907",

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/astropy__astropy-12907"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "astropy: table and unit handling in a large scientific codebase"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/django__django-10097"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "Django: the framework half of the benchmark, wide blast radius per change"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/matplotlib__matplotlib-13989"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "matplotlib: rendering state, where the test is the only oracle"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/pydata__xarray-2905"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "xarray: array semantics over pandas and numpy, dtype-sensitive"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/pytest-dev__pytest-10051"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "pytest: the test runner itself, so a fix has to be reentrant"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/scikit-learn__scikit-learn-10297"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "scikit-learn: estimator API conformance, contract-shaped"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/sphinx-doc__sphinx-10323"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "Sphinx: documentation tooling, heavy on configuration surface"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/sympy__sympy-11618"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "SymPy: symbolic evaluation, where a plausible-looking fix is usually wrong"
    }
  ]

        "issue": 21,
        "acceptedCommit": "586edf191fb221d1e1f5c600b4a714d40fc95820"
      },
      "environmentProven": false,
      "environmentAvailable": false,
      "rationale": "Add Ollama local model support to openagents coder"
    },
    {

        "issue": 24,
        "acceptedCommit": "f6366a80930ade654bc29b3b1217268011506061"
      },
      "environmentProven": false,
      "environmentAvailable": false,
      "rationale": "Resume a coder thread with --resume"
    },
    {

        "issue": 31,
        "acceptedCommit": "cf1861c9cbd2b2bf3e02593bf4c291db82c88e58"
      },
      "environmentProven": false,
      "environmentAvailable": false,
      "rationale": "Adopt the proxy's reasoning and tool-call fidelity in the coder"
    },
    {

        "issue": 36,
        "acceptedCommit": "2a631cf63107d1b706e7e8058f49c1bd283447ed"
      },
      "environmentProven": false,
      "environmentAvailable": false,
      "rationale": "Teach the coder's tools token economy, per model family"
    },
    {

        "issue": 40,
        "acceptedCommit": "fb705a506a2d56263b07698aa02b3223861d0df6"
      },
      "environmentProven": false,
      "environmentAvailable": false,
      "rationale": "Coder presents model tiers, never vendor model names"
    },
    {

        "issue": 41,
        "acceptedCommit": "a04c4fcf4708ff30b264970eb08acbebe5f20ed5"
      },
      "environmentProven": false,
      "environmentAvailable": false,
      "rationale": "Read a conversation on request: the read-conversation plugin, discovered, loaded, and streamed in the coder"
    }
  ]

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "fix-code-vulnerability"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "quick-shaped"
    },
    {

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "regex-log"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "quick-shaped"
    }
  ]

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/astropy__astropy-12907"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "astropy: table and unit handling in a large scientific codebase"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/django__django-10097"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "Django: the framework half of the benchmark, wide blast radius per change"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/matplotlib__matplotlib-13989"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "matplotlib: rendering state, where the test is the only oracle"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/pydata__xarray-2905"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "xarray: array semantics over pandas and numpy, dtype-sensitive"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/pytest-dev__pytest-10051"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "pytest: the test runner itself, so a fix has to be reentrant"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/scikit-learn__scikit-learn-10297"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "scikit-learn: estimator API conformance, contract-shaped"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/sphinx-doc__sphinx-10323"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "Sphinx: documentation tooling, heavy on configuration surface"
    },
    {

        "commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",
        "path": "datasets/swebench-verified/sympy__sympy-11618"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "SymPy: symbolic evaluation, where a plausible-looking fix is usually wrong"
    }
  ]

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "git-leak-recovery"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "sanitize-git-repo",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "sanitize-git-repo"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "merge-diff-arc-agi-task",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "merge-diff-arc-agi-task"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "build-cython-ext",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "build-cython-ext"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "sqlite-with-gcov",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "sqlite-with-gcov"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "fix-code-vulnerability",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "fix-code-vulnerability"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "regex-log",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "regex-log"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "count-dataset-tokens",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "count-dataset-tokens"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "password-recovery",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "password-recovery"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "openssl-selfsigned-cert",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "openssl-selfsigned-cert"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "nginx-request-logging",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "nginx-request-logging"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "schemelike-metacircular-eval",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "schemelike-metacircular-eval"
      },
      "environmentProven": true
      "environmentAvailable": true
    }
  ]
}

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "regex-log"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "near-zero tool surface: the suite's test of whether the agent can just answer"
    },
    {

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "openssl-selfsigned-cert"
      },
      "environmentProven": true,
      "environmentAvailable": true,
      "rationale": "a fully specified checklist: every command is known upfront, so round count is a tool habit rather than a reasoning result"
    }
  ]

    "issue": 31,
    "acceptedCommit": "cf1861c9cb..."
  },
  "environmentProven": false,
  "environmentAvailable": false,
  "rationale": "Adopt the proxy's reasoning and tool-call fidelity in the coder"
}
```


**No container exists that can grade any of these.** The pin is real and the
suite is real; the environment is not. Every owned task therefore carries
`environmentProven: false`, and `parseSuiteManifest` refuses to let an unproven
task into a `score`-tier suite — so `owned-closed-issues.suite.json` is
`smoke`, and `coder-effectiveness-v1` holds the twenty proven public tasks and
none of these.
`environmentAvailable: false`, and `parseSuiteManifest` refuses to let a task
with no gradeable environment into a `score`-tier suite — so
`owned-closed-issues.suite.json` is `smoke`, and `coder-effectiveness-v1` holds
the twenty registry-backed public tasks and none of these.

The registry tasks carry `environmentAvailable: true` because the dataset ships
their environment and verifier, which is what makes a public subset cost a
manifest entry rather than a harness. The flag says a gradeable definition
exists; it does not say the task has been run here, or that it passed, or that
its image builds on any particular machine.

That refusal is the point rather than a limitation. A score suite that included
a task nobody could run would report those trials as missing, and a missing

   recorded test pass; it does not mean the agent reproduced the recorded diff,
   and it should not.

Two things have to be true before the first of these is `environmentProven`,
and neither is cheap:
Two things have to be true before the first of these can flip
`environmentAvailable`, and neither is cheap:

- **The image.** `packages/openagents-cli` sits in a 117-package pnpm
  workspace, so the environment is a repo snapshot plus an install, not a

    "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
    "path": "regex-log"
  },
  "environmentProven": true
  "environmentAvailable": true
}
```


and a suite that pinned only names would report two different measurements
under one heading.

`environmentProven` is whether a container for this task has ever been built
and graded. A `score` suite may not hold an unproven task — a trial nobody
could run reads as the coder failing rather than as a missing environment — so
an owned task lives in a `smoke` suite until somebody proves it.
`environmentAvailable` is whether a container and verifier that can grade this
task exist. A registry task has them by construction; an owned task drawn from
a closed issue has neither until somebody writes them. A `score` suite may not
hold a task with no gradeable environment — a trial nobody can run reads as the
coder failing rather than as a missing environment — so the owned tasks live in
a `smoke` suite until their environments are written.

## A smoke run is never a published score, structurally



- **The owned lane's environments.** Six closed issues are pinned to their
  closing commits and none of them has a container that can grade it, so they
  are `environmentProven: false` and their suite is `smoke`.
  are `environmentAvailable: false` and their suite is `smoke`.
  `bench/tasks/owned/README.md` has the construction and the two things that
  have to be true before the first one is admitted to a score.
- **Per-model cost from the coder's own trajectory.** The ATIF exporter writes

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "build-cmake"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "fix-git",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "fix-git"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "parse-log",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "parse-log"
      },
      "environmentProven": true
      "environmentAvailable": true
    }
  ]
}

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "build-cmake"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "fix-git",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "fix-git"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "parse-log",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "parse-log"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "port-forward",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "port-forward"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "never-run",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "never-run"
      },
      "environmentProven": true
      "environmentAvailable": true
    }
  ]
}

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "build-cmake"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "fix-git",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "fix-git"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "parse-log",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "parse-log"
      },
      "environmentProven": true
      "environmentAvailable": true
    },
    {
      "id": "port-forward",

        "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        "path": "port-forward"
      },
      "environmentProven": true
      "environmentAvailable": true
    }
  ]
}

        commit: "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        path: trial.task,
      },
      environmentProven: true,
      environmentAvailable: true,
    })),
  });


    commit: overrides.commit ?? "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
    path: overrides.path ?? id,
  },
  environmentProven: true,
  environmentAvailable: true,
});

const suite = (overrides: Partial<SuiteManifest> = {}): SuiteManifest =>

    // The owned tasks are pinned and real and have no container yet. A score
    // over them would read their absence as the coder failing.
    expect(() =>
      suite({ tasks: [{ ...registryTask("regex-log"), environmentProven: false }] }),
    ).toThrow(/environment has never been built/u);
      suite({ tasks: [{ ...registryTask("regex-log"), environmentAvailable: false }] }),
    ).toThrow(/no environment that can grade/u);
  });

  test("allows an unproven task in a smoke suite", () => {
    const manifest = suite({
      tier: "smoke",
      tasks: [{ ...registryTask("regex-log"), environmentProven: false }],
      tasks: [{ ...registryTask("regex-log"), environmentAvailable: false }],
    });

    expect(manifest.tier).toBe("smoke");

  id: S.String,
  pin: SuiteTaskPinSchema,
  /**
   * Whether a container for this task has been built and graded at least once.
   * Whether a container and verifier that can grade this task exist.
   *
   * An owned task drawn from a closed issue is a real, pinned piece of work
   * long before anybody has built an image that can grade it, and writing it
   * into the manifest early is how the suite records the intent. But a score
   * cannot include a task nobody has ever run: its absence from the results
   * would read as a failure of the coder rather than an absence of an
   * environment. So {@link parseSuiteManifest} refuses a `score` manifest that
   * holds an unproven task, and the suite that holds them stays `smoke` until
   * somebody proves them.
   * A `harbor-registry` task has them by construction: the dataset ships the
   * environment and the verifier, which is the entire reason a public subset
   * costs a manifest entry rather than a harness. A `tracker-closed-issue` task
   * has neither until somebody writes them — it is a real, pinned piece of work
   * with a recorded accepted outcome and no way yet to run it, and writing it
   * into the manifest early is how the suite records the intent.
   *
   * Note what this field does NOT claim: that the task has been run here, or
   * that it passed, or that its image builds on this machine. It claims a
   * gradeable definition exists. That is the property a suite needs, because a
   * score cannot include a task nobody can run at all — its absence from the
   * results would read as a failure of the coder rather than an absence of an
   * environment, and the suite would get quietly worse the day somebody added a
   * task and forgot to build it. So {@link parseSuiteManifest} refuses a
   * `score` manifest that holds one, and the suite that holds them stays
   * `smoke` until the environments are written.
   */
  environmentProven: S.Boolean,
  environmentAvailable: S.Boolean,
  /** One line on why this task is in the suite. Not digested. */
  rationale: S.optional(S.String),
});

  }

  if (manifest.tier === "score") {
    const unproven = manifest.tasks
      .filter((task) => !task.environmentProven)
    const unavailable = manifest.tasks
      .filter((task) => !task.environmentAvailable)
      .map((task) => task.id);
    if (unproven.length > 0) {
    if (unavailable.length > 0) {
      throw new Error(
        `suite ${manifest.id} is tier score but holds ${String(unproven.length)} task(s) whose environment has never been built and graded (${unproven.join(", ")}); a task nobody has run would read as a failure of the coder rather than a missing environment`,
        `suite ${manifest.id} is tier score but holds ${String(unavailable.length)} task(s) with no environment that can grade them (${unavailable.join(", ")}); a task nobody can run would read as a failure of the coder rather than a missing environment`,
      );
    }
  }

 *
 * `id` is in because it is how a trial directory names itself, so a suite that
 * renamed a task is running against a different result shape. `rationale` and
 * `environmentProven` are out: prose about why a task was chosen, and whether
 * anybody has built it yet, are facts about the suite's bookkeeping rather than
 * about the work the coder is asked to do.
 * `environmentAvailable` are out: prose about why a task was chosen, and
 * whether anybody has written its environment yet, are facts about the suite's
 * bookkeeping rather than about the work the coder is asked to do.
 */
export const taskDigestOf = (task: SuiteTask): string =>
  `task:${createHash("sha256")

        commit: "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",
        path: id,
      },
      environmentProven: true,
      environmentAvailable: true,
    })),
  });


168	168		`return names[0];`
169	169		`};`
170	170
171		-	`const registryTask = (datasets, dataset, name, rationale, environmentProven) => ({`
	171	+	`const registryTask = (datasets, dataset, name, rationale, environmentAvailable) => ({`
172	172		`id: name,`
173	173		`pin: pinFor(datasets, dataset, name),`
174		-	`environmentProven,`
	174	+	`environmentAvailable,`
175	175		`...(rationale === undefined ? {} : { rationale }),`
176	176		`});`
177	177

184	184		`* be perfectly well closed, but without a commit there is nothing to grade`
185	185		`* against.`
186	186		`*`
187		-	* Every one of these is `environmentProven: false` today. The pin is real — the
	187	+	* Every one of these is `environmentAvailable: false` today. The pin is real — the
188	188		`* issue, its instruction, and the commit that satisfied it — and no container`
189	189		* has been built that can grade it, so `parseSuiteManifest` will refuse to let
190	190		* them into a score-tier suite until one has. See `bench/tasks/owned/README.md`.

213	213		`issue: issue.number,`
214	214		`acceptedCommit,`
215	215		`},`
216		-	`environmentProven: false,`
	216	+	`environmentAvailable: false,`
217	217		`rationale: issue.title,`
218	218		`});`
219	219		`}`

46	46		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
47	47		`"path": "build-cython-ext"`
48	48		`},`
49		-	`"environmentProven": true`
	49	+	`"environmentAvailable": true`
50	50		`},`
51	51		`{`
52	52		`"id": "sqlite-with-gcov",`

79	79		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
80	80		`"path": "regex-log"`
81	81		`},`
82		-	`"environmentProven": true`
	82	+	`"environmentAvailable": true`
83	83		`},`
84	84		`{`
85	85		`"id": "count-dataset-tokens",`

Commit

Rename environmentProven, which claimed more than it knew

Deploy story

Not deployed through the forge lane

Changed files

Diff

13	13		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
14	14		`"path": "git-leak-recovery"`
15	15		`},`
16		-	`"environmentProven": true`
	16	+	`"environmentAvailable": true`
17	17		`},`
18	18		`{`
19	19		`"id": "sanitize-git-repo",`

24	24		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
25	25		`"path": "sanitize-git-repo"`
26	26		`},`
27		-	`"environmentProven": true`
	27	+	`"environmentAvailable": true`
28	28		`},`
29	29		`{`
30	30		`"id": "merge-diff-arc-agi-task",`

35	35		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
36	36		`"path": "merge-diff-arc-agi-task"`
37	37		`},`
38		-	`"environmentProven": true`
	38	+	`"environmentAvailable": true`
39	39		`},`
40	40		`{`
41	41		`"id": "build-cython-ext",`

57	57		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
58	58		`"path": "sqlite-with-gcov"`
59	59		`},`
60		-	`"environmentProven": true`
	60	+	`"environmentAvailable": true`
61	61		`},`
62	62		`{`
63	63		`"id": "fix-code-vulnerability",`

68	68		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
69	69		`"path": "fix-code-vulnerability"`
70	70		`},`
71		-	`"environmentProven": true`
	71	+	`"environmentAvailable": true`
72	72		`},`
73	73		`{`
74	74		`"id": "regex-log",`

90	90		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
91	91		`"path": "count-dataset-tokens"`
92	92		`},`
93		-	`"environmentProven": true`
	93	+	`"environmentAvailable": true`
94	94		`},`
95	95		`{`
96	96		`"id": "password-recovery",`

101	101		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
102	102		`"path": "password-recovery"`
103	103		`},`
104		-	`"environmentProven": true`
	104	+	`"environmentAvailable": true`
105	105		`},`
106	106		`{`
107	107		`"id": "openssl-selfsigned-cert",`

112	112		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
113	113		`"path": "openssl-selfsigned-cert"`
114	114		`},`
115		-	`"environmentProven": true`
	115	+	`"environmentAvailable": true`
116	116		`},`
117	117		`{`
118	118		`"id": "nginx-request-logging",`

123	123		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
124	124		`"path": "nginx-request-logging"`
125	125		`},`
126		-	`"environmentProven": true`
	126	+	`"environmentAvailable": true`
127	127		`},`
128	128		`{`
129	129		`"id": "schemelike-metacircular-eval",`

134	134		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
135	135		`"path": "schemelike-metacircular-eval"`
136	136		`},`
137		-	`"environmentProven": true`
	137	+	`"environmentAvailable": true`
138	138		`},`
139	139		`{`
140	140		`"id": "astropy__astropy-12907",`

145	145		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
146	146		`"path": "datasets/swebench-verified/astropy__astropy-12907"`
147	147		`},`
148		-	`"environmentProven": true,`
	148	+	`"environmentAvailable": true,`
149	149		`"rationale": "astropy: table and unit handling in a large scientific codebase"`
150	150		`},`
151	151		`{`

157	157		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
158	158		`"path": "datasets/swebench-verified/django__django-10097"`
159	159		`},`
160		-	`"environmentProven": true,`
	160	+	`"environmentAvailable": true,`
161	161		`"rationale": "Django: the framework half of the benchmark, wide blast radius per change"`
162	162		`},`
163	163		`{`

193	193		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
194	194		`"path": "datasets/swebench-verified/pytest-dev__pytest-10051"`
195	195		`},`
196		-	`"environmentProven": true,`
	196	+	`"environmentAvailable": true,`
197	197		`"rationale": "pytest: the test runner itself, so a fix has to be reentrant"`
198	198		`},`
199	199		`{`

205	205		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
206	206		`"path": "datasets/swebench-verified/scikit-learn__scikit-learn-10297"`
207	207		`},`
208		-	`"environmentProven": true,`
	208	+	`"environmentAvailable": true,`
209	209		`"rationale": "scikit-learn: estimator API conformance, contract-shaped"`
210	210		`},`
211	211		`{`

169	169		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
170	170		`"path": "datasets/swebench-verified/matplotlib__matplotlib-13989"`
171	171		`},`
172		-	`"environmentProven": true,`
	172	+	`"environmentAvailable": true,`
173	173		`"rationale": "matplotlib: rendering state, where the test is the only oracle"`
174	174		`},`
175	175		`{`

181	181		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
182	182		`"path": "datasets/swebench-verified/pydata__xarray-2905"`
183	183		`},`
184		-	`"environmentProven": true,`
	184	+	`"environmentAvailable": true,`
185	185		`"rationale": "xarray: array semantics over pandas and numpy, dtype-sensitive"`
186	186		`},`
187	187		`{`

217	217		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
218	218		`"path": "datasets/swebench-verified/sphinx-doc__sphinx-10323"`
219	219		`},`
220		-	`"environmentProven": true,`
	220	+	`"environmentAvailable": true,`
221	221		`"rationale": "Sphinx: documentation tooling, heavy on configuration surface"`
222	222		`},`
223	223		`{`

229	229		`"commit": "86723674f04e4209ac479d0fb75d9d9f44b4377e",`
230	230		`"path": "datasets/swebench-verified/sympy__sympy-11618"`
231	231		`},`
232		-	`"environmentProven": true,`
	232	+	`"environmentAvailable": true,`
233	233		`"rationale": "SymPy: symbolic evaluation, where a plausible-looking fix is usually wrong"`
234	234		`}`
235	235		`]`

12	12		`"issue": 21,`
13	13		`"acceptedCommit": "586edf191fb221d1e1f5c600b4a714d40fc95820"`
14	14		`},`
15		-	`"environmentProven": false,`
	15	+	`"environmentAvailable": false,`
16	16		`"rationale": "Add Ollama local model support to openagents coder"`
17	17		`},`
18	18		`{`

23	23		`"issue": 24,`
24	24		`"acceptedCommit": "f6366a80930ade654bc29b3b1217268011506061"`
25	25		`},`
26		-	`"environmentProven": false,`
	26	+	`"environmentAvailable": false,`
27	27		`"rationale": "Resume a coder thread with --resume"`
28	28		`},`
29	29		`{`

34	34		`"issue": 31,`
35	35		`"acceptedCommit": "cf1861c9cbd2b2bf3e02593bf4c291db82c88e58"`
36	36		`},`
37		-	`"environmentProven": false,`
	37	+	`"environmentAvailable": false,`
38	38		`"rationale": "Adopt the proxy's reasoning and tool-call fidelity in the coder"`
39	39		`},`
40	40		`{`

45	45		`"issue": 36,`
46	46		`"acceptedCommit": "2a631cf63107d1b706e7e8058f49c1bd283447ed"`
47	47		`},`
48		-	`"environmentProven": false,`
	48	+	`"environmentAvailable": false,`
49	49		`"rationale": "Teach the coder's tools token economy, per model family"`
50	50		`},`
51	51		`{`

56	56		`"issue": 40,`
57	57		`"acceptedCommit": "fb705a506a2d56263b07698aa02b3223861d0df6"`
58	58		`},`
59		-	`"environmentProven": false,`
	59	+	`"environmentAvailable": false,`
60	60		`"rationale": "Coder presents model tiers, never vendor model names"`
61	61		`},`
62	62		`{`

67	67		`"issue": 41,`
68	68		`"acceptedCommit": "a04c4fcf4708ff30b264970eb08acbebe5f20ed5"`
69	69		`},`
70		-	`"environmentProven": false,`
	70	+	`"environmentAvailable": false,`
71	71		`"rationale": "Read a conversation on request: the read-conversation plugin, discovered, loaded, and streamed in the coder"`
72	72		`}`
73	73		`]`

23	23		`"issue": 31,`
24	24		`"acceptedCommit": "cf1861c9cb..."`
25	25		`},`
26		-	`"environmentProven": false,`
	26	+	`"environmentAvailable": false,`
27	27		`"rationale": "Adopt the proxy's reasoning and tool-call fidelity in the coder"`
28	28		`}`
29	29		```

44	44
45	45		`No container exists that can grade any of these. The pin is real and the`
46	46		`suite is real; the environment is not. Every owned task therefore carries`
47		-	`environmentProven: false`, and `parseSuiteManifest` refuses to let an unproven
48		-	task into a `score`-tier suite — so `owned-closed-issues.suite.json` is
49		-	`smoke`, and `coder-effectiveness-v1` holds the twenty proven public tasks and
50		-	`none of these.`
	47	+	`environmentAvailable: false`, and `parseSuiteManifest` refuses to let a task
	48	+	with no gradeable environment into a `score`-tier suite — so
	49	+	`owned-closed-issues.suite.json` is `smoke`, and `coder-effectiveness-v1` holds
	50	+	`the twenty registry-backed public tasks and none of these.`
	51	+
	52	+	The registry tasks carry `environmentAvailable: true` because the dataset ships
	53	+	`their environment and verifier, which is what makes a public subset cost a`
	54	+	`manifest entry rather than a harness. The flag says a gradeable definition`
	55	+	`exists; it does not say the task has been run here, or that it passed, or that`
	56	+	`its image builds on any particular machine.`
51	57
52	58		`That refusal is the point rather than a limitation. A score suite that included`
53	59		`a task nobody could run would report those trials as missing, and a missing`

68	74		`recorded test pass; it does not mean the agent reproduced the recorded diff,`
69	75		`and it should not.`
70	76
71		-	Two things have to be true before the first of these is `environmentProven`,
72		-	`and neither is cheap:`
	77	+	`Two things have to be true before the first of these can flip`
	78	+	`environmentAvailable`, and neither is cheap:
73	79
74	80		- The image. `packages/openagents-cli` sits in a 117-package pnpm
75	81		`workspace, so the environment is a repo snapshot plus an install, not a`

79	79		`and a suite that pinned only names would report two different measurements`
80	80		`under one heading.`
81	81
82		-	`environmentProven` is whether a container for this task has ever been built
83		-	and graded. A `score` suite may not hold an unproven task — a trial nobody
84		-	`could run reads as the coder failing rather than as a missing environment — so`
85		-	an owned task lives in a `smoke` suite until somebody proves it.
	82	+	`environmentAvailable` is whether a container and verifier that can grade this
	83	+	`task exist. A registry task has them by construction; an owned task drawn from`
	84	+	a closed issue has neither until somebody writes them. A `score` suite may not
	85	+	`hold a task with no gradeable environment — a trial nobody can run reads as the`
	86	+	`coder failing rather than as a missing environment — so the owned tasks live in`
	87	+	a `smoke` suite until their environments are written.
86	88
87	89		`## A smoke run is never a published score, structurally`
88	90

304	306
305	307		`- The owned lane's environments. Six closed issues are pinned to their`
306	308		`closing commits and none of them has a container that can grade it, so they`
307		-	are `environmentProven: false` and their suite is `smoke`.
	309	+	are `environmentAvailable: false` and their suite is `smoke`.
308	310		`bench/tasks/owned/README.md` has the construction and the two things that
309	311		`have to be true before the first one is admitted to a score.`
310	312		`- Per-model cost from the coder's own trajectory. The ATIF exporter writes`

13	13		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
14	14		`"path": "build-cmake"`
15	15		`},`
16		-	`"environmentProven": true`
	16	+	`"environmentAvailable": true`
17	17		`},`
18	18		`{`
19	19		`"id": "fix-git",`

24	24		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
25	25		`"path": "fix-git"`
26	26		`},`
27		-	`"environmentProven": true`
	27	+	`"environmentAvailable": true`
28	28		`},`
29	29		`{`
30	30		`"id": "parse-log",`

35	35		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
36	36		`"path": "parse-log"`
37	37		`},`
38		-	`"environmentProven": true`
	38	+	`"environmentAvailable": true`
39	39		`}`
40	40		`]`
41	41		`}`

46	46		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
47	47		`"path": "port-forward"`
48	48		`},`
49		-	`"environmentProven": true`
	49	+	`"environmentAvailable": true`
50	50		`},`
51	51		`{`
52	52		`"id": "never-run",`

57	57		`"commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c",`
58	58		`"path": "never-run"`
59	59		`},`
60		-	`"environmentProven": true`
	60	+	`"environmentAvailable": true`
61	61		`}`
62	62		`]`
63	63		`}`