tenant.openagents/omega

1#!/usr/bin/env node --redirect-warnings=/dev/null
2
3const fs = require("fs");
4const path = require("path");
5const { spawnSync } = require("child_process");
6
7const FAILING_SEED_REGEX = /failing seed: (\d+)/gi;
8const CARGO_TEST_ARGS = ["--release", "--lib", "--package", "collab"];
9
10if (require.main === module) {
11  if (process.argv.length < 4) {
12    process.stderr.write(
13      "usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n",
14    );
15    process.exit(1);
16  }
17
18  minimizeTestPlan(
19    process.argv[2],
20    process.argv[3],
21    parseInt(process.argv[4]) || 0,
22  );
23}
24
25function minimizeTestPlan(inputPlanPath, outputPlanPath, startIndex = 0) {
26  const tempPlanPath = inputPlanPath + ".try";
27
28  fs.copyFileSync(inputPlanPath, outputPlanPath);
29  let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8"));
30
31  process.stderr.write("minimizing failing test plan...\n");
32  for (let ix = startIndex; ix < testPlan.length; ix++) {
33    // Skip 'MutateClients' entries, since they themselves are not single operations.
34    if (testPlan[ix].MutateClients) {
35      continue;
36    }
37
38    // Remove a row from the test plan
39    const newTestPlan = testPlan.slice();
40    newTestPlan.splice(ix, 1);
41    fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), "utf8");
42
43    process.stderr.write(
44      `${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`,
45    );
46    const failingSeed = runTests({
47      SEED: "0",
48      LOAD_PLAN: tempPlanPath,
49      SAVE_PLAN: tempPlanPath,
50      ITERATIONS: "500",
51    });
52
53    // If the test failed, keep the test plan with the removed row. Reload the test
54    // plan from the JSON file, since the test itself will remove any operations
55    // which are no longer valid before saving the test plan.
56    if (failingSeed != null) {
57      process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`);
58      fs.copyFileSync(tempPlanPath, outputPlanPath);
59      testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8"));
60      ix--;
61    } else {
62      process.stderr.write(` - keep.\n`);
63    }
64  }
65
66  fs.unlinkSync(tempPlanPath);
67
68  // Re-run the final minimized plan to get the correct failing seed.
69  // This is a workaround for the fact that the execution order can
70  // slightly change when replaying a test plan after it has been
71  // saved and loaded.
72  const failingSeed = runTests({
73    SEED: "0",
74    ITERATIONS: "5000",
75    LOAD_PLAN: outputPlanPath,
76  });
77
78  process.stderr.write(`final test plan: ${outputPlanPath}\n`);
79  process.stderr.write(`final seed: ${failingSeed}\n`);
80  return failingSeed;
81}
82
83function buildTests() {
84  const { status } = spawnSync(
85    "cargo",
86    ["test", "--no-run", ...CARGO_TEST_ARGS],
87    {
88      stdio: "inherit",
89      encoding: "utf8",
90      env: {
91        ...process.env,
92      },
93    },
94  );
95  if (status !== 0) {
96    throw new Error("build failed");
97  }
98}
99
100function runTests(env) {
101  const { status, stdout } = spawnSync(
102    "cargo",
103    ["test", ...CARGO_TEST_ARGS, "random_project_collaboration"],
104    {
105      stdio: "pipe",
106      encoding: "utf8",
107      env: {
108        ...process.env,
109        ...env,
110      },
111    },
112  );
113
114  if (status !== 0) {
115    FAILING_SEED_REGEX.lastIndex = 0;
116    const match = FAILING_SEED_REGEX.exec(stdout);
117    if (!match) {
118      process.stderr.write("test failed, but no failing seed found:\n");
119      process.stderr.write(stdout);
120      process.stderr.write("\n");
121      process.exit(1);
122    }
123    return match[1];
124  } else {
125    return null;
126  }
127}
128
129function serializeTestPlan(plan) {
130  return "[\n" + plan.map((row) => JSON.stringify(row)).join(",\n") + "\n]\n";
131}
132
133exports.buildTests = buildTests;
134exports.runTests = runTests;
135exports.minimizeTestPlan = minimizeTestPlan;
136