fix(benchmarks): address review feedback on error handling and validation

- headless.ts: emit error field on tool_result when output starts with Error:
- test-multi-model.ts: errored/timed-out models now shown as RED and exit(1)
- test-multi-model.ts: validate --timeout arg (reject NaN/negative)
- test-edge-cases.ts: use exact match instead of trim() for whitespace test
- test-edge-cases.ts: skip file pre-creation for create-via-append test

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
minpeter
2026-02-27 01:44:51 +09:00
parent 3cff4c1bce
commit a2b2cb7e87
3 changed files with 28 additions and 10 deletions
+6 -2
View File
@@ -53,6 +53,7 @@ interface TestCase {
fileName: string;
name: string;
prompt: string;
skipFileCreate?: boolean;
validate: (content: string) => { passed: boolean; reason: string };
}
@@ -288,6 +289,7 @@ const TEST_CASES: TestCase[] = [
name: "7. Create new file via append",
fileName: "create-via-append.txt",
fileContent: "",
skipFileCreate: true,
prompt: [
"Create create-via-append.txt via edit_file append (do not call read_file first).",
"Use one call with edits: [{ op: 'append', lines: ['created line 1', 'created line 2'] }].",
@@ -597,7 +599,7 @@ const TEST_CASES: TestCase[] = [
reason: "non-target lines changed unexpectedly",
};
}
if (lines[1].trim() !== "middle-content") {
if (lines[1] !== "middle-content") {
return {
passed: false,
reason: `line 2 expected 'middle-content' but got ${JSON.stringify(lines[1])}`,
@@ -907,7 +909,9 @@ async function runTestCase(
duration: number;
}> {
const testFile = join(testDir, tc.fileName);
writeFileSync(testFile, tc.fileContent, "utf-8");
if (!tc.skipFileCreate) {
writeFileSync(testFile, tc.fileContent, "utf-8");
}
const headlessScript = resolve(import.meta.dir, "headless.ts");
const headlessArgs = [