luna/oh-my-opencode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(benchmarks): address review feedback on error handling and validation

Browse Source 
- headless.ts: emit error field on tool_result when output starts with Error:
- test-multi-model.ts: errored/timed-out models now shown as RED and exit(1)
- test-multi-model.ts: validate --timeout arg (reject NaN/negative)
- test-edge-cases.ts: use exact match instead of trim() for whitespace test
- test-edge-cases.ts: skip file pre-creation for create-via-append test

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
minpeter
2026-02-27 01:44:51 +09:00
parent 3cff4c1bce
commit a2b2cb7e87
3 changed files with 28 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/test-multi-model.ts
+17 -6
View File
@@ -25,9 +25,13 @@ let perModelTimeoutSec = 900; // 15 min default per model (5 tests)
const rawArgs = process.argv.slice(2);
for (let i = 0; i < rawArgs.length; i++) {
if (rawArgs[i] === "--timeout" && i + 1 < rawArgs.length) {
perModelTimeoutSec = Number.parseInt(rawArgs[i + 1], 10);
const parsed = Number.parseInt(rawArgs[i + 1], 10);
if (Number.isNaN(parsed) || parsed <= 0) {
console.error(`Invalid --timeout value: ${rawArgs[i + 1]}`);
process.exit(1);
}
perModelTimeoutSec = parsed;
i++;
}
}
// ── Colors ────────────────────────────────────────────────────
@@ -228,8 +232,9 @@ const main = async () => {
// Per-model results
for (const r of allResults) {
const timeStr = `${(r.durationMs / 1000).toFixed(0)}s`;
const color = r.totalPassed === r.totalTests ? GREEN : r.totalPassed > 0 ? YELLOW : RED;
console.log(` ${r.modelShort.padEnd(8)} ${color}${r.totalPassed}/${r.totalTests}${RESET} (${timeStr})`);
const color = r.error ? RED : r.totalPassed === r.totalTests ? GREEN : r.totalPassed > 0 ? YELLOW : RED;
const label = r.error ? `ERROR: ${r.error}` : `${r.totalPassed}/${r.totalTests}`;
console.log(` ${r.modelShort.padEnd(8)} ${color}${label}${RESET} (${timeStr})`);
for (const t of r.tests) {
const icon = t.passed ? `${GREEN}✓${RESET}` : `${RED}✗${RESET}`;
console.log(` ${icon} ${t.name}`);
@@ -240,8 +245,9 @@ const main = async () => {
// Overall
const totalModels = allResults.length;
const erroredModels = allResults.filter((r) => r.error).length;
const perfectModels = allResults.filter(
(r) => r.totalPassed === r.totalTests
(r) => !r.error && r.totalPassed === r.totalTests && r.totalTests > 0
).length;
console.log(
`${BOLD}Models with 100%: ${perfectModels}/${totalModels}${RESET}`
@@ -255,7 +261,12 @@ const main = async () => {
console.log();
if (perfectModels === totalModels) {
if (erroredModels > 0) {
console.log(
`${BOLD}${RED}${erroredModels} model(s) errored. See details above.${RESET}\n`
);
process.exit(1);
} else if (perfectModels === totalModels) {
console.log(`${BOLD}${GREEN}🎉 ALL MODELS PASSED ALL TESTS!${RESET}\n`);
process.exit(0);
} else {