1140080927
GPT models (5.3-codex, 5.4, etc.) frequently hang when using apply_patch due to verification loops. This adds: 1. Tool restriction: apply_patch is denied for GPT variants of Hephaestus, Sisyphus-Junior, and Sisyphus agents 2. Prompt guidance: GPT-specific prompts now explicitly instruct using edit/write tools instead of apply_patch 3. Removed the 'Always use apply_patch' instruction from sisyphus-junior/gpt-5-4.ts that contradicted the fix The deny is model-conditional — Claude variants retain apply_patch access since it works reliably there.
135 lines
3.8 KiB
TypeScript
135 lines
3.8 KiB
TypeScript
import { describe, test, expect } from "bun:test"
|
|
import { createOracleAgent } from "./oracle"
|
|
import { createLibrarianAgent } from "./librarian"
|
|
import { createExploreAgent } from "./explore"
|
|
import { createMomusAgent } from "./momus"
|
|
import { createMetisAgent } from "./metis"
|
|
import { createAtlasAgent } from "./atlas"
|
|
import { createSisyphusAgent } from "./sisyphus"
|
|
|
|
const TEST_MODEL = "anthropic/claude-sonnet-4-5"
|
|
|
|
describe("read-only agent tool restrictions", () => {
|
|
const FILE_WRITE_TOOLS = ["write", "edit", "apply_patch"]
|
|
|
|
describe("Oracle", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createOracleAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
|
|
test("denies task but allows call_omo_agent for research", () => {
|
|
// given
|
|
const agent = createOracleAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
expect(permission["task"]).toBe("deny")
|
|
expect(permission["call_omo_agent"]).toBeUndefined()
|
|
})
|
|
})
|
|
|
|
describe("Librarian", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createLibrarianAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Explore", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createExploreAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Momus", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createMomusAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Metis", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createMetisAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Atlas", () => {
|
|
test("allows delegation tools for orchestration", () => {
|
|
// given
|
|
const agent = createAtlasAgent({ model: TEST_MODEL })
|
|
|
|
// when
|
|
const permission = (agent.permission ?? {}) as Record<string, string>
|
|
|
|
// then
|
|
expect(permission["task"]).toBeUndefined()
|
|
expect(permission["call_omo_agent"]).toBeUndefined()
|
|
})
|
|
})
|
|
|
|
describe("Sisyphus GPT variants", () => {
|
|
test("deny apply_patch for GPT models but not Claude models", () => {
|
|
// given
|
|
const gpt54Agent = createSisyphusAgent("openai/gpt-5.4")
|
|
const gptGenericAgent = createSisyphusAgent("openai/gpt-5.2")
|
|
const claudeAgent = createSisyphusAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const gpt54Permission = (gpt54Agent.permission ?? {}) as Record<string, string>
|
|
const gptGenericPermission = (gptGenericAgent.permission ?? {}) as Record<string, string>
|
|
const claudePermission = (claudeAgent.permission ?? {}) as Record<string, string>
|
|
|
|
// then
|
|
expect(gpt54Permission["apply_patch"]).toBe("deny")
|
|
expect(gptGenericPermission["apply_patch"]).toBe("deny")
|
|
expect(claudePermission["apply_patch"]).toBeUndefined()
|
|
})
|
|
})
|
|
})
|