80791f10bc
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
183 lines
5.3 KiB
TypeScript
183 lines
5.3 KiB
TypeScript
/// <reference types="bun-types" />
|
|
|
|
import { describe, test, expect } from "bun:test"
|
|
import { createOracleAgent } from "./oracle"
|
|
import { createLibrarianAgent } from "./librarian"
|
|
import { createExploreAgent } from "./explore"
|
|
import { createMomusAgent } from "./momus"
|
|
import { createMetisAgent } from "./metis"
|
|
import { createAtlasAgent } from "./atlas"
|
|
import { createSisyphusAgent } from "./sisyphus"
|
|
import { createHephaestusAgent } from "./hephaestus"
|
|
|
|
const TEST_MODEL = "anthropic/claude-sonnet-4-5"
|
|
|
|
describe("read-only agent tool restrictions", () => {
|
|
const FILE_WRITE_TOOLS = ["write", "edit", "apply_patch"]
|
|
|
|
describe("Oracle", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createOracleAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
|
|
test("denies task but allows call_omo_agent for research", () => {
|
|
// given
|
|
const agent = createOracleAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
expect(permission["task"]).toBe("deny")
|
|
expect(permission["call_omo_agent"]).toBeUndefined()
|
|
})
|
|
})
|
|
|
|
describe("Librarian", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createLibrarianAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Explore", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createExploreAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Momus", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createMomusAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Metis", () => {
|
|
test("denies all file-writing tools", () => {
|
|
// given
|
|
const agent = createMetisAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const permission = agent.permission as Record<string, string>
|
|
|
|
// then
|
|
for (const tool of FILE_WRITE_TOOLS) {
|
|
expect(permission[tool]).toBe("deny")
|
|
}
|
|
})
|
|
})
|
|
|
|
describe("Atlas", () => {
|
|
test("allows delegation tools for orchestration", () => {
|
|
// given
|
|
const agent = createAtlasAgent({ model: TEST_MODEL })
|
|
|
|
// when
|
|
const permission = (agent.permission ?? {}) as Record<string, string>
|
|
|
|
// then
|
|
expect(permission["task"]).toBeUndefined()
|
|
expect(permission["call_omo_agent"]).toBeUndefined()
|
|
})
|
|
})
|
|
|
|
describe("Sisyphus GPT variants", () => {
|
|
test("deny apply_patch for GPT models but not Claude models", () => {
|
|
// given
|
|
const gpt54Agent = createSisyphusAgent("openai/gpt-5.4")
|
|
const gptGenericAgent = createSisyphusAgent("openai/gpt-5.2")
|
|
const claudeAgent = createSisyphusAgent(TEST_MODEL)
|
|
|
|
// when
|
|
const gpt54Permission = (gpt54Agent.permission ?? {}) as Record<string, string>
|
|
const gptGenericPermission = (gptGenericAgent.permission ?? {}) as Record<string, string>
|
|
const claudePermission = (claudeAgent.permission ?? {}) as Record<string, string>
|
|
|
|
// then
|
|
expect(gpt54Permission["apply_patch"]).toBe("deny")
|
|
expect(gptGenericPermission["apply_patch"]).toBe("deny")
|
|
expect(claudePermission["apply_patch"]).toBeUndefined()
|
|
})
|
|
})
|
|
|
|
describe("Sisyphus and Hephaestus frontier tool schema restrictions", () => {
|
|
test("deny grep and glob for Opus 4.7 and GPT 5.5 models", () => {
|
|
// given
|
|
const frontierAgents = [
|
|
createSisyphusAgent("anthropic/claude-opus-4-7"),
|
|
createSisyphusAgent("anthropic/claude-opus-4.7"),
|
|
createSisyphusAgent("openai/gpt-5.5"),
|
|
createHephaestusAgent("anthropic/claude-opus-4-7"),
|
|
createHephaestusAgent("anthropic/claude-opus-4.7"),
|
|
createHephaestusAgent("openai/gpt-5.5"),
|
|
]
|
|
|
|
// when
|
|
const permissions = frontierAgents.map(
|
|
(agent) => (agent.permission ?? {}) as Record<string, string>,
|
|
)
|
|
|
|
// then
|
|
for (const permission of permissions) {
|
|
expect(permission.grep).toBe("deny")
|
|
expect(permission.glob).toBe("deny")
|
|
}
|
|
})
|
|
|
|
test("keeps grep and glob available for other models", () => {
|
|
// given
|
|
const otherAgents = [
|
|
createSisyphusAgent("anthropic/claude-sonnet-4-5"),
|
|
createSisyphusAgent("openai/gpt-5.4"),
|
|
createHephaestusAgent("openai/gpt-5.4"),
|
|
]
|
|
|
|
// when
|
|
const permissions = otherAgents.map(
|
|
(agent) => (agent.permission ?? {}) as Record<string, string>,
|
|
)
|
|
|
|
// then
|
|
for (const permission of permissions) {
|
|
expect(permission.grep).toBeUndefined()
|
|
expect(permission.glob).toBeUndefined()
|
|
}
|
|
})
|
|
})
|
|
})
|