YeonGyu-Kim
11c3da752c
default-mode (system-transform): -e5463e2dbintroduced auto-activation of ultrawork+ralph-loop, anddc2e082acthen skipped the ultrawork system prompt whenever ralph_loop was also enabled. Net effect: the keyword-detector still showed 'Default ultrawork mode enabled' to the user, but the first turn had none of the ultrawork behavior. Loop continuation kept the ultrawork prefix, so the contract was honored only on later iterations. - Drop the skip so the initial turn matches what the toast advertises. New matrix test pins all four (ultrawork, ralph_loop) combinations. multimodal-looker: - Prompt claimed 'read' and 'call_omo_agent' were available, but the look_at invocation runtime explicitly disables both via READ_ENABLED and createAgentToolAllowlist([]). Small VL models trusted the prompt and looped on rejected tool calls (#4116). - Rewrite the agent prompt to describe direct-attachment analysis and forbid tool/agent calls. Add a consistency test that extracts the prompt's 'available tools' claim and compares it against the configured allowlist. delegate-task (skill-resolver): -088693697filtered per-agent restricted skills at the skill tool and builtin agent prompt layers, but delegate-task itself happily injected whatever skill name a caller passed. A target agent could be force-fed a skill marked agent: oracle just by listing it in load_skills. - Thread the target agent through resolveSkills and silently filter skills whose definition.agent does not include it. Public skills with no restriction are unaffected. Regression test pins the bypass.
78 lines
2.5 KiB
TypeScript
78 lines
2.5 KiB
TypeScript
import { describe, test, expect } from "bun:test"
|
|
import { createAgentToolAllowlist } from "../shared/permission-compat"
|
|
import { READ_ENABLED } from "../tools/look-at/look-at-prompt"
|
|
import { createMultimodalLookerAgent } from "./multimodal-looker"
|
|
|
|
function extractAvailableToolClaims(prompt: string): readonly string[] {
|
|
const availableToolsLine = prompt
|
|
.split("\n")
|
|
.find((line) => line.toLowerCase().includes("available tools"))
|
|
if (availableToolsLine === undefined) {
|
|
return []
|
|
}
|
|
|
|
const tools: string[] = []
|
|
for (const match of availableToolsLine.matchAll(/['`]([^'`]+)['`]/g)) {
|
|
const toolName = match[1]
|
|
if (toolName !== undefined) {
|
|
tools.push(toolName)
|
|
}
|
|
}
|
|
|
|
return [...new Set(tools)].sort()
|
|
}
|
|
|
|
function allowedToolNames(
|
|
toolAllowlist: ReturnType<typeof createAgentToolAllowlist>
|
|
): readonly string[] {
|
|
return Object.entries(toolAllowlist.permission)
|
|
.filter(([toolName, permission]) => toolName !== "*" && permission === "allow")
|
|
.map(([toolName]) => toolName)
|
|
.sort()
|
|
}
|
|
|
|
function createLookAtRuntimeToolAllowlist(): ReturnType<typeof createAgentToolAllowlist> {
|
|
return createAgentToolAllowlist(READ_ENABLED ? ["read"] : [])
|
|
}
|
|
|
|
describe("createMultimodalLookerAgent", () => {
|
|
test("prompt available tool claims match the look_at runtime allowlist", () => {
|
|
// given
|
|
const agent = createMultimodalLookerAgent("openai/gpt-5-nano")
|
|
const runtimeToolAllowlist = createLookAtRuntimeToolAllowlist()
|
|
|
|
// when
|
|
const prompt = typeof agent.prompt === "string" ? agent.prompt : ""
|
|
const promptToolClaims = extractAvailableToolClaims(prompt)
|
|
const runtimeToolNames = allowedToolNames(runtimeToolAllowlist)
|
|
|
|
// then
|
|
expect(promptToolClaims).toEqual(runtimeToolNames)
|
|
})
|
|
|
|
test("prompt denies tool use to prevent death loop on small VL models", () => {
|
|
// given
|
|
const agent = createMultimodalLookerAgent("openai/gpt-5-nano")
|
|
|
|
// when
|
|
const prompt = typeof agent.prompt === "string" ? agent.prompt : ""
|
|
const normalizedPrompt = prompt.toLowerCase()
|
|
|
|
// then
|
|
expect(normalizedPrompt).toContain("never")
|
|
expect(normalizedPrompt).toContain("tools")
|
|
expect(extractAvailableToolClaims(prompt)).toEqual([])
|
|
})
|
|
|
|
test("prompt instructs the agent never to call other tools", () => {
|
|
// given
|
|
const agent = createMultimodalLookerAgent("openai/gpt-5-nano")
|
|
|
|
// when
|
|
const prompt = typeof agent.prompt === "string" ? agent.prompt : ""
|
|
|
|
// then
|
|
expect(prompt.toLowerCase()).toContain("never")
|
|
})
|
|
})
|