fix(agents): declare multimodal-looker tool allowlist in prompt to prevent death loop on small VL models (fixes #4116)
The multimodal-looker prompt described what to do but never told the model which tools are available. Smaller VL models (e.g. Qwen3-VL-8B) would try to call non-existent tools and enter an infinite loop emitting:
Model tried to call unavailable tool 'invalid'. Available tools: call_omo_agent, read.
Add a single sentence at the top of the prompt that explicitly enumerates the only allowed tools ('read' and 'call_omo_agent') and forbids calling any other tool. This matches the runtime allowlist enforced by createAgentToolAllowlist(["read"]).
Regression test asserts the prompt contains the available-tools enumeration so future prompt rewrites don't regress.
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
import { describe, test, expect } from "bun:test"
|
||||
import { createMultimodalLookerAgent } from "./multimodal-looker"
|
||||
|
||||
describe("createMultimodalLookerAgent", () => {
|
||||
test("prompt explicitly enumerates the agent's available tools to prevent death loop on small VL models", () => {
|
||||
// given
|
||||
const agent = createMultimodalLookerAgent("openai/gpt-5-nano")
|
||||
|
||||
// when
|
||||
const prompt = typeof agent.prompt === "string" ? agent.prompt : ""
|
||||
|
||||
// then
|
||||
expect(prompt).toMatch(/available tools/i)
|
||||
expect(prompt).toContain("read")
|
||||
})
|
||||
|
||||
test("prompt instructs the agent never to call other tools", () => {
|
||||
// given
|
||||
const agent = createMultimodalLookerAgent("openai/gpt-5-nano")
|
||||
|
||||
// when
|
||||
const prompt = typeof agent.prompt === "string" ? agent.prompt : ""
|
||||
|
||||
// then
|
||||
expect(prompt.toLowerCase()).toContain("never")
|
||||
})
|
||||
})
|
||||
@@ -23,6 +23,8 @@ export function createMultimodalLookerAgent(model: string): AgentConfig {
|
||||
...restrictions,
|
||||
prompt: `You interpret media files that cannot be read as plain text.
|
||||
|
||||
Your only available tools are 'read' and 'call_omo_agent'. Always use 'read' to load the file first, then analyze the returned content. Never attempt to call any other tool.
|
||||
|
||||
Your job: examine the attached file and extract ONLY what was requested.
|
||||
|
||||
When to use you:
|
||||
|
||||
Reference in New Issue
Block a user