fix(agents): declare multimodal-looker tool allowlist in prompt to prevent death loop on small VL models (fixes #4116)

The multimodal-looker prompt described what to do but never told the model which tools are available. Smaller VL models (e.g. Qwen3-VL-8B) would try to call non-existent tools and enter an infinite loop emitting:

  Model tried to call unavailable tool 'invalid'. Available tools: call_omo_agent, read.

Add a single sentence at the top of the prompt that explicitly enumerates the only allowed tools ('read' and 'call_omo_agent') and forbids calling any other tool. This matches the runtime allowlist enforced by createAgentToolAllowlist(["read"]).

Regression test asserts the prompt contains the available-tools enumeration so future prompt rewrites don't regress.
This commit is contained in:
MoerAI
2026-05-19 10:13:52 +09:00
parent 6915f15299
commit 81ce512705
2 changed files with 29 additions and 0 deletions
+27
View File
@@ -0,0 +1,27 @@
import { describe, test, expect } from "bun:test"
import { createMultimodalLookerAgent } from "./multimodal-looker"
describe("createMultimodalLookerAgent", () => {
test("prompt explicitly enumerates the agent's available tools to prevent death loop on small VL models", () => {
// given
const agent = createMultimodalLookerAgent("openai/gpt-5-nano")
// when
const prompt = typeof agent.prompt === "string" ? agent.prompt : ""
// then
expect(prompt).toMatch(/available tools/i)
expect(prompt).toContain("read")
})
test("prompt instructs the agent never to call other tools", () => {
// given
const agent = createMultimodalLookerAgent("openai/gpt-5-nano")
// when
const prompt = typeof agent.prompt === "string" ? agent.prompt : ""
// then
expect(prompt.toLowerCase()).toContain("never")
})
})
+2
View File
@@ -23,6 +23,8 @@ export function createMultimodalLookerAgent(model: string): AgentConfig {
...restrictions,
prompt: `You interpret media files that cannot be read as plain text.
Your only available tools are 'read' and 'call_omo_agent'. Always use 'read' to load the file first, then analyze the returned content. Never attempt to call any other tool.
Your job: examine the attached file and extract ONLY what was requested.
When to use you: