fix(look-at): make multimodal-looker prompt conditional on Read availability (#3033)
When Read tool is disabled (attachment mode), the agent prompt now instructs to analyze the attached content directly instead of trying to use Read. 3 new tests, 55 total pass. Closes #3033
This commit is contained in:
@@ -659,4 +659,112 @@ describe("look-at tool", () => {
|
||||
expect(filePart.url).toContain("base64")
|
||||
})
|
||||
})
|
||||
|
||||
describe("createLookAt prompt conditional on Read availability", () => {
|
||||
const captureLastPromptBody = () => {
|
||||
const captured: { body: any } = { body: undefined }
|
||||
const mockClient = {
|
||||
app: {
|
||||
agents: async () => ({ data: [] }),
|
||||
},
|
||||
session: {
|
||||
get: async () => ({ data: { directory: "/project" } }),
|
||||
create: async () => ({ data: { id: "ses_prompt_conditional" } }),
|
||||
prompt: async (input: any) => {
|
||||
captured.body = input.body
|
||||
return { data: {} }
|
||||
},
|
||||
messages: async () => ({
|
||||
data: [
|
||||
{ info: { role: "assistant", time: { created: 1 } }, parts: [{ type: "text", text: "ok" }] },
|
||||
],
|
||||
}),
|
||||
},
|
||||
}
|
||||
return { mockClient, captured }
|
||||
}
|
||||
|
||||
const buildToolContext = (): ToolContext => ({
|
||||
sessionID: "parent-session",
|
||||
messageID: "parent-message",
|
||||
agent: "sisyphus",
|
||||
directory: "/project",
|
||||
worktree: "/project",
|
||||
abort: new AbortController().signal,
|
||||
metadata: () => {},
|
||||
ask: async () => {},
|
||||
})
|
||||
|
||||
// given file_path mode where Read tool is disabled in invocation
|
||||
// when LookAt tool sends prompt to multimodal-looker
|
||||
// then prompt instructs agent to analyze the attached file directly without using Read
|
||||
test("instructs agent to analyze attached file when Read is disabled (file_path mode)", async () => {
|
||||
const { mockClient, captured } = captureLastPromptBody()
|
||||
|
||||
const tool = createLookAt({
|
||||
client: mockClient,
|
||||
directory: "/project",
|
||||
} as any)
|
||||
|
||||
await tool.execute(
|
||||
{ file_path: "/test/file.png", goal: "describe contents" },
|
||||
buildToolContext(),
|
||||
)
|
||||
|
||||
expect(captured.body.tools.read).toBe(false)
|
||||
const promptPart = captured.body.parts.find((p: any) => p.type === "text")
|
||||
expect(promptPart).toBeDefined()
|
||||
const promptText: string = promptPart.text
|
||||
expect(promptText).toContain("attached")
|
||||
expect(promptText).not.toMatch(/\bRead\s+(?:the\s+)?file\b/i)
|
||||
expect(promptText).not.toMatch(/\buse\s+Read\b/i)
|
||||
})
|
||||
|
||||
// given image_data mode where no file path exists and Read is disabled
|
||||
// when LookAt tool sends prompt to multimodal-looker
|
||||
// then prompt instructs agent to analyze the attached image directly without referencing Read or file path
|
||||
test("instructs agent to analyze attached image when image_data is provided", async () => {
|
||||
const { mockClient, captured } = captureLastPromptBody()
|
||||
|
||||
const tool = createLookAt({
|
||||
client: mockClient,
|
||||
directory: "/project",
|
||||
} as any)
|
||||
|
||||
await tool.execute(
|
||||
{ image_data: "data:image/png;base64,iVBORw0KGgo=", goal: "describe image" },
|
||||
buildToolContext(),
|
||||
)
|
||||
|
||||
expect(captured.body.tools.read).toBe(false)
|
||||
const promptPart = captured.body.parts.find((p: any) => p.type === "text")
|
||||
expect(promptPart).toBeDefined()
|
||||
const promptText: string = promptPart.text
|
||||
expect(promptText).toContain("attached")
|
||||
expect(promptText).not.toMatch(/\bRead\s+(?:the\s+)?file\b/i)
|
||||
expect(promptText).not.toMatch(/\buse\s+Read\b/i)
|
||||
})
|
||||
|
||||
// given prompt is generated for any invocation where Read is denied
|
||||
// when LookAt tool sends prompt to multimodal-looker
|
||||
// then prompt explicitly tells the agent NOT to attempt Read tool
|
||||
test("explicitly warns the agent not to attempt Read when Read is disabled", async () => {
|
||||
const { mockClient, captured } = captureLastPromptBody()
|
||||
|
||||
const tool = createLookAt({
|
||||
client: mockClient,
|
||||
directory: "/project",
|
||||
} as any)
|
||||
|
||||
await tool.execute(
|
||||
{ file_path: "/test/file.pdf", goal: "extract text" },
|
||||
buildToolContext(),
|
||||
)
|
||||
|
||||
const promptPart = captured.body.parts.find((p: any) => p.type === "text")
|
||||
const promptText: string = promptPart.text
|
||||
// The prompt must mention the agent cannot use Read so the agent does not hallucinate
|
||||
expect(promptText.toLowerCase()).toContain("read tool")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -129,7 +129,15 @@ export function createLookAt(ctx: PluginInput): ToolDefinition {
|
||||
return "Error: Must provide either 'file_path' or 'image_data'."
|
||||
}
|
||||
|
||||
const prompt = `Analyze this ${isBase64Input ? "image" : "file"} and extract the requested information.
|
||||
const readEnabled = false
|
||||
const subjectNoun = isBase64Input ? "image" : "file"
|
||||
const sourceClause = readEnabled
|
||||
? `Use the Read tool on the provided file path to load its contents, then analyze it.`
|
||||
: `The ${subjectNoun} is already attached to this message. Analyze it directly from the attachment. Do NOT attempt to use the Read tool. The Read tool is disabled for this invocation and the ${subjectNoun} cannot be loaded by path.`
|
||||
|
||||
const prompt = `Analyze the attached ${subjectNoun} and extract the requested information.
|
||||
|
||||
${sourceClause}
|
||||
|
||||
Goal: ${args.goal}
|
||||
|
||||
@@ -182,7 +190,7 @@ Original error: ${createResult.error}`
|
||||
task: false,
|
||||
call_omo_agent: false,
|
||||
look_at: false,
|
||||
read: false,
|
||||
read: readEnabled,
|
||||
},
|
||||
parts: [
|
||||
{ type: "text", text: prompt },
|
||||
|
||||
Reference in New Issue
Block a user