diff --git a/src/tools/look-at/tools.test.ts b/src/tools/look-at/tools.test.ts index 63713041b..9067032de 100644 --- a/src/tools/look-at/tools.test.ts +++ b/src/tools/look-at/tools.test.ts @@ -659,4 +659,112 @@ describe("look-at tool", () => { expect(filePart.url).toContain("base64") }) }) + + describe("createLookAt prompt conditional on Read availability", () => { + const captureLastPromptBody = () => { + const captured: { body: any } = { body: undefined } + const mockClient = { + app: { + agents: async () => ({ data: [] }), + }, + session: { + get: async () => ({ data: { directory: "/project" } }), + create: async () => ({ data: { id: "ses_prompt_conditional" } }), + prompt: async (input: any) => { + captured.body = input.body + return { data: {} } + }, + messages: async () => ({ + data: [ + { info: { role: "assistant", time: { created: 1 } }, parts: [{ type: "text", text: "ok" }] }, + ], + }), + }, + } + return { mockClient, captured } + } + + const buildToolContext = (): ToolContext => ({ + sessionID: "parent-session", + messageID: "parent-message", + agent: "sisyphus", + directory: "/project", + worktree: "/project", + abort: new AbortController().signal, + metadata: () => {}, + ask: async () => {}, + }) + + // given file_path mode where Read tool is disabled in invocation + // when LookAt tool sends prompt to multimodal-looker + // then prompt instructs agent to analyze the attached file directly without using Read + test("instructs agent to analyze attached file when Read is disabled (file_path mode)", async () => { + const { mockClient, captured } = captureLastPromptBody() + + const tool = createLookAt({ + client: mockClient, + directory: "/project", + } as any) + + await tool.execute( + { file_path: "/test/file.png", goal: "describe contents" }, + buildToolContext(), + ) + + expect(captured.body.tools.read).toBe(false) + const promptPart = captured.body.parts.find((p: any) => p.type === "text") + expect(promptPart).toBeDefined() + const promptText: string = promptPart.text + expect(promptText).toContain("attached") + expect(promptText).not.toMatch(/\bRead\s+(?:the\s+)?file\b/i) + expect(promptText).not.toMatch(/\buse\s+Read\b/i) + }) + + // given image_data mode where no file path exists and Read is disabled + // when LookAt tool sends prompt to multimodal-looker + // then prompt instructs agent to analyze the attached image directly without referencing Read or file path + test("instructs agent to analyze attached image when image_data is provided", async () => { + const { mockClient, captured } = captureLastPromptBody() + + const tool = createLookAt({ + client: mockClient, + directory: "/project", + } as any) + + await tool.execute( + { image_data: "data:image/png;base64,iVBORw0KGgo=", goal: "describe image" }, + buildToolContext(), + ) + + expect(captured.body.tools.read).toBe(false) + const promptPart = captured.body.parts.find((p: any) => p.type === "text") + expect(promptPart).toBeDefined() + const promptText: string = promptPart.text + expect(promptText).toContain("attached") + expect(promptText).not.toMatch(/\bRead\s+(?:the\s+)?file\b/i) + expect(promptText).not.toMatch(/\buse\s+Read\b/i) + }) + + // given prompt is generated for any invocation where Read is denied + // when LookAt tool sends prompt to multimodal-looker + // then prompt explicitly tells the agent NOT to attempt Read tool + test("explicitly warns the agent not to attempt Read when Read is disabled", async () => { + const { mockClient, captured } = captureLastPromptBody() + + const tool = createLookAt({ + client: mockClient, + directory: "/project", + } as any) + + await tool.execute( + { file_path: "/test/file.pdf", goal: "extract text" }, + buildToolContext(), + ) + + const promptPart = captured.body.parts.find((p: any) => p.type === "text") + const promptText: string = promptPart.text + // The prompt must mention the agent cannot use Read so the agent does not hallucinate + expect(promptText.toLowerCase()).toContain("read tool") + }) + }) }) diff --git a/src/tools/look-at/tools.ts b/src/tools/look-at/tools.ts index 773d334d0..1296afd29 100644 --- a/src/tools/look-at/tools.ts +++ b/src/tools/look-at/tools.ts @@ -129,7 +129,15 @@ export function createLookAt(ctx: PluginInput): ToolDefinition { return "Error: Must provide either 'file_path' or 'image_data'." } - const prompt = `Analyze this ${isBase64Input ? "image" : "file"} and extract the requested information. + const readEnabled = false + const subjectNoun = isBase64Input ? "image" : "file" + const sourceClause = readEnabled + ? `Use the Read tool on the provided file path to load its contents, then analyze it.` + : `The ${subjectNoun} is already attached to this message. Analyze it directly from the attachment. Do NOT attempt to use the Read tool. The Read tool is disabled for this invocation and the ${subjectNoun} cannot be loaded by path.` + + const prompt = `Analyze the attached ${subjectNoun} and extract the requested information. + +${sourceClause} Goal: ${args.goal} @@ -182,7 +190,7 @@ Original error: ${createResult.error}` task: false, call_omo_agent: false, look_at: false, - read: false, + read: readEnabled, }, parts: [ { type: "text", text: prompt },