Merge pull request #3174 from code-yeongyu/fix/issue-3033

fix(look-at): make multimodal-looker prompt conditional on Read availability (#3033)

src/tools/look-at/tools.test.ts

@@ -659,4 +659,112 @@ describe("look-at tool", () => {
       expect(filePart.url).toContain("base64")
     })
   })
+
+  describe("createLookAt prompt conditional on Read availability", () => {
+    const captureLastPromptBody = () => {
+      const captured: { body: any } = { body: undefined }
+      const mockClient = {
+        app: {
+          agents: async () => ({ data: [] }),
+        },
+        session: {
+          get: async () => ({ data: { directory: "/project" } }),
+          create: async () => ({ data: { id: "ses_prompt_conditional" } }),
+          prompt: async (input: any) => {
+            captured.body = input.body
+            return { data: {} }
+          },
+          messages: async () => ({
+            data: [
+              { info: { role: "assistant", time: { created: 1 } }, parts: [{ type: "text", text: "ok" }] },
+            ],
+          }),
+        },
+      }
+      return { mockClient, captured }
+    }
+
+    const buildToolContext = (): ToolContext => ({
+      sessionID: "parent-session",
+      messageID: "parent-message",
+      agent: "sisyphus",
+      directory: "/project",
+      worktree: "/project",
+      abort: new AbortController().signal,
+      metadata: () => {},
+      ask: async () => {},
+    })
+
+    // given file_path mode where Read tool is disabled in invocation
+    // when LookAt tool sends prompt to multimodal-looker
+    // then prompt instructs agent to analyze the attached file directly without using Read
+    test("instructs agent to analyze attached file when Read is disabled (file_path mode)", async () => {
+      const { mockClient, captured } = captureLastPromptBody()
+
+      const tool = createLookAt({
+        client: mockClient,
+        directory: "/project",
+      } as any)
+
+      await tool.execute(
+        { file_path: "/test/file.png", goal: "describe contents" },
+        buildToolContext(),
+      )
+
+      expect(captured.body.tools.read).toBe(false)
+      const promptPart = captured.body.parts.find((p: any) => p.type === "text")
+      expect(promptPart).toBeDefined()
+      const promptText: string = promptPart.text
+      expect(promptText).toContain("attached")
+      expect(promptText).not.toMatch(/\bRead\s+(?:the\s+)?file\b/i)
+      expect(promptText).not.toMatch(/\buse\s+Read\b/i)
+    })
+
+    // given image_data mode where no file path exists and Read is disabled
+    // when LookAt tool sends prompt to multimodal-looker
+    // then prompt instructs agent to analyze the attached image directly without referencing Read or file path
+    test("instructs agent to analyze attached image when image_data is provided", async () => {
+      const { mockClient, captured } = captureLastPromptBody()
+
+      const tool = createLookAt({
+        client: mockClient,
+        directory: "/project",
+      } as any)
+
+      await tool.execute(
+        { image_data: "data:image/png;base64,iVBORw0KGgo=", goal: "describe image" },
+        buildToolContext(),
+      )
+
+      expect(captured.body.tools.read).toBe(false)
+      const promptPart = captured.body.parts.find((p: any) => p.type === "text")
+      expect(promptPart).toBeDefined()
+      const promptText: string = promptPart.text
+      expect(promptText).toContain("attached")
+      expect(promptText).not.toMatch(/\bRead\s+(?:the\s+)?file\b/i)
+      expect(promptText).not.toMatch(/\buse\s+Read\b/i)
+    })
+
+    // given prompt is generated for any invocation where Read is denied
+    // when LookAt tool sends prompt to multimodal-looker
+    // then prompt explicitly tells the agent NOT to attempt Read tool
+    test("explicitly warns the agent not to attempt Read when Read is disabled", async () => {
+      const { mockClient, captured } = captureLastPromptBody()
+
+      const tool = createLookAt({
+        client: mockClient,
+        directory: "/project",
+      } as any)
+
+      await tool.execute(
+        { file_path: "/test/file.pdf", goal: "extract text" },
+        buildToolContext(),
+      )
+
+      const promptPart = captured.body.parts.find((p: any) => p.type === "text")
+      const promptText: string = promptPart.text
+      // The prompt must mention the agent cannot use Read so the agent does not hallucinate
+      expect(promptText.toLowerCase()).toContain("read tool")
+    })
+  })
 })

src/tools/look-at/tools.ts

@@ -129,7 +129,15 @@ export function createLookAt(ctx: PluginInput): ToolDefinition {
         return "Error: Must provide either 'file_path' or 'image_data'."
       }
 
-      const prompt = `Analyze this ${isBase64Input ? "image" : "file"} and extract the requested information.
+      const readEnabled = false
+      const subjectNoun = isBase64Input ? "image" : "file"
+      const sourceClause = readEnabled
+        ? `Use the Read tool on the provided file path to load its contents, then analyze it.`
+        : `The ${subjectNoun} is already attached to this message. Analyze it directly from the attachment. Do NOT attempt to use the Read tool. The Read tool is disabled for this invocation and the ${subjectNoun} cannot be loaded by path.`
+
+      const prompt = `Analyze the attached ${subjectNoun} and extract the requested information.
+
+${sourceClause}
 
 Goal: ${args.goal}
 
@@ -182,7 +190,7 @@ Original error: ${createResult.error}`
               task: false,
               call_omo_agent: false,
               look_at: false,
-              read: false,
+              read: readEnabled,
             },
             parts: [
               { type: "text", text: prompt },