YeonGyu-Kim
790b80f7d9
Add a new tool and agent for analyzing media files (PDFs, images, diagrams) that require visual interpretation beyond raw text. - Add `multimodal-looker` agent using Gemini 2.5 Flash model - Add `look_at` tool that spawns multimodal-looker sessions - Restrict multimodal-looker from calling task/call_omo_agent/look_at tools Inspired by Sourcegraph Ampcode's look_at tool design. 🤖 GENERATED WITH ASSISTANCE OF [OhMyOpenCode](https://github.com/code-yeongyu/oh-my-opencode)
92 lines
3.3 KiB
TypeScript
92 lines
3.3 KiB
TypeScript
import { tool, type PluginInput } from "@opencode-ai/plugin"
|
|
import { LOOK_AT_DESCRIPTION, MULTIMODAL_LOOKER_AGENT } from "./constants"
|
|
import type { LookAtArgs } from "./types"
|
|
import { log } from "../../shared/logger"
|
|
|
|
export function createLookAt(ctx: PluginInput) {
|
|
return tool({
|
|
description: LOOK_AT_DESCRIPTION,
|
|
args: {
|
|
file_path: tool.schema.string().describe("Absolute path to the file to analyze"),
|
|
goal: tool.schema.string().describe("What specific information to extract from the file"),
|
|
},
|
|
async execute(args: LookAtArgs, toolContext) {
|
|
log(`[look_at] Analyzing file: ${args.file_path}, goal: ${args.goal}`)
|
|
|
|
const prompt = `Analyze this file and extract the requested information.
|
|
|
|
File path: ${args.file_path}
|
|
Goal: ${args.goal}
|
|
|
|
Read the file using the Read tool, then provide ONLY the extracted information that matches the goal.
|
|
Be thorough on what was requested, concise on everything else.
|
|
If the requested information is not found, clearly state what is missing.`
|
|
|
|
log(`[look_at] Creating session with parent: ${toolContext.sessionID}`)
|
|
const createResult = await ctx.client.session.create({
|
|
body: {
|
|
parentID: toolContext.sessionID,
|
|
title: `look_at: ${args.goal.substring(0, 50)}`,
|
|
},
|
|
})
|
|
|
|
if (createResult.error) {
|
|
log(`[look_at] Session create error:`, createResult.error)
|
|
return `Error: Failed to create session: ${createResult.error}`
|
|
}
|
|
|
|
const sessionID = createResult.data.id
|
|
log(`[look_at] Created session: ${sessionID}`)
|
|
|
|
log(`[look_at] Sending prompt to session ${sessionID}`)
|
|
await ctx.client.session.prompt({
|
|
path: { id: sessionID },
|
|
body: {
|
|
agent: MULTIMODAL_LOOKER_AGENT,
|
|
tools: {
|
|
task: false,
|
|
call_omo_agent: false,
|
|
look_at: false,
|
|
},
|
|
parts: [{ type: "text", text: prompt }],
|
|
},
|
|
})
|
|
|
|
log(`[look_at] Prompt sent, fetching messages...`)
|
|
|
|
const messagesResult = await ctx.client.session.messages({
|
|
path: { id: sessionID },
|
|
})
|
|
|
|
if (messagesResult.error) {
|
|
log(`[look_at] Messages error:`, messagesResult.error)
|
|
return `Error: Failed to get messages: ${messagesResult.error}`
|
|
}
|
|
|
|
const messages = messagesResult.data
|
|
log(`[look_at] Got ${messages.length} messages`)
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
const lastAssistantMessage = messages
|
|
.filter((m: any) => m.info.role === "assistant")
|
|
.sort((a: any, b: any) => (b.info.time?.created || 0) - (a.info.time?.created || 0))[0]
|
|
|
|
if (!lastAssistantMessage) {
|
|
log(`[look_at] No assistant message found`)
|
|
return `Error: No response from multimodal-looker agent`
|
|
}
|
|
|
|
log(`[look_at] Found assistant message with ${lastAssistantMessage.parts.length} parts`)
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
const textParts = lastAssistantMessage.parts.filter((p: any) => p.type === "text")
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
const responseText = textParts.map((p: any) => p.text).join("\n")
|
|
|
|
log(`[look_at] Got response, length: ${responseText.length}`)
|
|
|
|
return responseText
|
|
},
|
|
})
|
|
}
|