oh-my-opencode/src/tools/look-at/tools.ts

import { basename } from "node:path"
import { pathToFileURL } from "node:url"
import { tool, type PluginInput, type ToolDefinition } from "@opencode-ai/plugin"
import { LOOK_AT_DESCRIPTION, MULTIMODAL_LOOKER_AGENT } from "./constants"
import type { LookAtArgs } from "./types"
import { log, promptSyncWithModelSuggestionRetry } from "../../shared"
import { extractLatestAssistantText } from "./assistant-message-extractor"
import type { LookAtArgsWithAlias } from "./look-at-arguments"
import { normalizeArgs, validateArgs } from "./look-at-arguments"
import {
  extractBase64Data,
  inferMimeTypeFromBase64,
  inferMimeTypeFromFilePath,
} from "./mime-type-inference"
import { resolveMultimodalLookerAgentMetadata } from "./multimodal-agent-metadata"
import {
  needsConversion,
  convertImageToJpeg,
  convertBase64ImageToJpeg,
  cleanupConvertedImage,
} from "./image-converter"

function getTemporaryConversionPath(error: unknown): string | null {
  if (!(error instanceof Error)) {
    return null
  }

  const temporaryOutputPath = Reflect.get(error, "temporaryOutputPath")
  if (typeof temporaryOutputPath === "string" && temporaryOutputPath.length > 0) {
    return temporaryOutputPath
  }

  const temporaryDirectory = Reflect.get(error, "temporaryDirectory")
  if (typeof temporaryDirectory === "string" && temporaryDirectory.length > 0) {
    return temporaryDirectory
  }

  return null
}


export { normalizeArgs, validateArgs } from "./look-at-arguments"

export function createLookAt(ctx: PluginInput): ToolDefinition {
  return tool({
    description: LOOK_AT_DESCRIPTION,
    args: {
      file_path: tool.schema.string().optional().describe("Absolute path to the file to analyze"),
      image_data: tool.schema.string().optional().describe("Base64 encoded image data (for clipboard/pasted images)"),
      goal: tool.schema.string().describe("What specific information to extract from the file"),
    },
    async execute(rawArgs: LookAtArgs, toolContext) {
      const args = normalizeArgs(rawArgs as LookAtArgsWithAlias)
      const validationError = validateArgs(args)
      if (validationError) {
        log(`[look_at] Validation failed: ${validationError}`)
        return validationError
      }

      const isBase64Input = Boolean(args.image_data)
      const sourceDescription = isBase64Input ? "clipboard/pasted image" : args.file_path
      log(`[look_at] Analyzing ${sourceDescription}, goal: ${args.goal}`)

      const imageData = args.image_data
      const filePath = args.file_path

      let mimeType: string
      let filePart: { type: "file"; mime: string; url: string; filename: string }
      let tempFilePath: string | null = null
      let tempConversionPath: string | null = null
      let tempFilesToCleanup: string[] = []

      try {
        if (imageData) {
          mimeType = inferMimeTypeFromBase64(imageData)
          
          let finalBase64Data = extractBase64Data(imageData)
          let finalMimeType = mimeType
          
          if (needsConversion(mimeType)) {
            log(`[look_at] Detected unsupported Base64 format: ${mimeType}, converting to JPEG...`)
            try {
              const { base64, tempFiles } = convertBase64ImageToJpeg(finalBase64Data, mimeType)
              finalBase64Data = base64
              finalMimeType = "image/jpeg"
              tempFilesToCleanup = tempFiles
              log(`[look_at] Base64 conversion successful`)
            } catch (conversionError) {
              log(`[look_at] Base64 conversion failed: ${conversionError}`)
              return `Error: Failed to convert Base64 image format. ${conversionError}`
            }
          }
          
          filePart = {
            type: "file",
            mime: finalMimeType,
            url: `data:${finalMimeType};base64,${finalBase64Data}`,
            filename: `clipboard-image.${finalMimeType.split("/")[1] || "png"}`,
          }
        } else if (filePath) {
        mimeType = inferMimeTypeFromFilePath(filePath)
        
        let actualFilePath = filePath
        if (needsConversion(mimeType)) {
          log(`[look_at] Detected unsupported format: ${mimeType}, converting to JPEG...`)
          try {
            tempFilePath = convertImageToJpeg(filePath, mimeType)
            tempConversionPath = tempFilePath
            actualFilePath = tempFilePath
            mimeType = "image/jpeg"
            log(`[look_at] Conversion successful: ${tempFilePath}`)
          } catch (conversionError) {
            const failedConversionPath = getTemporaryConversionPath(conversionError)
            if (failedConversionPath) {
              tempConversionPath = failedConversionPath
            }
            log(`[look_at] Conversion failed: ${conversionError}`)
            return `Error: Failed to convert image format. ${conversionError}`
          }
        }

        filePart = {
          type: "file",
          mime: mimeType,
          url: pathToFileURL(actualFilePath).href,
          filename: basename(actualFilePath),
        }
      } else {
        return "Error: Must provide either 'file_path' or 'image_data'."
      }

      const prompt = `Analyze this ${isBase64Input ? "image" : "file"} and extract the requested information.

Goal: ${args.goal}

Provide ONLY the extracted information that matches the goal.
Be thorough on what was requested, concise on everything else.
If the requested information is not found, clearly state what is missing.`

      const { agentModel, agentVariant } = await resolveMultimodalLookerAgentMetadata(ctx)

      log(`[look_at] Creating session with parent: ${toolContext.sessionID}`)
      const parentSession = await ctx.client.session.get({
        path: { id: toolContext.sessionID },
      }).catch(() => null)
      const parentDirectory = parentSession?.data?.directory ?? ctx.directory

      const createResult = await ctx.client.session.create({
        body: {
          parentID: toolContext.sessionID,
          title: `look_at: ${args.goal.substring(0, 50)}`,
        },
        query: { directory: parentDirectory },
      })

      if (createResult.error) {
        log(`[look_at] Session create error:`, createResult.error)
        const errorStr = String(createResult.error)
        if (errorStr.toLowerCase().includes("unauthorized")) {
          return `Error: Failed to create session (Unauthorized). This may be due to:
1. OAuth token restrictions (e.g., Claude Code credentials are restricted to Claude Code only)
2. Provider authentication issues
3. Session permission inheritance problems

Try using a different provider or API key authentication.

Original error: ${createResult.error}`
        }
        return `Error: Failed to create session: ${createResult.error}`
      }

      const sessionID = createResult.data.id
      log(`[look_at] Created session: ${sessionID}`)

      log(`[look_at] Sending prompt with ${isBase64Input ? "base64 image" : "file"} to session ${sessionID}`)
      try {
        await promptSyncWithModelSuggestionRetry(ctx.client, {
          path: { id: sessionID },
          body: {
            agent: MULTIMODAL_LOOKER_AGENT,
            tools: {
              task: false,
              call_omo_agent: false,
              look_at: false,
              read: false,
            },
            parts: [
              { type: "text", text: prompt },
              filePart,
            ],
            ...(agentModel ? { model: { providerID: agentModel.providerID, modelID: agentModel.modelID } } : {}),
            ...(agentVariant ? { variant: agentVariant } : {}),
          },
        })
      } catch (promptError) {
        log(`[look_at] Prompt error (ignored, will still fetch messages):`, promptError)
      }

      log(`[look_at] Fetching messages from session ${sessionID}...`)

      const messagesResult = await ctx.client.session.messages({
        path: { id: sessionID },
      })

      if (messagesResult.error) {
        log(`[look_at] Messages error:`, messagesResult.error)
        return `Error: Failed to get messages: ${messagesResult.error}`
      }

      const messages = messagesResult.data
      log(`[look_at] Got ${messages.length} messages`)

      const responseText = extractLatestAssistantText(messages)
      if (!responseText) {
        log("[look_at] No assistant message found")
        return "Error: No response from multimodal-looker agent"
      }

        log(`[look_at] Got response, length: ${responseText.length}`)
        return responseText
      } catch (error) {
        const errorMessage = error instanceof Error ? error.message : String(error)
        log(`[look_at] Unexpected error analyzing ${sourceDescription}:`, error)
        return `Error: Failed to analyze ${sourceDescription}: ${errorMessage}`
      } finally {
        if (tempConversionPath) {
          cleanupConvertedImage(tempConversionPath)
        } else if (tempFilePath) {
          cleanupConvertedImage(tempFilePath)
        }
        tempFilesToCleanup.forEach(file => {
          cleanupConvertedImage(file)
        })
      }
    },
  })
}