diff --git a/src/tools/look-at/image-converter.test.ts b/src/tools/look-at/image-converter.test.ts index 12efaacc3..ad70bd641 100644 --- a/src/tools/look-at/image-converter.test.ts +++ b/src/tools/look-at/image-converter.test.ts @@ -211,3 +211,104 @@ describe("image-converter command execution safety", () => { }) }) }) + +describe("image resizing for API dimension limits", () => { + let execFileSyncSpy: ReturnType + + beforeEach(() => { + execFileSyncSpy = spyOn(childProcess, "execFileSync").mockImplementation( + ((_command: string, _args: string[], _options?: unknown) => "") as typeof childProcess.execFileSync, + ) + }) + + afterEach(() => { + execFileSyncSpy.mockRestore() + }) + + test("#given oversized image #when resizeImageIfNeeded called #then resizes to fit max dimension", async () => { + const testDir = mkdtempSync(join(tmpdir(), "img-resize-test-")) + const inputPath = join(testDir, "large.jpg") + writeFileSync(inputPath, "fake-jpeg-data") + + const { resizeImageIfNeeded, MAX_IMAGE_DIMENSION } = await loadImageConverter() + + execFileSyncSpy.mockImplementation( + ((command: string, args: string[]) => { + // Mock sips -g pixelWidth/pixelHeight returning oversized dimensions + if (command === "sips" && args.includes("-g")) { + return " pixelWidth: 4000\n pixelHeight: 3000\n" + } + // Mock sips --resampleHeightWidthMax writing output + if (command === "sips" && args.includes("--resampleHeightWidthMax")) { + const outIndex = args.indexOf("--out") + if (outIndex >= 0) writeFileSync(args[outIndex + 1], "resized-jpeg") + return "" + } + return "" + }) as typeof childProcess.execFileSync, + ) + + const result = resizeImageIfNeeded(inputPath) + + expect(result).not.toBe(inputPath) + expect(existsSync(result)).toBe(true) + expect(MAX_IMAGE_DIMENSION).toBe(2000) + + // Verify sips was called with resize args + const resizeCalls = execFileSyncSpy.mock.calls.filter( + (call: unknown[]) => call[0] === "sips" && (call[1] as string[]).includes("--resampleHeightWidthMax") + ) + expect(resizeCalls.length).toBe(1) + + if (existsSync(result)) unlinkSync(result) + rmSync(dirname(result), { recursive: true, force: true }) + rmSync(testDir, { recursive: true, force: true }) + }) + + test("#given image within limits #when resizeImageIfNeeded called #then returns original path", async () => { + const testDir = mkdtempSync(join(tmpdir(), "img-no-resize-test-")) + const inputPath = join(testDir, "small.jpg") + writeFileSync(inputPath, "fake-jpeg-data") + + const { resizeImageIfNeeded } = await loadImageConverter() + + execFileSyncSpy.mockImplementation( + ((command: string, args: string[]) => { + if (command === "sips" && args.includes("-g")) { + return " pixelWidth: 1024\n pixelHeight: 768\n" + } + return "" + }) as typeof childProcess.execFileSync, + ) + + const result = resizeImageIfNeeded(inputPath) + + expect(result).toBe(inputPath) + + // Verify no resize was attempted + const resizeCalls = execFileSyncSpy.mock.calls.filter( + (call: unknown[]) => call[0] === "sips" && (call[1] as string[]).includes("--resampleHeightWidthMax") + ) + expect(resizeCalls.length).toBe(0) + + rmSync(testDir, { recursive: true, force: true }) + }) + + test("#given dimension check fails #when resizeImageIfNeeded called #then returns original path gracefully", async () => { + const testDir = mkdtempSync(join(tmpdir(), "img-dim-fail-test-")) + const inputPath = join(testDir, "unknown.jpg") + writeFileSync(inputPath, "fake-jpeg-data") + + const { resizeImageIfNeeded } = await loadImageConverter() + + execFileSyncSpy.mockImplementation((() => { + throw new Error("sips not found") + }) as typeof childProcess.execFileSync) + + const result = resizeImageIfNeeded(inputPath) + + expect(result).toBe(inputPath) + + rmSync(testDir, { recursive: true, force: true }) + }) +}) diff --git a/src/tools/look-at/image-converter.ts b/src/tools/look-at/image-converter.ts index 163e35e28..e16295e99 100644 --- a/src/tools/look-at/image-converter.ts +++ b/src/tools/look-at/image-converter.ts @@ -34,6 +34,112 @@ const UNSUPPORTED_FORMATS = new Set([ const CONVERSION_TIMEOUT_MS = 30_000 +/** + * Maximum image dimension (width or height) in pixels. + * Anthropic's API rejects images exceeding 2000px in many-image requests. + */ +export const MAX_IMAGE_DIMENSION = 2000 + +function getImageDimensions(imagePath: string): { width: number; height: number } | null { + try { + if (process.platform === "darwin") { + // sips outputs: pixelWidth: 3000\n pixelHeight: 2000 + const output = childProcess.execFileSync("sips", ["-g", "pixelWidth", "-g", "pixelHeight", "--", imagePath], { + stdio: "pipe", + encoding: "utf-8", + timeout: CONVERSION_TIMEOUT_MS, + }) + const widthMatch = output.match(/pixelWidth:\s*(\d+)/) + const heightMatch = output.match(/pixelHeight:\s*(\d+)/) + if (widthMatch && heightMatch) { + return { width: parseInt(widthMatch[1], 10), height: parseInt(heightMatch[1], 10) } + } + } + + // ImageMagick identify: "3000x2000" + const identifyCmd = process.platform === "darwin" ? "identify" : "magick" + const identifyArgs = process.platform === "darwin" ? ["-format", "%wx%h", "--", imagePath] : ["identify", "-format", "%wx%h", "--", imagePath] + const output = childProcess.execFileSync(identifyCmd, identifyArgs, { + stdio: "pipe", + encoding: "utf-8", + timeout: CONVERSION_TIMEOUT_MS, + }) + const match = output.trim().match(/^(\d+)x(\d+)/) + if (match) { + return { width: parseInt(match[1], 10), height: parseInt(match[2], 10) } + } + } catch (error) { + log(`[image-converter] Failed to get dimensions: ${error}`) + } + return null +} + +/** + * Resize an image if either dimension exceeds MAX_IMAGE_DIMENSION. + * Preserves aspect ratio. Returns the path to the resized image (may be a new temp file) + * or the original path if no resize was needed. + */ +export function resizeImageIfNeeded(imagePath: string, maxDimension: number = MAX_IMAGE_DIMENSION): string { + const dimensions = getImageDimensions(imagePath) + if (!dimensions) { + log("[image-converter] Could not determine dimensions, skipping resize") + return imagePath + } + + if (dimensions.width <= maxDimension && dimensions.height <= maxDimension) { + log(`[image-converter] Image ${dimensions.width}x${dimensions.height} within limits, no resize needed`) + return imagePath + } + + log(`[image-converter] Image ${dimensions.width}x${dimensions.height} exceeds ${maxDimension}px, resizing`) + + const tempDir = mkdtempSync(join(tmpdir(), "opencode-resize-")) + const ext = imagePath.split(".").pop() || "jpg" + const outputPath = join(tempDir, `resized.${ext}`) + + try { + if (process.platform === "darwin") { + try { + childProcess.execFileSync("sips", ["--resampleHeightWidthMax", String(maxDimension), "--", imagePath, "--out", outputPath], { + stdio: "pipe", + encoding: "utf-8", + timeout: CONVERSION_TIMEOUT_MS, + }) + + if (existsSync(outputPath)) { + log(`[image-converter] Resized using sips: ${outputPath}`) + return outputPath + } + } catch (sipsError) { + log(`[image-converter] sips resize failed: ${sipsError}`) + } + } + + // ImageMagick fallback + try { + const magickCmd = process.platform === "darwin" ? "convert" : "magick" + childProcess.execFileSync(magickCmd, ["--", imagePath, "-resize", `${maxDimension}x${maxDimension}>`, outputPath], { + stdio: "pipe", + encoding: "utf-8", + timeout: CONVERSION_TIMEOUT_MS, + }) + + if (existsSync(outputPath)) { + log(`[image-converter] Resized using ImageMagick: ${outputPath}`) + return outputPath + } + } catch (convertError) { + log(`[image-converter] ImageMagick resize failed: ${convertError}`) + } + + log("[image-converter] No resize tool available, returning original") + return imagePath + } catch (error) { + log(`[image-converter] Resize failed: ${error}`) + return imagePath + } +} + export function needsConversion(mimeType: string): boolean { if (SUPPORTED_FORMATS.has(mimeType)) { return false @@ -67,7 +173,7 @@ export function convertImageToJpeg(inputPath: string, mimeType: string): string if (existsSync(outputPath)) { log(`[image-converter] Converted using sips: ${outputPath}`) - return outputPath + return resizeImageIfNeeded(outputPath) } } catch (sipsError) { log(`[image-converter] sips failed: ${sipsError}`) @@ -84,7 +190,7 @@ export function convertImageToJpeg(inputPath: string, mimeType: string): string if (existsSync(outputPath)) { log(`[image-converter] Converted using ImageMagick: ${outputPath}`) - return outputPath + return resizeImageIfNeeded(outputPath) } } catch (convertError) { log(`[image-converter] ImageMagick convert failed: ${convertError}`) @@ -150,6 +256,16 @@ export function convertBase64ImageToJpeg( const convertedBuffer = readFileSync(outputPath) const convertedBase64 = convertedBuffer.toString("base64") + // Resize if needed before encoding back to base64 + const resizedPath = resizeImageIfNeeded(outputPath) + if (resizedPath !== outputPath) { + tempFiles.push(resizedPath) + const resizedBuffer = readFileSync(resizedPath) + const resizedBase64 = resizedBuffer.toString("base64") + log(`[image-converter] Base64 conversion + resize successful`) + return { base64: resizedBase64, tempFiles } + } + log(`[image-converter] Base64 conversion successful`) return { base64: convertedBase64, tempFiles } diff --git a/src/tools/look-at/tools.ts b/src/tools/look-at/tools.ts index 773d334d0..7b0d841fc 100644 --- a/src/tools/look-at/tools.ts +++ b/src/tools/look-at/tools.ts @@ -18,6 +18,8 @@ import { convertImageToJpeg, convertBase64ImageToJpeg, cleanupConvertedImage, + resizeImageIfNeeded, + MAX_IMAGE_DIMENSION, } from "./image-converter" function getTemporaryConversionPath(error: unknown): string | null { @@ -91,6 +93,26 @@ export function createLookAt(ctx: PluginInput): ToolDefinition { } } + // Resize oversized base64 images to fit API dimension limits (issue #3032) + try { + const { mkdtempSync, writeFileSync, readFileSync, existsSync } = require("node:fs") + const { tmpdir } = require("node:os") + const { join } = require("node:path") + const resizeTmpDir = mkdtempSync(join(tmpdir(), "opencode-resize-b64-")) + const ext = finalMimeType.split("/")[1] || "png" + const resizeTmpFile = join(resizeTmpDir, `check.${ext}`) + writeFileSync(resizeTmpFile, Buffer.from(finalBase64Data, "base64")) + const resizedPath = resizeImageIfNeeded(resizeTmpFile) + if (resizedPath !== resizeTmpFile && existsSync(resizedPath)) { + finalBase64Data = readFileSync(resizedPath).toString("base64") + tempFilesToCleanup.push(resizedPath) + log(`[look_at] Base64 image resized to fit API limits`) + } + tempFilesToCleanup.push(resizeTmpFile, resizeTmpDir) + } catch (resizeError) { + log(`[look_at] Base64 resize check failed (non-fatal): ${resizeError}`) + } + filePart = { type: "file", mime: finalMimeType,