fix: auto-resize oversized images to fit API dimension limits
Images exceeding 2000px in either dimension cause Anthropic's API to reject the request with 'At least one of the image dimensions exceed max allowed size for many-image requests: 2000 pixels'. Fix: - Add getImageDimensions() to check image size via sips (macOS) or ImageMagick identify - Add resizeImageIfNeeded() that resizes to fit within MAX_IMAGE_DIMENSION (2000px) while preserving aspect ratio - Wire resizing into convertImageToJpeg(), convertBase64ImageToJpeg(), AND the direct base64 pasted-image path in tools.ts (which bypasses the converter for supported formats like PNG/JPEG) - Uses sips --resampleHeightWidthMax on macOS, magick -resize NxN> on Linux Tests: 8 pass, 0 fail (5 existing + 3 new resize tests) Closes #3032
This commit is contained in:
@@ -211,3 +211,104 @@ describe("image-converter command execution safety", () => {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("image resizing for API dimension limits", () => {
|
||||
let execFileSyncSpy: ReturnType<typeof spyOn>
|
||||
|
||||
beforeEach(() => {
|
||||
execFileSyncSpy = spyOn(childProcess, "execFileSync").mockImplementation(
|
||||
((_command: string, _args: string[], _options?: unknown) => "") as typeof childProcess.execFileSync,
|
||||
)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
execFileSyncSpy.mockRestore()
|
||||
})
|
||||
|
||||
test("#given oversized image #when resizeImageIfNeeded called #then resizes to fit max dimension", async () => {
|
||||
const testDir = mkdtempSync(join(tmpdir(), "img-resize-test-"))
|
||||
const inputPath = join(testDir, "large.jpg")
|
||||
writeFileSync(inputPath, "fake-jpeg-data")
|
||||
|
||||
const { resizeImageIfNeeded, MAX_IMAGE_DIMENSION } = await loadImageConverter()
|
||||
|
||||
execFileSyncSpy.mockImplementation(
|
||||
((command: string, args: string[]) => {
|
||||
// Mock sips -g pixelWidth/pixelHeight returning oversized dimensions
|
||||
if (command === "sips" && args.includes("-g")) {
|
||||
return " pixelWidth: 4000\n pixelHeight: 3000\n"
|
||||
}
|
||||
// Mock sips --resampleHeightWidthMax writing output
|
||||
if (command === "sips" && args.includes("--resampleHeightWidthMax")) {
|
||||
const outIndex = args.indexOf("--out")
|
||||
if (outIndex >= 0) writeFileSync(args[outIndex + 1], "resized-jpeg")
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
}) as typeof childProcess.execFileSync,
|
||||
)
|
||||
|
||||
const result = resizeImageIfNeeded(inputPath)
|
||||
|
||||
expect(result).not.toBe(inputPath)
|
||||
expect(existsSync(result)).toBe(true)
|
||||
expect(MAX_IMAGE_DIMENSION).toBe(2000)
|
||||
|
||||
// Verify sips was called with resize args
|
||||
const resizeCalls = execFileSyncSpy.mock.calls.filter(
|
||||
(call: unknown[]) => call[0] === "sips" && (call[1] as string[]).includes("--resampleHeightWidthMax")
|
||||
)
|
||||
expect(resizeCalls.length).toBe(1)
|
||||
|
||||
if (existsSync(result)) unlinkSync(result)
|
||||
rmSync(dirname(result), { recursive: true, force: true })
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test("#given image within limits #when resizeImageIfNeeded called #then returns original path", async () => {
|
||||
const testDir = mkdtempSync(join(tmpdir(), "img-no-resize-test-"))
|
||||
const inputPath = join(testDir, "small.jpg")
|
||||
writeFileSync(inputPath, "fake-jpeg-data")
|
||||
|
||||
const { resizeImageIfNeeded } = await loadImageConverter()
|
||||
|
||||
execFileSyncSpy.mockImplementation(
|
||||
((command: string, args: string[]) => {
|
||||
if (command === "sips" && args.includes("-g")) {
|
||||
return " pixelWidth: 1024\n pixelHeight: 768\n"
|
||||
}
|
||||
return ""
|
||||
}) as typeof childProcess.execFileSync,
|
||||
)
|
||||
|
||||
const result = resizeImageIfNeeded(inputPath)
|
||||
|
||||
expect(result).toBe(inputPath)
|
||||
|
||||
// Verify no resize was attempted
|
||||
const resizeCalls = execFileSyncSpy.mock.calls.filter(
|
||||
(call: unknown[]) => call[0] === "sips" && (call[1] as string[]).includes("--resampleHeightWidthMax")
|
||||
)
|
||||
expect(resizeCalls.length).toBe(0)
|
||||
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test("#given dimension check fails #when resizeImageIfNeeded called #then returns original path gracefully", async () => {
|
||||
const testDir = mkdtempSync(join(tmpdir(), "img-dim-fail-test-"))
|
||||
const inputPath = join(testDir, "unknown.jpg")
|
||||
writeFileSync(inputPath, "fake-jpeg-data")
|
||||
|
||||
const { resizeImageIfNeeded } = await loadImageConverter()
|
||||
|
||||
execFileSyncSpy.mockImplementation((() => {
|
||||
throw new Error("sips not found")
|
||||
}) as typeof childProcess.execFileSync)
|
||||
|
||||
const result = resizeImageIfNeeded(inputPath)
|
||||
|
||||
expect(result).toBe(inputPath)
|
||||
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -34,6 +34,112 @@ const UNSUPPORTED_FORMATS = new Set([
|
||||
|
||||
const CONVERSION_TIMEOUT_MS = 30_000
|
||||
|
||||
/**
|
||||
* Maximum image dimension (width or height) in pixels.
|
||||
* Anthropic's API rejects images exceeding 2000px in many-image requests.
|
||||
*/
|
||||
export const MAX_IMAGE_DIMENSION = 2000
|
||||
|
||||
function getImageDimensions(imagePath: string): { width: number; height: number } | null {
|
||||
try {
|
||||
if (process.platform === "darwin") {
|
||||
// sips outputs: pixelWidth: 3000\n pixelHeight: 2000
|
||||
const output = childProcess.execFileSync("sips", ["-g", "pixelWidth", "-g", "pixelHeight", "--", imagePath], {
|
||||
stdio: "pipe",
|
||||
encoding: "utf-8",
|
||||
timeout: CONVERSION_TIMEOUT_MS,
|
||||
})
|
||||
const widthMatch = output.match(/pixelWidth:\s*(\d+)/)
|
||||
const heightMatch = output.match(/pixelHeight:\s*(\d+)/)
|
||||
if (widthMatch && heightMatch) {
|
||||
return { width: parseInt(widthMatch[1], 10), height: parseInt(heightMatch[1], 10) }
|
||||
}
|
||||
}
|
||||
|
||||
// ImageMagick identify: "3000x2000"
|
||||
const identifyCmd = process.platform === "darwin" ? "identify" : "magick"
|
||||
const identifyArgs = process.platform === "darwin" ? ["-format", "%wx%h", "--", imagePath] : ["identify", "-format", "%wx%h", "--", imagePath]
|
||||
const output = childProcess.execFileSync(identifyCmd, identifyArgs, {
|
||||
stdio: "pipe",
|
||||
encoding: "utf-8",
|
||||
timeout: CONVERSION_TIMEOUT_MS,
|
||||
})
|
||||
const match = output.trim().match(/^(\d+)x(\d+)/)
|
||||
if (match) {
|
||||
return { width: parseInt(match[1], 10), height: parseInt(match[2], 10) }
|
||||
}
|
||||
} catch (error) {
|
||||
log(`[image-converter] Failed to get dimensions: ${error}`)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Resize an image if either dimension exceeds MAX_IMAGE_DIMENSION.
|
||||
* Preserves aspect ratio. Returns the path to the resized image (may be a new temp file)
|
||||
* or the original path if no resize was needed.
|
||||
*/
|
||||
export function resizeImageIfNeeded(imagePath: string, maxDimension: number = MAX_IMAGE_DIMENSION): string {
|
||||
const dimensions = getImageDimensions(imagePath)
|
||||
if (!dimensions) {
|
||||
log("[image-converter] Could not determine dimensions, skipping resize")
|
||||
return imagePath
|
||||
}
|
||||
|
||||
if (dimensions.width <= maxDimension && dimensions.height <= maxDimension) {
|
||||
log(`[image-converter] Image ${dimensions.width}x${dimensions.height} within limits, no resize needed`)
|
||||
return imagePath
|
||||
}
|
||||
|
||||
log(`[image-converter] Image ${dimensions.width}x${dimensions.height} exceeds ${maxDimension}px, resizing`)
|
||||
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "opencode-resize-"))
|
||||
const ext = imagePath.split(".").pop() || "jpg"
|
||||
const outputPath = join(tempDir, `resized.${ext}`)
|
||||
|
||||
try {
|
||||
if (process.platform === "darwin") {
|
||||
try {
|
||||
childProcess.execFileSync("sips", ["--resampleHeightWidthMax", String(maxDimension), "--", imagePath, "--out", outputPath], {
|
||||
stdio: "pipe",
|
||||
encoding: "utf-8",
|
||||
timeout: CONVERSION_TIMEOUT_MS,
|
||||
})
|
||||
|
||||
if (existsSync(outputPath)) {
|
||||
log(`[image-converter] Resized using sips: ${outputPath}`)
|
||||
return outputPath
|
||||
}
|
||||
} catch (sipsError) {
|
||||
log(`[image-converter] sips resize failed: ${sipsError}`)
|
||||
}
|
||||
}
|
||||
|
||||
// ImageMagick fallback
|
||||
try {
|
||||
const magickCmd = process.platform === "darwin" ? "convert" : "magick"
|
||||
childProcess.execFileSync(magickCmd, ["--", imagePath, "-resize", `${maxDimension}x${maxDimension}>`, outputPath], {
|
||||
stdio: "pipe",
|
||||
encoding: "utf-8",
|
||||
timeout: CONVERSION_TIMEOUT_MS,
|
||||
})
|
||||
|
||||
if (existsSync(outputPath)) {
|
||||
log(`[image-converter] Resized using ImageMagick: ${outputPath}`)
|
||||
return outputPath
|
||||
}
|
||||
} catch (convertError) {
|
||||
log(`[image-converter] ImageMagick resize failed: ${convertError}`)
|
||||
}
|
||||
|
||||
log("[image-converter] No resize tool available, returning original")
|
||||
return imagePath
|
||||
} catch (error) {
|
||||
log(`[image-converter] Resize failed: ${error}`)
|
||||
return imagePath
|
||||
}
|
||||
}
|
||||
|
||||
export function needsConversion(mimeType: string): boolean {
|
||||
if (SUPPORTED_FORMATS.has(mimeType)) {
|
||||
return false
|
||||
@@ -67,7 +173,7 @@ export function convertImageToJpeg(inputPath: string, mimeType: string): string
|
||||
|
||||
if (existsSync(outputPath)) {
|
||||
log(`[image-converter] Converted using sips: ${outputPath}`)
|
||||
return outputPath
|
||||
return resizeImageIfNeeded(outputPath)
|
||||
}
|
||||
} catch (sipsError) {
|
||||
log(`[image-converter] sips failed: ${sipsError}`)
|
||||
@@ -84,7 +190,7 @@ export function convertImageToJpeg(inputPath: string, mimeType: string): string
|
||||
|
||||
if (existsSync(outputPath)) {
|
||||
log(`[image-converter] Converted using ImageMagick: ${outputPath}`)
|
||||
return outputPath
|
||||
return resizeImageIfNeeded(outputPath)
|
||||
}
|
||||
} catch (convertError) {
|
||||
log(`[image-converter] ImageMagick convert failed: ${convertError}`)
|
||||
@@ -150,6 +256,16 @@ export function convertBase64ImageToJpeg(
|
||||
const convertedBuffer = readFileSync(outputPath)
|
||||
const convertedBase64 = convertedBuffer.toString("base64")
|
||||
|
||||
// Resize if needed before encoding back to base64
|
||||
const resizedPath = resizeImageIfNeeded(outputPath)
|
||||
if (resizedPath !== outputPath) {
|
||||
tempFiles.push(resizedPath)
|
||||
const resizedBuffer = readFileSync(resizedPath)
|
||||
const resizedBase64 = resizedBuffer.toString("base64")
|
||||
log(`[image-converter] Base64 conversion + resize successful`)
|
||||
return { base64: resizedBase64, tempFiles }
|
||||
}
|
||||
|
||||
log(`[image-converter] Base64 conversion successful`)
|
||||
|
||||
return { base64: convertedBase64, tempFiles }
|
||||
|
||||
@@ -18,6 +18,8 @@ import {
|
||||
convertImageToJpeg,
|
||||
convertBase64ImageToJpeg,
|
||||
cleanupConvertedImage,
|
||||
resizeImageIfNeeded,
|
||||
MAX_IMAGE_DIMENSION,
|
||||
} from "./image-converter"
|
||||
|
||||
function getTemporaryConversionPath(error: unknown): string | null {
|
||||
@@ -91,6 +93,26 @@ export function createLookAt(ctx: PluginInput): ToolDefinition {
|
||||
}
|
||||
}
|
||||
|
||||
// Resize oversized base64 images to fit API dimension limits (issue #3032)
|
||||
try {
|
||||
const { mkdtempSync, writeFileSync, readFileSync, existsSync } = require("node:fs")
|
||||
const { tmpdir } = require("node:os")
|
||||
const { join } = require("node:path")
|
||||
const resizeTmpDir = mkdtempSync(join(tmpdir(), "opencode-resize-b64-"))
|
||||
const ext = finalMimeType.split("/")[1] || "png"
|
||||
const resizeTmpFile = join(resizeTmpDir, `check.${ext}`)
|
||||
writeFileSync(resizeTmpFile, Buffer.from(finalBase64Data, "base64"))
|
||||
const resizedPath = resizeImageIfNeeded(resizeTmpFile)
|
||||
if (resizedPath !== resizeTmpFile && existsSync(resizedPath)) {
|
||||
finalBase64Data = readFileSync(resizedPath).toString("base64")
|
||||
tempFilesToCleanup.push(resizedPath)
|
||||
log(`[look_at] Base64 image resized to fit API limits`)
|
||||
}
|
||||
tempFilesToCleanup.push(resizeTmpFile, resizeTmpDir)
|
||||
} catch (resizeError) {
|
||||
log(`[look_at] Base64 resize check failed (non-fatal): ${resizeError}`)
|
||||
}
|
||||
|
||||
filePart = {
|
||||
type: "file",
|
||||
mime: finalMimeType,
|
||||
|
||||
Reference in New Issue
Block a user