feat(look_at): add image_data parameter for clipboard/pasted image support
Closes #704 Add support for base64-encoded image data in the look_at tool, enabling analysis of clipboard/pasted images without requiring a file path. Changes: - Add optional image_data parameter to LookAtArgs type - Update validateArgs to accept either file_path or image_data - Add inferMimeTypeFromBase64 function to detect image format - Add try/catch around atob() to handle invalid base64 gracefully - Update execute to handle both file path and data URL inputs - Add comprehensive tests for image_data functionality
This commit is contained in:
lihaitao
@@ -31,25 +31,52 @@ describe("look-at tool", () => {
|
||||
const normalized = normalizeArgs(args as any)
|
||||
expect(normalized.file_path).toBe("/preferred.png")
|
||||
})
|
||||
|
||||
// given image_data provided
|
||||
// when called with base64 image data
|
||||
// then preserve image_data in normalized args
|
||||
test("preserves image_data when provided", () => {
|
||||
const args = { image_data: "data:image/png;base64,iVBORw0KGgo=", goal: "analyze" }
|
||||
const normalized = normalizeArgs(args as any)
|
||||
expect(normalized.image_data).toBe("data:image/png;base64,iVBORw0KGgo=")
|
||||
expect(normalized.file_path).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe("validateArgs", () => {
|
||||
// given valid arguments
|
||||
// given valid arguments with file_path
|
||||
// when validated
|
||||
// then return null (no error)
|
||||
test("returns null for valid args", () => {
|
||||
test("returns null for valid args with file_path", () => {
|
||||
const args = { file_path: "/valid/path.png", goal: "analyze" }
|
||||
expect(validateArgs(args)).toBeNull()
|
||||
})
|
||||
|
||||
// given file_path missing
|
||||
// given valid arguments with image_data
|
||||
// when validated
|
||||
// then return null (no error)
|
||||
test("returns null for valid args with image_data", () => {
|
||||
const args = { image_data: "data:image/png;base64,iVBORw0KGgo=", goal: "analyze" }
|
||||
expect(validateArgs(args)).toBeNull()
|
||||
})
|
||||
|
||||
// given neither file_path nor image_data
|
||||
// when validated
|
||||
// then clear error message
|
||||
test("returns error when file_path is missing", () => {
|
||||
test("returns error when neither file_path nor image_data provided", () => {
|
||||
const args = { goal: "analyze" } as any
|
||||
const error = validateArgs(args)
|
||||
expect(error).toContain("file_path")
|
||||
expect(error).toContain("required")
|
||||
expect(error).toContain("image_data")
|
||||
})
|
||||
|
||||
// given both file_path and image_data
|
||||
// when validated
|
||||
// then return error (mutually exclusive)
|
||||
test("returns error when both file_path and image_data provided", () => {
|
||||
const args = { file_path: "/path.png", image_data: "base64data", goal: "analyze" }
|
||||
const error = validateArgs(args)
|
||||
expect(error).toContain("only one")
|
||||
})
|
||||
|
||||
// given goal missing
|
||||
@@ -69,6 +96,17 @@ describe("look-at tool", () => {
|
||||
const args = { file_path: "", goal: "analyze" }
|
||||
const error = validateArgs(args)
|
||||
expect(error).toContain("file_path")
|
||||
expect(error).toContain("image_data")
|
||||
})
|
||||
|
||||
// given image_data is empty string
|
||||
// when validated
|
||||
// then return error
|
||||
test("returns error when image_data is empty string", () => {
|
||||
const args = { image_data: "", goal: "analyze" }
|
||||
const error = validateArgs(args)
|
||||
expect(error).toContain("file_path")
|
||||
expect(error).toContain("image_data")
|
||||
})
|
||||
})
|
||||
|
||||
@@ -109,7 +147,7 @@ describe("look-at tool", () => {
|
||||
toolContext
|
||||
)
|
||||
|
||||
expect(result).toContain("Error: Failed to analyze file")
|
||||
expect(result).toContain("Error: Failed to analyze")
|
||||
expect(result).toContain("malformed response")
|
||||
expect(result).toContain("multimodal-looker")
|
||||
expect(result).toContain("image/png")
|
||||
@@ -217,4 +255,111 @@ describe("look-at tool", () => {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("createLookAt with image_data", () => {
|
||||
// given base64 image data is provided
|
||||
// when LookAt tool executed
|
||||
// then should send data URL to session.prompt
|
||||
test("sends data URL when image_data provided", async () => {
|
||||
let promptBody: any
|
||||
|
||||
const mockClient = {
|
||||
app: {
|
||||
agents: async () => ({ data: [] }),
|
||||
},
|
||||
session: {
|
||||
get: async () => ({ data: { directory: "/project" } }),
|
||||
create: async () => ({ data: { id: "ses_image_data_test" } }),
|
||||
prompt: async (input: any) => {
|
||||
promptBody = input.body
|
||||
return { data: {} }
|
||||
},
|
||||
messages: async () => ({
|
||||
data: [
|
||||
{ info: { role: "assistant", time: { created: 1 } }, parts: [{ type: "text", text: "analyzed" }] },
|
||||
],
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
const tool = createLookAt({
|
||||
client: mockClient,
|
||||
directory: "/project",
|
||||
} as any)
|
||||
|
||||
const toolContext: ToolContext = {
|
||||
sessionID: "parent-session",
|
||||
messageID: "parent-message",
|
||||
agent: "sisyphus",
|
||||
directory: "/project",
|
||||
worktree: "/project",
|
||||
abort: new AbortController().signal,
|
||||
metadata: () => {},
|
||||
ask: async () => {},
|
||||
}
|
||||
|
||||
await tool.execute(
|
||||
{ image_data: "data:image/png;base64,iVBORw0KGgo=", goal: "describe this image" },
|
||||
toolContext
|
||||
)
|
||||
|
||||
const filePart = promptBody.parts.find((p: any) => p.type === "file")
|
||||
expect(filePart).toBeDefined()
|
||||
expect(filePart.url).toContain("data:image/png;base64")
|
||||
expect(filePart.mime).toBe("image/png")
|
||||
expect(filePart.filename).toContain("clipboard-image")
|
||||
})
|
||||
|
||||
// given raw base64 without data URI prefix
|
||||
// when LookAt tool executed
|
||||
// then should detect mime type and create proper data URL
|
||||
test("handles raw base64 without data URI prefix", async () => {
|
||||
let promptBody: any
|
||||
|
||||
const mockClient = {
|
||||
app: {
|
||||
agents: async () => ({ data: [] }),
|
||||
},
|
||||
session: {
|
||||
get: async () => ({ data: { directory: "/project" } }),
|
||||
create: async () => ({ data: { id: "ses_raw_base64_test" } }),
|
||||
prompt: async (input: any) => {
|
||||
promptBody = input.body
|
||||
return { data: {} }
|
||||
},
|
||||
messages: async () => ({
|
||||
data: [
|
||||
{ info: { role: "assistant", time: { created: 1 } }, parts: [{ type: "text", text: "analyzed" }] },
|
||||
],
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
const tool = createLookAt({
|
||||
client: mockClient,
|
||||
directory: "/project",
|
||||
} as any)
|
||||
|
||||
const toolContext: ToolContext = {
|
||||
sessionID: "parent-session",
|
||||
messageID: "parent-message",
|
||||
agent: "sisyphus",
|
||||
directory: "/project",
|
||||
worktree: "/project",
|
||||
abort: new AbortController().signal,
|
||||
metadata: () => {},
|
||||
ask: async () => {},
|
||||
}
|
||||
|
||||
await tool.execute(
|
||||
{ image_data: "iVBORw0KGgo=", goal: "analyze" },
|
||||
toolContext
|
||||
)
|
||||
|
||||
const filePart = promptBody.parts.find((p: any) => p.type === "file")
|
||||
expect(filePart).toBeDefined()
|
||||
expect(filePart.url).toContain("data:")
|
||||
expect(filePart.url).toContain("base64")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user