feat: add save_image tool and list image MCP tools

Expose save_image across direct model, MCP gateway, DeepSeek Harness and remote Agents (written on the remote host via chunked reads), and show the image MCP tool list in settings.
This commit is contained in:
mesalogo
2026-10-02 10:08:42 +08:00
parent b02ce31886
commit dd29f309f1
20 changed files with 479 additions and 33 deletions
+43 -3
View File
@@ -1,6 +1,9 @@
// @vitest-environment node
import { randomUUID } from 'node:crypto'
import { randomBytes, randomUUID } from 'node:crypto'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { createServer } from 'node:http'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
import { afterEach, describe, expect, it, vi } from 'vitest'
@@ -46,7 +49,7 @@ async function setup() {
service.initialize()
cleanup.push(() => service.dispose())
const frames: Uint8Array[] = []
const connect = async (mode: 'ask' | 'execute' = 'execute') => {
const connect = async (mode: 'ask' | 'execute' = 'execute', options: { save?: boolean } = {}) => {
const binding = service.bind({ ...context, requestId: randomUUID(), workMode: mode })
const queue: RuntimeProtocolBinaryFrame[] = []
let receiver: ((frame: RuntimeProtocolBinaryFrame) => void) | undefined
@@ -62,7 +65,7 @@ async function setup() {
}
const wait = new AbortController()
const main = new MainImageToolSession(channel, binding, wait.signal)
const adapter = new AgentImageToolMcp({ channelId: channel.channelId, channelEpoch: '1', description: await binding.describe() ?? 'Not available' }, async payload => {
const adapter = new AgentImageToolMcp({ channelId: channel.channelId, channelEpoch: '1', description: await binding.describe() ?? 'Not available', ...(options.save ? { saveDescription: 'Save image' } : {}) }, async payload => {
if (closed) throw new Error('disconnected')
frames.push(payload)
const frame = { payload, sequence: '1', consume: async () => {} }
@@ -130,6 +133,43 @@ describe('remote image MCP to Main service', () => {
expect((await execute.client.listTools()).tools[0]!.description).toContain('Changed next prompt')
})
it('saves a conversation image on the Agent host by pulling bounded chunks from Desktop', async () => {
const h = await setup()
const plain = await h.connect()
expect((await plain.client.listTools()).tools.map(tool => tool.name)).not.toContain('save_image')
const remote = await h.connect('execute', { save: true })
expect((await remote.client.listTools()).tools.map(tool => tool.name)).toContain('save_image')
// Larger than one 128 KiB chunk so the Agent must reassemble several replies.
const large = Buffer.concat([Buffer.from(png, 'base64'), randomBytes(300 * 1024)])
const [artifactId] = h.service.persistUploads(h.context, [{ name: 'large.png', mediaType: 'image/png', data: large.toString('base64') }])
const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-remote-save-'))
cleanup.push(() => rm(directory, { recursive: true, force: true }))
const target = join(directory, 'nested', 'out.png')
const saved = await remote.client.callTool({ name: 'save_image', arguments: { artifactId, path: target } })
expect(saved.isError).not.toBe(true)
expect(JSON.parse((saved.content as { text: string }[])[0]!.text)).toMatchObject({ artifactId, path: target, mimeType: 'image/png', byteSize: large.byteLength })
expect((await readFile(target)).equals(large)).toBe(true)
expect(h.frames.length).toBeGreaterThan(4)
expect(h.frames.every(frame => frame.byteLength <= 256 * 1024)).toBe(true)
const again = await remote.client.callTool({ name: 'save_image', arguments: { artifactId, path: target } })
expect(again.isError).toBe(true)
expect(JSON.stringify(again.content)).toContain('already exists')
const foreign = await remote.client.callTool({ name: 'save_image', arguments: { artifactId: randomUUID(), path: join(directory, 'x.png') } })
expect(foreign.isError).toBe(true)
expect(JSON.stringify(foreign.content)).toContain('not available in this conversation')
})
it('rejects remote saves in Ask mode at the desktop binding', async () => {
const h = await setup()
const [artifactId] = h.service.persistUploads(h.context, [{ name: 'source.png', mediaType: 'image/png', data: png }])
const ask = await h.connect('ask', { save: true })
const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-remote-save-'))
cleanup.push(() => rm(directory, { recursive: true, force: true }))
const denied = await ask.client.callTool({ name: 'save_image', arguments: { artifactId, path: join(directory, 'out.png') } })
expect(denied.isError).toBe(true)
await expect(readFile(join(directory, 'out.png'))).rejects.toThrow()
})
it('returns unknown delivery without sending a second request for the same MCP call', async () => {
const send = vi.fn(async () => { throw new Error('connection lost after send') })
const adapter = new AgentImageToolMcp({ channelId: randomUUID(), channelEpoch: '1', description: 'Images' }, send)
+56 -5
View File
@@ -1,14 +1,19 @@
import { randomUUID } from 'node:crypto'
import { mkdir, writeFile } from 'node:fs/promises'
import { createServer, type Server } from 'node:http'
import { dirname, isAbsolute } from 'node:path'
import { Server as McpServer } from '@modelcontextprotocol/sdk/server/index.js'
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'
import { CallToolRequestSchema, ListToolsRequestSchema, type CallToolResult } from '@modelcontextprotocol/sdk/types.js'
import { z } from 'zod'
import { imageToolInputSchema, imageToolName } from '../shared/image-generation-contracts'
import {
imageSaveMimeTypeForPath, imageSaveToolInputSchema, imageSaveToolName,
imageToolInputSchema, imageToolName, type ImageSaveResult, type ImageSaveToolInput
} from '../shared/image-generation-contracts'
import { storyGraphTools, type StoryGraphToolName } from '../shared/story-graph-tools'
import {
decodeRemoteImageToolMessage, encodeRemoteImageToolMessage,
remoteImageToolMaximumBytes, remoteImageToolReplySchema, type remoteImageToolSchema
remoteImageSaveChunkSchema, remoteImageToolMaximumBytes, remoteImageToolReplySchema, type remoteImageToolSchema
} from '../shared/remote-image-tool-contracts'
/** One prompt's local MCP adapter. It never holds provider credentials or image bytes. */
@@ -31,6 +36,8 @@ export class AgentImageToolMcp {
const path = `/${randomUUID()}/mcp`
const schema = z.toJSONSchema(imageToolInputSchema, { target: 'draft-7', io: 'input' })
Reflect.deleteProperty(schema, '$schema')
const saveSchema = z.toJSONSchema(imageSaveToolInputSchema, { target: 'draft-7', io: 'input' })
Reflect.deleteProperty(saveSchema, '$schema')
this.server = createServer((request, response) => {
void (async () => {
if (this.closed || request.url !== path) { response.writeHead(404).end(); return }
@@ -50,6 +57,8 @@ export class AgentImageToolMcp {
const graphEnabled = available?.content.some(part => part.type === 'text' && JSON.parse(part.text).available === true)
return { tools: [
...(this.descriptor.description ? [{ name: imageToolName, description: this.descriptor.description, inputSchema: schema as { type: 'object' } }] : []),
...(this.descriptor.saveDescription ? [{ name: imageSaveToolName, description: this.descriptor.saveDescription, inputSchema: saveSchema as { type: 'object' },
annotations: { readOnlyHint: false, destructiveHint: true } }] : []),
...(graphEnabled ? storyGraphTools.map(tool => ({ name: tool.name, description: tool.description,
inputSchema: z.toJSONSchema(tool.inputSchema, { target: 'draft-7', io: 'input' }) as { type: 'object' },
annotations: { readOnlyHint: true, destructiveHint: false } })) : [])
@@ -61,6 +70,14 @@ export class AgentImageToolMcp {
try { return await this.call(call.params.arguments ?? {}, graphTool.name) }
catch (error) { return { isError: true, content: [{ type: 'text', text: error instanceof Error ? error.message : 'Story Graph read failed' }] } }
}
if (call.params.name === imageSaveToolName && this.descriptor.saveDescription) {
try {
const saved = await this.saveImage(imageSaveToolInputSchema.parse(call.params.arguments ?? {}))
return { content: [{ type: 'text', text: JSON.stringify(saved) }] }
} catch (error) {
return { isError: true, content: [{ type: 'text', text: error instanceof Error ? error.message : 'Saving image failed' }] }
}
}
if (call.params.name !== imageToolName || !this.descriptor.description) throw new Error('Unknown image tool')
const input = imageToolInputSchema.parse(call.params.arguments)
// Repeated delivery of an MCP request returns the same outcome, including unknown delivery.
@@ -99,8 +116,42 @@ export class AgentImageToolMcp {
const reply = remoteImageToolReplySchema.parse(decodeRemoteImageToolMessage(payload))
const pending = this.pending.get(reply.callId)
if (!pending) return
if (reply.error || (!reply.result && !reply.storyGraphResult)) pending.reject(new Error(reply.error ?? 'Tool response has no result'))
else pending.resolve({ content: [{ type: 'text', text: JSON.stringify(reply.storyGraphResult ?? reply.result) }] })
if (reply.error || (!reply.result && !reply.storyGraphResult && !reply.imageChunk)) pending.reject(new Error(reply.error ?? 'Tool response has no result'))
else pending.resolve({ content: [{ type: 'text', text: JSON.stringify(reply.imageChunk ?? reply.storyGraphResult ?? reply.result) }] })
}
/** Pulls the encoded image from Desktop in bounded chunks and writes it on this host. */
private async saveImage(input: ImageSaveToolInput): Promise<ImageSaveResult> {
const target = input.path
if (!isAbsolute(target)) throw new Error('path must be an absolute file path')
const mimeType = imageSaveMimeTypeForPath(target)
if (!mimeType) throw new Error('path must end with .png, .jpg, .jpeg or .webp')
const chunks: Buffer[] = []
let offset = 0
let totalBytes: number | undefined
do {
const result = await this.call({ artifactId: input.artifactId, mimeType, offset }, 'save_image_read')
const part = result.content[0]
if (part?.type !== 'text') throw new Error('Image chunk response is invalid')
const chunk = remoteImageSaveChunkSchema.parse(JSON.parse(part.text))
if (chunk.offset !== offset || chunk.mimeType !== mimeType || (totalBytes !== undefined && chunk.totalBytes !== totalBytes)) {
throw new Error('Image chunk response is inconsistent')
}
totalBytes = chunk.totalBytes
const bytes = Buffer.from(chunk.data, 'base64')
if (bytes.byteLength === 0 || offset + bytes.byteLength > totalBytes) throw new Error('Image chunk response is inconsistent')
chunks.push(bytes)
offset += bytes.byteLength
} while (offset < totalBytes)
const data = Buffer.concat(chunks)
await mkdir(dirname(target), { recursive: true })
try {
await writeFile(target, data, { flag: input.overwrite ? 'w' : 'wx' })
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'EEXIST') throw new Error('File already exists; choose another path or set overwrite=true', { cause: error })
throw error
}
return { artifactId: input.artifactId, path: target, mimeType, byteSize: data.byteLength }
}
close(): void {
@@ -112,7 +163,7 @@ export class AgentImageToolMcp {
this.server?.close()
}
private async call(input: Record<string, unknown>, name?: StoryGraphToolName | 'story_graph_list'): Promise<CallToolResult> {
private async call(input: Record<string, unknown>, name?: StoryGraphToolName | 'story_graph_list' | 'save_image_read'): Promise<CallToolResult> {
if (this.closed) throw new Error('Image tool prompt has ended')
const callId = randomUUID()
let timer: NodeJS.Timeout | undefined
+1 -1
View File
@@ -1173,7 +1173,7 @@ export class RuntimeAcpBackend {
}
}
}
if (preparation.imageTool && ((preparation.imageTool.description && preparation.workMode !== 'execute') || !this.#options.blobSink)) {
if (preparation.imageTool && (((preparation.imageTool.description || preparation.imageTool.saveDescription) && preparation.workMode !== 'execute') || !this.#options.blobSink)) {
throw new RuntimeAcpBackendError('Image tools require Execute and a desktop transport', 'identity')
}
if (requestedModelBridgePolicy !== undefined) {