From dd29f309f17f0a2f7ca4e04315dab44bffa6185d Mon Sep 17 00:00:00 2001 From: mesalogo Date: Fri, 2 Oct 2026 10:08:42 +0800 Subject: [PATCH] feat: add save_image tool and list image MCP tools Expose save_image across direct model, MCP gateway, DeepSeek Harness and remote Agents (written on the remote host via chunked reads), and show the image MCP tool list in settings. --- src/agent-daemon/image-tool-mcp.test.ts | 46 ++++++++++- src/agent-daemon/image-tool-mcp.ts | 61 +++++++++++++-- src/agent-daemon/runtime-acp-backend.ts | 2 +- src/main/agent/deepseek-harness-runtime.ts | 12 ++- .../agent/image-generation-service.test.ts | 57 +++++++++++++- src/main/agent/image-generation-service.ts | 77 ++++++++++++++++++- src/main/agent/image-tool-binding.ts | 21 ++++- src/main/agent/knowledge-mcp-gateway.ts | 16 +++- src/main/agent/model-tool-provider.ts | 20 ++++- src/main/ipc.ts | 4 + .../remote-agent/main-image-tool-session.ts | 22 +++++- .../protocol-remote-runtime-channel.ts | 6 +- src/renderer/src/McpSettingsSection.test.tsx | 16 ++++ src/renderer/src/McpSettingsSection.tsx | 66 +++++++++++++++- .../src/i18n/locales/en-US/integrations.ts | 6 +- .../src/i18n/locales/zh-CN/integrations.ts | 6 +- src/shared/builtin-model-tools.test.ts | 9 ++- src/shared/builtin-model-tools.ts | 7 ++ src/shared/image-generation-contracts.ts | 31 ++++++++ src/shared/remote-image-tool-contracts.ts | 27 ++++++- 20 files changed, 479 insertions(+), 33 deletions(-) diff --git a/src/agent-daemon/image-tool-mcp.test.ts b/src/agent-daemon/image-tool-mcp.test.ts index a078b45e..bd5909ba 100644 --- a/src/agent-daemon/image-tool-mcp.test.ts +++ b/src/agent-daemon/image-tool-mcp.test.ts @@ -1,6 +1,9 @@ // @vitest-environment node -import { randomUUID } from 'node:crypto' +import { randomBytes, randomUUID } from 'node:crypto' +import { mkdtemp, readFile, rm } from 'node:fs/promises' import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' import { afterEach, describe, expect, it, vi } from 'vitest' @@ -46,7 +49,7 @@ async function setup() { service.initialize() cleanup.push(() => service.dispose()) const frames: Uint8Array[] = [] - const connect = async (mode: 'ask' | 'execute' = 'execute') => { + const connect = async (mode: 'ask' | 'execute' = 'execute', options: { save?: boolean } = {}) => { const binding = service.bind({ ...context, requestId: randomUUID(), workMode: mode }) const queue: RuntimeProtocolBinaryFrame[] = [] let receiver: ((frame: RuntimeProtocolBinaryFrame) => void) | undefined @@ -62,7 +65,7 @@ async function setup() { } const wait = new AbortController() const main = new MainImageToolSession(channel, binding, wait.signal) - const adapter = new AgentImageToolMcp({ channelId: channel.channelId, channelEpoch: '1', description: await binding.describe() ?? 'Not available' }, async payload => { + const adapter = new AgentImageToolMcp({ channelId: channel.channelId, channelEpoch: '1', description: await binding.describe() ?? 'Not available', ...(options.save ? { saveDescription: 'Save image' } : {}) }, async payload => { if (closed) throw new Error('disconnected') frames.push(payload) const frame = { payload, sequence: '1', consume: async () => {} } @@ -130,6 +133,43 @@ describe('remote image MCP to Main service', () => { expect((await execute.client.listTools()).tools[0]!.description).toContain('Changed next prompt') }) + it('saves a conversation image on the Agent host by pulling bounded chunks from Desktop', async () => { + const h = await setup() + const plain = await h.connect() + expect((await plain.client.listTools()).tools.map(tool => tool.name)).not.toContain('save_image') + const remote = await h.connect('execute', { save: true }) + expect((await remote.client.listTools()).tools.map(tool => tool.name)).toContain('save_image') + // Larger than one 128 KiB chunk so the Agent must reassemble several replies. + const large = Buffer.concat([Buffer.from(png, 'base64'), randomBytes(300 * 1024)]) + const [artifactId] = h.service.persistUploads(h.context, [{ name: 'large.png', mediaType: 'image/png', data: large.toString('base64') }]) + const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-remote-save-')) + cleanup.push(() => rm(directory, { recursive: true, force: true })) + const target = join(directory, 'nested', 'out.png') + const saved = await remote.client.callTool({ name: 'save_image', arguments: { artifactId, path: target } }) + expect(saved.isError).not.toBe(true) + expect(JSON.parse((saved.content as { text: string }[])[0]!.text)).toMatchObject({ artifactId, path: target, mimeType: 'image/png', byteSize: large.byteLength }) + expect((await readFile(target)).equals(large)).toBe(true) + expect(h.frames.length).toBeGreaterThan(4) + expect(h.frames.every(frame => frame.byteLength <= 256 * 1024)).toBe(true) + const again = await remote.client.callTool({ name: 'save_image', arguments: { artifactId, path: target } }) + expect(again.isError).toBe(true) + expect(JSON.stringify(again.content)).toContain('already exists') + const foreign = await remote.client.callTool({ name: 'save_image', arguments: { artifactId: randomUUID(), path: join(directory, 'x.png') } }) + expect(foreign.isError).toBe(true) + expect(JSON.stringify(foreign.content)).toContain('not available in this conversation') + }) + + it('rejects remote saves in Ask mode at the desktop binding', async () => { + const h = await setup() + const [artifactId] = h.service.persistUploads(h.context, [{ name: 'source.png', mediaType: 'image/png', data: png }]) + const ask = await h.connect('ask', { save: true }) + const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-remote-save-')) + cleanup.push(() => rm(directory, { recursive: true, force: true })) + const denied = await ask.client.callTool({ name: 'save_image', arguments: { artifactId, path: join(directory, 'out.png') } }) + expect(denied.isError).toBe(true) + await expect(readFile(join(directory, 'out.png'))).rejects.toThrow() + }) + it('returns unknown delivery without sending a second request for the same MCP call', async () => { const send = vi.fn(async () => { throw new Error('connection lost after send') }) const adapter = new AgentImageToolMcp({ channelId: randomUUID(), channelEpoch: '1', description: 'Images' }, send) diff --git a/src/agent-daemon/image-tool-mcp.ts b/src/agent-daemon/image-tool-mcp.ts index 0aa15558..6ee9bba8 100644 --- a/src/agent-daemon/image-tool-mcp.ts +++ b/src/agent-daemon/image-tool-mcp.ts @@ -1,14 +1,19 @@ import { randomUUID } from 'node:crypto' +import { mkdir, writeFile } from 'node:fs/promises' import { createServer, type Server } from 'node:http' +import { dirname, isAbsolute } from 'node:path' import { Server as McpServer } from '@modelcontextprotocol/sdk/server/index.js' import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js' import { CallToolRequestSchema, ListToolsRequestSchema, type CallToolResult } from '@modelcontextprotocol/sdk/types.js' import { z } from 'zod' -import { imageToolInputSchema, imageToolName } from '../shared/image-generation-contracts' +import { + imageSaveMimeTypeForPath, imageSaveToolInputSchema, imageSaveToolName, + imageToolInputSchema, imageToolName, type ImageSaveResult, type ImageSaveToolInput +} from '../shared/image-generation-contracts' import { storyGraphTools, type StoryGraphToolName } from '../shared/story-graph-tools' import { decodeRemoteImageToolMessage, encodeRemoteImageToolMessage, - remoteImageToolMaximumBytes, remoteImageToolReplySchema, type remoteImageToolSchema + remoteImageSaveChunkSchema, remoteImageToolMaximumBytes, remoteImageToolReplySchema, type remoteImageToolSchema } from '../shared/remote-image-tool-contracts' /** One prompt's local MCP adapter. It never holds provider credentials or image bytes. */ @@ -31,6 +36,8 @@ export class AgentImageToolMcp { const path = `/${randomUUID()}/mcp` const schema = z.toJSONSchema(imageToolInputSchema, { target: 'draft-7', io: 'input' }) Reflect.deleteProperty(schema, '$schema') + const saveSchema = z.toJSONSchema(imageSaveToolInputSchema, { target: 'draft-7', io: 'input' }) + Reflect.deleteProperty(saveSchema, '$schema') this.server = createServer((request, response) => { void (async () => { if (this.closed || request.url !== path) { response.writeHead(404).end(); return } @@ -50,6 +57,8 @@ export class AgentImageToolMcp { const graphEnabled = available?.content.some(part => part.type === 'text' && JSON.parse(part.text).available === true) return { tools: [ ...(this.descriptor.description ? [{ name: imageToolName, description: this.descriptor.description, inputSchema: schema as { type: 'object' } }] : []), + ...(this.descriptor.saveDescription ? [{ name: imageSaveToolName, description: this.descriptor.saveDescription, inputSchema: saveSchema as { type: 'object' }, + annotations: { readOnlyHint: false, destructiveHint: true } }] : []), ...(graphEnabled ? storyGraphTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: z.toJSONSchema(tool.inputSchema, { target: 'draft-7', io: 'input' }) as { type: 'object' }, annotations: { readOnlyHint: true, destructiveHint: false } })) : []) @@ -61,6 +70,14 @@ export class AgentImageToolMcp { try { return await this.call(call.params.arguments ?? {}, graphTool.name) } catch (error) { return { isError: true, content: [{ type: 'text', text: error instanceof Error ? error.message : 'Story Graph read failed' }] } } } + if (call.params.name === imageSaveToolName && this.descriptor.saveDescription) { + try { + const saved = await this.saveImage(imageSaveToolInputSchema.parse(call.params.arguments ?? {})) + return { content: [{ type: 'text', text: JSON.stringify(saved) }] } + } catch (error) { + return { isError: true, content: [{ type: 'text', text: error instanceof Error ? error.message : 'Saving image failed' }] } + } + } if (call.params.name !== imageToolName || !this.descriptor.description) throw new Error('Unknown image tool') const input = imageToolInputSchema.parse(call.params.arguments) // Repeated delivery of an MCP request returns the same outcome, including unknown delivery. @@ -99,8 +116,42 @@ export class AgentImageToolMcp { const reply = remoteImageToolReplySchema.parse(decodeRemoteImageToolMessage(payload)) const pending = this.pending.get(reply.callId) if (!pending) return - if (reply.error || (!reply.result && !reply.storyGraphResult)) pending.reject(new Error(reply.error ?? 'Tool response has no result')) - else pending.resolve({ content: [{ type: 'text', text: JSON.stringify(reply.storyGraphResult ?? reply.result) }] }) + if (reply.error || (!reply.result && !reply.storyGraphResult && !reply.imageChunk)) pending.reject(new Error(reply.error ?? 'Tool response has no result')) + else pending.resolve({ content: [{ type: 'text', text: JSON.stringify(reply.imageChunk ?? reply.storyGraphResult ?? reply.result) }] }) + } + + /** Pulls the encoded image from Desktop in bounded chunks and writes it on this host. */ + private async saveImage(input: ImageSaveToolInput): Promise { + const target = input.path + if (!isAbsolute(target)) throw new Error('path must be an absolute file path') + const mimeType = imageSaveMimeTypeForPath(target) + if (!mimeType) throw new Error('path must end with .png, .jpg, .jpeg or .webp') + const chunks: Buffer[] = [] + let offset = 0 + let totalBytes: number | undefined + do { + const result = await this.call({ artifactId: input.artifactId, mimeType, offset }, 'save_image_read') + const part = result.content[0] + if (part?.type !== 'text') throw new Error('Image chunk response is invalid') + const chunk = remoteImageSaveChunkSchema.parse(JSON.parse(part.text)) + if (chunk.offset !== offset || chunk.mimeType !== mimeType || (totalBytes !== undefined && chunk.totalBytes !== totalBytes)) { + throw new Error('Image chunk response is inconsistent') + } + totalBytes = chunk.totalBytes + const bytes = Buffer.from(chunk.data, 'base64') + if (bytes.byteLength === 0 || offset + bytes.byteLength > totalBytes) throw new Error('Image chunk response is inconsistent') + chunks.push(bytes) + offset += bytes.byteLength + } while (offset < totalBytes) + const data = Buffer.concat(chunks) + await mkdir(dirname(target), { recursive: true }) + try { + await writeFile(target, data, { flag: input.overwrite ? 'w' : 'wx' }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') throw new Error('File already exists; choose another path or set overwrite=true', { cause: error }) + throw error + } + return { artifactId: input.artifactId, path: target, mimeType, byteSize: data.byteLength } } close(): void { @@ -112,7 +163,7 @@ export class AgentImageToolMcp { this.server?.close() } - private async call(input: Record, name?: StoryGraphToolName | 'story_graph_list'): Promise { + private async call(input: Record, name?: StoryGraphToolName | 'story_graph_list' | 'save_image_read'): Promise { if (this.closed) throw new Error('Image tool prompt has ended') const callId = randomUUID() let timer: NodeJS.Timeout | undefined diff --git a/src/agent-daemon/runtime-acp-backend.ts b/src/agent-daemon/runtime-acp-backend.ts index df85209a..85fe7d8b 100644 --- a/src/agent-daemon/runtime-acp-backend.ts +++ b/src/agent-daemon/runtime-acp-backend.ts @@ -1173,7 +1173,7 @@ export class RuntimeAcpBackend { } } } - if (preparation.imageTool && ((preparation.imageTool.description && preparation.workMode !== 'execute') || !this.#options.blobSink)) { + if (preparation.imageTool && (((preparation.imageTool.description || preparation.imageTool.saveDescription) && preparation.workMode !== 'execute') || !this.#options.blobSink)) { throw new RuntimeAcpBackendError('Image tools require Execute and a desktop transport', 'identity') } if (requestedModelBridgePolicy !== undefined) { diff --git a/src/main/agent/deepseek-harness-runtime.ts b/src/main/agent/deepseek-harness-runtime.ts index 6248d329..d2149992 100644 --- a/src/main/agent/deepseek-harness-runtime.ts +++ b/src/main/agent/deepseek-harness-runtime.ts @@ -358,11 +358,15 @@ function proxyToolInputSchema( delete scope.anyOf return dshCompatibleMainInputSchema(schema) } - return isMainWebTool(tool) || tool.name === 'generate_image' + return isMainWebTool(tool) || isMainImageTool(tool.name) ? dshCompatibleMainInputSchema(tool.inputSchema) : tool.inputSchema } +function isMainImageTool(name: string): boolean { + return name === 'generate_image' || name === 'save_image' +} + function boundedProxyToolCatalog( tools: Awaited< ReturnType @@ -376,7 +380,7 @@ function boundedProxyToolCatalog( const catalog = tools.filter( (tool) => isMainWebTool(tool) || isStoryGraphTool(tool.name) || - (workMode === 'execute' && (tool.source === 'mcp' || tool.name === 'generate_image')) + (workMode === 'execute' && (tool.source === 'mcp' || isMainImageTool(tool.name))) ) if (catalog.length > MAX_MCP_PROXY_TOOLS) { throw new Error( @@ -864,7 +868,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime { ) const catalog = boundedProxyToolCatalog(tools, context.workMode) this.proxyToolCatalogs.set(params.sessionId, tools.filter( - (tool) => tool.source === 'mcp' || isMainWebTool(tool) || isStoryGraphTool(tool.name) || tool.name === 'generate_image' + (tool) => tool.source === 'mcp' || isMainWebTool(tool) || isStoryGraphTool(tool.name) || isMainImageTool(tool.name) )) return { tools: catalog } } @@ -905,7 +909,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime { const tool = tools.find( (candidate) => candidate.name === name && - (candidate.source === 'mcp' || candidate.name === 'generate_image' || isStoryGraphTool(candidate.name) || + (candidate.source === 'mcp' || isMainImageTool(candidate.name) || isStoryGraphTool(candidate.name) || isMainWebTool(candidate)) ) if (!tool) { diff --git a/src/main/agent/image-generation-service.test.ts b/src/main/agent/image-generation-service.test.ts index 5decce14..a4ded2e2 100644 --- a/src/main/agent/image-generation-service.test.ts +++ b/src/main/agent/image-generation-service.test.ts @@ -1,4 +1,7 @@ import { randomUUID } from 'node:crypto' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' @@ -8,9 +11,14 @@ import { imageOperationSchema, type ImageOperation } from '../../shared/image-ge import { KnowledgeMcpGateway } from './knowledge-mcp-gateway' import { ModelToolProvider } from './model-tool-provider' -vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64') }) } })) +vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64'), toJPEG: () => Buffer.from('jpeg-bytes') }) } })) const png = 'iVBORw0KGgo=' const cleanups: (() => void | Promise)[] = [] +async function tempDir(): Promise { + const dir = await mkdtemp(join(tmpdir(), 'goodbuddy-save-image-')) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + return dir +} afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup() }) function setup(fetcher = vi.fn(async () => Response.json({ data: [{ b64_json: png }] })), observers: Pick = {}) { @@ -222,10 +230,55 @@ describe('Main conversation image service', () => { const client = new Client({ name: 'image-test', version: '1' }) cleanups.push(() => client.close()) await client.connect(new StreamableHTTPClientTransport(new URL(gateway.getEndpoint()!), { requestInit: { headers: { Authorization: `Bearer ${token}` } } })) - expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image']) + expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image', 'save_image']) const result = await client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } }) expect(JSON.stringify(result)).toContain('completed') expect(h.fetcher).toHaveBeenCalledOnce() + const operation = JSON.parse((result.content as { text: string }[])[0]!.text) as ImageOperation + const target = join(await tempDir(), 'mcp.png') + const saved = await client.callTool({ name: 'save_image', arguments: { artifactId: operation.artifactIds[0], path: target } }) + expect(saved.isError).toBeFalsy() + expect((await readFile(target)).toString('base64')).toBe(png) gateway.revoke(token) }) + + it('saves conversation images to local files with conversion, overwrite protection and scoping', async () => { + const h = setup() + const created = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'save') + const artifactId = created.artifactIds[0]! + const binding = h.service.bind(h.context) + const dir = await tempDir() + const pngPath = join(dir, 'nested', 'out.png') + expect(await binding.save!({ artifactId, path: pngPath })).toMatchObject({ path: pngPath, mimeType: 'image/png' }) + expect((await readFile(pngPath)).toString('base64')).toBe(png) + await expect(binding.save!({ artifactId, path: pngPath })).rejects.toThrow('already exists') + await expect(binding.save!({ artifactId, path: pngPath, overwrite: true })).resolves.toMatchObject({ path: pngPath }) + const jpgPath = join(dir, 'out.jpg') + expect(await binding.save!({ artifactId, path: jpgPath })).toMatchObject({ mimeType: 'image/jpeg' }) + expect((await readFile(jpgPath)).toString()).toBe('jpeg-bytes') + await expect(binding.save!({ artifactId, path: join(dir, 'out.webp') })).rejects.toThrow('WebP') + await expect(binding.save!({ artifactId, path: join(dir, 'out.gif') })).rejects.toThrow('.png') + await expect(binding.save!({ artifactId, path: 'relative.png' })).rejects.toThrow('absolute') + await expect(binding.save!({ artifactId: randomUUID(), path: join(dir, 'x.png') })).rejects.toThrow('not available') + await expect(h.service.bind(h.context, () => 'ask').save!({ artifactId, path: join(dir, 'ask.png') })).rejects.toThrow('Execute') + const provider = new ModelToolProvider(process.cwd()) + cleanups.push(() => provider.dispose()) + const context = { conversationId: h.context.conversationId, workMode: 'execute' as const, imageToolBinding: binding } + const native = await provider.callTool('save_image', { artifactId, path: join(dir, 'native.png') }, new AbortController().signal, context) + expect(JSON.stringify(native)).toContain('native.png') + await expect(provider.callTool('save_image', { artifactId, path: join(dir, 'ask2.png') }, new AbortController().signal, { ...context, workMode: 'ask' })).rejects.toThrow() + expect(h.fetcher).toHaveBeenCalledOnce() + }) + + it('offers save_image only when the conversation has images or can generate them', async () => { + const h = setup() + const binding = h.service.bind(h.context) + expect(await binding.describeSave!()).toMatch(/none yet/) + h.profile.allowConversationInvocation = false + expect(await binding.describeSave!()).toBeUndefined() + const [uploadId] = h.service.persistUploads(h.context, [{ name: 'upload.png', mediaType: 'image/png', data: png }]) + // IDs are listed so the model can save images even when it cannot generate. + expect(await binding.describeSave!()).toContain(uploadId) + expect(await h.service.bind(h.context, () => 'ask').describeSave!()).toBeUndefined() + }) }) diff --git a/src/main/agent/image-generation-service.ts b/src/main/agent/image-generation-service.ts index cde81885..b888e361 100644 --- a/src/main/agent/image-generation-service.ts +++ b/src/main/agent/image-generation-service.ts @@ -1,7 +1,9 @@ import { randomUUID } from 'node:crypto' +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname, isAbsolute } from 'node:path' import { - imageToolDescription, imageToolInputSchema, - type ImageOperation, type ImageRequestContext, type ImageToolInput + imageSaveMaximumBytes, imageSaveMimeTypeForPath, imageSaveToolDescription, imageSaveToolInputSchema, imageToolDescription, imageToolInputSchema, + type ImageOperation, type ImageRequestContext, type ImageSaveResult, type ImageSaveToolInput, type ImageToolInput } from '../../shared/image-generation-contracts' import type { AssistantDatabase } from '../assistant/assistant-database' import { ModelAgentRuntime, type ModelRuntimeOptions } from './model-runtime' @@ -89,10 +91,81 @@ export class ImageGenerationService { return active.operation } return this.getOperation(bound.conversationId, operation.id) + }, + describeSave: async () => { + if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') return undefined + const conversation = this.options.database.getConversation(bound.conversationId) + // The model can only save IDs it has seen, so list them even when generate_image is unavailable. + const images = conversation.messages.flatMap(message => [ + ...(message.imageSourceArtifactIds ?? []).map(id => ({ artifactId: id, kind: 'upload' })), + ...(message.imageOperations ?? []).flatMap(operation => operation.artifactIds.map(id => ({ artifactId: id, kind: 'generated' }))) + ]).slice(-32) + if (!images.length) { + const settings = await this.options.getSettings() + if (!settings.modelProfiles.some(profile => profile.protocol === 'openai-images-generations' && profile.allowConversationInvocation === true)) return undefined + } + return `${imageSaveToolDescription}\nConversation images (oldest first): ${images.length ? JSON.stringify(images) : 'none yet; images returned by generate_image can be saved'}` + }, + save: async (input, signal) => { + if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') throw new Error('Saving images requires Execute mode') + return this.saveImage(bound.conversationId, imageSaveToolInputSchema.parse(input), signal) + }, + readForSave: async (artifactId, mimeType) => { + if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') throw new Error('Saving images requires Execute mode') + return this.readImageForSave(bound.conversationId, artifactId, mimeType) } } } + /** Writes a conversation-visible image artifact to an absolute local path. */ + async saveImage(conversationId: string, input: ImageSaveToolInput, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + const target = input.path + if (!isAbsolute(target)) throw new Error('path must be an absolute file path') + const targetMimeType = imageSaveMimeTypeForPath(target) + if (!targetMimeType) throw new Error('path must end with .png, .jpg, .jpeg or .webp') + const bytes = await this.readImageForSave(conversationId, input.artifactId, targetMimeType) + signal?.throwIfAborted() + await mkdir(dirname(target), { recursive: true }) + try { + await writeFile(target, bytes, { flag: input.overwrite ? 'w' : 'wx' }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') throw new Error('File already exists; choose another path or set overwrite=true', { cause: error }) + throw error + } + return { artifactId: input.artifactId, path: target, mimeType: targetMimeType, byteSize: bytes.byteLength } + } + + /** Returns a conversation-visible image encoded as the requested format; used for local and remote saves. */ + async readImageForSave(conversationId: string, artifactId: string, targetMimeType: ImageSaveResult['mimeType']): Promise { + const conversation = this.options.database.getConversation(conversationId) + const allowed = new Set(conversation.messages.flatMap(message => [ + ...(message.artifactIds ?? []), ...(message.imageSourceArtifactIds ?? []), + ...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? []) + ])) + if (!allowed.has(artifactId)) throw new Error('Image is not available in this conversation') + let artifact: ReturnType + try { + artifact = this.options.database.getArtifact(artifactId) + } catch { + throw new Error('Image is not available in this conversation') + } + const match = artifact.kind === 'image' + ? artifact.content?.match(/^data:(image\/(?:png|jpeg|webp));base64,([\s\S]+)$/u) + : undefined + if (!match) throw new Error('Artifact is not a saved image') + let bytes: Buffer = Buffer.from(match[2]!, 'base64') + if (targetMimeType !== match[1]) { + if (targetMimeType === 'image/webp') throw new Error('Converting to WebP is not supported; use .png or .jpg') + const { nativeImage } = await import('electron') + const image = nativeImage.createFromBuffer(bytes) + if (image.isEmpty()) throw new Error('Image could not be decoded for conversion') + bytes = targetMimeType === 'image/png' ? image.toPNG() : image.toJPEG(92) + } + if (bytes.byteLength > imageSaveMaximumBytes) throw new Error('Image is too large to save') + return bytes + } + persistUploads(context: Pick, images: readonly AgentImage[]): string[] { return this.options.database.saveConversationImageSources({ ...context, images }) } diff --git a/src/main/agent/image-tool-binding.ts b/src/main/agent/image-tool-binding.ts index d494d2c1..168c15f9 100644 --- a/src/main/agent/image-tool-binding.ts +++ b/src/main/agent/image-tool-binding.ts @@ -1,10 +1,20 @@ import { z } from 'zod' -import { imageToolInputSchema, imageToolName, type ImageOperation, type ImageRequestContext } from '../../shared/image-generation-contracts' +import { + imageSaveToolDescription, imageSaveToolInputSchema, imageSaveToolName, + imageToolInputSchema, imageToolName, + type ImageOperation, type ImageRequestContext, type ImageSaveResult +} from '../../shared/image-generation-contracts' export type ImageToolBinding = { context: Readonly describe(): Promise call(input: unknown, callId: string, waitSignal?: AbortSignal): Promise + /** Writes an existing conversation image to a local file. Absent for bindings without local file access. */ + save?(input: unknown, signal?: AbortSignal): Promise + /** Returns image bytes in the requested format so a remote Agent can write them on its own host. */ + readForSave?(artifactId: string, mimeType: ImageSaveResult['mimeType']): Promise + /** save_image description with this conversation's image IDs; undefined when saving is not meaningful. */ + describeSave?(): Promise } export async function imageToolDefinition(binding: ImageToolBinding | undefined) { @@ -14,3 +24,12 @@ export async function imageToolDefinition(binding: ImageToolBinding | undefined) Reflect.deleteProperty(inputSchema, '$schema') return { name: imageToolName, displayName: 'Generate or edit image', description, inputSchema, source: 'builtin' as const } } + +export async function imageSaveToolDefinition(binding: ImageToolBinding | undefined) { + if (!binding?.save || binding.context.workMode !== 'execute') return undefined + const description = binding.describeSave ? await binding.describeSave() : imageSaveToolDescription + if (!description) return undefined + const inputSchema = z.toJSONSchema(imageSaveToolInputSchema, { target: 'draft-7', io: 'input' }) + Reflect.deleteProperty(inputSchema, '$schema') + return { name: imageSaveToolName, displayName: 'Save image to file', description, inputSchema, source: 'builtin' as const } +} diff --git a/src/main/agent/knowledge-mcp-gateway.ts b/src/main/agent/knowledge-mcp-gateway.ts index 5a54dcc4..848b0c5b 100644 --- a/src/main/agent/knowledge-mcp-gateway.ts +++ b/src/main/agent/knowledge-mcp-gateway.ts @@ -118,6 +118,7 @@ export { export type GoodBuddyBuiltinToolName = | 'generate_image' + | 'save_image' | ScopedDataToolName | BrowserToolName @@ -834,6 +835,7 @@ export class KnowledgeMcpGateway { return [ ...(capability.storyGraph ? storyGraphToolNames : []), ...(capability.imageToolBinding ? ['generate_image' as const] : []), + ...(capability.imageToolBinding?.save ? ['save_image' as const] : []), ...(capability.libraryIds.length > 0 ? knowledgeToolNames : []), @@ -1769,6 +1771,7 @@ export class KnowledgeMcpGateway { ) const capability = this.getCapability(token) const imageTool = await imageToolDefinition(capability.imageToolBinding) + const imageSaveTool = await imageSaveToolDefinition(capability.imageToolBinding) const browserTools = capability.browserConversationId && capability.browserTabId ? new BrowserModelTools({ service: this.browserService!, @@ -1796,6 +1799,7 @@ export class KnowledgeMcpGateway { return { tools: [ ...(imageTool ? [{ name: imageTool.name, description: imageTool.description, inputSchema: imageTool.inputSchema as Tool['inputSchema'], annotations: { readOnlyHint: false, destructiveHint: false } }] : []), + ...(imageSaveTool ? [{ name: imageSaveTool.name, title: imageSaveTool.displayName, description: imageSaveTool.description, inputSchema: imageSaveTool.inputSchema as Tool['inputSchema'], annotations: { readOnlyHint: false, destructiveHint: true } }] : []), ...scopedTools, ...browserDefinitions, ...[...customBindings.values()].map( @@ -1828,6 +1832,16 @@ export class KnowledgeMcpGateway { const operation = await capability.imageToolBinding.call(input, `${session.registryKey}:${extra.requestId}`, AbortSignal.any([extra.signal, capability.signal])) return { content: [{ type: 'text' as const, text: JSON.stringify(operation) }] } } + if (name === 'save_image') { + const capability = this.getCapability(token) + if (!capability.imageToolBinding?.save) throw new Error('Image save capability is unavailable') + try { + const saved = await capability.imageToolBinding.save(input, AbortSignal.any([extra.signal, capability.signal])) + return { content: [{ type: 'text' as const, text: JSON.stringify(saved) }] } + } catch (error) { + return { isError: true, content: [{ type: 'text' as const, text: error instanceof Error ? error.message : 'Saving image failed' }] } + } + } if (availableTools.has(name as ScopedDataToolName)) { const definition = scopedDataToolByName.get( name as ScopedDataToolName @@ -2123,4 +2137,4 @@ export class KnowledgeMcpGateway { } } } -import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding' +import { imageSaveToolDefinition, imageToolDefinition, type ImageToolBinding } from './image-tool-binding' diff --git a/src/main/agent/model-tool-provider.ts b/src/main/agent/model-tool-provider.ts index 5dc8e0ec..f7fff575 100644 --- a/src/main/agent/model-tool-provider.ts +++ b/src/main/agent/model-tool-provider.ts @@ -1257,6 +1257,7 @@ export class ModelToolProvider implements ModelToolProviderLike { ? await this.knowledgeGateway.isStoryGraphAvailable(context.knowledgeCapabilityToken) : false const scopedTools = grantedTools.filter(tool => !tool.name.startsWith('story_graph_') || storyGraphAvailable) const imageTool = context.workMode === 'execute' ? await imageToolDefinition(context.imageToolBinding) : undefined + const imageSaveTool = context.workMode === 'execute' ? await imageSaveToolDefinition(context.imageToolBinding) : undefined const webTools = this.webSearchEnabled ? this.getWebSearchDefinitions() : [] @@ -1280,6 +1281,7 @@ export class ModelToolProvider implements ModelToolProviderLike { return [ ...workspaceTools, ...(imageTool ? [imageTool] : []), + ...(imageSaveTool ? [imageSaveTool] : []), ...processTools, ...subagentTools, ...outputTools, @@ -1357,6 +1359,18 @@ export class ModelToolProvider implements ModelToolProviderLike { allowPermanent: false } } + if (tool.name === 'save_image') { + return { + scopeKey: 'model:builtin:save_image', + title: '允许将图片保存到本地文件？', + description: path + ? `会使用当前用户权限写入：${path}` + : '会使用当前用户权限将会话图片写入本地文件。', + toolName: tool.displayName, + argumentSummary, + allowPermanent: false + } + } if (tool.name === 'output_read') { return { scopeKey: 'model:builtin:output_read', @@ -1430,6 +1444,10 @@ export class ModelToolProvider implements ModelToolProviderLike { if (!context.imageToolBinding || !context.toolCallId) throw new Error('Image tool request binding is unavailable') return createTextToolResult(JSON.stringify(await context.imageToolBinding.call(argumentsValue, context.toolCallId, signal))) } + if (name === 'save_image') { + if (!context.imageToolBinding?.save) throw new Error('Image save capability is unavailable') + return createTextToolResult(JSON.stringify(await context.imageToolBinding.save(argumentsValue, signal))) + } if (name === 'knowledge_list') { if ( !this.knowledgeGateway || @@ -2017,4 +2035,4 @@ function emitDirectModelSubagentUsage( ) }) } -import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding' +import { imageSaveToolDefinition, imageToolDefinition, type ImageToolBinding } from './image-tool-binding' diff --git a/src/main/ipc.ts b/src/main/ipc.ts index 4dba815e..ed1fe3d7 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -3740,6 +3740,9 @@ export function registerIpcHandlers( }, () => normalizeInteractiveWorkMode(assistantDatabase.getConversation(enrichedRequest.conversationId).workMode)) : undefined const imageToolAvailable = Boolean(await imageToolBinding?.describe()) + // Remote Agents write the file on the remote host; Desktop streams the bytes on demand. + const imageSaveAvailable = Boolean(imageToolBinding?.save && normalizedWorkMode === 'execute' && + await imageToolBinding.describeSave?.()) const scopedCapability = await grantScopedDataCapability({ storyGraph: selectedRuntimeTarget && applicationSettings?.heartbeatEnabled ? { runtimeTarget: selectedRuntimeTarget, projectId: enrichedRequest.projectId } : undefined, @@ -3773,6 +3776,7 @@ export function registerIpcHandlers( const knowledgeCapabilityToken = scopedCapability.token const availableTools = [ ...(imageToolAvailable ? ['generate_image'] : []), + ...(imageSaveAvailable ? ['save_image'] : []), ...(webSearchEnabled ? ['web_search', 'web_fetch'] : []), ...(!agentRuntimeSelected && !imageGeneration ? ['workspace_rg', 'workspace_read_text', 'output_read', 'subagent_delegate'] diff --git a/src/main/remote-agent/main-image-tool-session.ts b/src/main/remote-agent/main-image-tool-session.ts index 0c38e6fa..fced876c 100644 --- a/src/main/remote-agent/main-image-tool-session.ts +++ b/src/main/remote-agent/main-image-tool-session.ts @@ -2,13 +2,15 @@ import type { ImageToolBinding } from '../agent/image-tool-binding' import type { RuntimeProtocolBinaryChannel } from './protocol-remote-runtime-channel' import { decodeRemoteImageToolMessage, encodeRemoteImageToolMessage, - remoteImageToolCallSchema, remoteImageToolReplySchema + remoteImageSaveChunkBytes, remoteImageToolCallSchema, remoteImageToolReplySchema } from '../../shared/remote-image-tool-contracts' export class MainImageToolSession { private readonly wait = new AbortController() private readonly unsubscribe: () => void private readonly removeAbort: () => void + /** Encoded bytes of the image currently being streamed to the Agent; one save at a time per prompt. */ + private saveCache?: { key: string; bytes: Buffer } constructor( private readonly channel: RuntimeProtocolBinaryChannel, @@ -28,6 +30,7 @@ export class MainImageToolSession { if (this.wait.signal.aborted) return // This aborts binding.call's wait, never the Main-owned image operation. this.wait.abort() + this.saveCache = undefined this.removeAbort() this.unsubscribe() this.channel.close() @@ -46,7 +49,22 @@ export class MainImageToolSession { let reply: ReturnType try { this.wait.signal.throwIfAborted() - if ('name' in call) { + if ('name' in call && call.name === 'save_image_read') { + if (!this.binding?.readForSave || this.binding.context.workMode !== 'execute') throw new Error('Saving images is unavailable in Ask mode') + const { artifactId, mimeType, offset } = call.input + const cacheKey = `${artifactId}:${mimeType}` + let bytes = this.saveCache?.key === cacheKey ? this.saveCache.bytes : undefined + if (!bytes || offset === 0) { + bytes = await this.binding.readForSave(artifactId, mimeType) + this.saveCache = { key: cacheKey, bytes } + } + if (offset >= bytes.byteLength) throw new Error('Image chunk offset is out of range') + const end = Math.min(offset + remoteImageSaveChunkBytes, bytes.byteLength) + if (end === bytes.byteLength) this.saveCache = undefined + reply = remoteImageToolReplySchema.parse({ callId: call.callId, imageChunk: { + mimeType, totalBytes: bytes.byteLength, offset, data: bytes.subarray(offset, end).toString('base64') + } }) + } else if ('name' in call) { if (!this.storyGraph) throw new Error('Story Graph capability unavailable') const storyGraphResult = call.name === 'story_graph_list' ? { available: await this.storyGraph.available() } diff --git a/src/main/remote-agent/protocol-remote-runtime-channel.ts b/src/main/remote-agent/protocol-remote-runtime-channel.ts index 0d659cfe..8e17af80 100644 --- a/src/main/remote-agent/protocol-remote-runtime-channel.ts +++ b/src/main/remote-agent/protocol-remote-runtime-channel.ts @@ -597,13 +597,15 @@ export class ProtocolRemoteRuntimeChannel this.#imageTool = undefined const description = preparation.workMode === 'execute' ? await imageToolBinding?.describe() : undefined const storyGraph = await storyGraphBinding?.available() + const saveDescription = preparation.workMode === 'execute' && imageToolBinding?.readForSave + ? await imageToolBinding.describeSave?.() : undefined waitSignal?.throwIfAborted() let imageTool: z.infer['imageTool'] - if ((description && imageToolBinding) || storyGraph) { + if ((description && imageToolBinding) || storyGraph || saveDescription) { this.#assertCurrent() const binary = this.#state.client.allocateBinaryChannel?.({ kind: 'blob' }) if (!binary) throw new Error('Remote image tool transport is unavailable') - imageTool = { channelId: binary.channelId, channelEpoch: binary.channelEpoch, ...(description ? { description } : {}), ...(storyGraph ? { storyGraph: true } : {}) } + imageTool = { channelId: binary.channelId, channelEpoch: binary.channelEpoch, ...(description ? { description } : {}), ...(storyGraph ? { storyGraph: true } : {}), ...(saveDescription ? { saveDescription } : {}) } this.#imageTool = new MainImageToolSession(binary, imageToolBinding, waitSignal, storyGraphBinding) } try { diff --git a/src/renderer/src/McpSettingsSection.test.tsx b/src/renderer/src/McpSettingsSection.test.tsx index b2e876f1..b5f07851 100644 --- a/src/renderer/src/McpSettingsSection.test.tsx +++ b/src/renderer/src/McpSettingsSection.test.tsx @@ -148,6 +148,22 @@ it('preserves the Obsidian draft after failures and folder-picker cancellation', }) }) +it('lists the image generation and save tools on the managed image card', async () => { + const getSnapshot = vi.fn().mockResolvedValue({ mcpServers: [], skills: [] }) + vi.stubGlobal('goodbuddy', { capabilities: { getSnapshot } }) + render() + await waitFor(() => expect(getSnapshot).toHaveBeenCalledOnce()) + const name = i18n.t('integrations:conversationImages.title') + const toggle = screen.getByRole('button', { name: i18n.t('integrations:mcp.builtin.expandServer', { name }) }) + expect(toggle).toHaveTextContent(i18n.t('integrations:mcp.builtin.toolCount', { count: 2 })) + expect(screen.queryByText('save_image')).not.toBeInTheDocument() + fireEvent.click(toggle) + const tools = screen.getByRole('region', { name: i18n.t('integrations:mcp.builtin.toolsAriaLabel', { name }) }) + expect(tools).toHaveTextContent('generate_image') + expect(tools).toHaveTextContent('save_image') + expect(tools).toHaveTextContent(i18n.t('integrations:conversationImages.tools.save_image')) +}) + it('removes the redundant MCP title and places the custom action in content', async () => { const getSnapshot = vi.fn().mockResolvedValue({ mcpServers: [], skills: [] }) vi.stubGlobal('goodbuddy', { capabilities: { getSnapshot } }) diff --git a/src/renderer/src/McpSettingsSection.tsx b/src/renderer/src/McpSettingsSection.tsx index b152506c..d3c03e58 100644 --- a/src/renderer/src/McpSettingsSection.tsx +++ b/src/renderer/src/McpSettingsSection.tsx @@ -16,7 +16,10 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { createPortal } from 'react-dom' import { useTranslation } from 'react-i18next' import { builtinMcpServers } from '../../shared/builtin-mcp-servers' -import { builtinModelToolGroups } from '../../shared/builtin-model-tools' +import { + builtinModelToolGroups, + builtinModelTools +} from '../../shared/builtin-model-tools' import type { BuiltinMcpServerId, CapabilityDiagnosticReport, @@ -40,6 +43,8 @@ const configurableMcpTargets: RuntimeTarget[] = [ 'continue', 'deepseek-harness' ] +const imageTools = builtinModelTools.filter((tool) => tool.group === 'image') +const imageExpansionId = 'builtin:image-generation' type McpSettingsTab = | 'builtin' | 'custom' @@ -647,6 +652,65 @@ export function McpSettingsSection({ > {t('conversationImages.openSettings')} + + {expandedItemIds.has(imageExpansionId) && ( +
+
+
+ {t('mcp.builtin.tools')} + + {t('mcp.profiles.count', { count: imageTools.length })} + +
+
    + {imageTools.map((tool) => ( +
  • +
    + {tool.name} + + {tool.access === 'write' + ? t('mcp.builtin.write') + : t('mcp.builtin.readOnly')} + +
    +

    {t(`conversationImages.tools.${tool.name}`)}

    +
    • + ))} +
+
+
+ )} {builtinMcpServers.map((server) => { const expansionId = `builtin:${server.id}` diff --git a/src/renderer/src/i18n/locales/en-US/integrations.ts b/src/renderer/src/i18n/locales/en-US/integrations.ts index b9112a12..6bb2b1a3 100644 --- a/src/renderer/src/i18n/locales/en-US/integrations.ts +++ b/src/renderer/src/i18n/locales/en-US/integrations.ts @@ -7,7 +7,11 @@ export const integrations = { managed: 'Builtin · Automatically managed', description: 'Automatically managed by the image models’ “Allow AI to invoke in conversations” setting.', runtimeScope: 'Assignments include applicable remote Runtimes through GoodBuddy Agent. Generation requires Execute and a chat model with tool calling; assignments do not indicate runtime readiness.', - openSettings: 'Go to image model settings' + openSettings: 'Go to image model settings', + tools: { + generate_image: 'Generate a new image with an image model allowed in conversations, or edit images from this conversation. Results are saved in the conversation.', + save_image: 'Save a generated or uploaded image from this conversation to a PNG, JPEG or WebP file. Requires Execute mode; in remote projects the file is written on the remote host.' + } }, channels: { tabs: { diff --git a/src/renderer/src/i18n/locales/zh-CN/integrations.ts b/src/renderer/src/i18n/locales/zh-CN/integrations.ts index 6fe66cac..64b67183 100644 --- a/src/renderer/src/i18n/locales/zh-CN/integrations.ts +++ b/src/renderer/src/i18n/locales/zh-CN/integrations.ts @@ -4,7 +4,11 @@ export const integrations = { managed: '内置 · 自动管理', description: '由图片模型的「允许 AI 在会话中调用」设置自动管理。', runtimeScope: '分配包含 GoodBuddy Agent 远程路径上的适用 Runtime。生成需要 Execute 模式及支持工具调用的聊天模型；勾选不代表 Runtime 已就绪。', - openSettings: '前往图片模型设置' + openSettings: '前往图片模型设置', + tools: { + generate_image: '使用已允许会话调用的图片模型生成新图片，或基于会话中的图片进行编辑，结果保存在当前会话。', + save_image: '将当前会话中已生成或已上传的图片保存为 PNG、JPEG 或 WebP 文件；需要 Execute 模式，远程项目中文件写入远程主机。' + } }, channels: { tabs: { diff --git a/src/shared/builtin-model-tools.test.ts b/src/shared/builtin-model-tools.test.ts index 2da7edba..bf164a80 100644 --- a/src/shared/builtin-model-tools.test.ts +++ b/src/shared/builtin-model-tools.test.ts @@ -25,9 +25,12 @@ describe('built-in model tool catalog', () => { expect.arrayContaining(directModelTools.map((tool) => tool.name)) ) expect(groupedNames).not.toContain('generate_image') - expect(builtinModelTools.find((tool) => tool.name === 'generate_image')).toMatchObject({ - group: 'image', access: 'write' - }) + expect(groupedNames).not.toContain('save_image') + expect( + builtinModelTools + .filter((tool) => tool.group === 'image') + .map((tool) => [tool.name, tool.access]) + ).toEqual([['generate_image', 'write'], ['save_image', 'write']]) for (const group of builtinModelToolGroups) { expect(group.tools.every((tool) => tool.group === group.id)).toBe(true) diff --git a/src/shared/builtin-model-tools.ts b/src/shared/builtin-model-tools.ts index 01feae80..a63bfcc3 100644 --- a/src/shared/builtin-model-tools.ts +++ b/src/shared/builtin-model-tools.ts @@ -14,6 +14,13 @@ export const builtinModelTools = [ access: 'write', group: 'image' }, + { + name: 'save_image', + displayName: 'Save image to file', + description: 'Save a generated or uploaded image from this conversation to a local PNG, JPEG or WebP file.', + access: 'write', + group: 'image' + }, { name: 'workspace_rg', displayName: '搜索工作区', diff --git a/src/shared/image-generation-contracts.ts b/src/shared/image-generation-contracts.ts index 8bfface3..a45361be 100644 --- a/src/shared/image-generation-contracts.ts +++ b/src/shared/image-generation-contracts.ts @@ -40,6 +40,37 @@ export type ImageRequestContext = { workMode: 'ask' | 'execute' } +export const imageSaveToolInputSchema = z.object({ + artifactId: z.string().uuid(), + path: z.string().trim().min(1).max(4_096), + overwrite: z.boolean().default(false) +}).strict() +export type ImageSaveToolInput = z.infer + +export type ImageSaveResult = { + artifactId: string + path: string + mimeType: 'image/png' | 'image/jpeg' | 'image/webp' + byteSize: number +} + +export const imageSaveMimeTypeSchema = z.enum(['image/png', 'image/jpeg', 'image/webp']) + +/** Target format from a save path extension; undefined when unsupported. Pure so the remote Agent can share it. */ +export function imageSaveMimeTypeForPath(path: string): ImageSaveResult['mimeType'] | undefined { + const extension = /\.([A-Za-z0-9]+)$/u.exec(path)?.[1]?.toLowerCase() + return extension === 'png' ? 'image/png' + : extension === 'jpg' || extension === 'jpeg' ? 'image/jpeg' + : extension === 'webp' ? 'image/webp' + : undefined +} + +/** Upper bound for bytes written by save_image, including chunked remote delivery. */ +export const imageSaveMaximumBytes = 48 * 1024 * 1024 + +export const imageSaveToolName = 'save_image' as const +export const imageSaveToolDescription = 'Save an existing conversation image (generated result or uploaded source) to a local file. Use an artifactId listed in this conversation; never invent IDs. path must be absolute and end with .png, .jpg, .jpeg or .webp; the image is converted when the extension differs from its stored format (conversion to .webp is not supported). Missing parent folders are created. Existing files are kept unless overwrite=true. The file is written on the machine where this tool runs (the remote host for remote projects).' + export const imageToolName = 'generate_image' as const export const imageToolDescriptionLimit = 16_000 export const imageToolDescription = 'Generate or edit an image using a configured image model. Use intent=create for a new unrelated image, or intent=edit with explicit sourceArtifactIds. Never invent profile or artifact IDs. Clarify ambiguous references. A failed edit must not be replaced by generation. Results contain operation and artifact references, not image bytes. Do not retry a paid request automatically. A running operation continues after this chat stops; inspect its conversation card for completion.' diff --git a/src/shared/remote-image-tool-contracts.ts b/src/shared/remote-image-tool-contracts.ts index 71e87a03..40a21a5f 100644 --- a/src/shared/remote-image-tool-contracts.ts +++ b/src/shared/remote-image-tool-contracts.ts @@ -1,14 +1,19 @@ import { z } from 'zod' import { agentIdentifierSchema, positiveAgentSequenceSchema } from './agent-protocol/contracts' -import { imageOperationSchema, imageToolDescriptionLimit, imageToolInputSchema } from './image-generation-contracts' +import { imageOperationSchema, imageSaveMaximumBytes, imageSaveMimeTypeSchema, imageToolDescriptionLimit, imageToolInputSchema } from './image-generation-contracts' export const remoteImageToolSchema = z.object({ channelId: agentIdentifierSchema, channelEpoch: positiveAgentSequenceSchema, description: z.string().min(1).max(imageToolDescriptionLimit).optional(), - storyGraph: z.boolean().optional() + storyGraph: z.boolean().optional(), + /** When present, the Agent exposes save_image with this description and writes files on its own host. */ + saveDescription: z.string().min(1).max(imageToolDescriptionLimit).optional() }).strict() +/** Raw bytes per save_image chunk; base64 plus JSON framing stays below remoteImageToolMaximumBytes. */ +export const remoteImageSaveChunkBytes = 128 * 1024 + export const remoteImageToolCallSchema = z.union([z.object({ callId: z.string().uuid(), input: imageToolInputSchema @@ -16,16 +21,32 @@ export const remoteImageToolCallSchema = z.union([z.object({ callId: z.string().uuid(), name: z.enum(['story_graph_list', 'story_graph_search', 'story_graph_get_context', 'story_graph_read_source']), input: z.record(z.string(), z.unknown()) +}).strict(), z.object({ + callId: z.string().uuid(), + name: z.literal('save_image_read'), + input: z.object({ + artifactId: z.string().uuid(), + mimeType: imageSaveMimeTypeSchema, + offset: z.number().int().min(0).max(imageSaveMaximumBytes) + }).strict() }).strict()]) +export const remoteImageSaveChunkSchema = z.object({ + mimeType: imageSaveMimeTypeSchema, + totalBytes: z.number().int().min(1).max(imageSaveMaximumBytes), + offset: z.number().int().min(0).max(imageSaveMaximumBytes), + data: z.string().max(Math.ceil(remoteImageSaveChunkBytes / 3) * 4) +}).strict() + export const remoteImageToolReplySchema = z.object({ callId: z.string().uuid(), result: imageOperationSchema.optional(), storyGraphResult: z.record(z.string(), z.unknown()).optional(), + imageChunk: remoteImageSaveChunkSchema.optional(), error: z.string().max(4_000).optional() }).strict() -// Bounded graph evidence or image-operation metadata. Image bytes never cross this channel. +// Bounded graph evidence, image-operation metadata, or one chunk of an image the model explicitly asked to save. export const remoteImageToolMaximumBytes = 256 * 1024 export function encodeRemoteImageToolMessage(value: unknown): Uint8Array {