feat: add save_image tool and list image MCP tools
Expose save_image across direct model, MCP gateway, DeepSeek Harness and remote Agents (written on the remote host via chunked reads), and show the image MCP tool list in settings.
This commit is contained in:
@@ -358,11 +358,15 @@ function proxyToolInputSchema(
|
||||
delete scope.anyOf
|
||||
return dshCompatibleMainInputSchema(schema)
|
||||
}
|
||||
return isMainWebTool(tool) || tool.name === 'generate_image'
|
||||
return isMainWebTool(tool) || isMainImageTool(tool.name)
|
||||
? dshCompatibleMainInputSchema(tool.inputSchema)
|
||||
: tool.inputSchema
|
||||
}
|
||||
|
||||
function isMainImageTool(name: string): boolean {
|
||||
return name === 'generate_image' || name === 'save_image'
|
||||
}
|
||||
|
||||
function boundedProxyToolCatalog(
|
||||
tools: Awaited<
|
||||
ReturnType<ModelToolProviderLike['listTools']>
|
||||
@@ -376,7 +380,7 @@ function boundedProxyToolCatalog(
|
||||
const catalog = tools.filter(
|
||||
(tool) =>
|
||||
isMainWebTool(tool) || isStoryGraphTool(tool.name) ||
|
||||
(workMode === 'execute' && (tool.source === 'mcp' || tool.name === 'generate_image'))
|
||||
(workMode === 'execute' && (tool.source === 'mcp' || isMainImageTool(tool.name)))
|
||||
)
|
||||
if (catalog.length > MAX_MCP_PROXY_TOOLS) {
|
||||
throw new Error(
|
||||
@@ -864,7 +868,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
)
|
||||
const catalog = boundedProxyToolCatalog(tools, context.workMode)
|
||||
this.proxyToolCatalogs.set(params.sessionId, tools.filter(
|
||||
(tool) => tool.source === 'mcp' || isMainWebTool(tool) || isStoryGraphTool(tool.name) || tool.name === 'generate_image'
|
||||
(tool) => tool.source === 'mcp' || isMainWebTool(tool) || isStoryGraphTool(tool.name) || isMainImageTool(tool.name)
|
||||
))
|
||||
return { tools: catalog }
|
||||
}
|
||||
@@ -905,7 +909,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
const tool = tools.find(
|
||||
(candidate) =>
|
||||
candidate.name === name &&
|
||||
(candidate.source === 'mcp' || candidate.name === 'generate_image' || isStoryGraphTool(candidate.name) ||
|
||||
(candidate.source === 'mcp' || isMainImageTool(candidate.name) || isStoryGraphTool(candidate.name) ||
|
||||
isMainWebTool(candidate))
|
||||
)
|
||||
if (!tool) {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
|
||||
@@ -8,9 +11,14 @@ import { imageOperationSchema, type ImageOperation } from '../../shared/image-ge
|
||||
import { KnowledgeMcpGateway } from './knowledge-mcp-gateway'
|
||||
import { ModelToolProvider } from './model-tool-provider'
|
||||
|
||||
vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64') }) } }))
|
||||
vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64'), toJPEG: () => Buffer.from('jpeg-bytes') }) } }))
|
||||
const png = 'iVBORw0KGgo='
|
||||
const cleanups: (() => void | Promise<void>)[] = []
|
||||
async function tempDir(): Promise<string> {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'goodbuddy-save-image-'))
|
||||
cleanups.push(() => rm(dir, { recursive: true, force: true }))
|
||||
return dir
|
||||
}
|
||||
afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup() })
|
||||
|
||||
function setup(fetcher = vi.fn<typeof fetch>(async () => Response.json({ data: [{ b64_json: png }] })), observers: Pick<ImageGenerationServiceOptions, 'onOperation' | 'onUsage' | 'onError'> = {}) {
|
||||
@@ -222,10 +230,55 @@ describe('Main conversation image service', () => {
|
||||
const client = new Client({ name: 'image-test', version: '1' })
|
||||
cleanups.push(() => client.close())
|
||||
await client.connect(new StreamableHTTPClientTransport(new URL(gateway.getEndpoint()!), { requestInit: { headers: { Authorization: `Bearer ${token}` } } }))
|
||||
expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image'])
|
||||
expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image', 'save_image'])
|
||||
const result = await client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } })
|
||||
expect(JSON.stringify(result)).toContain('completed')
|
||||
expect(h.fetcher).toHaveBeenCalledOnce()
|
||||
const operation = JSON.parse((result.content as { text: string }[])[0]!.text) as ImageOperation
|
||||
const target = join(await tempDir(), 'mcp.png')
|
||||
const saved = await client.callTool({ name: 'save_image', arguments: { artifactId: operation.artifactIds[0], path: target } })
|
||||
expect(saved.isError).toBeFalsy()
|
||||
expect((await readFile(target)).toString('base64')).toBe(png)
|
||||
gateway.revoke(token)
|
||||
})
|
||||
|
||||
it('saves conversation images to local files with conversion, overwrite protection and scoping', async () => {
|
||||
const h = setup()
|
||||
const created = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'save')
|
||||
const artifactId = created.artifactIds[0]!
|
||||
const binding = h.service.bind(h.context)
|
||||
const dir = await tempDir()
|
||||
const pngPath = join(dir, 'nested', 'out.png')
|
||||
expect(await binding.save!({ artifactId, path: pngPath })).toMatchObject({ path: pngPath, mimeType: 'image/png' })
|
||||
expect((await readFile(pngPath)).toString('base64')).toBe(png)
|
||||
await expect(binding.save!({ artifactId, path: pngPath })).rejects.toThrow('already exists')
|
||||
await expect(binding.save!({ artifactId, path: pngPath, overwrite: true })).resolves.toMatchObject({ path: pngPath })
|
||||
const jpgPath = join(dir, 'out.jpg')
|
||||
expect(await binding.save!({ artifactId, path: jpgPath })).toMatchObject({ mimeType: 'image/jpeg' })
|
||||
expect((await readFile(jpgPath)).toString()).toBe('jpeg-bytes')
|
||||
await expect(binding.save!({ artifactId, path: join(dir, 'out.webp') })).rejects.toThrow('WebP')
|
||||
await expect(binding.save!({ artifactId, path: join(dir, 'out.gif') })).rejects.toThrow('.png')
|
||||
await expect(binding.save!({ artifactId, path: 'relative.png' })).rejects.toThrow('absolute')
|
||||
await expect(binding.save!({ artifactId: randomUUID(), path: join(dir, 'x.png') })).rejects.toThrow('not available')
|
||||
await expect(h.service.bind(h.context, () => 'ask').save!({ artifactId, path: join(dir, 'ask.png') })).rejects.toThrow('Execute')
|
||||
const provider = new ModelToolProvider(process.cwd())
|
||||
cleanups.push(() => provider.dispose())
|
||||
const context = { conversationId: h.context.conversationId, workMode: 'execute' as const, imageToolBinding: binding }
|
||||
const native = await provider.callTool('save_image', { artifactId, path: join(dir, 'native.png') }, new AbortController().signal, context)
|
||||
expect(JSON.stringify(native)).toContain('native.png')
|
||||
await expect(provider.callTool('save_image', { artifactId, path: join(dir, 'ask2.png') }, new AbortController().signal, { ...context, workMode: 'ask' })).rejects.toThrow()
|
||||
expect(h.fetcher).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('offers save_image only when the conversation has images or can generate them', async () => {
|
||||
const h = setup()
|
||||
const binding = h.service.bind(h.context)
|
||||
expect(await binding.describeSave!()).toMatch(/none yet/)
|
||||
h.profile.allowConversationInvocation = false
|
||||
expect(await binding.describeSave!()).toBeUndefined()
|
||||
const [uploadId] = h.service.persistUploads(h.context, [{ name: 'upload.png', mediaType: 'image/png', data: png }])
|
||||
// IDs are listed so the model can save images even when it cannot generate.
|
||||
expect(await binding.describeSave!()).toContain(uploadId)
|
||||
expect(await h.service.bind(h.context, () => 'ask').describeSave!()).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { mkdir, writeFile } from 'node:fs/promises'
|
||||
import { dirname, isAbsolute } from 'node:path'
|
||||
import {
|
||||
imageToolDescription, imageToolInputSchema,
|
||||
type ImageOperation, type ImageRequestContext, type ImageToolInput
|
||||
imageSaveMaximumBytes, imageSaveMimeTypeForPath, imageSaveToolDescription, imageSaveToolInputSchema, imageToolDescription, imageToolInputSchema,
|
||||
type ImageOperation, type ImageRequestContext, type ImageSaveResult, type ImageSaveToolInput, type ImageToolInput
|
||||
} from '../../shared/image-generation-contracts'
|
||||
import type { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import { ModelAgentRuntime, type ModelRuntimeOptions } from './model-runtime'
|
||||
@@ -89,10 +91,81 @@ export class ImageGenerationService {
|
||||
return active.operation
|
||||
}
|
||||
return this.getOperation(bound.conversationId, operation.id)
|
||||
},
|
||||
describeSave: async () => {
|
||||
if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') return undefined
|
||||
const conversation = this.options.database.getConversation(bound.conversationId)
|
||||
// The model can only save IDs it has seen, so list them even when generate_image is unavailable.
|
||||
const images = conversation.messages.flatMap(message => [
|
||||
...(message.imageSourceArtifactIds ?? []).map(id => ({ artifactId: id, kind: 'upload' })),
|
||||
...(message.imageOperations ?? []).flatMap(operation => operation.artifactIds.map(id => ({ artifactId: id, kind: 'generated' })))
|
||||
]).slice(-32)
|
||||
if (!images.length) {
|
||||
const settings = await this.options.getSettings()
|
||||
if (!settings.modelProfiles.some(profile => profile.protocol === 'openai-images-generations' && profile.allowConversationInvocation === true)) return undefined
|
||||
}
|
||||
return `${imageSaveToolDescription}\nConversation images (oldest first): ${images.length ? JSON.stringify(images) : 'none yet; images returned by generate_image can be saved'}`
|
||||
},
|
||||
save: async (input, signal) => {
|
||||
if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') throw new Error('Saving images requires Execute mode')
|
||||
return this.saveImage(bound.conversationId, imageSaveToolInputSchema.parse(input), signal)
|
||||
},
|
||||
readForSave: async (artifactId, mimeType) => {
|
||||
if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') throw new Error('Saving images requires Execute mode')
|
||||
return this.readImageForSave(bound.conversationId, artifactId, mimeType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes a conversation-visible image artifact to an absolute local path. */
|
||||
async saveImage(conversationId: string, input: ImageSaveToolInput, signal?: AbortSignal): Promise<ImageSaveResult> {
|
||||
signal?.throwIfAborted()
|
||||
const target = input.path
|
||||
if (!isAbsolute(target)) throw new Error('path must be an absolute file path')
|
||||
const targetMimeType = imageSaveMimeTypeForPath(target)
|
||||
if (!targetMimeType) throw new Error('path must end with .png, .jpg, .jpeg or .webp')
|
||||
const bytes = await this.readImageForSave(conversationId, input.artifactId, targetMimeType)
|
||||
signal?.throwIfAborted()
|
||||
await mkdir(dirname(target), { recursive: true })
|
||||
try {
|
||||
await writeFile(target, bytes, { flag: input.overwrite ? 'w' : 'wx' })
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'EEXIST') throw new Error('File already exists; choose another path or set overwrite=true', { cause: error })
|
||||
throw error
|
||||
}
|
||||
return { artifactId: input.artifactId, path: target, mimeType: targetMimeType, byteSize: bytes.byteLength }
|
||||
}
|
||||
|
||||
/** Returns a conversation-visible image encoded as the requested format; used for local and remote saves. */
|
||||
async readImageForSave(conversationId: string, artifactId: string, targetMimeType: ImageSaveResult['mimeType']): Promise<Buffer> {
|
||||
const conversation = this.options.database.getConversation(conversationId)
|
||||
const allowed = new Set(conversation.messages.flatMap(message => [
|
||||
...(message.artifactIds ?? []), ...(message.imageSourceArtifactIds ?? []),
|
||||
...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? [])
|
||||
]))
|
||||
if (!allowed.has(artifactId)) throw new Error('Image is not available in this conversation')
|
||||
let artifact: ReturnType<AssistantDatabase['getArtifact']>
|
||||
try {
|
||||
artifact = this.options.database.getArtifact(artifactId)
|
||||
} catch {
|
||||
throw new Error('Image is not available in this conversation')
|
||||
}
|
||||
const match = artifact.kind === 'image'
|
||||
? artifact.content?.match(/^data:(image\/(?:png|jpeg|webp));base64,([\s\S]+)$/u)
|
||||
: undefined
|
||||
if (!match) throw new Error('Artifact is not a saved image')
|
||||
let bytes: Buffer = Buffer.from(match[2]!, 'base64')
|
||||
if (targetMimeType !== match[1]) {
|
||||
if (targetMimeType === 'image/webp') throw new Error('Converting to WebP is not supported; use .png or .jpg')
|
||||
const { nativeImage } = await import('electron')
|
||||
const image = nativeImage.createFromBuffer(bytes)
|
||||
if (image.isEmpty()) throw new Error('Image could not be decoded for conversion')
|
||||
bytes = targetMimeType === 'image/png' ? image.toPNG() : image.toJPEG(92)
|
||||
}
|
||||
if (bytes.byteLength > imageSaveMaximumBytes) throw new Error('Image is too large to save')
|
||||
return bytes
|
||||
}
|
||||
|
||||
persistUploads(context: Pick<ImageRequestContext, 'conversationId' | 'messageId'>, images: readonly AgentImage[]): string[] {
|
||||
return this.options.database.saveConversationImageSources({ ...context, images })
|
||||
}
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
import { z } from 'zod'
|
||||
import { imageToolInputSchema, imageToolName, type ImageOperation, type ImageRequestContext } from '../../shared/image-generation-contracts'
|
||||
import {
|
||||
imageSaveToolDescription, imageSaveToolInputSchema, imageSaveToolName,
|
||||
imageToolInputSchema, imageToolName,
|
||||
type ImageOperation, type ImageRequestContext, type ImageSaveResult
|
||||
} from '../../shared/image-generation-contracts'
|
||||
|
||||
export type ImageToolBinding = {
|
||||
context: Readonly<ImageRequestContext>
|
||||
describe(): Promise<string | undefined>
|
||||
call(input: unknown, callId: string, waitSignal?: AbortSignal): Promise<ImageOperation>
|
||||
/** Writes an existing conversation image to a local file. Absent for bindings without local file access. */
|
||||
save?(input: unknown, signal?: AbortSignal): Promise<ImageSaveResult>
|
||||
/** Returns image bytes in the requested format so a remote Agent can write them on its own host. */
|
||||
readForSave?(artifactId: string, mimeType: ImageSaveResult['mimeType']): Promise<Buffer>
|
||||
/** save_image description with this conversation's image IDs; undefined when saving is not meaningful. */
|
||||
describeSave?(): Promise<string | undefined>
|
||||
}
|
||||
|
||||
export async function imageToolDefinition(binding: ImageToolBinding | undefined) {
|
||||
@@ -14,3 +24,12 @@ export async function imageToolDefinition(binding: ImageToolBinding | undefined)
|
||||
Reflect.deleteProperty(inputSchema, '$schema')
|
||||
return { name: imageToolName, displayName: 'Generate or edit image', description, inputSchema, source: 'builtin' as const }
|
||||
}
|
||||
|
||||
export async function imageSaveToolDefinition(binding: ImageToolBinding | undefined) {
|
||||
if (!binding?.save || binding.context.workMode !== 'execute') return undefined
|
||||
const description = binding.describeSave ? await binding.describeSave() : imageSaveToolDescription
|
||||
if (!description) return undefined
|
||||
const inputSchema = z.toJSONSchema(imageSaveToolInputSchema, { target: 'draft-7', io: 'input' })
|
||||
Reflect.deleteProperty(inputSchema, '$schema')
|
||||
return { name: imageSaveToolName, displayName: 'Save image to file', description, inputSchema, source: 'builtin' as const }
|
||||
}
|
||||
|
||||
@@ -118,6 +118,7 @@ export {
|
||||
|
||||
export type GoodBuddyBuiltinToolName =
|
||||
| 'generate_image'
|
||||
| 'save_image'
|
||||
| ScopedDataToolName
|
||||
| BrowserToolName
|
||||
|
||||
@@ -834,6 +835,7 @@ export class KnowledgeMcpGateway {
|
||||
return [
|
||||
...(capability.storyGraph ? storyGraphToolNames : []),
|
||||
...(capability.imageToolBinding ? ['generate_image' as const] : []),
|
||||
...(capability.imageToolBinding?.save ? ['save_image' as const] : []),
|
||||
...(capability.libraryIds.length > 0
|
||||
? knowledgeToolNames
|
||||
: []),
|
||||
@@ -1769,6 +1771,7 @@ export class KnowledgeMcpGateway {
|
||||
)
|
||||
const capability = this.getCapability(token)
|
||||
const imageTool = await imageToolDefinition(capability.imageToolBinding)
|
||||
const imageSaveTool = await imageSaveToolDefinition(capability.imageToolBinding)
|
||||
const browserTools = capability.browserConversationId && capability.browserTabId
|
||||
? new BrowserModelTools({
|
||||
service: this.browserService!,
|
||||
@@ -1796,6 +1799,7 @@ export class KnowledgeMcpGateway {
|
||||
return {
|
||||
tools: [
|
||||
...(imageTool ? [{ name: imageTool.name, description: imageTool.description, inputSchema: imageTool.inputSchema as Tool['inputSchema'], annotations: { readOnlyHint: false, destructiveHint: false } }] : []),
|
||||
...(imageSaveTool ? [{ name: imageSaveTool.name, title: imageSaveTool.displayName, description: imageSaveTool.description, inputSchema: imageSaveTool.inputSchema as Tool['inputSchema'], annotations: { readOnlyHint: false, destructiveHint: true } }] : []),
|
||||
...scopedTools,
|
||||
...browserDefinitions,
|
||||
...[...customBindings.values()].map(
|
||||
@@ -1828,6 +1832,16 @@ export class KnowledgeMcpGateway {
|
||||
const operation = await capability.imageToolBinding.call(input, `${session.registryKey}:${extra.requestId}`, AbortSignal.any([extra.signal, capability.signal]))
|
||||
return { content: [{ type: 'text' as const, text: JSON.stringify(operation) }] }
|
||||
}
|
||||
if (name === 'save_image') {
|
||||
const capability = this.getCapability(token)
|
||||
if (!capability.imageToolBinding?.save) throw new Error('Image save capability is unavailable')
|
||||
try {
|
||||
const saved = await capability.imageToolBinding.save(input, AbortSignal.any([extra.signal, capability.signal]))
|
||||
return { content: [{ type: 'text' as const, text: JSON.stringify(saved) }] }
|
||||
} catch (error) {
|
||||
return { isError: true, content: [{ type: 'text' as const, text: error instanceof Error ? error.message : 'Saving image failed' }] }
|
||||
}
|
||||
}
|
||||
if (availableTools.has(name as ScopedDataToolName)) {
|
||||
const definition = scopedDataToolByName.get(
|
||||
name as ScopedDataToolName
|
||||
@@ -2123,4 +2137,4 @@ export class KnowledgeMcpGateway {
|
||||
}
|
||||
}
|
||||
}
|
||||
import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding'
|
||||
import { imageSaveToolDefinition, imageToolDefinition, type ImageToolBinding } from './image-tool-binding'
|
||||
|
||||
@@ -1257,6 +1257,7 @@ export class ModelToolProvider implements ModelToolProviderLike {
|
||||
? await this.knowledgeGateway.isStoryGraphAvailable(context.knowledgeCapabilityToken) : false
|
||||
const scopedTools = grantedTools.filter(tool => !tool.name.startsWith('story_graph_') || storyGraphAvailable)
|
||||
const imageTool = context.workMode === 'execute' ? await imageToolDefinition(context.imageToolBinding) : undefined
|
||||
const imageSaveTool = context.workMode === 'execute' ? await imageSaveToolDefinition(context.imageToolBinding) : undefined
|
||||
const webTools = this.webSearchEnabled
|
||||
? this.getWebSearchDefinitions()
|
||||
: []
|
||||
@@ -1280,6 +1281,7 @@ export class ModelToolProvider implements ModelToolProviderLike {
|
||||
return [
|
||||
...workspaceTools,
|
||||
...(imageTool ? [imageTool] : []),
|
||||
...(imageSaveTool ? [imageSaveTool] : []),
|
||||
...processTools,
|
||||
...subagentTools,
|
||||
...outputTools,
|
||||
@@ -1357,6 +1359,18 @@ export class ModelToolProvider implements ModelToolProviderLike {
|
||||
allowPermanent: false
|
||||
}
|
||||
}
|
||||
if (tool.name === 'save_image') {
|
||||
return {
|
||||
scopeKey: 'model:builtin:save_image',
|
||||
title: '允许将图片保存到本地文件?',
|
||||
description: path
|
||||
? `会使用当前用户权限写入:${path}`
|
||||
: '会使用当前用户权限将会话图片写入本地文件。',
|
||||
toolName: tool.displayName,
|
||||
argumentSummary,
|
||||
allowPermanent: false
|
||||
}
|
||||
}
|
||||
if (tool.name === 'output_read') {
|
||||
return {
|
||||
scopeKey: 'model:builtin:output_read',
|
||||
@@ -1430,6 +1444,10 @@ export class ModelToolProvider implements ModelToolProviderLike {
|
||||
if (!context.imageToolBinding || !context.toolCallId) throw new Error('Image tool request binding is unavailable')
|
||||
return createTextToolResult(JSON.stringify(await context.imageToolBinding.call(argumentsValue, context.toolCallId, signal)))
|
||||
}
|
||||
if (name === 'save_image') {
|
||||
if (!context.imageToolBinding?.save) throw new Error('Image save capability is unavailable')
|
||||
return createTextToolResult(JSON.stringify(await context.imageToolBinding.save(argumentsValue, signal)))
|
||||
}
|
||||
if (name === 'knowledge_list') {
|
||||
if (
|
||||
!this.knowledgeGateway ||
|
||||
@@ -2017,4 +2035,4 @@ function emitDirectModelSubagentUsage(
|
||||
)
|
||||
})
|
||||
}
|
||||
import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding'
|
||||
import { imageSaveToolDefinition, imageToolDefinition, type ImageToolBinding } from './image-tool-binding'
|
||||
|
||||
@@ -3740,6 +3740,9 @@ export function registerIpcHandlers(
|
||||
}, () => normalizeInteractiveWorkMode(assistantDatabase.getConversation(enrichedRequest.conversationId).workMode))
|
||||
: undefined
|
||||
const imageToolAvailable = Boolean(await imageToolBinding?.describe())
|
||||
// Remote Agents write the file on the remote host; Desktop streams the bytes on demand.
|
||||
const imageSaveAvailable = Boolean(imageToolBinding?.save && normalizedWorkMode === 'execute' &&
|
||||
await imageToolBinding.describeSave?.())
|
||||
const scopedCapability = await grantScopedDataCapability({
|
||||
storyGraph: selectedRuntimeTarget && applicationSettings?.heartbeatEnabled
|
||||
? { runtimeTarget: selectedRuntimeTarget, projectId: enrichedRequest.projectId } : undefined,
|
||||
@@ -3773,6 +3776,7 @@ export function registerIpcHandlers(
|
||||
const knowledgeCapabilityToken = scopedCapability.token
|
||||
const availableTools = [
|
||||
...(imageToolAvailable ? ['generate_image'] : []),
|
||||
...(imageSaveAvailable ? ['save_image'] : []),
|
||||
...(webSearchEnabled ? ['web_search', 'web_fetch'] : []),
|
||||
...(!agentRuntimeSelected && !imageGeneration
|
||||
? ['workspace_rg', 'workspace_read_text', 'output_read', 'subagent_delegate']
|
||||
|
||||
@@ -2,13 +2,15 @@ import type { ImageToolBinding } from '../agent/image-tool-binding'
|
||||
import type { RuntimeProtocolBinaryChannel } from './protocol-remote-runtime-channel'
|
||||
import {
|
||||
decodeRemoteImageToolMessage, encodeRemoteImageToolMessage,
|
||||
remoteImageToolCallSchema, remoteImageToolReplySchema
|
||||
remoteImageSaveChunkBytes, remoteImageToolCallSchema, remoteImageToolReplySchema
|
||||
} from '../../shared/remote-image-tool-contracts'
|
||||
|
||||
export class MainImageToolSession {
|
||||
private readonly wait = new AbortController()
|
||||
private readonly unsubscribe: () => void
|
||||
private readonly removeAbort: () => void
|
||||
/** Encoded bytes of the image currently being streamed to the Agent; one save at a time per prompt. */
|
||||
private saveCache?: { key: string; bytes: Buffer }
|
||||
|
||||
constructor(
|
||||
private readonly channel: RuntimeProtocolBinaryChannel,
|
||||
@@ -28,6 +30,7 @@ export class MainImageToolSession {
|
||||
if (this.wait.signal.aborted) return
|
||||
// This aborts binding.call's wait, never the Main-owned image operation.
|
||||
this.wait.abort()
|
||||
this.saveCache = undefined
|
||||
this.removeAbort()
|
||||
this.unsubscribe()
|
||||
this.channel.close()
|
||||
@@ -46,7 +49,22 @@ export class MainImageToolSession {
|
||||
let reply: ReturnType<typeof remoteImageToolReplySchema.parse>
|
||||
try {
|
||||
this.wait.signal.throwIfAborted()
|
||||
if ('name' in call) {
|
||||
if ('name' in call && call.name === 'save_image_read') {
|
||||
if (!this.binding?.readForSave || this.binding.context.workMode !== 'execute') throw new Error('Saving images is unavailable in Ask mode')
|
||||
const { artifactId, mimeType, offset } = call.input
|
||||
const cacheKey = `${artifactId}:${mimeType}`
|
||||
let bytes = this.saveCache?.key === cacheKey ? this.saveCache.bytes : undefined
|
||||
if (!bytes || offset === 0) {
|
||||
bytes = await this.binding.readForSave(artifactId, mimeType)
|
||||
this.saveCache = { key: cacheKey, bytes }
|
||||
}
|
||||
if (offset >= bytes.byteLength) throw new Error('Image chunk offset is out of range')
|
||||
const end = Math.min(offset + remoteImageSaveChunkBytes, bytes.byteLength)
|
||||
if (end === bytes.byteLength) this.saveCache = undefined
|
||||
reply = remoteImageToolReplySchema.parse({ callId: call.callId, imageChunk: {
|
||||
mimeType, totalBytes: bytes.byteLength, offset, data: bytes.subarray(offset, end).toString('base64')
|
||||
} })
|
||||
} else if ('name' in call) {
|
||||
if (!this.storyGraph) throw new Error('Story Graph capability unavailable')
|
||||
const storyGraphResult = call.name === 'story_graph_list'
|
||||
? { available: await this.storyGraph.available() }
|
||||
|
||||
@@ -597,13 +597,15 @@ export class ProtocolRemoteRuntimeChannel
|
||||
this.#imageTool = undefined
|
||||
const description = preparation.workMode === 'execute' ? await imageToolBinding?.describe() : undefined
|
||||
const storyGraph = await storyGraphBinding?.available()
|
||||
const saveDescription = preparation.workMode === 'execute' && imageToolBinding?.readForSave
|
||||
? await imageToolBinding.describeSave?.() : undefined
|
||||
waitSignal?.throwIfAborted()
|
||||
let imageTool: z.infer<typeof remotePromptOperationPreparationSchema>['imageTool']
|
||||
if ((description && imageToolBinding) || storyGraph) {
|
||||
if ((description && imageToolBinding) || storyGraph || saveDescription) {
|
||||
this.#assertCurrent()
|
||||
const binary = this.#state.client.allocateBinaryChannel?.({ kind: 'blob' })
|
||||
if (!binary) throw new Error('Remote image tool transport is unavailable')
|
||||
imageTool = { channelId: binary.channelId, channelEpoch: binary.channelEpoch, ...(description ? { description } : {}), ...(storyGraph ? { storyGraph: true } : {}) }
|
||||
imageTool = { channelId: binary.channelId, channelEpoch: binary.channelEpoch, ...(description ? { description } : {}), ...(storyGraph ? { storyGraph: true } : {}), ...(saveDescription ? { saveDescription } : {}) }
|
||||
this.#imageTool = new MainImageToolSession(binary, imageToolBinding, waitSignal, storyGraphBinding)
|
||||
}
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user