feat: generate and edit conversation images across runtimes
Add model-controlled image invocation and read-only runtime assignments. Route image tools through Main so accepted operations retain their originating conversation when chat runtimes change, with saved references, cancellation, and manual recovery. Keep Node hashing outside browser-shared contracts to restore production builds. Validation includes the production build, typecheck, lint, focused regressions, real Host transport checks, and two successful real image requests. Full UI-driven runtime switching acceptance remains documented as pending. Release note: 开启图片模型的会话调用后，聊天助手可调用图片生成与编辑工具，Runtime 分配自动管理。图片成果保存在原会话，可继续修改，并独立显示生成状态和取消入口。
This commit is contained in:
mesalogo
@@ -87,12 +87,13 @@ export class AgentAcpConnection {
|
||||
sessionId: string,
|
||||
operationId: string,
|
||||
socketPath?: string,
|
||||
workMode?: 'ask' | 'execute'
|
||||
workMode?: 'ask' | 'execute',
|
||||
imageToolName?: string
|
||||
): Promise<void> {
|
||||
if (!this.bridgeOrigin) throw new Error('Shared model bridge is not ready')
|
||||
const response = await fetch(`${this.bridgeOrigin}/session`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ sessionId, operationId, ...(socketPath ? { socketPath, workMode } : { release: true }) }),
|
||||
body: JSON.stringify({ sessionId, operationId, ...(socketPath ? { socketPath, workMode, imageToolName } : { release: true }) }),
|
||||
signal: AbortSignal.timeout(10_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`Shared model bridge route failed (${response.status})`)
|
||||
|
||||
@@ -30,23 +30,27 @@ describe('AgentOwnedAcpPrompt', () => {
|
||||
let client!: Client
|
||||
let finish!: (response: PromptResponse) => void
|
||||
const newSession = vi.fn(async () => ({ sessionId: 'session-1' }))
|
||||
const resumeSession = vi.fn(async () => ({}))
|
||||
let imageUrl: string | undefined
|
||||
const mcpServers = () => imageUrl ? [{ type: 'http' as const, name: 'images', url: imageUrl, headers: [] }] : []
|
||||
const prepareSession = vi.fn(async () => undefined)
|
||||
const completePrompt = vi.fn(async () => undefined)
|
||||
const owner = new AgentOwnedAcpPrompt({
|
||||
bindingId: 'binding-1', controllerId: 'controller-1',
|
||||
workspaceDirectory: '/workspace', process: new MemoryProcess(),
|
||||
transcript, prepareSession, completePrompt,
|
||||
transcript, prepareSession, completePrompt, mcpServers,
|
||||
createConnection: factory => {
|
||||
client = factory()
|
||||
return {
|
||||
initialize: async () => ({ protocolVersion: 1, agentCapabilities: {} }),
|
||||
newSession,
|
||||
newSession, resumeSession,
|
||||
prompt: () => new Promise<PromptResponse>(resolve => { finish = resolve })
|
||||
} as unknown as ClientSideConnection
|
||||
}
|
||||
})
|
||||
try {
|
||||
for (const [index, workMode] of (['execute', 'ask', 'execute'] as const).entries()) {
|
||||
for (const [index, workMode] of (['execute', 'execute', 'ask', 'execute', 'ask'] as const).entries()) {
|
||||
imageUrl = workMode === 'execute' && index !== 1 ? `http://127.0.0.1:1234/prompt-${index}` : undefined
|
||||
const operationId = `operation-${index}`
|
||||
transcript.prepare({
|
||||
bindingId: 'binding-1', operationId, requestId: operationId,
|
||||
@@ -58,6 +62,9 @@ describe('AgentOwnedAcpPrompt', () => {
|
||||
prompt: [{ type: 'text', text: 'Continue with the selected mode' }]
|
||||
}, workMode)
|
||||
expect(prepareSession).toHaveBeenLastCalledWith('session-1', operationId, workMode)
|
||||
if (index !== 2) {
|
||||
expect(index === 0 ? newSession : resumeSession).toHaveBeenLastCalledWith(expect.objectContaining({ mcpServers: mcpServers() }))
|
||||
}
|
||||
const result = await client.requestPermission!({
|
||||
sessionId: 'session-1',
|
||||
toolCall: { toolCallId: `tool-${index}`, kind: 'execute', title: 'Run a test' },
|
||||
@@ -74,7 +81,8 @@ describe('AgentOwnedAcpPrompt', () => {
|
||||
.toMatchObject({ state: 'completed' }))
|
||||
}
|
||||
expect(newSession).toHaveBeenCalledOnce()
|
||||
expect(completePrompt).toHaveBeenCalledTimes(3)
|
||||
expect(resumeSession).toHaveBeenCalledTimes(3)
|
||||
expect(completePrompt).toHaveBeenCalledTimes(5)
|
||||
} finally {
|
||||
owner.close()
|
||||
transcript.close()
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
PROTOCOL_VERSION,
|
||||
ndJsonStream,
|
||||
type Client,
|
||||
type McpServer,
|
||||
type PromptResponse,
|
||||
type RequestPermissionRequest,
|
||||
type RequestPermissionResponse,
|
||||
@@ -26,6 +27,7 @@ export type AgentOwnedAcpPromptOptions = {
|
||||
bindingId: string
|
||||
controllerId: string
|
||||
workspaceDirectory: string
|
||||
mcpServers?: () => McpServer[]
|
||||
expectedModel?: string
|
||||
process: RuntimeAcpProcessOwner
|
||||
transport?: AgentAcpConnection
|
||||
@@ -60,6 +62,7 @@ export class AgentOwnedAcpPrompt {
|
||||
#promptSettled?: Promise<void>
|
||||
#closed = false
|
||||
#initialized = false
|
||||
#hasMcpServers = false
|
||||
readonly #questions = new Map<string, {
|
||||
endpoint: string; operationId: string; questionCount: number; notification: SessionNotification
|
||||
}>()
|
||||
@@ -109,6 +112,7 @@ export class AgentOwnedAcpPrompt {
|
||||
)
|
||||
}
|
||||
|
||||
const mcpServers = this.#options.mcpServers?.() ?? []
|
||||
if (!this.#initialized) {
|
||||
const initialization = await this.#whileProcessAlive(
|
||||
this.#transport.initialize({
|
||||
@@ -135,7 +139,7 @@ export class AgentOwnedAcpPrompt {
|
||||
this.#connection.loadSession({
|
||||
sessionId: request.acpSessionId,
|
||||
cwd: this.#options.workspaceDirectory,
|
||||
mcpServers: []
|
||||
mcpServers
|
||||
})
|
||||
)
|
||||
} else if (
|
||||
@@ -145,7 +149,7 @@ export class AgentOwnedAcpPrompt {
|
||||
this.#connection.resumeSession({
|
||||
sessionId: request.acpSessionId,
|
||||
cwd: this.#options.workspaceDirectory,
|
||||
mcpServers: []
|
||||
mcpServers
|
||||
})
|
||||
)
|
||||
} else {
|
||||
@@ -156,7 +160,7 @@ export class AgentOwnedAcpPrompt {
|
||||
const created = await this.#whileProcessAlive(
|
||||
this.#connection.newSession({
|
||||
cwd: this.#options.workspaceDirectory,
|
||||
mcpServers: []
|
||||
mcpServers
|
||||
})
|
||||
)
|
||||
this.#sessionId = created.sessionId
|
||||
@@ -186,7 +190,13 @@ export class AgentOwnedAcpPrompt {
|
||||
request.acpSessionId !== this.#sessionId
|
||||
) {
|
||||
throw new Error('ACP session identity cannot change within a binding')
|
||||
} else if (mcpServers.length > 0 || this.#hasMcpServers) {
|
||||
await this.#whileProcessAlive(this.#connection.resumeSession({
|
||||
sessionId: this.#sessionId!, cwd: this.#options.workspaceDirectory,
|
||||
mcpServers
|
||||
}))
|
||||
}
|
||||
this.#hasMcpServers = mcpServers.length > 0
|
||||
await this.#options.prepareSession?.(this.#sessionId!, request.operationId, workMode)
|
||||
this.#active = {
|
||||
operationId: request.operationId,
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// @vitest-environment node
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createServer } from 'node:http'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { AgentImageToolMcp } from './image-tool-mcp'
|
||||
import { imageOperationSchema } from '../shared/image-generation-contracts'
|
||||
import { AssistantDatabase } from '../main/assistant/assistant-database'
|
||||
import { ImageGenerationService } from '../main/agent/image-generation-service'
|
||||
import { MainImageToolSession } from '../main/remote-agent/main-image-tool-session'
|
||||
import type { RuntimeProtocolBinaryChannel, RuntimeProtocolBinaryFrame } from '../main/remote-agent/protocol-remote-runtime-channel'
|
||||
|
||||
const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII='
|
||||
vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64') }) } }))
|
||||
const cleanup: (() => void | Promise<void>)[] = []
|
||||
afterEach(async () => { for (const close of cleanup.splice(0).reverse()) await close() })
|
||||
|
||||
async function setup() {
|
||||
const received: { path: string; body: string; authorization?: string }[] = []
|
||||
let release: (() => void) | undefined
|
||||
let hold = false
|
||||
const provider = createServer((req, res) => {
|
||||
void (async () => {
|
||||
let body = ''
|
||||
for await (const chunk of req) body += chunk
|
||||
received.push({ path: req.url!, body, authorization: req.headers.authorization })
|
||||
if (hold) await new Promise<void>(resolve => { release = resolve })
|
||||
res.setHeader('content-type', 'application/json')
|
||||
res.end(JSON.stringify({ data: [{ b64_json: png }] }))
|
||||
})()
|
||||
})
|
||||
await new Promise<void>(resolve => provider.listen(0, '127.0.0.1', resolve))
|
||||
cleanup.push(() => { release?.(); provider.closeAllConnections(); provider.close() })
|
||||
const address = provider.address() as { port: number }
|
||||
const database = new AssistantDatabase(':memory:')
|
||||
database.initialize(process.cwd())
|
||||
cleanup.push(() => database.close())
|
||||
const context = { conversationId: randomUUID(), messageId: randomUUID(), requestId: randomUUID(), workMode: 'execute' as 'ask' | 'execute' }
|
||||
database.saveLocalConversations([{ header: { id: context.conversationId, title: 'Remote images', updatedAt: 1 },
|
||||
messages: [{ id: context.messageId, role: 'assistant', content: '', createdAt: 1, state: 'complete' }] }])
|
||||
const profile = { id: randomUUID(), name: 'Controlled image provider', modelName: 'controlled-image',
|
||||
protocol: 'openai-images-generations', allowConversationInvocation: true,
|
||||
baseUrl: `http://127.0.0.1:${address.port}/v1`, authentication: 'api-key' as const, apiKey: 'controlled-main-only' }
|
||||
const service = new ImageGenerationService({ database, getSettings: async () => ({ modelProfiles: [profile], defaultImageModelProfileId: profile.id }) })
|
||||
service.initialize()
|
||||
cleanup.push(() => service.dispose())
|
||||
const frames: Uint8Array[] = []
|
||||
const connect = async (mode: 'ask' | 'execute' = 'execute') => {
|
||||
const binding = service.bind({ ...context, requestId: randomUUID(), workMode: mode })
|
||||
const queue: RuntimeProtocolBinaryFrame[] = []
|
||||
let receiver: ((frame: RuntimeProtocolBinaryFrame) => void) | undefined
|
||||
let rejectReceive: ((error: unknown) => void) | undefined
|
||||
const listeners = new Set<() => void>()
|
||||
let closed = false
|
||||
const channel: RuntimeProtocolBinaryChannel = {
|
||||
channelId: randomUUID(), channelEpoch: '1',
|
||||
send: async payload => { frames.push(payload); adapter.onReply(payload) },
|
||||
receive: async () => queue.shift() ?? await new Promise((resolve, reject) => { receiver = resolve; rejectReceive = reject }),
|
||||
close: () => { if (closed) return; closed = true; rejectReceive?.(new Error('closed')); for (const fn of listeners) fn() },
|
||||
onClose: fn => { listeners.add(fn); return () => { listeners.delete(fn) } }
|
||||
}
|
||||
const wait = new AbortController()
|
||||
const main = new MainImageToolSession(channel, binding, wait.signal)
|
||||
const adapter = new AgentImageToolMcp({ channelId: channel.channelId, channelEpoch: '1', description: await binding.describe() ?? 'Not available' }, async payload => {
|
||||
if (closed) throw new Error('disconnected')
|
||||
frames.push(payload)
|
||||
const frame = { payload, sequence: '1', consume: async () => {} }
|
||||
if (receiver) { const resolve = receiver; receiver = undefined; resolve(frame) } else queue.push(frame)
|
||||
})
|
||||
await adapter.start()
|
||||
const client = new Client({ name: 'remote-image-test', version: '1' })
|
||||
await client.connect(new StreamableHTTPClientTransport(new URL(adapter.url!)))
|
||||
cleanup.push(async () => { main.close(); adapter.close(); await client.close() })
|
||||
return { client, adapter, wait, main, binding }
|
||||
}
|
||||
return { received, context, profile, service, database, connect, frames,
|
||||
hold: () => { hold = true }, release: () => release?.() }
|
||||
}
|
||||
|
||||
describe('remote image MCP to Main service', () => {
|
||||
it('generates, edits persisted uploads and history through a controlled HTTP provider', async () => {
|
||||
const h = await setup()
|
||||
const remote = await h.connect()
|
||||
expect((await remote.client.listTools()).tools[0]!.description).toContain(h.profile.id)
|
||||
const call = async (args: Record<string, unknown>) => {
|
||||
const result = await remote.client.callTool({ name: 'generate_image', arguments: args })
|
||||
expect(result.isError).not.toBe(true)
|
||||
return imageOperationSchema.parse(JSON.parse((result.content as { text: string }[])[0]!.text))
|
||||
}
|
||||
const first = await call({ intent: 'create', prompt: 'A blue square' })
|
||||
const uploaded = h.service.persistUploads(h.context, [{ name: 'source.png', mediaType: 'image/png', data: png }])
|
||||
const uploadEdit = await call({ intent: 'edit', prompt: 'Make the uploaded square red', sourceArtifactIds: uploaded })
|
||||
const historyEdit = await call({ intent: 'edit', prompt: 'Make the previous result green', sourceArtifactIds: first.artifactIds })
|
||||
expect([first, uploadEdit, historyEdit].map(op => op.state)).toEqual(['completed', 'completed', 'completed'])
|
||||
expect(h.received.map(call => call.path)).toEqual(['/v1/images/generations', '/v1/images/edits', '/v1/images/edits'])
|
||||
expect(h.received.every(call => call.authorization === 'Bearer controlled-main-only')).toBe(true)
|
||||
expect(h.received[1]!.body).toContain('name="image"')
|
||||
expect(h.database.getArtifact(historyEdit.artifactIds[0]!).content).toContain(png)
|
||||
expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageOperations).toHaveLength(3)
|
||||
const wire = Buffer.concat(h.frames).toString()
|
||||
expect(wire).not.toContain(png)
|
||||
expect(wire).not.toContain('controlled-main-only')
|
||||
expect(wire).not.toContain(h.profile.baseUrl)
|
||||
})
|
||||
|
||||
it.each(['chat cancellation', 'disconnect'])('preserves Main operation after %s without replay', async reason => {
|
||||
const h = await setup()
|
||||
h.hold()
|
||||
const remote = await h.connect()
|
||||
const waiting = remote.client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } }).catch(() => undefined)
|
||||
await vi.waitFor(() => expect(h.received).toHaveLength(1))
|
||||
if (reason === 'chat cancellation') remote.wait.abort()
|
||||
else remote.main.close()
|
||||
remote.adapter.close()
|
||||
h.release()
|
||||
await vi.waitFor(() => expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageOperations?.[0]?.state).toBe('completed'))
|
||||
await waiting
|
||||
expect(h.received).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('rejects Ask calls at the desktop binding and refreshes a new prompt description', async () => {
|
||||
const h = await setup()
|
||||
const ask = await h.connect('ask')
|
||||
const denied = await ask.client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } })
|
||||
expect(denied.isError).toBe(true)
|
||||
expect(h.received).toHaveLength(0)
|
||||
h.profile.name = 'Changed next prompt'
|
||||
const execute = await h.connect()
|
||||
expect((await execute.client.listTools()).tools[0]!.description).toContain('Changed next prompt')
|
||||
})
|
||||
|
||||
it('returns unknown delivery without sending a second request for the same MCP call', async () => {
|
||||
const send = vi.fn(async () => { throw new Error('connection lost after send') })
|
||||
const adapter = new AgentImageToolMcp({ channelId: randomUUID(), channelEpoch: '1', description: 'Images' }, send)
|
||||
await adapter.start()
|
||||
cleanup.push(() => adapter.close())
|
||||
const request = { jsonrpc: '2.0', id: 7, method: 'tools/call', params: { name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } } }
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const response = await fetch(adapter.url!, { method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream' }, body: JSON.stringify(request) })
|
||||
const result = await response.json()
|
||||
expect(JSON.stringify(result)).toContain('unknown')
|
||||
}
|
||||
expect(send).toHaveBeenCalledOnce()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,121 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createServer, type Server } from 'node:http'
|
||||
import { Server as McpServer } from '@modelcontextprotocol/sdk/server/index.js'
|
||||
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'
|
||||
import { CallToolRequestSchema, ListToolsRequestSchema, type CallToolResult } from '@modelcontextprotocol/sdk/types.js'
|
||||
import { z } from 'zod'
|
||||
import { imageToolInputSchema, imageToolName } from '../shared/image-generation-contracts'
|
||||
import {
|
||||
decodeRemoteImageToolMessage, encodeRemoteImageToolMessage,
|
||||
remoteImageToolMaximumBytes, remoteImageToolReplySchema, type remoteImageToolSchema
|
||||
} from '../shared/remote-image-tool-contracts'
|
||||
|
||||
/** One prompt's local MCP adapter. It never holds provider credentials or image bytes. */
|
||||
export class AgentImageToolMcp {
|
||||
private readonly calls = new Map<string, { input: string; result: Promise<CallToolResult> }>()
|
||||
private readonly pending = new Map<string, {
|
||||
resolve: (value: CallToolResult) => void
|
||||
reject: (error: Error) => void
|
||||
}>()
|
||||
private server?: Server
|
||||
private closed = false
|
||||
url?: string
|
||||
|
||||
constructor(
|
||||
readonly descriptor: z.infer<typeof remoteImageToolSchema>,
|
||||
private readonly send: (payload: Uint8Array) => Promise<void>
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
const path = `/${randomUUID()}/mcp`
|
||||
const schema = z.toJSONSchema(imageToolInputSchema, { target: 'draft-7', io: 'input' })
|
||||
Reflect.deleteProperty(schema, '$schema')
|
||||
this.server = createServer((request, response) => {
|
||||
void (async () => {
|
||||
if (this.closed || request.url !== path) { response.writeHead(404).end(); return }
|
||||
if (request.method !== 'POST') { response.writeHead(405).end(); return }
|
||||
const chunks: Buffer[] = []
|
||||
let size = 0
|
||||
for await (const chunk of request) {
|
||||
const bytes = Buffer.from(chunk)
|
||||
size += bytes.length
|
||||
if (size > remoteImageToolMaximumBytes) { response.writeHead(413).end(); return }
|
||||
chunks.push(bytes)
|
||||
}
|
||||
const body: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
||||
const mcp = new McpServer({ name: 'goodbuddy-image', version: '1' }, { capabilities: { tools: {} } })
|
||||
mcp.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: [{
|
||||
name: imageToolName, description: this.descriptor.description,
|
||||
inputSchema: schema as { type: 'object' }
|
||||
}] }))
|
||||
mcp.setRequestHandler(CallToolRequestSchema, async (call, extra) => {
|
||||
if (call.params.name !== imageToolName) throw new Error('Unknown image tool')
|
||||
const input = imageToolInputSchema.parse(call.params.arguments)
|
||||
// Repeated delivery of an MCP request returns the same outcome, including unknown delivery.
|
||||
const key = `${typeof extra.requestId}:${extra.requestId}`
|
||||
let existing = this.calls.get(key)
|
||||
if (existing && existing.input !== JSON.stringify(input)) throw new Error('Image tool request ID reused with different input')
|
||||
if (!existing) {
|
||||
const result = this.call(input).catch((error: unknown) => ({
|
||||
isError: true,
|
||||
content: [{ type: 'text' as const, text: error instanceof Error ? error.message : 'Image tool delivery failed; do not retry automatically' }]
|
||||
}))
|
||||
existing = { input: JSON.stringify(input), result }
|
||||
this.calls.set(key, existing)
|
||||
}
|
||||
return await existing.result
|
||||
})
|
||||
const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined, enableJsonResponse: true })
|
||||
response.once('close', () => { void mcp.close().catch(() => undefined) })
|
||||
await mcp.connect(transport)
|
||||
await transport.handleRequest(request, response, body)
|
||||
})().catch(() => {
|
||||
if (!response.headersSent) response.writeHead(400)
|
||||
response.end()
|
||||
})
|
||||
})
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
this.server!.once('error', reject)
|
||||
this.server!.listen(0, '127.0.0.1', resolve)
|
||||
})
|
||||
const address = this.server.address()
|
||||
if (!address || typeof address === 'string') throw new Error('Image MCP endpoint did not start')
|
||||
this.url = `http://127.0.0.1:${address.port}${path}`
|
||||
}
|
||||
|
||||
onReply(payload: Uint8Array): void {
|
||||
const reply = remoteImageToolReplySchema.parse(decodeRemoteImageToolMessage(payload))
|
||||
const pending = this.pending.get(reply.callId)
|
||||
if (!pending) return
|
||||
if (reply.error || !reply.result) pending.reject(new Error(reply.error ?? 'Image tool response has no result'))
|
||||
else pending.resolve({ content: [{ type: 'text', text: JSON.stringify(reply.result) }] })
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.closed) return
|
||||
this.closed = true
|
||||
for (const pending of this.pending.values()) pending.reject(new Error('Desktop image tool disconnected; delivery may be unknown. Do not retry automatically.'))
|
||||
this.pending.clear()
|
||||
this.server?.closeAllConnections()
|
||||
this.server?.close()
|
||||
}
|
||||
|
||||
private async call(input: z.infer<typeof imageToolInputSchema>): Promise<CallToolResult> {
|
||||
if (this.closed) throw new Error('Image tool prompt has ended')
|
||||
const callId = randomUUID()
|
||||
let timer: NodeJS.Timeout | undefined
|
||||
const result = new Promise<CallToolResult>((resolve, reject) => {
|
||||
this.pending.set(callId, { resolve, reject })
|
||||
timer = setTimeout(() => reject(new Error('Image tool wait timed out; delivery may be unknown. Do not retry automatically.')), 5 * 60_000)
|
||||
timer.unref()
|
||||
})
|
||||
// Register the wait before sending; send failures must not cause a retry.
|
||||
void this.send(encodeRemoteImageToolMessage({ callId, input })).catch(() => {
|
||||
this.pending.get(callId)?.reject(new Error('Image tool delivery may be unknown. Do not retry automatically.'))
|
||||
})
|
||||
try { return await result } finally {
|
||||
clearTimeout(timer)
|
||||
this.pending.delete(callId)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -133,7 +133,7 @@ const FORWARDED_HEADER_NAMES = [
|
||||
export class ModelBridgeLoopbackProxy {
|
||||
readonly #exchange: ModelBridgeExchange
|
||||
readonly #sharedSessions: boolean
|
||||
readonly #sessionRoutes = new Map<string, { operationId: string; socketPath: string; workMode: 'ask' | 'execute' }>()
|
||||
readonly #sessionRoutes = new Map<string, { operationId: string; socketPath: string; workMode: 'ask' | 'execute'; imageToolName?: string }>()
|
||||
readonly #routeToken: string
|
||||
readonly #maximumConnections: number
|
||||
readonly #requestTimeoutMs: number
|
||||
@@ -317,12 +317,12 @@ export class ModelBridgeLoopbackProxy {
|
||||
const route = this.#sessionRoutes.get(url.searchParams.get('sessionId') ?? '')
|
||||
if (!route) throw new HttpRequestError(409, 'session-inactive')
|
||||
outgoing.setHeader('content-type', 'application/json')
|
||||
outgoing.end(JSON.stringify({ operationId: route.operationId, workMode: route.workMode }))
|
||||
outgoing.end(JSON.stringify({ operationId: route.operationId, workMode: route.workMode, imageToolName: route.imageToolName }))
|
||||
return
|
||||
}
|
||||
if (incoming.method !== 'POST') throw new HttpRequestError(405, 'method-not-allowed')
|
||||
const value = JSON.parse((await readBoundedBody(incoming)).toString('utf8')) as {
|
||||
sessionId?: string; operationId?: string; socketPath?: string; release?: boolean; workMode?: 'ask' | 'execute'
|
||||
sessionId?: string; operationId?: string; socketPath?: string; release?: boolean; workMode?: 'ask' | 'execute'; imageToolName?: string
|
||||
}
|
||||
if (
|
||||
typeof value.sessionId !== 'string' || value.sessionId.length > 128 ||
|
||||
@@ -335,7 +335,8 @@ export class ModelBridgeLoopbackProxy {
|
||||
} else {
|
||||
if (value.workMode !== 'ask' && value.workMode !== 'execute') throw new HttpRequestError(400, 'request-invalid')
|
||||
const socketPath = normalizedAbsolutePath(value.socketPath ?? '', 'Model bridge socket')
|
||||
this.#sessionRoutes.set(value.sessionId, { operationId: value.operationId, socketPath, workMode: value.workMode })
|
||||
if (value.imageToolName !== undefined && !/^goodbuddy_image_[a-f0-9]{24}$/.test(value.imageToolName)) throw new HttpRequestError(400, 'request-invalid')
|
||||
this.#sessionRoutes.set(value.sessionId, { operationId: value.operationId, socketPath, workMode: value.workMode, imageToolName: value.imageToolName })
|
||||
}
|
||||
outgoing.end('{}')
|
||||
return
|
||||
|
||||
@@ -5,6 +5,29 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { openCodeSubagentPluginSource } from './opencode-subagent-plugin'
|
||||
|
||||
describe('OpenCode ACP child event plugin', () => {
|
||||
it('hides foreign image tools and removes the current image tool in Ask on a reused Session', async () => {
|
||||
let mode = 'execute'
|
||||
const imageToolName = `goodbuddy_image_${'a'.repeat(24)}`
|
||||
const patch = vi.fn()
|
||||
const factory = runInNewContext(
|
||||
`(${openCodeSubagentPluginSource('http://127.0.0.1:12345/fixture').replace(/^import .*\n/gmu, '').replace('export default ', '')})`,
|
||||
{ process: { stdout: { write: vi.fn() } }, createServer, randomBytes,
|
||||
fetch: async () => ({ ok: true, json: async () => ({ operationId: mode, workMode: mode, imageToolName }) }) }
|
||||
)
|
||||
const hooks = await factory({ client: {
|
||||
_client: { get: vi.fn(), post: vi.fn(), patch },
|
||||
session: { get: async () => ({ data: { id: 'root' } }) }
|
||||
} })
|
||||
try {
|
||||
for (mode of ['execute', 'ask', 'execute']) {
|
||||
await hooks.event({ event: { type: 'session.status', properties: { sessionID: 'root', status: { type: 'idle' } } } })
|
||||
await hooks['chat.message']({ sessionID: 'root' }, { message: { id: mode } })
|
||||
const rules = patch.mock.calls.at(-1)![0].body.permission
|
||||
expect(rules).toContainEqual({ permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' })
|
||||
expect(rules.some((rule: { permission: string; action: string }) => rule.permission === `${imageToolName}_*` && rule.action === 'allow')).toBe(mode === 'execute')
|
||||
}
|
||||
} finally { await hooks.dispose() }
|
||||
})
|
||||
it('attributes child and compaction model rounds to the frozen root operation', async () => {
|
||||
const operations = new Map([['one', 'operation-1'], ['two', 'operation-2']])
|
||||
const fetchRoute = vi.fn(async (url: string) => {
|
||||
@@ -33,7 +56,10 @@ describe('OpenCode ACP child event plugin', () => {
|
||||
try {
|
||||
await hooks['chat.message']({ sessionID: 'one' }, { message: { id: 'first' } })
|
||||
expect(patch).toHaveBeenCalledWith({
|
||||
url: '/session/one', body: { permission: [{ permission: '*', pattern: '*', action: 'allow' }] }, throwOnError: true
|
||||
url: '/session/one', body: { permission: [
|
||||
{ permission: '*', pattern: '*', action: 'allow' },
|
||||
{ permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' }
|
||||
] }, throwOnError: true
|
||||
})
|
||||
expect(await headers('child', 'child-message')).toEqual({
|
||||
'x-goodbuddy-session': 'one', 'x-goodbuddy-operation': 'operation-1'
|
||||
@@ -42,7 +68,8 @@ describe('OpenCode ACP child event plugin', () => {
|
||||
expect(patch).toHaveBeenLastCalledWith({
|
||||
url: '/session/child', body: { permission: [
|
||||
{ permission: '*', pattern: '*', action: 'allow' },
|
||||
{ permission: 'task', pattern: '*', action: 'deny' }
|
||||
{ permission: 'task', pattern: '*', action: 'deny' },
|
||||
{ permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' }
|
||||
] }, throwOnError: true
|
||||
})
|
||||
expect(await headers('two', 'compact')).toEqual({
|
||||
|
||||
@@ -26,7 +26,7 @@ async function plugin(input?: {
|
||||
}) {
|
||||
const tasks = new Map<string, { sessionId: string; callId: string }>()
|
||||
const pending = new Map<string, { sessionId: string; root: string }>()
|
||||
const modelMessages = new Map<string, { sessionId: string; operationId: string; workMode: 'ask' | 'execute' }>()
|
||||
const modelMessages = new Map<string, { sessionId: string; operationId: string; workMode: 'ask' | 'execute'; imageToolName?: string }>()
|
||||
const modelRoute = async (sessionId: string, messageId: string) => {
|
||||
const key = `${sessionId}\0${messageId}`
|
||||
const previous = modelMessages.get(key)
|
||||
@@ -43,8 +43,8 @@ async function plugin(input?: {
|
||||
}
|
||||
const response = await fetch(`${modules.modelBridgeOrigin}/session?sessionId=${encodeURIComponent(root)}`)
|
||||
if (!response.ok) throw new Error('GoodBuddy model operation is no longer active')
|
||||
const { operationId, workMode } = await response.json() as { operationId: string; workMode: 'ask' | 'execute' }
|
||||
const route = { sessionId: root, operationId, workMode }
|
||||
const { operationId, workMode, imageToolName } = await response.json() as { operationId: string; workMode: 'ask' | 'execute'; imageToolName?: string }
|
||||
const route = { sessionId: root, operationId, workMode, imageToolName }
|
||||
modelMessages.set(key, route)
|
||||
return route
|
||||
}
|
||||
@@ -131,6 +131,10 @@ async function plugin(input?: {
|
||||
// Keep its explicit subagent restrictions after the request rules.
|
||||
permission.push(...(child.data.permission ?? []).filter(rule => rule.permission !== '*'))
|
||||
}
|
||||
permission.push({ permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' })
|
||||
if (route.workMode === 'execute' && route.imageToolName) {
|
||||
permission.push({ permission: `${route.imageToolName}_*`, pattern: '*', action: 'allow' })
|
||||
}
|
||||
await input!.client._client.patch({
|
||||
url: `/session/${encodeURIComponent(request.sessionID)}`, body: { permission }, throwOnError: true
|
||||
})
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
|
||||
import { decodeRemoteImageToolMessage, encodeRemoteImageToolMessage, remoteImageToolCallSchema } from '../shared/remote-image-tool-contracts'
|
||||
import type { AgentFrame } from '../shared/agent-protocol'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -309,7 +313,7 @@ function harness(input: {
|
||||
throw new Error('broker close failed')
|
||||
}
|
||||
})
|
||||
const blobSink = vi.fn(async () => undefined)
|
||||
const blobSink = vi.fn<(frame: AgentFrame) => Promise<void>>(async () => undefined)
|
||||
let bridgeDispatch: ModelBridgeBrokerDispatch | undefined
|
||||
const launchModelBridges: unknown[] = []
|
||||
const resolved = resolvedBundle()
|
||||
@@ -451,6 +455,49 @@ function harness(input: {
|
||||
}
|
||||
|
||||
describe('RuntimeAcpBackend', () => {
|
||||
it('binds MCP image calls to authenticated blob framing and closes each prompt endpoint', async () => {
|
||||
const fixture = harness({ workMode: 'execute' })
|
||||
const client = new Client({ name: 'image-backend-test', version: '1' })
|
||||
try {
|
||||
await open(fixture)
|
||||
const accepted = await invoke(fixture, 'runtime/preparePrompt', fixture.preparation({
|
||||
imageTool: { channelId: 'image-channel', channelEpoch: '1', description: 'Image catalog' }
|
||||
})) as { imageToolUrl: string }
|
||||
await client.connect(new StreamableHTTPClientTransport(new URL(accepted.imageToolUrl)))
|
||||
expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image'])
|
||||
const result = client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } })
|
||||
await vi.waitFor(() => expect(fixture.blobSink).toHaveBeenCalledOnce())
|
||||
const frame = fixture.blobSink.mock.calls[0]![0]
|
||||
expect(frame.header).toMatchObject({ channelId: 'image-channel', channelEpoch: '1', sequence: '1', direction: 'agent-to-main' })
|
||||
expect(fixture.backend.authorizeBlobFrame(frame, fixture.context.controller)).toBe(true)
|
||||
expect(fixture.backend.authorizeBlobFrame(frame, { ...fixture.context.controller, connectionId: 'foreign' })).toBe(false)
|
||||
const call = remoteImageToolCallSchema.parse(decodeRemoteImageToolMessage(frame.payload))
|
||||
const payload = encodeRemoteImageToolMessage({ callId: call.callId, error: 'Controlled test failure' })
|
||||
const reply = { header: { ...frame.header, direction: 'main-to-agent' as const, payloadLength: payload.byteLength }, payload }
|
||||
await expect(fixture.backend.onBlobFrame(reply, { ...fixture.context, channelId: 'image-channel', controller: { ...fixture.context.controller, connectionId: 'foreign' } })).rejects.toThrow('authority')
|
||||
await fixture.backend.onBlobFrame(reply, { ...fixture.context, channelId: 'image-channel' })
|
||||
expect(await result).toMatchObject({ isError: true })
|
||||
await invoke(fixture, 'runtime/completePrompt', { bindingId: 'binding-1', operationId: 'request-1', requestId: 'request-1' })
|
||||
await expect(fetch(accepted.imageToolUrl)).rejects.toThrow()
|
||||
const next = await invoke(fixture, 'runtime/preparePrompt', fixture.preparation({
|
||||
requestId: 'request-2', operationId: 'request-2', promptSequence: 1,
|
||||
imageTool: { channelId: 'image-channel-2', channelEpoch: '2', description: 'Refreshed image catalog' }
|
||||
})) as { imageToolUrl: string }
|
||||
expect(next.imageToolUrl).not.toBe(accepted.imageToolUrl)
|
||||
} finally { await client.close(); await fixture.backend.dispose() }
|
||||
})
|
||||
|
||||
it('rejects image tool preparation in Ask before launching a process or bridge', async () => {
|
||||
const fixture = harness()
|
||||
try {
|
||||
await open(fixture)
|
||||
await expect(invoke(fixture, 'runtime/preparePrompt', fixture.preparation({
|
||||
imageTool: { channelId: 'image-channel', channelEpoch: '1', description: 'Images' }
|
||||
}))).rejects.toThrow('Execute')
|
||||
expect(fixture.launches).toHaveLength(0)
|
||||
expect(fixture.lifecycle).not.toContain('bridge-listen')
|
||||
} finally { await fixture.backend.dispose() }
|
||||
})
|
||||
it.each([false, true])('allows mode changes only on a shared Session (shared=%s)', async (shared) => {
|
||||
const fixture = harness({ agentOwned: true, shareOwnedProcesses: shared, workMode: 'execute' })
|
||||
try {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { z } from 'zod'
|
||||
import { AgentImageToolMcp } from './image-tool-mcp'
|
||||
import { remoteImageToolMcpName } from '../shared/remote-image-tool-node'
|
||||
import { remoteQuestionResponseSchema } from '../shared/remote-question-contracts'
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
@@ -338,6 +340,7 @@ type BindingState = {
|
||||
ownedPromptStartTimer?: NodeJS.Timeout
|
||||
modelBridgePolicy?: ModelBridgePolicy
|
||||
modelBridgeClient?: ModelBridgeBlobClient
|
||||
imageTool?: AgentImageToolMcp
|
||||
modelBridgeBroker?: ModelBridgeBrokerServer
|
||||
ownedAcp?: AgentOwnedAcpPrompt
|
||||
workspaceDirectory?: string
|
||||
@@ -558,6 +561,18 @@ export class RuntimeAcpBackend {
|
||||
'identity'
|
||||
)
|
||||
}
|
||||
const imageBinding = [...this.#bindings.values()].find(binding =>
|
||||
binding.controllerId === context.controller.controllerId &&
|
||||
binding.controllerGeneration === context.controller.generation &&
|
||||
binding.connectionId === context.controller.connectionId &&
|
||||
binding.activeOperationId !== undefined &&
|
||||
binding.imageTool?.descriptor.channelId === frame.header.channelId &&
|
||||
binding.imageTool.descriptor.channelEpoch === frame.header.channelEpoch &&
|
||||
context.channelId === frame.header.channelId)
|
||||
if (imageBinding) {
|
||||
imageBinding.imageTool!.onReply(frame.payload)
|
||||
return
|
||||
}
|
||||
const matches = [...this.#bindings.values()].filter(
|
||||
(binding) =>
|
||||
binding.controllerId === context.controller.controllerId &&
|
||||
@@ -613,10 +628,10 @@ export class RuntimeAcpBackend {
|
||||
binding.controllerId === controller.controllerId &&
|
||||
binding.controllerGeneration === controller.generation &&
|
||||
binding.connectionId === controller.connectionId &&
|
||||
binding.modelBridgeClient?.binding.channelId ===
|
||||
frame.header.channelId &&
|
||||
binding.modelBridgeClient.binding.channelEpoch ===
|
||||
frame.header.channelEpoch &&
|
||||
((binding.modelBridgeClient?.binding.channelId === frame.header.channelId &&
|
||||
binding.modelBridgeClient.binding.channelEpoch === frame.header.channelEpoch) ||
|
||||
(binding.imageTool?.descriptor.channelId === frame.header.channelId &&
|
||||
binding.imageTool.descriptor.channelEpoch === frame.header.channelEpoch)) &&
|
||||
frame.header.connectionId === binding.connectionId &&
|
||||
frame.header.generation === binding.controllerGeneration
|
||||
)
|
||||
@@ -1140,11 +1155,37 @@ export class RuntimeAcpBackend {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (preparation.imageTool && (preparation.workMode !== 'execute' || !this.#options.blobSink)) {
|
||||
throw new RuntimeAcpBackendError('Image tools require Execute and a desktop transport', 'identity')
|
||||
}
|
||||
if (requestedModelBridgePolicy !== undefined) {
|
||||
await this.#startModelBridge(binding, preparation, workspace)
|
||||
}
|
||||
const acceptedAt = new Date(this.#now()).toISOString()
|
||||
binding.imageTool?.close()
|
||||
binding.imageTool = undefined
|
||||
if (preparation.imageTool) {
|
||||
let sequence = 0n
|
||||
const imageTool = new AgentImageToolMcp(preparation.imageTool, async payload => {
|
||||
if (binding.activeOperationId !== preparation.operationId || binding.transportState === 'detached') {
|
||||
throw new Error('Image tool prompt is no longer connected')
|
||||
}
|
||||
await this.#options.blobSink!({
|
||||
header: {
|
||||
protocolMajor: AGENT_PROTOCOL_VERSION.major, protocolMinor: AGENT_PROTOCOL_VERSION.minor,
|
||||
kind: 'blob', direction: 'agent-to-main',
|
||||
connectionId: binding.connectionId, generation: binding.controllerGeneration,
|
||||
channelId: preparation.imageTool!.channelId, channelEpoch: preparation.imageTool!.channelEpoch,
|
||||
sequence: String(++sequence), payloadLength: payload.byteLength
|
||||
}, payload
|
||||
}, { bindingId: binding.request.bindingId, controllerId: binding.controllerId,
|
||||
controllerGeneration: binding.controllerGeneration })
|
||||
})
|
||||
binding.imageTool = imageTool
|
||||
await imageTool.start()
|
||||
}
|
||||
const acceptance = remotePromptOperationAcceptanceSchema.parse({
|
||||
...(binding.imageTool?.url ? { imageToolUrl: binding.imageTool.url } : {}),
|
||||
bindingId: preparation.bindingId,
|
||||
operationId: preparation.operationId,
|
||||
requestId: preparation.requestId,
|
||||
@@ -1448,6 +1489,9 @@ export class RuntimeAcpBackend {
|
||||
bindingId: binding.request.bindingId,
|
||||
controllerId: binding.controllerId,
|
||||
workspaceDirectory: binding.workspaceDirectory,
|
||||
mcpServers: () => binding.imageTool?.url ? [{
|
||||
type: 'http', name: remoteImageToolMcpName(binding.request.bindingId), url: binding.imageTool.url, headers: []
|
||||
}] : [],
|
||||
...(binding.modelBridgePolicy === undefined
|
||||
? {}
|
||||
: {
|
||||
@@ -1461,7 +1505,8 @@ export class RuntimeAcpBackend {
|
||||
transport: binding.sharedProcess.transport,
|
||||
prepareSession: async (sessionId: string, operationId: string, workMode: 'ask' | 'execute') => {
|
||||
await binding.sharedProcess!.transport.setModelRoute(
|
||||
sessionId, operationId, binding.modelBridgeBroker!.socketPath, workMode
|
||||
sessionId, operationId, binding.modelBridgeBroker!.socketPath, workMode,
|
||||
binding.imageTool ? remoteImageToolMcpName(binding.request.bindingId) : undefined
|
||||
)
|
||||
}
|
||||
} : {}),
|
||||
@@ -1816,6 +1861,8 @@ export class RuntimeAcpBackend {
|
||||
poisonIfActive: boolean,
|
||||
preserveRuntime = false
|
||||
): Promise<void> {
|
||||
binding.imageTool?.close()
|
||||
binding.imageTool = undefined
|
||||
const broker = binding.modelBridgeBroker
|
||||
const client = binding.modelBridgeClient
|
||||
binding.modelBridgeBroker = undefined
|
||||
@@ -2141,6 +2188,8 @@ export class RuntimeAcpBackend {
|
||||
return
|
||||
}
|
||||
binding.transportState = 'detached'
|
||||
binding.imageTool?.close()
|
||||
binding.imageTool = undefined
|
||||
if (binding.modelBridgeClient !== undefined) {
|
||||
await this.#closeModelBridge(binding, false, true).catch(
|
||||
() => undefined
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { remoteImageToolMcpName } from '../../shared/remote-image-tool-node'
|
||||
import { remoteQuestionSchema } from '../../shared/remote-question-contracts'
|
||||
import type { AgentQuestionAnswer } from '../../shared/contracts'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
@@ -191,6 +192,7 @@ type SessionRecord = {
|
||||
sessionId: string
|
||||
context: ChannelContext
|
||||
modelBridge?: RemoteModelBridgeSession
|
||||
imageToolUrl?: string
|
||||
sendHistoryWithNextPrompt: boolean
|
||||
ownedPromptAttached?: boolean
|
||||
}
|
||||
@@ -1183,7 +1185,9 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
raw = await this.awaitOperation(
|
||||
context,
|
||||
'启动远端请求',
|
||||
context.channel.preparePrompt(preparation),
|
||||
request.imageToolBinding
|
||||
? context.channel.preparePrompt(preparation, request.imageToolBinding, signal)
|
||||
: context.channel.preparePrompt(preparation),
|
||||
signal
|
||||
)
|
||||
} catch (error) {
|
||||
@@ -1596,7 +1600,7 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
state.agent.loadSession({
|
||||
sessionId: binding.acpSessionId,
|
||||
cwd: this.options.workspacePath,
|
||||
mcpServers: []
|
||||
mcpServers: this.imageMcpServers(binding.bindingId, prepared.acceptance.imageToolUrl)
|
||||
}),
|
||||
signal
|
||||
)
|
||||
@@ -1607,7 +1611,7 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
state.agent.resumeSession({
|
||||
sessionId: binding.acpSessionId,
|
||||
cwd: this.options.workspacePath,
|
||||
mcpServers: []
|
||||
mcpServers: this.imageMcpServers(binding.bindingId, prepared.acceptance.imageToolUrl)
|
||||
}),
|
||||
signal
|
||||
)
|
||||
@@ -1619,6 +1623,7 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
sessionId: binding.acpSessionId,
|
||||
context,
|
||||
modelBridge: prepared.modelBridge,
|
||||
imageToolUrl: prepared.acceptance.imageToolUrl,
|
||||
sendHistoryWithNextPrompt: false
|
||||
}
|
||||
pendingModelBridge = undefined
|
||||
@@ -1668,7 +1673,7 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
'创建会话',
|
||||
state.agent.newSession({
|
||||
cwd: this.options.workspacePath,
|
||||
mcpServers: []
|
||||
mcpServers: this.imageMcpServers(binding.bindingId, prepared.acceptance.imageToolUrl)
|
||||
}),
|
||||
signal
|
||||
)
|
||||
@@ -1697,6 +1702,7 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
sessionId: response.sessionId,
|
||||
context,
|
||||
modelBridge: prepared.modelBridge,
|
||||
imageToolUrl: prepared.acceptance.imageToolUrl,
|
||||
sendHistoryWithNextPrompt: true
|
||||
}
|
||||
pendingModelBridge = undefined
|
||||
@@ -1725,6 +1731,10 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
private imageMcpServers(bindingId: string, url?: string) {
|
||||
return url ? [{ type: 'http' as const, name: remoteImageToolMcpName(bindingId), url, headers: [] }] : []
|
||||
}
|
||||
|
||||
private async getSession(
|
||||
conversationId: string,
|
||||
request: AgentExecutionRequest,
|
||||
@@ -1782,6 +1792,13 @@ export class AcpRemoteRuntime implements AgentRuntime {
|
||||
current.modelBridge = prepared.modelBridge
|
||||
if (this.options.modelProfile !== undefined) {
|
||||
current.ownedPromptAttached = false
|
||||
} else if (prepared.acceptance.imageToolUrl || current.imageToolUrl) {
|
||||
const state = await this.getState(current.context, signal)
|
||||
await this.awaitOperation(current.context, 'Refresh image tools', state.agent.resumeSession({
|
||||
sessionId: current.sessionId, cwd: this.options.workspacePath,
|
||||
mcpServers: this.imageMcpServers(current.binding.bindingId, prepared.acceptance.imageToolUrl)
|
||||
}), signal)
|
||||
current.imageToolUrl = prepared.acceptance.imageToolUrl
|
||||
}
|
||||
return current
|
||||
} catch (error) {
|
||||
|
||||
@@ -464,6 +464,34 @@ describe('ContinueAgentRuntime', () => {
|
||||
expect(gateway.grantCustomMcp).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([undefined, 'existing-capability'])('cleans up image capabilities after custom MCP preparation fails (existing=%s)', async existingToken => {
|
||||
const gateway = {
|
||||
getEndpoint: () => 'http://127.0.0.1:4567/mcp',
|
||||
bindImageTool: vi.fn(() => existingToken ?? 'image-capability'),
|
||||
grantCustomMcp: vi.fn(() => 'custom-capability'),
|
||||
prepareCustomMcpTools: vi.fn(async () => { throw new Error('Custom MCP unavailable') }),
|
||||
revoke: vi.fn()
|
||||
}
|
||||
const context = { requestId: randomUUID(), conversationId: 'conversation-1', messageId: 'message-1', workMode: 'execute' as const }
|
||||
const runtime = new ContinueAgentRuntime({
|
||||
binaryPath: '', configPath: 'C:\\safe config\\continue.yaml', defaultWorkspace: process.cwd(),
|
||||
hostCacheRoot: 'C:\\safe\\continue-host',
|
||||
knowledgeGateway: gateway as unknown as KnowledgeMcpGateway,
|
||||
mcpServers: [{ id: randomUUID(), name: 'Custom', description: '', enabled: true, allowDynamicTools: false,
|
||||
assignments: ['continue'], secretConfigured: false, transport: 'http', url: 'https://mcp.test' }],
|
||||
createHostAdapter: () => ({ getPreparedHost: mocks.prepareHost, run: mocks.runHost, dispose: mocks.disposeHost })
|
||||
})
|
||||
try {
|
||||
await expect((async () => {
|
||||
for await (const event of runtime.run({ ...context, prompt: 'Draw an image', knowledgeCapabilityToken: existingToken,
|
||||
imageToolBinding: { context, describe: async () => 'Images', call: vi.fn() }
|
||||
}, new AbortController().signal)) void event
|
||||
})()).rejects.toThrow('Custom MCP unavailable')
|
||||
expect(gateway.revoke.mock.calls).toEqual(existingToken ? [['custom-capability']] : [['custom-capability'], ['image-capability']])
|
||||
expect(mocks.runHost).not.toHaveBeenCalled()
|
||||
} finally { await runtime.dispose() }
|
||||
})
|
||||
|
||||
it('adds assigned Skill instructions to the Continue prompt', async () => {
|
||||
let hostOptions: ContinueHostAdapterOptions | undefined
|
||||
const runtime = new ContinueAgentRuntime({
|
||||
|
||||
@@ -455,48 +455,46 @@ export class ContinueAgentRuntime implements AgentRuntime {
|
||||
}
|
||||
|
||||
const execute = request.workMode === 'execute'
|
||||
const imageCapabilityToken = request.imageToolBinding && execute
|
||||
? this.options.knowledgeGateway?.bindImageTool(request.imageToolBinding, signal, request.knowledgeCapabilityToken)
|
||||
: undefined
|
||||
const knowledgeEndpoint = this.options.knowledgeGateway?.getEndpoint()
|
||||
const knowledgeCapability =
|
||||
request.knowledgeCapabilityToken && knowledgeEndpoint
|
||||
(imageCapabilityToken ?? request.knowledgeCapabilityToken) && knowledgeEndpoint
|
||||
? {
|
||||
endpoint: knowledgeEndpoint,
|
||||
token: request.knowledgeCapabilityToken
|
||||
token: (imageCapabilityToken ?? request.knowledgeCapabilityToken)!
|
||||
}
|
||||
: undefined
|
||||
let customMcpCapability:
|
||||
| { endpoint: string; token: string }
|
||||
| undefined
|
||||
if (
|
||||
execute &&
|
||||
knowledgeEndpoint &&
|
||||
this.options.mcpServers?.length
|
||||
) {
|
||||
const token = this.options.knowledgeGateway?.grantCustomMcp(
|
||||
request.requestId,
|
||||
this.options.mcpServers,
|
||||
signal
|
||||
)
|
||||
if (token) {
|
||||
customMcpCapability = {
|
||||
endpoint: knowledgeEndpoint,
|
||||
token
|
||||
}
|
||||
try {
|
||||
await this.options.knowledgeGateway?.prepareCustomMcpTools(
|
||||
token,
|
||||
signal
|
||||
)
|
||||
} catch (error) {
|
||||
this.options.knowledgeGateway?.revoke(token)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
let result: ContinueHostRunResult
|
||||
const emittedTools = new Map<string, ContinueHostTool>()
|
||||
let emittedText = ''
|
||||
const requestQuestionIds = new Set<string>()
|
||||
try {
|
||||
if (
|
||||
execute &&
|
||||
knowledgeEndpoint &&
|
||||
this.options.mcpServers?.length
|
||||
) {
|
||||
const token = this.options.knowledgeGateway?.grantCustomMcp(
|
||||
request.requestId,
|
||||
this.options.mcpServers,
|
||||
signal
|
||||
)
|
||||
if (token) {
|
||||
customMcpCapability = {
|
||||
endpoint: knowledgeEndpoint,
|
||||
token
|
||||
}
|
||||
await this.options.knowledgeGateway?.prepareCustomMcpTools(
|
||||
token,
|
||||
signal
|
||||
)
|
||||
}
|
||||
}
|
||||
const host = this.getHostAdapter(
|
||||
binaryPath,
|
||||
execute || knowledgeCapability ? 'agent' : 'chat'
|
||||
@@ -643,6 +641,7 @@ export class ContinueAgentRuntime implements AgentRuntime {
|
||||
customMcpCapability.token
|
||||
)
|
||||
}
|
||||
if (imageCapabilityToken && imageCapabilityToken !== request.knowledgeCapabilityToken) this.options.knowledgeGateway?.revoke(imageCapabilityToken)
|
||||
}
|
||||
if (!result.text) {
|
||||
throw new Error('Continue CLI 未返回内容')
|
||||
|
||||
@@ -41,6 +41,64 @@ import { LocalRuntimeRegistry } from './local-runtime-registry'
|
||||
import { createAgentRuntime } from './create-runtime'
|
||||
import { defaultRuntimeSettings } from '../../shared/contracts'
|
||||
import type { ResolvedRuntimeSettings } from '../runtime-settings-store'
|
||||
import { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import { ImageGenerationService } from './image-generation-service'
|
||||
|
||||
vi.mock('electron', () => ({ nativeImage: {} }))
|
||||
|
||||
it('generates and edits durable images through the controlled Harness Main proxy and refreshes it in Ask', async () => {
|
||||
const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-images-')))
|
||||
const database = new AssistantDatabase(join(root, 'assistant.sqlite'))
|
||||
database.initialize(root)
|
||||
const conversationId = crypto.randomUUID()
|
||||
const profileId = crypto.randomUUID()
|
||||
const fetcher = vi.fn<typeof fetch>(async () => Response.json({ data: [{ b64_json: 'iVBORw0KGgo=' }] }))
|
||||
const service = new ImageGenerationService({ database, fetcher, getSettings: async () => ({ defaultImageModelProfileId: profileId, modelProfiles: [{
|
||||
id: profileId, name: 'Image fixture', modelName: 'fixture', protocol: 'openai-images-generations',
|
||||
allowConversationInvocation: true, baseUrl: 'https://image.test/v1', authentication: 'none'
|
||||
}] }) })
|
||||
let input: Record<string, unknown> = { intent: 'create', prompt: 'Blue circle' }
|
||||
const inProcess = createInProcessLaunch(root, {
|
||||
stream(options) {
|
||||
const prompt = latestUserText(options)
|
||||
const tool = options.tools?.find(tool => tool.name === 'generate_image')
|
||||
if (prompt === 'ask-image') {
|
||||
expect(tool).toBeUndefined()
|
||||
return textResponse('Ask is read-only')
|
||||
}
|
||||
expect(tool).toBeDefined()
|
||||
expect(tool?.description).toContain(profileId)
|
||||
if (!toolResultText(options, prompt)) return toolCall(prompt, 'generate_image', input)
|
||||
expect(toolResultText(options, prompt)).toContain('completed')
|
||||
return textResponse('IMAGE_SAVED')
|
||||
}
|
||||
})
|
||||
const runtime = new DeepSeekHarnessRuntime({ defaultWorkspace: root, baseUrl: 'https://chat.test/v1', model: 'fixture',
|
||||
launch: inProcess.launch, credentialRefs: { [CREDENTIAL_REF]: 'fixture' }, toolProvider: new ModelToolProvider(root),
|
||||
initializationTimeoutMs: 20_000, promptTimeoutMs: 20_000, shutdownTimeoutMs: 5_000 })
|
||||
try {
|
||||
for (const intent of ['create', 'edit'] as const) {
|
||||
const messageId = crypto.randomUUID()
|
||||
const requestId = crypto.randomUUID()
|
||||
database.saveLocalConversations([{ header: { id: conversationId, title: 'Images', updatedAt: Date.now() },
|
||||
messages: [{ id: messageId, role: 'assistant', content: '', createdAt: Date.now(), state: 'complete' }] }])
|
||||
const events = await collect(runtime.run({ requestId, conversationId, prompt: `${intent}-${requestId}`, workMode: 'execute',
|
||||
imageToolBinding: service.bind({ conversationId, messageId, requestId, workMode: 'execute' }) }, new AbortController().signal, async () => 'once'))
|
||||
expect(events.at(-1)?.type).toBe('done')
|
||||
const operation = database.getConversation(conversationId).messages.find(message => message.id === messageId)!.imageOperations![0]!
|
||||
expect(operation.state).toBe('completed')
|
||||
input = { intent: 'edit', prompt: 'Turn red', sourceArtifactIds: operation.artifactIds }
|
||||
}
|
||||
await collect(runtime.run({ requestId: crypto.randomUUID(), conversationId, prompt: 'ask-image', workMode: 'ask' }, new AbortController().signal))
|
||||
expect(fetcher).toHaveBeenCalledTimes(2)
|
||||
expect(String(fetcher.mock.calls[1]![0])).toMatch(/images\/edits$/u)
|
||||
} finally {
|
||||
await runtime.dispose()
|
||||
await service.dispose()
|
||||
database.close()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
}, 60_000)
|
||||
|
||||
const CREDENTIAL_REF = 'GOODBUDDY_HARNESS_MODEL_API_KEY'
|
||||
const SKILL_CALL_ID = 'e2e-skill-call'
|
||||
|
||||
@@ -290,7 +290,7 @@ function isMainWebTool(
|
||||
)
|
||||
}
|
||||
|
||||
function dshCompatibleWebInputSchema(
|
||||
function dshCompatibleMainInputSchema(
|
||||
schema: Record<string, unknown>
|
||||
): Record<string, unknown> {
|
||||
const compatible: Record<string, unknown> = {}
|
||||
@@ -311,7 +311,7 @@ function dshCompatibleWebInputSchema(
|
||||
propertySchema &&
|
||||
typeof propertySchema === 'object' &&
|
||||
!Array.isArray(propertySchema)
|
||||
? dshCompatibleWebInputSchema(
|
||||
? dshCompatibleMainInputSchema(
|
||||
propertySchema as Record<string, unknown>
|
||||
)
|
||||
: propertySchema
|
||||
@@ -325,7 +325,7 @@ function dshCompatibleWebInputSchema(
|
||||
typeof value === 'object' &&
|
||||
!Array.isArray(value)
|
||||
) {
|
||||
compatible.items = dshCompatibleWebInputSchema(
|
||||
compatible.items = dshCompatibleMainInputSchema(
|
||||
value as Record<string, unknown>
|
||||
)
|
||||
continue
|
||||
@@ -335,7 +335,7 @@ function dshCompatibleWebInputSchema(
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
!Array.isArray(candidate)
|
||||
? dshCompatibleWebInputSchema(
|
||||
? dshCompatibleMainInputSchema(
|
||||
candidate as Record<string, unknown>
|
||||
)
|
||||
: candidate
|
||||
@@ -350,8 +350,8 @@ function proxyToolInputSchema(
|
||||
ReturnType<ModelToolProviderLike['listTools']>
|
||||
>[number]
|
||||
): Record<string, unknown> {
|
||||
return isMainWebTool(tool)
|
||||
? dshCompatibleWebInputSchema(tool.inputSchema)
|
||||
return isMainWebTool(tool) || tool.name === 'generate_image'
|
||||
? dshCompatibleMainInputSchema(tool.inputSchema)
|
||||
: tool.inputSchema
|
||||
}
|
||||
|
||||
@@ -368,7 +368,7 @@ function boundedProxyToolCatalog(
|
||||
const catalog = tools.filter(
|
||||
(tool) =>
|
||||
isMainWebTool(tool) ||
|
||||
(workMode === 'execute' && tool.source === 'mcp')
|
||||
(workMode === 'execute' && (tool.source === 'mcp' || tool.name === 'generate_image'))
|
||||
)
|
||||
if (catalog.length > MAX_MCP_PROXY_TOOLS) {
|
||||
throw new Error(
|
||||
@@ -386,7 +386,7 @@ function boundedProxyToolCatalog(
|
||||
names.add(tool.name)
|
||||
const description = tool.description.slice(
|
||||
0,
|
||||
MAX_MCP_TOOL_DESCRIPTION_CHARACTERS
|
||||
tool.name === 'generate_image' ? imageToolDescriptionLimit : MAX_MCP_TOOL_DESCRIPTION_CHARACTERS
|
||||
)
|
||||
const inputSchema = proxyToolInputSchema(tool)
|
||||
let serialized: string
|
||||
@@ -838,6 +838,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
const run = this.activeRuns.get(params.sessionId)
|
||||
if (!run?.toolProvider || run.closed) return { tools: [] }
|
||||
const context = {
|
||||
imageToolBinding: run.request.imageToolBinding,
|
||||
conversationId:
|
||||
run?.request.conversationId ??
|
||||
'deepseek-harness-tool-catalog',
|
||||
@@ -855,7 +856,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
)
|
||||
const catalog = boundedProxyToolCatalog(tools, context.workMode)
|
||||
this.proxyToolCatalogs.set(params.sessionId, tools.filter(
|
||||
(tool) => tool.source === 'mcp' || isMainWebTool(tool)
|
||||
(tool) => tool.source === 'mcp' || isMainWebTool(tool) || tool.name === 'generate_image'
|
||||
))
|
||||
return { tools: catalog }
|
||||
}
|
||||
@@ -881,6 +882,8 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
)
|
||||
}
|
||||
const context = {
|
||||
imageToolBinding: run.request.imageToolBinding,
|
||||
...(name === 'generate_image' ? { toolCallId: typeof params.callId === 'string' ? params.callId : crypto.randomUUID() } : {}),
|
||||
conversationId: run.request.conversationId,
|
||||
browserTabId: run.request.browserTabId,
|
||||
workMode:
|
||||
@@ -894,7 +897,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
const tool = tools.find(
|
||||
(candidate) =>
|
||||
candidate.name === name &&
|
||||
(candidate.source === 'mcp' ||
|
||||
(candidate.source === 'mcp' || candidate.name === 'generate_image' ||
|
||||
isMainWebTool(candidate))
|
||||
)
|
||||
if (!tool) {
|
||||
@@ -1754,3 +1757,4 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
).catch(() => undefined)
|
||||
}
|
||||
}
|
||||
import { imageToolDescriptionLimit } from '../../shared/image-generation-contracts'
|
||||
|
||||
@@ -221,7 +221,7 @@ function parseProxyToolCatalog(
|
||||
!/^[a-zA-Z0-9_-]{1,64}$/u.test(tool.name) ||
|
||||
names.has(tool.name) ||
|
||||
typeof tool.description !== 'string' ||
|
||||
tool.description.length > 1_000 ||
|
||||
tool.description.length > (tool.name === 'generate_image' ? imageToolDescriptionLimit : 1_000) ||
|
||||
!tool.inputSchema ||
|
||||
typeof tool.inputSchema !== 'object' ||
|
||||
Array.isArray(tool.inputSchema)
|
||||
@@ -885,6 +885,7 @@ export class GoodBuddyHarnessControlPlane {
|
||||
{
|
||||
sessionId,
|
||||
name: tool.name,
|
||||
callId: randomUUID(),
|
||||
arguments: argumentsValue
|
||||
}
|
||||
)
|
||||
@@ -1450,3 +1451,4 @@ export class GoodBuddyHarnessControlPlane {
|
||||
await this.disposing
|
||||
}
|
||||
}
|
||||
import { imageToolDescriptionLimit } from '../../shared/image-generation-contracts'
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
|
||||
import { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import { ImageGenerationService, type ImageServiceSettings, type ImageGenerationServiceOptions } from './image-generation-service'
|
||||
import { imageOperationSchema, type ImageOperation } from '../../shared/image-generation-contracts'
|
||||
import { KnowledgeMcpGateway } from './knowledge-mcp-gateway'
|
||||
import { ModelToolProvider } from './model-tool-provider'
|
||||
|
||||
vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64') }) } }))
|
||||
const png = 'iVBORw0KGgo='
|
||||
const cleanups: (() => void | Promise<void>)[] = []
|
||||
afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup() })
|
||||
|
||||
function setup(fetcher = vi.fn<typeof fetch>(async () => Response.json({ data: [{ b64_json: png }] })), observers: Pick<ImageGenerationServiceOptions, 'onOperation' | 'onUsage' | 'onError'> = {}) {
|
||||
const database = new AssistantDatabase(':memory:')
|
||||
database.initialize(process.cwd())
|
||||
cleanups.push(() => database.close())
|
||||
const context = { conversationId: randomUUID(), messageId: randomUUID(), requestId: randomUUID(), workMode: 'execute' as const }
|
||||
const message = { id: context.messageId, role: 'assistant' as const, content: '', createdAt: Date.now(), state: 'complete' as const }
|
||||
const header = { id: context.conversationId, title: 'Images', updatedAt: Date.now() }
|
||||
database.saveLocalConversations([{ header, messages: [message] }])
|
||||
const profile = { id: randomUUID(), name: 'Image A', modelName: 'image-a', protocol: 'openai-images-generations',
|
||||
allowConversationInvocation: true, baseUrl: 'https://provider.test/v1', authentication: 'api-key' as const, apiKey: 'test-secret' }
|
||||
const settings: ImageServiceSettings = { modelProfiles: [profile], defaultImageModelProfileId: profile.id }
|
||||
const events: ImageOperation[] = []
|
||||
const service = new ImageGenerationService({ database, getSettings: async () => settings, fetcher,
|
||||
onOperation: operation => { imageOperationSchema.parse(operation); events.push(operation); observers.onOperation?.(operation) }, onUsage: observers.onUsage, onError: observers.onError })
|
||||
service.initialize()
|
||||
cleanups.push(() => service.dispose())
|
||||
return { database, service, context, fetcher, settings, profile, events, header, message }
|
||||
}
|
||||
|
||||
describe('Main conversation image service', () => {
|
||||
it('does not let a destroyed Renderer or usage observer prevent submission or overwrite completion', async () => {
|
||||
const onError = vi.fn()
|
||||
const h = setup(vi.fn<typeof fetch>(async () => Response.json({ data: [{ b64_json: png }], usage: { input_tokens: 2, output_tokens: 3 } })), {
|
||||
onOperation: () => { throw new Error('Renderer is destroyed') },
|
||||
onUsage: () => { throw new Error('Usage observer failed') }, onError
|
||||
})
|
||||
const operation = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'observer')
|
||||
expect(operation.state).toBe('completed')
|
||||
expect(h.service.getOperation(h.context.conversationId, operation.id).state).toBe('completed')
|
||||
expect(h.fetcher).toHaveBeenCalledOnce()
|
||||
expect(onError).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns the known persistence failure instead of a stale running state when the database stops accepting writes', async () => {
|
||||
const onError = vi.fn()
|
||||
const h = setup(undefined, { onError })
|
||||
h.fetcher.mockImplementation(async () => {
|
||||
vi.spyOn(h.database, 'saveConversationImageOperation').mockImplementation(() => { throw new Error('Disk full') })
|
||||
return Response.json({ data: [{ b64_json: png }] })
|
||||
})
|
||||
const operation = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'disk-full')
|
||||
expect(operation).toMatchObject({ state: 'failed', error: 'Disk full', artifactIds: [] })
|
||||
expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'Disk full' }))
|
||||
expect(h.fetcher).toHaveBeenCalledOnce()
|
||||
})
|
||||
it('saves before publishing, preserves metadata against stale renderer saves, and edits durable history', async () => {
|
||||
const h = setup()
|
||||
const first = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue circle' }, 'first')
|
||||
expect(first.state).toBe('completed')
|
||||
expect(h.database.getArtifact(first.artifactIds[0]!).content).toContain(png)
|
||||
h.database.saveLocalConversations([{ header: h.header, messages: [h.message] }])
|
||||
expect(h.service.getOperation(h.context.conversationId, first.id)).toEqual(first)
|
||||
expect(h.database.getConversation(h.context.conversationId).messages[0]!.artifactIds).toEqual(first.artifactIds)
|
||||
h.settings.defaultImageModelProfileId = randomUUID()
|
||||
const second = await h.service.bind({ ...h.context, requestId: randomUUID() }).call({ intent: 'edit', prompt: 'Turn red', sourceArtifactIds: first.artifactIds }, 'edit')
|
||||
expect(second.modelProfileId).toBe(first.modelProfileId)
|
||||
expect(String(h.fetcher.mock.calls[1]![0])).toMatch(/images\/edits$/u)
|
||||
expect((h.fetcher.mock.calls[1]![1]!.body as FormData).get('image')).toBeInstanceOf(Blob)
|
||||
expect(h.events.map(event => event.state)).toEqual(['running', 'saving', 'completed', 'running', 'saving', 'completed'])
|
||||
})
|
||||
|
||||
it('persists upload bytes and rejects cross-conversation or missing source references', async () => {
|
||||
const h = setup()
|
||||
const ids = h.service.persistUploads(h.context, [{ name: 'upload.png', mediaType: 'image/png', data: png }])
|
||||
h.database.saveLocalConversations([{ header: h.header, messages: [h.message] }])
|
||||
expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageSourceArtifactIds).toEqual(ids)
|
||||
const binding = h.service.bind(h.context)
|
||||
expect((await binding.call({ intent: 'edit', prompt: 'Blue', sourceArtifactIds: ids }, 'upload')).state).toBe('completed')
|
||||
await expect(binding.call({ intent: 'edit', prompt: 'Blue', sourceArtifactIds: [randomUUID()] }, 'foreign')).rejects.toThrow('not available')
|
||||
await expect(binding.call({ intent: 'edit', prompt: 'Blue' }, 'missing')).rejects.toThrow('requires source')
|
||||
expect(h.fetcher).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('rejects absent, disabled, deleted defaults without silent fallback and rechecks current mode', async () => {
|
||||
const h = setup()
|
||||
let mode: 'ask' | 'execute' = 'execute'
|
||||
const binding = h.service.bind(h.context, () => mode)
|
||||
expect(await binding.describe()).toContain(h.profile.id)
|
||||
h.profile.allowConversationInvocation = false
|
||||
await expect(binding.call({ intent: 'create', prompt: 'Blue' }, 'disabled')).rejects.toThrow('unavailable')
|
||||
h.profile.allowConversationInvocation = true
|
||||
h.settings.defaultImageModelProfileId = randomUUID()
|
||||
await expect(binding.call({ intent: 'create', prompt: 'Blue' }, 'deleted')).rejects.toThrow('unavailable')
|
||||
h.settings.defaultImageModelProfileId = null
|
||||
await expect(binding.call({ intent: 'create', prompt: 'Blue' }, 'absent')).rejects.toThrow('default')
|
||||
mode = 'ask'
|
||||
await expect(binding.call({ intent: 'create', prompt: 'Blue', modelProfileId: h.profile.id }, 'ask')).rejects.toThrow('Execute')
|
||||
expect(h.fetcher).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('continues after chat abort and pins the accepted profile while a replacement binding starts', async () => {
|
||||
let resolve!: (response: Response) => void
|
||||
const h = setup(vi.fn<typeof fetch>(() => new Promise(done => { resolve = done })))
|
||||
const controller = new AbortController()
|
||||
const pending = h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'call', controller.signal)
|
||||
await vi.waitFor(() => expect(h.fetcher).toHaveBeenCalledOnce())
|
||||
const providerSignal = h.fetcher.mock.calls[0]![1]!.signal!
|
||||
controller.abort()
|
||||
expect((await pending).state).toBe('running')
|
||||
expect(providerSignal.aborted).toBe(false)
|
||||
h.profile.modelName = 'replacement'
|
||||
h.profile.allowConversationInvocation = false
|
||||
resolve(Response.json({ data: [{ b64_json: png }] }))
|
||||
await vi.waitFor(() => expect(h.events.at(-1)?.state).toBe('completed'))
|
||||
expect(h.events.at(-1)).toMatchObject({ modelName: 'image-a', modelProfileName: 'Image A' })
|
||||
expect(JSON.parse(h.fetcher.mock.calls[0]![1]!.body as string).model).toBe('image-a')
|
||||
await expect(h.service.bind({ ...h.context, requestId: randomUUID() }).call({ intent: 'create', prompt: 'Other' }, 'other')).rejects.toThrow('unavailable')
|
||||
})
|
||||
|
||||
it('deduplicates one invocation, creates a new operation on manual regenerate, and retains late cancellation output', async () => {
|
||||
let resolve!: (response: Response) => void
|
||||
const h = setup(vi.fn<typeof fetch>(() => new Promise(done => { resolve = done })))
|
||||
const binding = h.service.bind(h.context)
|
||||
const input = { intent: 'create', prompt: 'Blue' }
|
||||
const one = binding.call(input, 'same')
|
||||
const two = binding.call(input, 'same')
|
||||
await vi.waitFor(() => expect(h.events).toHaveLength(1))
|
||||
h.service.cancel(h.context.conversationId, h.events[0]!.id)
|
||||
resolve(Response.json({ data: [{ b64_json: png }] }))
|
||||
const completed = await one
|
||||
expect(await two).toEqual(completed)
|
||||
expect(completed).toMatchObject({ state: 'completed', cancellationRequested: true })
|
||||
expect(h.fetcher).toHaveBeenCalledOnce()
|
||||
h.fetcher.mockImplementation(async () => Response.json({ data: [{ b64_json: png }] }))
|
||||
const regenerated = await h.service.regenerate(h.context, completed.id)
|
||||
expect(regenerated.id).not.toBe(completed.id)
|
||||
expect(h.fetcher).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('never falls back to a paid generation after an edit rejection and redacts provider errors', async () => {
|
||||
const h = setup(vi.fn<typeof fetch>(async () => Response.json({ error: { message: 'Image editing not supported Authorization: Bearer private-key' } }, { status: 405 })))
|
||||
const ids = h.service.persistUploads(h.context, [{ name: 'source', mediaType: 'image/png', data: png }])
|
||||
const result = await h.service.bind(h.context).call({ intent: 'edit', prompt: 'Blue', sourceArtifactIds: ids }, 'edit')
|
||||
expect(result.state).toBe('failed')
|
||||
expect(result.error).toContain('405')
|
||||
expect(result.error).not.toContain('private-key')
|
||||
expect(h.fetcher).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('does not resurrect a deleted conversation or retain its late artifact', async () => {
|
||||
let resolve!: (response: Response) => void
|
||||
const h = setup(vi.fn<typeof fetch>(() => new Promise(done => { resolve = done })))
|
||||
const pending = h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'deleted')
|
||||
await vi.waitFor(() => expect(h.fetcher).toHaveBeenCalledOnce())
|
||||
h.database.deleteLocalConversation(h.context.conversationId)
|
||||
resolve(Response.json({ data: [{ b64_json: png }] }))
|
||||
await pending
|
||||
expect(() => h.database.getConversation(h.context.conversationId)).toThrow()
|
||||
expect(h.database.listArtifacts()).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('marks unfinished operations unconfirmed on initialization without issuing requests', () => {
|
||||
const h = setup()
|
||||
h.database.saveConversationImageOperation({ id: randomUUID(), conversationId: h.context.conversationId, messageId: h.context.messageId,
|
||||
requestId: h.context.requestId, callId: 'old', modelProfileId: h.profile.id, modelName: h.profile.name,
|
||||
input: { intent: 'create', prompt: 'Blue', sourceArtifactIds: [] }, state: 'running', artifactIds: [], createdAt: 1, updatedAt: 1 })
|
||||
h.service.initialize()
|
||||
expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageOperations![0]!.state).toBe('unconfirmed')
|
||||
expect(h.fetcher).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports provider-side uncertainty after network loss and local cancellation without retrying', async () => {
|
||||
const h = setup(vi.fn<typeof fetch>(async () => { throw new TypeError('fetch failed') }))
|
||||
const uncertain = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'network')
|
||||
expect(uncertain.state).toBe('unconfirmed')
|
||||
h.fetcher.mockImplementation((_url, init) => new Promise((_resolve, reject) => {
|
||||
init!.signal!.addEventListener('abort', () => reject(init!.signal!.reason), { once: true })
|
||||
}))
|
||||
const pending = h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'cancel')
|
||||
await vi.waitFor(() => expect(h.fetcher).toHaveBeenCalledTimes(2))
|
||||
h.service.cancel(h.context.conversationId, h.events.at(-1)!.id)
|
||||
expect(await pending).toMatchObject({ state: 'stopped', cancellationRequested: true })
|
||||
expect(h.fetcher).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('reuses an accepted invocation across bindings and does not skip a disabled historical model', async () => {
|
||||
const h = setup()
|
||||
const created = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'same')
|
||||
expect((await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'same')).id).toBe(created.id)
|
||||
const other = { ...h.profile, id: randomUUID(), name: 'Other', modelName: 'other' }
|
||||
h.settings.modelProfiles = [h.profile, other]
|
||||
h.settings.defaultImageModelProfileId = other.id
|
||||
h.profile.allowConversationInvocation = false
|
||||
await expect(h.service.bind(h.context).call({ intent: 'edit', prompt: 'Red', sourceArtifactIds: created.artifactIds }, 'blocked-edit')).rejects.toThrow('unavailable')
|
||||
const edited = await h.service.bind(h.context).call({ intent: 'edit', prompt: 'Red', sourceArtifactIds: created.artifactIds, modelProfileId: other.id }, 'explicit-edit')
|
||||
expect(edited.modelProfileId).toBe(other.id)
|
||||
expect(h.fetcher).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('dispatches the native Model tool through its scoped binding and denies Ask', async () => {
|
||||
const h = setup()
|
||||
const provider = new ModelToolProvider(process.cwd())
|
||||
cleanups.push(() => provider.dispose())
|
||||
const context = { conversationId: h.context.conversationId, workMode: 'execute' as const, imageToolBinding: h.service.bind(h.context), toolCallId: 'native' }
|
||||
const result = await provider.callTool('generate_image', { intent: 'create', prompt: 'Blue' }, new AbortController().signal, context)
|
||||
expect(JSON.stringify(result)).toContain('completed')
|
||||
expect(JSON.stringify(result)).not.toContain(png)
|
||||
await expect(provider.callTool('generate_image', { intent: 'create', prompt: 'Blue' }, new AbortController().signal, { ...context, workMode: 'ask' })).rejects.toThrow()
|
||||
})
|
||||
|
||||
it('round trips image-only MCP discovery/call and does not abort accepted work on capability revocation', async () => {
|
||||
const h = setup()
|
||||
const gateway = new KnowledgeMcpGateway({} as never)
|
||||
await gateway.start()
|
||||
cleanups.push(() => gateway.dispose())
|
||||
const token = gateway.bindImageTool(h.service.bind(h.context), new AbortController().signal)
|
||||
const client = new Client({ name: 'image-test', version: '1' })
|
||||
cleanups.push(() => client.close())
|
||||
await client.connect(new StreamableHTTPClientTransport(new URL(gateway.getEndpoint()!), { requestInit: { headers: { Authorization: `Bearer ${token}` } } }))
|
||||
expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image'])
|
||||
const result = await client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } })
|
||||
expect(JSON.stringify(result)).toContain('completed')
|
||||
expect(h.fetcher).toHaveBeenCalledOnce()
|
||||
gateway.revoke(token)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,209 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
imageToolDescription, imageToolInputSchema,
|
||||
type ImageOperation, type ImageRequestContext, type ImageToolInput
|
||||
} from '../../shared/image-generation-contracts'
|
||||
import type { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import { ModelAgentRuntime, type ModelRuntimeOptions } from './model-runtime'
|
||||
import { withImageConversationContext } from './image-conversation-context'
|
||||
import type { AgentImage, RuntimeEvent } from './runtime'
|
||||
import { redactSensitiveText, safeToolErrorDetail } from './approval-summary'
|
||||
import type { ImageToolBinding } from './image-tool-binding'
|
||||
export type { ImageToolBinding } from './image-tool-binding'
|
||||
|
||||
export type ConversationImageProfile = {
|
||||
id: string
|
||||
name: string
|
||||
protocol: string
|
||||
modelName: string
|
||||
allowConversationInvocation?: boolean
|
||||
} & Pick<ModelRuntimeOptions, 'baseUrl' | 'apiKey' | 'authentication' | 'requestHeaders' | 'requestBody' | 'imageGenerationQuality'>
|
||||
|
||||
export type ImageServiceSettings = {
|
||||
modelProfiles: readonly ConversationImageProfile[]
|
||||
defaultImageModelProfileId?: string | null
|
||||
}
|
||||
export type ImageGenerationServiceOptions = {
|
||||
database: Pick<AssistantDatabase, 'getConversation' | 'getArtifact' | 'saveConversationImageOperation' | 'saveConversationImageSources' | 'markUnfinishedImageOperationsUnconfirmed'>
|
||||
getSettings(): Promise<ImageServiceSettings>
|
||||
onOperation?(operation: ImageOperation): void
|
||||
onUsage?(event: Extract<RuntimeEvent, { type: 'model-usage' }>): void
|
||||
/** Diagnostic sink for observer or terminal-persistence failures; must not throw. */
|
||||
onError?(error: Error): void
|
||||
fetcher?: typeof fetch
|
||||
requestTimeoutMs?: number
|
||||
toolWaitMs?: number
|
||||
}
|
||||
|
||||
/** Main owns requests; request-scoped bindings own only admission and waiting. */
|
||||
export class ImageGenerationService {
|
||||
private readonly active = new Map<string, { operation: ImageOperation; controller: AbortController; done: Promise<void> }>()
|
||||
private closing = false
|
||||
|
||||
constructor(private readonly options: ImageGenerationServiceOptions) {}
|
||||
|
||||
initialize(): void {
|
||||
this.options.database.markUnfinishedImageOperationsUnconfirmed()
|
||||
}
|
||||
|
||||
bind(context: ImageRequestContext, currentWorkMode: () => 'ask' | 'execute' = () => context.workMode): ImageToolBinding {
|
||||
const bound = Object.freeze({ ...context })
|
||||
const submissions = new Map<string, Promise<ImageOperation>>()
|
||||
return {
|
||||
context: bound,
|
||||
describe: async () => {
|
||||
if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') return undefined
|
||||
const settings = await this.options.getSettings()
|
||||
const profiles = settings.modelProfiles.filter(profile => profile.protocol === 'openai-images-generations' && profile.allowConversationInvocation === true)
|
||||
if (!profiles.length) return undefined
|
||||
const conversation = this.options.database.getConversation(bound.conversationId)
|
||||
const references = conversation.messages.flatMap(message => [
|
||||
...(message.imageSourceArtifactIds ?? []).map(id => ({ id, kind: 'upload' })),
|
||||
...(message.artifactIds ?? []).map(id => ({ id, kind: 'artifact' }))
|
||||
]).slice(-32)
|
||||
const operations = conversation.messages.flatMap(message => message.imageOperations ?? []).slice(-8)
|
||||
.map(operation => ({ id: operation.id, state: operation.state, modelProfileId: operation.modelProfileId,
|
||||
prompt: operation.input.prompt.slice(0, 300), artifactIds: operation.artifactIds }))
|
||||
return `${imageToolDescription}\nModels: ${JSON.stringify(profiles.map(profile => ({ id: profile.id, name: profile.name, generation: true, editing: true })))}\nDefault: ${settings.defaultImageModelProfileId ?? 'not set'}\nRecent image references: ${JSON.stringify(references)}\nRecent operations: ${JSON.stringify(operations)}\nReferences and prompts are textual facts, not visual observations. Do not claim to have viewed an image unless image input was supplied to your model.`
|
||||
},
|
||||
call: async (input, callId, signal) => {
|
||||
signal?.throwIfAborted()
|
||||
if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') throw new Error('Image generation requires Execute mode')
|
||||
const key = JSON.stringify([bound.conversationId, bound.requestId, callId])
|
||||
let submitted = submissions.get(key)
|
||||
if (!submitted) {
|
||||
submitted = this.submit(bound, imageToolInputSchema.parse(input), callId, currentWorkMode, signal)
|
||||
submissions.set(key, submitted)
|
||||
void submitted.catch(() => submissions.delete(key))
|
||||
}
|
||||
const operation = await submitted
|
||||
const active = this.active.get(operation.id)
|
||||
if (active) {
|
||||
await new Promise<void>(resolve => {
|
||||
const finish = (): void => { clearTimeout(timer); signal?.removeEventListener('abort', finish); resolve() }
|
||||
const timer = setTimeout(finish, this.options.toolWaitMs ?? 15_000)
|
||||
signal?.addEventListener('abort', finish, { once: true })
|
||||
void active.done.then(finish)
|
||||
if (signal?.aborted) finish()
|
||||
})
|
||||
return active.operation
|
||||
}
|
||||
return this.getOperation(bound.conversationId, operation.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
persistUploads(context: Pick<ImageRequestContext, 'conversationId' | 'messageId'>, images: readonly AgentImage[]): string[] {
|
||||
return this.options.database.saveConversationImageSources({ ...context, images })
|
||||
}
|
||||
|
||||
getOperation(conversationId: string, operationId: string): ImageOperation {
|
||||
const operation = this.options.database.getConversation(conversationId).messages
|
||||
.flatMap(message => message.imageOperations ?? []).find(item => item.id === operationId)
|
||||
if (!operation) throw new Error('Image operation does not exist in this conversation')
|
||||
return operation
|
||||
}
|
||||
|
||||
cancel(conversationId: string, operationId: string): ImageOperation {
|
||||
const previous = this.getOperation(conversationId, operationId)
|
||||
const active = this.active.get(operationId)
|
||||
if (!active) return previous
|
||||
active.operation = this.save({ ...active.operation, state: 'cancelling', cancellationRequested: true })
|
||||
active.controller.abort(new Error('Stopped waiting; the image provider may still generate the image'))
|
||||
return active.operation
|
||||
}
|
||||
|
||||
regenerate(context: ImageRequestContext, operationId: string, currentWorkMode?: () => 'ask' | 'execute'): Promise<ImageOperation> {
|
||||
const previous = this.getOperation(context.conversationId, operationId)
|
||||
return this.bind(context, currentWorkMode).call({ ...previous.input, modelProfileId: previous.modelProfileId }, randomUUID())
|
||||
}
|
||||
|
||||
cancelConversation(conversationId: string): void {
|
||||
for (const active of this.active.values()) {
|
||||
if (active.operation.conversationId === conversationId) this.cancel(conversationId, active.operation.id)
|
||||
}
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
this.closing = true
|
||||
for (const active of this.active.values()) active.controller.abort(new Error('Application is closing; image result is unconfirmed'))
|
||||
await Promise.all([...this.active.values()].map(active => active.done))
|
||||
}
|
||||
|
||||
private save(operation: ImageOperation, image?: { mimeType: 'image/png' | 'image/jpeg' | 'image/webp'; data: string }): ImageOperation {
|
||||
const saved = this.options.database.saveConversationImageOperation({ ...operation, updatedAt: Date.now() }, image)
|
||||
try {
|
||||
this.options.onOperation?.(saved)
|
||||
} catch (error) {
|
||||
// A closed Renderer must not stop an accepted provider request or change its outcome.
|
||||
this.report(error)
|
||||
}
|
||||
return saved
|
||||
}
|
||||
|
||||
private report(error: unknown): void {
|
||||
this.options.onError?.(new Error(redactSensitiveText(safeToolErrorDetail(error) ?? 'Image operation notification or persistence failed')))
|
||||
}
|
||||
|
||||
private async submit(context: ImageRequestContext, input: ImageToolInput, callId: string, currentWorkMode: () => 'ask' | 'execute', signal?: AbortSignal): Promise<ImageOperation> {
|
||||
if (this.closing) throw new Error('Image service is closing')
|
||||
const settings = await this.options.getSettings()
|
||||
const conversation = this.options.database.getConversation(context.conversationId)
|
||||
const existing = conversation.messages.flatMap(message => message.imageOperations ?? [])
|
||||
.find(operation => operation.requestId === context.requestId && operation.callId === callId)
|
||||
if (existing) return existing
|
||||
if (!conversation.messages.some(message => message.id === context.messageId)) throw new Error('Originating image message must be persisted before submission')
|
||||
const allowed = new Set(conversation.messages.flatMap(message => [
|
||||
...(message.artifactIds ?? []), ...(message.imageSourceArtifactIds ?? []),
|
||||
...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? [])
|
||||
]))
|
||||
if (input.sourceArtifactIds.some(id => !allowed.has(id))) throw new Error('Source image is not available in this conversation')
|
||||
const origins = new Set(conversation.messages.flatMap(message => message.imageOperations ?? [])
|
||||
.filter(operation => operation.artifactIds.some(id => input.sourceArtifactIds.includes(id)))
|
||||
.map(operation => operation.modelProfileId))
|
||||
if (!input.modelProfileId && origins.size > 1) throw new Error('Source images use different models; specify the image model')
|
||||
const profileId = input.modelProfileId ?? (input.intent === 'edit' ? [...origins][0] : undefined) ?? settings.defaultImageModelProfileId
|
||||
if (!profileId) throw new Error('Specify an image model or set the default image model')
|
||||
const profile = settings.modelProfiles.find(item => item.id === profileId)
|
||||
if (!profile || !profile.allowConversationInvocation || profile.protocol !== 'openai-images-generations') throw new Error('Selected image model is unavailable; select another model')
|
||||
if (profile.authentication === 'api-key' && !profile.apiKey) throw new Error('Selected image model has no API key configured')
|
||||
const connection = structuredClone(profile)
|
||||
const request = withImageConversationContext({ requestId: context.requestId, conversationId: context.conversationId, prompt: input.prompt, imageContextArtifactIds: input.sourceArtifactIds }, id => this.options.database.getArtifact(id))
|
||||
if (input.intent === 'edit' && (request.imageContextNotice || request.images?.length !== input.sourceArtifactIds.length)) throw new Error('Source image is missing or unsupported; select the image again')
|
||||
signal?.throwIfAborted()
|
||||
if (this.closing) throw new Error('Image service is closing')
|
||||
if (currentWorkMode() !== 'execute') throw new Error('Image generation requires Execute mode')
|
||||
const operation = this.save({ id: randomUUID(), conversationId: context.conversationId, messageId: context.messageId, requestId: context.requestId,
|
||||
callId, modelProfileId: connection.id, modelName: connection.modelName, modelProfileName: connection.name,
|
||||
input: { ...input, quality: input.quality ?? connection.imageGenerationQuality ?? 'auto' },
|
||||
state: 'running', artifactIds: [], createdAt: Date.now(), updatedAt: Date.now() })
|
||||
const runtime = new ModelAgentRuntime({ ...connection, model: connection.modelName, protocol: 'openai-images-generations',
|
||||
imageGenerationQuality: input.quality ?? connection.imageGenerationQuality, fetcher: this.options.fetcher, requestTimeoutMs: this.options.requestTimeoutMs })
|
||||
const active = { operation, controller: new AbortController(), done: Promise.resolve() }
|
||||
this.active.set(operation.id, active)
|
||||
active.done = (async () => {
|
||||
try {
|
||||
for await (const event of runtime.generateImage(request, active.controller.signal)) {
|
||||
if (event.type === 'model-usage') {
|
||||
try { this.options.onUsage?.(event) } catch (error) { this.report(error) }
|
||||
}
|
||||
if (event.type === 'generated-image') {
|
||||
active.operation = this.save({ ...active.operation, state: 'saving' })
|
||||
active.operation = this.save(active.operation, event)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const state = this.closing ? 'unconfirmed' : active.controller.signal.aborted ? 'stopped' : /timeout|timed out|fetch failed|network|超时/iu.test(String(error)) ? 'unconfirmed' : 'failed'
|
||||
const detail = safeToolErrorDetail(error) ?? 'Image request failed'
|
||||
const suffix = detail.match(/（HTTP [^）]+）$/u)?.[0] ?? ''
|
||||
const publicDetail = redactSensitiveText(suffix ? detail.slice(0, -suffix.length) : detail) + suffix
|
||||
active.operation = { ...active.operation, state, error: publicDetail.slice(0, 2_000), updatedAt: Date.now() }
|
||||
try { active.operation = this.save(active.operation) } catch (storageError) { this.report(storageError) }
|
||||
} finally {
|
||||
await runtime.dispose()
|
||||
this.active.delete(operation.id)
|
||||
}
|
||||
})()
|
||||
return operation
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// @vitest-environment node
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { createServer } from 'node:http'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import { ImageGenerationService } from './image-generation-service'
|
||||
import { KnowledgeMcpGateway } from './knowledge-mcp-gateway'
|
||||
import { ModelAgentRuntime } from './model-runtime'
|
||||
import { OpenCodeRuntime } from './opencode-runtime'
|
||||
import { ContinueAgentRuntime } from './continue-runtime'
|
||||
|
||||
vi.mock('electron', () => ({ nativeImage: {} }))
|
||||
const openCodeBinary = join(process.cwd(), '.runtime-resources', process.arch, process.platform === 'win32' ? 'opencode.exe' : 'opencode')
|
||||
const continueBinary = join(process.cwd(), 'node_modules', '@continuedev', 'cli', 'dist', 'cn.js')
|
||||
|
||||
it.for(['model', 'opencode', 'continue'] as const)('generates and edits through the real local %s tool loop', { timeout: 120_000 }, async (name, { skip }) => {
|
||||
if ((name === 'opencode' && !existsSync(openCodeBinary)) || (name === 'continue' && !existsSync(continueBinary))) return skip()
|
||||
const root = await mkdtemp(join(tmpdir(), 'goodbuddy-image-runtime-'))
|
||||
const database = new AssistantDatabase(join(root, 'assistant.sqlite'))
|
||||
database.initialize(root)
|
||||
const conversationId = randomUUID()
|
||||
const imageProfileId = randomUUID()
|
||||
let toolInput: Record<string, unknown> = { intent: 'create', prompt: 'Blue circle' }
|
||||
const textRequests: unknown[] = []
|
||||
const model = createServer((request, response) => {
|
||||
void (async () => {
|
||||
let content = ''
|
||||
for await (const chunk of request) content += String(chunk)
|
||||
const body = JSON.parse(content) as { messages?: { role: string; content: unknown }[]; tools?: { function: { name: string } }[] }
|
||||
textRequests.push(body)
|
||||
let lastUser = -1
|
||||
body.messages?.forEach((message, index) => { if (message.role === 'user') lastUser = index })
|
||||
const results = body.messages?.slice(lastUser + 1).filter(message => message.role === 'tool') ?? []
|
||||
const tool = body.tools?.find(tool => tool.function.name.endsWith('generate_image'))
|
||||
if (!tool) throw new Error('Image tool was not exposed')
|
||||
const call = results.length === 0
|
||||
const delta = call ? { role: 'assistant', tool_calls: [{ index: 0, id: randomUUID(), type: 'function',
|
||||
function: { name: tool.function.name, arguments: JSON.stringify(toolInput) } }] } : { role: 'assistant', content: 'IMAGE_SAVED' }
|
||||
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
||||
for (const [value, finish] of [[delta, null], [{}, call ? 'tool_calls' : 'stop']]) {
|
||||
response.write(`data: ${JSON.stringify({ id: randomUUID(), object: 'chat.completion.chunk', created: 1, model: 'fixture',
|
||||
choices: [{ index: 0, delta: value, finish_reason: finish }] })}\n\n`)
|
||||
}
|
||||
response.end('data: [DONE]\n\n')
|
||||
})().catch(error => { response.writeHead(500).end(String(error)) })
|
||||
})
|
||||
await new Promise<void>(resolve => model.listen(0, '127.0.0.1', resolve))
|
||||
const port = (model.address() as { port: number }).port
|
||||
const fetcher = vi.fn<typeof fetch>(async () => Response.json({ data: [{ b64_json: 'iVBORw0KGgo=' }] }))
|
||||
const service = new ImageGenerationService({ database, fetcher, getSettings: async () => ({ defaultImageModelProfileId: imageProfileId, modelProfiles: [{
|
||||
id: imageProfileId, name: 'Image fixture', modelName: 'fixture', protocol: 'openai-images-generations',
|
||||
allowConversationInvocation: true, authentication: 'none', baseUrl: 'https://image.test/v1'
|
||||
}] }) })
|
||||
const gateway = new KnowledgeMcpGateway({} as never)
|
||||
await gateway.start()
|
||||
const profile = { id: randomUUID(), name: 'Chat fixture', modelName: 'fixture', protocol: 'openai-chat-completions' as const,
|
||||
authentication: 'none' as const, baseUrl: `http://127.0.0.1:${port}/v1` }
|
||||
const common = { defaultWorkspace: root, knowledgeGateway: gateway, modelProfile: profile }
|
||||
const runtime = name === 'model'
|
||||
? new ModelAgentRuntime({ ...profile, model: profile.modelName, defaultWorkspace: root })
|
||||
: name === 'opencode'
|
||||
? new OpenCodeRuntime({ ...common, embedded: true, binaryPath: openCodeBinary, configPath: '' })
|
||||
: new ContinueAgentRuntime({ ...common, binaryPath: '', bundledBinaryPath: continueBinary, configPath: '', hostCacheRoot: join(root, 'continue') })
|
||||
try {
|
||||
for (const intent of ['create', 'edit'] as const) {
|
||||
const messageId = randomUUID()
|
||||
const requestId = randomUUID()
|
||||
database.saveLocalConversations([{ header: { id: conversationId, title: 'Image integration', updatedAt: Date.now() },
|
||||
messages: [{ id: messageId, role: 'assistant', content: '', createdAt: Date.now(), state: 'complete' }] }])
|
||||
const binding = service.bind({ conversationId, messageId, requestId, workMode: 'execute' })
|
||||
const events = []
|
||||
for await (const event of runtime.run({ requestId, conversationId, prompt: `${intent} image ${requestId}`, workMode: 'execute', imageToolBinding: binding }, new AbortController().signal, async () => 'once')) events.push(event)
|
||||
expect(events.some(event => event.type === 'done')).toBe(true)
|
||||
const operation = database.getConversation(conversationId).messages.find(message => message.id === messageId)!.imageOperations?.[0]
|
||||
expect(operation?.state).toBe('completed')
|
||||
expect(database.getArtifact(operation!.artifactIds[0]!).content).toContain('iVBORw0KGgo=')
|
||||
toolInput = { intent: 'edit', prompt: 'Turn red', sourceArtifactIds: operation!.artifactIds }
|
||||
}
|
||||
expect(fetcher).toHaveBeenCalledTimes(2)
|
||||
expect(String(fetcher.mock.calls[1]![0])).toMatch(/images\/edits$/u)
|
||||
expect(textRequests.length).toBeGreaterThanOrEqual(4)
|
||||
} finally {
|
||||
await runtime.dispose()
|
||||
await gateway.dispose()
|
||||
await service.dispose()
|
||||
model.closeAllConnections()
|
||||
await new Promise<void>(resolve => model.close(() => resolve()))
|
||||
database.close()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,16 @@
|
||||
import { z } from 'zod'
|
||||
import { imageToolInputSchema, imageToolName, type ImageOperation, type ImageRequestContext } from '../../shared/image-generation-contracts'
|
||||
|
||||
export type ImageToolBinding = {
|
||||
context: Readonly<ImageRequestContext>
|
||||
describe(): Promise<string | undefined>
|
||||
call(input: unknown, callId: string, waitSignal?: AbortSignal): Promise<ImageOperation>
|
||||
}
|
||||
|
||||
export async function imageToolDefinition(binding: ImageToolBinding | undefined) {
|
||||
const description = await binding?.describe()
|
||||
if (!description) return undefined
|
||||
const inputSchema = z.toJSONSchema(imageToolInputSchema, { target: 'draft-7', io: 'input' })
|
||||
Reflect.deleteProperty(inputSchema, '$schema')
|
||||
return { name: imageToolName, displayName: 'Generate or edit image', description, inputSchema, source: 'builtin' as const }
|
||||
}
|
||||
@@ -102,6 +102,7 @@ export {
|
||||
}
|
||||
|
||||
export type GoodBuddyBuiltinToolName =
|
||||
| 'generate_image'
|
||||
| ScopedDataToolName
|
||||
| BrowserToolName
|
||||
|
||||
@@ -184,6 +185,7 @@ export type KnowledgeLibraryListItem = {
|
||||
}
|
||||
|
||||
type Capability = {
|
||||
imageToolBinding?: ImageToolBinding
|
||||
requestId: string
|
||||
libraryIds: readonly string[]
|
||||
magicNotesAccess: MagicNotesCapabilityAccess
|
||||
@@ -530,6 +532,20 @@ export class KnowledgeMcpGateway {
|
||||
}
|
||||
}
|
||||
|
||||
bindImageTool(binding: ImageToolBinding, signal: AbortSignal, existingToken?: string): string {
|
||||
signal.throwIfAborted()
|
||||
if (existingToken) {
|
||||
const capability = this.getCapability(existingToken)
|
||||
if (capability.requestId !== binding.context.requestId) throw new Error('Image capability request does not match')
|
||||
capability.imageToolBinding = binding
|
||||
return existingToken
|
||||
}
|
||||
return this.storeCapability({
|
||||
requestId: binding.context.requestId, libraryIds: [], magicNotesAccess: 'none', configAccess: 'none',
|
||||
customMcpServers: [], signal, imageToolBinding: binding
|
||||
})
|
||||
}
|
||||
|
||||
grantCustomMcp(
|
||||
requestId: string,
|
||||
servers: readonly ResolvedMcpServer[],
|
||||
@@ -760,6 +776,7 @@ export class KnowledgeMcpGateway {
|
||||
getAvailableToolNames(token: string): GoodBuddyBuiltinToolName[] {
|
||||
const capability = this.getCapability(token)
|
||||
return [
|
||||
...(capability.imageToolBinding ? ['generate_image' as const] : []),
|
||||
...(capability.libraryIds.length > 0
|
||||
? knowledgeToolNames
|
||||
: []),
|
||||
@@ -1570,6 +1587,7 @@ export class KnowledgeMcpGateway {
|
||||
}
|
||||
)
|
||||
const capability = this.getCapability(token)
|
||||
const imageTool = await imageToolDefinition(capability.imageToolBinding)
|
||||
const browserTools = capability.browserConversationId && capability.browserTabId
|
||||
? new BrowserModelTools({
|
||||
service: this.browserService!,
|
||||
@@ -1596,6 +1614,7 @@ export class KnowledgeMcpGateway {
|
||||
session.listedTools = true
|
||||
return {
|
||||
tools: [
|
||||
...(imageTool ? [{ name: imageTool.name, description: imageTool.description, inputSchema: imageTool.inputSchema as Tool['inputSchema'], annotations: { readOnlyHint: false, destructiveHint: false } }] : []),
|
||||
...scopedTools,
|
||||
...browserDefinitions,
|
||||
...[...customBindings.values()].map(
|
||||
@@ -1610,6 +1629,12 @@ export class KnowledgeMcpGateway {
|
||||
async (call, extra) => {
|
||||
const name = call.params.name
|
||||
const input = call.params.arguments ?? {}
|
||||
if (name === 'generate_image') {
|
||||
const capability = this.getCapability(token)
|
||||
if (!capability.imageToolBinding) throw new Error('Image capability is unavailable')
|
||||
const operation = await capability.imageToolBinding.call(input, `${session.registryKey}:${extra.requestId}`, AbortSignal.any([extra.signal, capability.signal]))
|
||||
return { content: [{ type: 'text' as const, text: JSON.stringify(operation) }] }
|
||||
}
|
||||
if (availableTools.has(name as ScopedDataToolName)) {
|
||||
const definition = scopedDataToolByName.get(
|
||||
name as ScopedDataToolName
|
||||
@@ -1868,3 +1893,4 @@ export class KnowledgeMcpGateway {
|
||||
}
|
||||
}
|
||||
}
|
||||
import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding'
|
||||
|
||||
@@ -2681,9 +2681,15 @@ export class ModelAgentRuntime implements AgentRuntime {
|
||||
return history
|
||||
}
|
||||
|
||||
/** Strict image service entry point; direct-image conversations retain their fallback policy. */
|
||||
generateImage(request: AgentExecutionRequest, signal: AbortSignal): AsyncGenerator<RuntimeEvent, void, void> {
|
||||
return this.runImageGeneration(request, signal, false)
|
||||
}
|
||||
|
||||
private async *runImageGeneration(
|
||||
request: AgentExecutionRequest,
|
||||
signal: AbortSignal
|
||||
signal: AbortSignal,
|
||||
allowEditingFallback = true
|
||||
): AsyncGenerator<RuntimeEvent, void, void> {
|
||||
const images = request.images ?? []
|
||||
const prompt = imageConversationPrompt(request)
|
||||
@@ -2744,7 +2750,7 @@ export class ModelAgentRuntime implements AgentRuntime {
|
||||
)
|
||||
const providerMessage = getErrorMessage(errorPayload)
|
||||
if (
|
||||
images.length > 0 &&
|
||||
allowEditingFallback && images.length > 0 &&
|
||||
isImageEditingUnavailable(response.status, providerMessage)
|
||||
) {
|
||||
signal.throwIfAborted()
|
||||
@@ -3417,6 +3423,7 @@ export class ModelAgentRuntime implements AgentRuntime {
|
||||
wakeNestedEvents = undefined
|
||||
}
|
||||
const toolContext: ModelToolCallContext = {
|
||||
imageToolBinding: request.imageToolBinding,
|
||||
conversationId: request.conversationId,
|
||||
browserTabId: request.browserTabId,
|
||||
browserConversationId: request.browserConversationId,
|
||||
@@ -3743,6 +3750,7 @@ export class ModelAgentRuntime implements AgentRuntime {
|
||||
let roundContextBytes = 0
|
||||
const roundCallIds = new Set<string>()
|
||||
for (const call of response.toolCalls) {
|
||||
toolContext.toolCallId = call.id
|
||||
signal.throwIfAborted()
|
||||
if (roundCallIds.has(call.id)) {
|
||||
throw new Error('模型重复使用了工具调用 ID')
|
||||
|
||||
@@ -290,6 +290,8 @@ export type ModelToolResult = {
|
||||
}
|
||||
|
||||
export type ModelToolCallContext = {
|
||||
imageToolBinding?: ImageToolBinding
|
||||
toolCallId?: string
|
||||
conversationId: string
|
||||
browserTabId?: BrowserTabId
|
||||
browserConversationId?: string
|
||||
@@ -1290,6 +1292,7 @@ export class ModelToolProvider implements ModelToolProviderLike {
|
||||
): Promise<ModelToolDefinition[]> {
|
||||
signal.throwIfAborted()
|
||||
const scopedTools = this.getScopedTools(context)
|
||||
const imageTool = context.workMode === 'execute' ? await imageToolDefinition(context.imageToolBinding) : undefined
|
||||
const webTools = this.webSearchEnabled
|
||||
? this.getWebSearchDefinitions()
|
||||
: []
|
||||
@@ -1312,6 +1315,7 @@ export class ModelToolProvider implements ModelToolProviderLike {
|
||||
const browserTools = this.getBrowserTools(context)
|
||||
return [
|
||||
...workspaceTools,
|
||||
...(imageTool ? [imageTool] : []),
|
||||
...processTools,
|
||||
...subagentTools,
|
||||
...outputTools,
|
||||
@@ -1439,6 +1443,10 @@ export class ModelToolProvider implements ModelToolProviderLike {
|
||||
): Promise<ModelToolResult> {
|
||||
signal.throwIfAborted()
|
||||
assertToolAuthorizedForWorkMode(name, context)
|
||||
if (name === 'generate_image') {
|
||||
if (!context.imageToolBinding || !context.toolCallId) throw new Error('Image tool request binding is unavailable')
|
||||
return createTextToolResult(JSON.stringify(await context.imageToolBinding.call(argumentsValue, context.toolCallId, signal)))
|
||||
}
|
||||
if (name === 'knowledge_list') {
|
||||
if (
|
||||
!this.knowledgeGateway ||
|
||||
@@ -2020,3 +2028,4 @@ function emitDirectModelSubagentUsage(
|
||||
)
|
||||
})
|
||||
}
|
||||
import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding'
|
||||
|
||||
@@ -2161,9 +2161,13 @@ export class OpenCodeRuntime implements AgentRuntime {
|
||||
let customMcpToken: string | undefined;
|
||||
const attemptedMcpNames: string[] = [];
|
||||
const subscriptionController = new AbortController();
|
||||
const imageCapabilityToken = request.imageToolBinding && request.workMode === 'execute'
|
||||
? this.options.knowledgeGateway?.bindImageTool(request.imageToolBinding, signal, request.knowledgeCapabilityToken)
|
||||
: undefined;
|
||||
const scopedCapabilityToken = imageCapabilityToken ?? request.knowledgeCapabilityToken;
|
||||
try {
|
||||
if (
|
||||
request.knowledgeCapabilityToken &&
|
||||
scopedCapabilityToken &&
|
||||
this.usesEmbeddedPermissionMediation() &&
|
||||
this.options.knowledgeGateway?.getEndpoint()
|
||||
) {
|
||||
@@ -2186,7 +2190,7 @@ export class OpenCodeRuntime implements AgentRuntime {
|
||||
url: knowledgeEndpoint,
|
||||
enabled: true,
|
||||
headers: {
|
||||
Authorization: `Bearer ${request.knowledgeCapabilityToken}`,
|
||||
Authorization: `Bearer ${scopedCapabilityToken}`,
|
||||
},
|
||||
oauth: false,
|
||||
},
|
||||
@@ -2212,7 +2216,7 @@ export class OpenCodeRuntime implements AgentRuntime {
|
||||
// experimental/tool/ids. Its model tool namespace is deterministic:
|
||||
// "<MCP server name>_<declared tool name>".
|
||||
knowledgeToolIds = this.options.knowledgeGateway
|
||||
.getAvailableToolNames(request.knowledgeCapabilityToken)
|
||||
.getAvailableToolNames(scopedCapabilityToken)
|
||||
.map((toolName) => `${knowledgeMcpName}_${toolName}`);
|
||||
}
|
||||
if (
|
||||
@@ -2996,6 +3000,7 @@ export class OpenCodeRuntime implements AgentRuntime {
|
||||
}
|
||||
} finally {
|
||||
subscriptionController.abort();
|
||||
if (imageCapabilityToken && imageCapabilityToken !== request.knowledgeCapabilityToken) this.options.knowledgeGateway?.revoke(imageCapabilityToken);
|
||||
for (const name of attemptedMcpNames) {
|
||||
await this.mutateMcp(new AbortController().signal, () => {
|
||||
// Queue wait must not consume the disconnect timeout.
|
||||
|
||||
@@ -18,6 +18,7 @@ import type {
|
||||
remoteSemanticTranscriptPageResultSchema,
|
||||
} from '../../shared/remote-agent-contracts'
|
||||
import type { z } from 'zod'
|
||||
import type { ImageToolBinding } from './image-tool-binding'
|
||||
import type { RemoteQuestionResponse } from '../../shared/remote-question-contracts'
|
||||
import type {
|
||||
ModelBridgePolicy
|
||||
@@ -111,7 +112,9 @@ export interface RemoteRuntimeChannel {
|
||||
): Promise<RuntimeSessionBindingCursors>
|
||||
/** Accepts the prompt and starts its Ask or Execute Runtime process. */
|
||||
preparePrompt(
|
||||
preparation: RemotePromptOperationPreparation
|
||||
preparation: RemotePromptOperationPreparation,
|
||||
imageToolBinding?: ImageToolBinding,
|
||||
waitSignal?: AbortSignal
|
||||
): Promise<RemotePromptOperationAcceptance>
|
||||
/** Starts an Agent-owned ACP prompt exactly once for its stable operation. */
|
||||
startOwnedPrompt?(
|
||||
|
||||
@@ -146,6 +146,7 @@ export type AgentExecutionRequest = Omit<AgentRequest, 'workMode'> & {
|
||||
trustedInstructions?: string
|
||||
/** Main-process-only request-scoped authorization for built-in data tools. */
|
||||
knowledgeCapabilityToken?: string
|
||||
imageToolBinding?: ImageToolBinding
|
||||
/** Main-process-only browser tab fixed for this request. */
|
||||
browserTabId?: BrowserTabId
|
||||
/**
|
||||
@@ -160,3 +161,4 @@ export type AgentExecutionRequest = Omit<AgentRequest, 'workMode'> & {
|
||||
/** Main-only durable native subagent state for partial recovery updates. */
|
||||
remoteRecoveredSubagents?: readonly RemoteRecoveredSubagent[]
|
||||
}
|
||||
import type { ImageToolBinding } from './image-tool-binding'
|
||||
|
||||
@@ -15,8 +15,43 @@ import {
|
||||
ASSISTANT_DATABASE_SCHEMA_VERSION
|
||||
} from './assistant-database'
|
||||
import { agentRuntimeSelectionKey } from '../../shared/runtime-selection-contracts'
|
||||
import type { ImageOperation } from '../../shared/image-generation-contracts'
|
||||
|
||||
const temporaryDirectories: string[] = []
|
||||
|
||||
it('atomically completes conversation images and retains operations and upload references across saves and reopening', async () => {
|
||||
const database = await createDatabase()
|
||||
const conversationId = randomUUID()
|
||||
const messageId = randomUUID()
|
||||
const header = { id: conversationId, title: 'Image operation', updatedAt: Date.now() }
|
||||
const message = { id: messageId, role: 'assistant' as const, content: '', state: 'complete' as const, createdAt: Date.now() }
|
||||
database.saveLocalConversations([{ header, messages: [message] }])
|
||||
const operation: ImageOperation = { id: randomUUID(), conversationId, messageId, requestId: randomUUID(), callId: 'image-call',
|
||||
modelProfileId: randomUUID(), modelName: 'Image model', input: { intent: 'create', prompt: 'Blue', sourceArtifactIds: [] },
|
||||
state: 'running', createdAt: Date.now(), updatedAt: Date.now(), artifactIds: [] }
|
||||
database.saveConversationImageOperation(operation)
|
||||
const sources = database.saveConversationImageSources({ conversationId, messageId,
|
||||
images: [{ name: 'source.png', mediaType: 'image/png', data: 'iVBORw0KGgo=' }] })
|
||||
const completed = database.saveConversationImageOperation(operation, { mimeType: 'image/png', data: 'iVBORw0KGgo=' })
|
||||
expect(database.saveConversationImageOperation(operation, { mimeType: 'image/png', data: 'iVBORw0KGgo=' })).toEqual(completed)
|
||||
expect(database.listArtifacts()).toHaveLength(2)
|
||||
database.saveLocalConversations([{ header, messages: [message] }])
|
||||
database.close()
|
||||
database.initialize(process.cwd())
|
||||
const restored = database.getConversation(conversationId).messages[0]!
|
||||
expect(restored.imageOperations).toEqual([completed])
|
||||
expect(restored.artifactIds).toEqual(completed.artifactIds)
|
||||
expect(restored.imageSourceArtifactIds).toEqual(sources)
|
||||
expect(database.getArtifact(sources[0]!).content).toContain('iVBORw0KGgo=')
|
||||
const failed = { ...operation, id: randomUUID() }
|
||||
database.saveConversationImageOperation(failed)
|
||||
expect(() => database.saveConversationImageOperation(failed, { mimeType: 'image/png', data: 'a'.repeat(5 * 1024 * 1024) })).toThrow('5MB')
|
||||
expect(database.listArtifacts()).toHaveLength(2)
|
||||
expect(database.getConversation(conversationId).messages[0]!.imageOperations?.find(item => item.id === failed.id)?.state).toBe('running')
|
||||
database.deleteLocalConversation(conversationId)
|
||||
expect(() => database.saveConversationImageOperation(failed, { mimeType: 'image/png', data: 'iVBORw0KGgo=' })).toThrow('no longer exists')
|
||||
database.close()
|
||||
})
|
||||
const channelDefaultProfileId =
|
||||
'00000000-0000-4000-8000-000000000001'
|
||||
|
||||
|
||||
@@ -279,6 +279,8 @@ type MessageMetadata = {
|
||||
sourceReferences?: ConversationSnapshot['messages'][number]['sourceReferences']
|
||||
knowledgeRetrieval?: ConversationSnapshot['messages'][number]['knowledgeRetrieval']
|
||||
artifactIds?: string[]
|
||||
imageOperations?: ImageOperation[]
|
||||
imageSourceArtifactIds?: string[]
|
||||
imageContextNotice?: ConversationSnapshot['messages'][number]['imageContextNotice']
|
||||
task?: ConversationSnapshot['messages'][number]['task']
|
||||
attachments?: ConversationSnapshot['messages'][number]['attachments']
|
||||
@@ -1177,6 +1179,8 @@ function toConversationSnapshot(
|
||||
sourceReferences: metadata.sourceReferences,
|
||||
knowledgeRetrieval: metadata.knowledgeRetrieval,
|
||||
artifactIds: metadata.artifactIds,
|
||||
imageOperations: metadata.imageOperations,
|
||||
imageSourceArtifactIds: metadata.imageSourceArtifactIds,
|
||||
imageContextNotice: metadata.imageContextNotice,
|
||||
task: metadata.task,
|
||||
attachments: metadata.attachments,
|
||||
@@ -1204,6 +1208,8 @@ function serializeConversationMessageMetadata(
|
||||
sourceReferences: message.sourceReferences,
|
||||
knowledgeRetrieval: message.knowledgeRetrieval,
|
||||
artifactIds: message.artifactIds,
|
||||
imageOperations: message.imageOperations,
|
||||
imageSourceArtifactIds: message.imageSourceArtifactIds,
|
||||
imageContextNotice: message.imageContextNotice,
|
||||
task: message.task,
|
||||
attachments: message.attachments,
|
||||
@@ -2862,7 +2868,7 @@ export class AssistantDatabase {
|
||||
WHERE id = ? AND channel IS NULL`
|
||||
)
|
||||
const findMessage = database.prepare(
|
||||
`SELECT conversation_id, role,
|
||||
`SELECT conversation_id, role, metadata_json,
|
||||
EXISTS(
|
||||
SELECT 1 FROM tasks
|
||||
WHERE tasks.id = messages.request_id
|
||||
@@ -2941,6 +2947,7 @@ export class AssistantDatabase {
|
||||
| {
|
||||
conversation_id: string
|
||||
role: MessageRow['role']
|
||||
metadata_json: string
|
||||
remote_recoverable: number
|
||||
}
|
||||
| undefined
|
||||
@@ -2957,7 +2964,20 @@ export class AssistantDatabase {
|
||||
updateMessage.run(
|
||||
message.content,
|
||||
message.state,
|
||||
serializeConversationMessageMetadata(message),
|
||||
serializeConversationMessageMetadata({
|
||||
...message,
|
||||
...(() => {
|
||||
const stored = JSON.parse(existingMessage.metadata_json) as MessageMetadata
|
||||
return {
|
||||
imageOperations: stored.imageOperations,
|
||||
imageSourceArtifactIds: stored.imageSourceArtifactIds,
|
||||
artifactIds: stored.imageOperations?.some(operation => operation.artifactIds.length) ? [...new Set([
|
||||
...(message.artifactIds ?? []),
|
||||
...(stored.imageOperations?.flatMap(operation => operation.artifactIds) ?? [])
|
||||
])].slice(-8) : message.artifactIds
|
||||
}
|
||||
})()
|
||||
}),
|
||||
message.id
|
||||
)
|
||||
continue
|
||||
@@ -5705,6 +5725,101 @@ export class AssistantDatabase {
|
||||
})
|
||||
}
|
||||
|
||||
saveConversationImageOperation(
|
||||
operation: ImageOperation,
|
||||
image?: { mimeType: 'image/png' | 'image/jpeg' | 'image/webp'; data: string }
|
||||
): ImageOperation {
|
||||
const database = this.requireDatabase()
|
||||
database.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const row = database.prepare(`SELECT m.metadata_json, c.project_id FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id
|
||||
WHERE m.id = ? AND m.conversation_id = ?`).get(operation.messageId, operation.conversationId) as
|
||||
{ metadata_json: string; project_id: string | null } | undefined
|
||||
if (!row) throw new Error('Image operation conversation or message no longer exists')
|
||||
const metadata = JSON.parse(row.metadata_json) as MessageMetadata
|
||||
const previous = metadata.imageOperations?.find(item => item.id === operation.id)
|
||||
if (previous?.state === 'completed') {
|
||||
database.exec('COMMIT')
|
||||
return previous
|
||||
}
|
||||
let saved = operation
|
||||
if (image) {
|
||||
const artifact = this.createImageArtifact({
|
||||
projectId: row.project_id ?? undefined,
|
||||
title: operation.input.prompt.slice(0, 120),
|
||||
mimeType: image.mimeType,
|
||||
base64: image.data
|
||||
})
|
||||
saved = { ...operation, state: 'completed', artifactIds: [artifact.id] }
|
||||
metadata.artifactIds = [...new Set([...(metadata.artifactIds ?? []), artifact.id])].slice(-8)
|
||||
}
|
||||
metadata.imageOperations = [...(metadata.imageOperations ?? []).filter(item => item.id !== saved.id), saved]
|
||||
database.prepare('UPDATE messages SET metadata_json = ? WHERE id = ? AND conversation_id = ?')
|
||||
.run(JSON.stringify(metadata), saved.messageId, saved.conversationId)
|
||||
database.prepare('UPDATE conversations SET updated_at = MAX(updated_at, ?) WHERE id = ?')
|
||||
.run(new Date(saved.updatedAt).toISOString(), saved.conversationId)
|
||||
database.exec('COMMIT')
|
||||
return saved
|
||||
} catch (error) {
|
||||
database.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
saveConversationImageSources(input: {
|
||||
conversationId: string
|
||||
messageId: string
|
||||
images: readonly { name: string; mediaType: 'image/png' | 'image/jpeg'; data: string }[]
|
||||
}): string[] {
|
||||
if (input.images.length > 8) throw new Error('At most eight source images are supported')
|
||||
const database = this.requireDatabase()
|
||||
database.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const row = database.prepare(`SELECT m.metadata_json, c.project_id FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id
|
||||
WHERE m.id = ? AND m.conversation_id = ?`).get(input.messageId, input.conversationId) as
|
||||
{ metadata_json: string; project_id: string | null } | undefined
|
||||
if (!row) throw new Error('Image source conversation or message no longer exists')
|
||||
const metadata = JSON.parse(row.metadata_json) as MessageMetadata
|
||||
if (!metadata.imageSourceArtifactIds?.length) {
|
||||
metadata.imageSourceArtifactIds = input.images.map(image => this.createImageArtifact({
|
||||
projectId: row.project_id ?? undefined, title: image.name,
|
||||
mimeType: image.mediaType, base64: image.data
|
||||
}).id)
|
||||
database.prepare('UPDATE messages SET metadata_json = ? WHERE id = ?').run(JSON.stringify(metadata), input.messageId)
|
||||
}
|
||||
database.exec('COMMIT')
|
||||
return metadata.imageSourceArtifactIds
|
||||
} catch (error) {
|
||||
database.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
markUnfinishedImageOperationsUnconfirmed(): void {
|
||||
const database = this.requireDatabase()
|
||||
database.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const rows = database.prepare("SELECT id, metadata_json FROM messages WHERE json_type(metadata_json, '$.imageOperations') = 'array'")
|
||||
.all() as { id: string; metadata_json: string }[]
|
||||
for (const row of rows) {
|
||||
const metadata = JSON.parse(row.metadata_json) as MessageMetadata
|
||||
let changed = false
|
||||
metadata.imageOperations = metadata.imageOperations?.map(operation => {
|
||||
if (!['running', 'saving', 'cancelling'].includes(operation.state)) return operation
|
||||
changed = true
|
||||
return { ...operation, state: 'unconfirmed', updatedAt: Date.now() }
|
||||
})
|
||||
if (changed) database.prepare('UPDATE messages SET metadata_json = ? WHERE id = ?').run(JSON.stringify(metadata), row.id)
|
||||
}
|
||||
database.exec('COMMIT')
|
||||
} catch (error) {
|
||||
database.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
createImageArtifact(input: {
|
||||
projectId?: string
|
||||
taskId?: string
|
||||
@@ -9764,3 +9879,4 @@ export class AssistantDatabase {
|
||||
return this.database
|
||||
}
|
||||
}
|
||||
import type { ImageOperation } from '../../shared/image-generation-contracts'
|
||||
|
||||
@@ -137,6 +137,30 @@ afterEach(async () => {
|
||||
})
|
||||
|
||||
describe('CapabilityService', () => {
|
||||
it('derives image assignments from saved profiles without persisting or exposing connection data', async () => {
|
||||
const image = { id: '00000000-0000-4000-8000-000000000001', name: 'Image', protocol: 'openai-images-generations', allowConversationInvocation: true, apiKey: 'must-not-leak' }
|
||||
let profiles = [image]
|
||||
const getSavedModelProfiles = vi.fn(async () => profiles)
|
||||
const { service, filePath, builtinRoot, importedRoot } = await createService({ getSavedModelProfiles })
|
||||
const all = ['model', 'opencode', 'continue', 'deepseek-harness']
|
||||
expect((await service.getSnapshot()).imageGeneration).toEqual({ modelProfiles: [{ id: image.id, name: image.name }], assignments: all })
|
||||
profiles = [image, { ...image, id: '00000000-0000-4000-8000-000000000002' }]
|
||||
expect((await service.getSnapshot()).imageGeneration?.modelProfiles).toHaveLength(2)
|
||||
profiles = [{ ...image, allowConversationInvocation: false }, profiles[1]!]
|
||||
expect((await service.getSnapshot()).imageGeneration).toMatchObject({ assignments: all, modelProfiles: [{ id: profiles[1]!.id }] })
|
||||
profiles = [{ ...image, allowConversationInvocation: false }, { ...image, protocol: 'openai-chat-completions' }]
|
||||
expect((await service.getSnapshot()).imageGeneration).toEqual({ modelProfiles: [], assignments: [] })
|
||||
profiles = []
|
||||
expect((await service.getSnapshot()).imageGeneration?.assignments).toEqual([])
|
||||
profiles = [image]
|
||||
const reloaded = new CapabilityService(filePath, builtinRoot, importedRoot, cipher, { getSavedModelProfiles })
|
||||
expect((await reloaded.getSnapshot()).imageGeneration?.assignments).toEqual(all)
|
||||
await service.setBuiltinMcpServerEnabled('knowledge-base', false)
|
||||
expect(await readFile(filePath, 'utf8')).not.toContain('imageGeneration')
|
||||
expect((await service.getSnapshot()).imageGeneration?.assignments).toEqual(all)
|
||||
expect((await new CapabilityService(filePath, builtinRoot, importedRoot, cipher).getSnapshot()).imageGeneration).toEqual({ modelProfiles: [], assignments: [] })
|
||||
})
|
||||
|
||||
it('memoizes concurrent loads and retries after a failed load', async () => {
|
||||
const initialStore = {
|
||||
load: vi.fn(async () => undefined),
|
||||
|
||||
@@ -273,6 +273,12 @@ export type SkillImportInspection = {
|
||||
}
|
||||
|
||||
export type CapabilityServiceOptions = Readonly<{
|
||||
getSavedModelProfiles?: () => Promise<ReadonlyArray<{
|
||||
id: string
|
||||
name: string
|
||||
protocol: string
|
||||
allowConversationInvocation?: boolean
|
||||
}>>
|
||||
platform?: NodeJS.Platform
|
||||
architecture?: string
|
||||
electronTarget?: boolean
|
||||
@@ -702,6 +708,7 @@ async function parseSkillZip(archivePath: string): Promise<ParsedSkillZip> {
|
||||
}
|
||||
|
||||
export class CapabilityService {
|
||||
private readonly getSavedModelProfiles: NonNullable<CapabilityServiceOptions['getSavedModelProfiles']>
|
||||
private state?: StoredCapabilities
|
||||
private loadPromise?: Promise<StoredCapabilities>
|
||||
private warnings: SettingsWarning[] = []
|
||||
@@ -721,6 +728,8 @@ export class CapabilityService {
|
||||
private readonly cipher: CapabilityCipher,
|
||||
options: CapabilityServiceOptions = {}
|
||||
) {
|
||||
this.getSavedModelProfiles =
|
||||
options.getSavedModelProfiles ?? (async () => [])
|
||||
this.platform = options.platform ?? process.platform
|
||||
this.architecture = options.architecture ?? process.arch
|
||||
this.electronTarget =
|
||||
@@ -971,12 +980,25 @@ export class CapabilityService {
|
||||
}
|
||||
|
||||
async getSnapshot(): Promise<CapabilitySnapshot> {
|
||||
const [state, catalog, browserProfileState] = await Promise.all([
|
||||
const [state, catalog, browserProfileState, savedModelProfiles] = await Promise.all([
|
||||
this.load(),
|
||||
this.getSkillCatalog(),
|
||||
this.browserProfiles.getSnapshot()
|
||||
this.browserProfiles.getSnapshot(),
|
||||
this.getSavedModelProfiles()
|
||||
])
|
||||
const imageProfiles = savedModelProfiles
|
||||
.filter((profile) =>
|
||||
profile.protocol === 'openai-images-generations' &&
|
||||
profile.allowConversationInvocation === true
|
||||
)
|
||||
.map(({ id, name }) => ({ id, name }))
|
||||
return {
|
||||
imageGeneration: {
|
||||
modelProfiles: imageProfiles,
|
||||
assignments: imageProfiles.length > 0
|
||||
? ['model', 'opencode', 'continue', 'deepseek-harness']
|
||||
: []
|
||||
},
|
||||
skills: catalog
|
||||
.map((skill) => ({
|
||||
...skill,
|
||||
|
||||
+25
-2
@@ -31,6 +31,7 @@ import {
|
||||
type SelectedRuntimeTarget
|
||||
} from './agent/runtime-selection'
|
||||
import { CapabilityService } from './capabilities/capability-service'
|
||||
import { ImageGenerationService } from './agent/image-generation-service'
|
||||
import { ContextManager } from './context-manager'
|
||||
import {
|
||||
registerIpcHandlers,
|
||||
@@ -211,6 +212,7 @@ const localRuntimeRegistry = new LocalRuntimeRegistry()
|
||||
let knowledgeService: KnowledgeService | undefined
|
||||
let knowledgeGateway: KnowledgeMcpGateway | undefined
|
||||
let assistantDatabase: AssistantDatabase | undefined
|
||||
let imageGenerationService: ImageGenerationService | undefined
|
||||
let browserService: BrowserService | undefined
|
||||
let globalTlsPolicy: GlobalTlsPolicy | undefined
|
||||
let feedbackService: FeedbackService | undefined
|
||||
@@ -750,7 +752,8 @@ if (hasSingleInstanceLock) {
|
||||
? join(process.resourcesPath, 'skills')
|
||||
: join(app.getAppPath(), 'resources', 'skills'),
|
||||
join(app.getPath('userData'), 'skills', 'imported'),
|
||||
secureCipher
|
||||
secureCipher,
|
||||
{ getSavedModelProfiles: () => settingsStore.getSavedModelProfiles() }
|
||||
)
|
||||
const channelSettingsStore = new ChannelSettingsStore(
|
||||
join(app.getPath('userData'), 'channel-settings.json'),
|
||||
@@ -910,6 +913,23 @@ if (hasSingleInstanceLock) {
|
||||
}
|
||||
)
|
||||
assistantDatabase = startupAssistantDatabase
|
||||
imageGenerationService = new ImageGenerationService({
|
||||
database: startupAssistantDatabase,
|
||||
getSettings: () => settingsStore.getResolvedSettings(),
|
||||
onError: (error) => {
|
||||
void desktopDiagnostics.recordFailure({ component: 'desktop', stage: 'image-generation', code: 'image-generation.observer.failed', error }).catch(() => undefined)
|
||||
},
|
||||
onOperation: (operation) => {
|
||||
if (mainWindow && !mainWindow.isDestroyed() && !mainWindow.webContents.isDestroyed()) {
|
||||
mainWindow.webContents.send(ipcChannels.imageOperationChanged, operation)
|
||||
}
|
||||
},
|
||||
onUsage: (event) => startupAssistantDatabase.upsertModelUsageCall({
|
||||
requestId: event.requestId, callId: event.callId, runtime: event.runtime,
|
||||
provider: event.provider, model: event.model, input: event.inputTokens,
|
||||
output: event.outputTokens, cacheRead: event.cacheReadTokens, cacheWrite: event.cacheWriteTokens
|
||||
})
|
||||
})
|
||||
directModelSubagentScheduler = new SubagentScheduler({
|
||||
concurrency: 3,
|
||||
queueLimit: 20,
|
||||
@@ -1130,6 +1150,7 @@ if (hasSingleInstanceLock) {
|
||||
createConfiguredRuntime(initialResolvedSettings),
|
||||
initializeAssistant: () => {
|
||||
startupAssistantDatabase.initialize(defaultWorkspace)
|
||||
imageGenerationService!.initialize()
|
||||
startupAssistantDatabase.ensureChannelProjects(
|
||||
defaultWorkspace,
|
||||
initialRuntimeSettings.defaultModelProfileId
|
||||
@@ -1426,7 +1447,8 @@ if (hasSingleInstanceLock) {
|
||||
agentPackageManager,
|
||||
startupRemoteAgentServices.connectionManager,
|
||||
terminalSessionManager,
|
||||
startupLocalToolEnvironmentService
|
||||
startupLocalToolEnvironmentService,
|
||||
imageGenerationService
|
||||
)
|
||||
removeFeedbackIpcHandler = registerFeedbackIpcHandler(
|
||||
mainWindow,
|
||||
@@ -1522,6 +1544,7 @@ app.on('before-quit', (event) => {
|
||||
() => feedbackService?.dispose(),
|
||||
() => removeFeedbackIpcHandler?.(),
|
||||
() => dshExtensionInstaller?.dispose(),
|
||||
() => imageGenerationService?.dispose(),
|
||||
() => removeIpcHandlers?.()
|
||||
],
|
||||
[() => stopRuntimeReconfiguration?.()],
|
||||
|
||||
+89
-4
@@ -28,6 +28,8 @@ import type {
|
||||
} from '../shared/ssh-host-contracts'
|
||||
import { defaultKnowledgeOntologySettings } from '../shared/knowledge-ontology'
|
||||
import { AssistantDatabase } from './assistant/assistant-database'
|
||||
import { ImageGenerationService } from './agent/image-generation-service'
|
||||
import type { AgentExecutionRequest } from './agent/runtime'
|
||||
import { KnowledgeService } from './knowledge/knowledge-service'
|
||||
import { BrowserNavigationStoppedError } from './browser/browser-service'
|
||||
import { RemotePromptCancelledError } from './agent/acp-remote-runtime'
|
||||
@@ -1310,6 +1312,7 @@ describe('registerIpcHandlers model download source routing', () => {
|
||||
})
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
nativeImage: { createFromBuffer: (buffer: Buffer) => ({ isEmpty: () => false, toPNG: () => buffer }) },
|
||||
app: {
|
||||
getName: vi.fn(() => 'GoodBuddy'),
|
||||
getVersion: vi.fn(() => '0.1.0')
|
||||
@@ -5058,7 +5061,9 @@ describe('registerIpcHandlers agent terminal state', () => {
|
||||
magicNotesEnabled = false,
|
||||
goodbuddyConfigService?: Record<string, unknown>,
|
||||
capabilityServiceOverride?: Record<string, unknown>,
|
||||
browserControl?: Record<string, unknown>
|
||||
browserControl?: Record<string, unknown>,
|
||||
imageService?: ImageGenerationService,
|
||||
imageDatabase?: AssistantDatabase
|
||||
) {
|
||||
const assistantDatabase = {
|
||||
createTask: vi.fn(),
|
||||
@@ -5229,7 +5234,7 @@ describe('registerIpcHandlers agent terminal state', () => {
|
||||
})
|
||||
)
|
||||
const onRuntimeSettingsChanged = vi.fn(async () => {})
|
||||
const dispose = registerIpcHandlers(
|
||||
const args: Parameters<typeof registerIpcHandlers> = [
|
||||
window as never,
|
||||
runtime as never,
|
||||
'CommandOrControl+Shift+Space',
|
||||
@@ -5256,7 +5261,7 @@ describe('registerIpcHandlers agent terminal state', () => {
|
||||
(knowledgeServiceOverride ?? {
|
||||
database: { listKnowledgeBases: vi.fn(() => []) }
|
||||
}) as never,
|
||||
assistantDatabase as never,
|
||||
(imageDatabase ?? assistantDatabase) as never,
|
||||
approvalBroker as never,
|
||||
{} as never,
|
||||
onRuntimeSettingsChanged,
|
||||
@@ -5279,7 +5284,9 @@ describe('registerIpcHandlers agent terminal state', () => {
|
||||
undefined,
|
||||
undefined,
|
||||
goodbuddyConfigService as never
|
||||
)
|
||||
]
|
||||
args[43] = imageService
|
||||
const dispose = registerIpcHandlers(...args)
|
||||
return {
|
||||
approvalBroker,
|
||||
assistantDatabase,
|
||||
@@ -5370,6 +5377,84 @@ describe('registerIpcHandlers agent terminal state', () => {
|
||||
knowledgeLibraryIds: []
|
||||
})
|
||||
|
||||
it('binds the production image service before text dispatch and routes regenerate, Ask rejection, and deletion through IPC', async () => {
|
||||
const database = new AssistantDatabase(':memory:')
|
||||
database.initialize(process.cwd())
|
||||
const conversationId = crypto.randomUUID()
|
||||
const userId = crypto.randomUUID()
|
||||
const messageId = crypto.randomUUID()
|
||||
const header = { id: conversationId, title: 'Image IPC', updatedAt: Date.now(), workMode: 'execute' as const }
|
||||
database.saveLocalConversations([{ header, messages: [] }])
|
||||
const profile = { id: crypto.randomUUID(), name: 'Image model', modelName: 'image-test', protocol: 'openai-images-generations',
|
||||
baseUrl: 'https://image.test/v1', authentication: 'none' as const, allowConversationInvocation: true }
|
||||
const fetcher = vi.fn<typeof fetch>(async () => Response.json({ data: [{ b64_json: 'iVBORw0KGgo=' }] }))
|
||||
const service = new ImageGenerationService({ database, getSettings: async () => ({ modelProfiles: [profile], defaultImageModelProfileId: profile.id }), fetcher })
|
||||
service.initialize()
|
||||
const run = vi.fn(async function* (request: AgentExecutionRequest) {
|
||||
expect(database.getConversation(conversationId).messages.map(message => message.id)).toEqual([userId, messageId])
|
||||
expect(request.imageToolBinding?.context.messageId).toBe(messageId)
|
||||
expect(await request.imageToolBinding?.describe()).toContain(profile.id)
|
||||
expect((await request.imageToolBinding!.call({ intent: 'create', prompt: 'A blue circle' }, 'image-call')).state).toBe('completed')
|
||||
yield { type: 'done' as const, requestId: request.requestId }
|
||||
})
|
||||
const harness = createHarness({ runtimeId: 'model', capability: 'text', supportsToolExecution: true, run },
|
||||
undefined, 'always', undefined, false, undefined, undefined, undefined, false, undefined, undefined, undefined, service, database)
|
||||
const event = trustedEvent(harness.webContents)
|
||||
try {
|
||||
const requestId = crypto.randomUUID()
|
||||
await harness.handler!(event, { requestId, conversationId, prompt: 'Draw a circle', workMode: 'execute',
|
||||
currentUserMessageId: userId, currentAssistantMessageId: messageId })
|
||||
await vi.waitFor(() => expect(database.getTask(requestId).status).toBe('completed'))
|
||||
const original = database.getConversation(conversationId).messages[1]!.imageOperations![0]!
|
||||
expect(database.getArtifact(original.artifactIds[0]!).content).toContain('iVBORw0KGgo=')
|
||||
const regenerate = electronMocks.handlers.get(ipcChannels.imageOperationRegenerate)!
|
||||
const cancel = electronMocks.handlers.get(ipcChannels.imageOperationCancel)!
|
||||
expect(() => cancel(event, { conversationId, operationId: original.id, prompt: 'injected' })).toThrow()
|
||||
const regenerated = await regenerate(event, { conversationId, operationId: original.id }) as typeof original
|
||||
expect(regenerated.id).not.toBe(original.id)
|
||||
expect(regenerated.messageId).toBe(messageId)
|
||||
expect(fetcher).toHaveBeenCalledTimes(2)
|
||||
database.saveLocalConversations([{ header: { ...header, workMode: 'ask' }, messages: [] }])
|
||||
await expect(regenerate(event, { conversationId, operationId: original.id })).rejects.toThrow('Execute')
|
||||
expect(fetcher).toHaveBeenCalledTimes(2)
|
||||
const cancelConversation = vi.spyOn(service, 'cancelConversation')
|
||||
await electronMocks.handlers.get(ipcChannels.conversationsDeleteLocal)!(event, conversationId)
|
||||
expect(cancelConversation).toHaveBeenCalledWith(conversationId)
|
||||
expect(() => database.getConversation(conversationId)).toThrow()
|
||||
} finally {
|
||||
await harness.dispose()
|
||||
await service.dispose()
|
||||
database.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('persists uploads before dispatch while withholding image bytes from a nonvisual text model', async () => {
|
||||
const database = new AssistantDatabase(':memory:')
|
||||
database.initialize(process.cwd())
|
||||
const conversationId = crypto.randomUUID()
|
||||
const userId = crypto.randomUUID()
|
||||
database.saveLocalConversations([{ header: { id: conversationId, title: 'Upload', updatedAt: Date.now() }, messages: [] }])
|
||||
const service = new ImageGenerationService({ database, getSettings: async () => ({ modelProfiles: [] }) })
|
||||
const run = vi.fn(async function* (request: AgentExecutionRequest) {
|
||||
expect(request.images).toBeUndefined()
|
||||
expect(request.trustedInstructions).toContain('cannot view')
|
||||
const sources = database.getConversation(conversationId).messages.find(message => message.id === userId)!.imageSourceArtifactIds!
|
||||
expect(sources).toHaveLength(1)
|
||||
expect(database.getArtifact(sources[0]!).content).toContain('iVBORw0KGgo=')
|
||||
yield { type: 'done' as const, requestId: request.requestId }
|
||||
})
|
||||
const harness = createHarness({ runtimeId: 'model', capability: 'text', supportsToolExecution: true, run },
|
||||
undefined, 'always', undefined, false, undefined, undefined, undefined, false, undefined, undefined, undefined, service, database)
|
||||
harness.contextManager.enrichRequest.mockImplementation(request => ({ ...request, images: [{ name: 'Upload', mediaType: 'image/png', data: 'iVBORw0KGgo=' }] }))
|
||||
try {
|
||||
const requestId = crypto.randomUUID()
|
||||
await harness.handler!(trustedEvent(harness.webContents), { requestId, conversationId, prompt: 'Turn it red', workMode: 'execute',
|
||||
currentUserMessageId: userId, currentAssistantMessageId: crypto.randomUUID() })
|
||||
await vi.waitFor(() => expect(database.getTask(requestId).status).toBe('completed'))
|
||||
expect(run).toHaveBeenCalledOnce()
|
||||
} finally { await harness.dispose(); await service.dispose(); database.close() }
|
||||
})
|
||||
|
||||
it('restores image context at the IPC boundary and persists the quiet notice on its artifact event', async () => {
|
||||
const png = 'iVBORw0KGgo='
|
||||
const artifactId = '00000000-0000-4000-8000-000000000301'
|
||||
|
||||
+84
-2
@@ -382,6 +382,8 @@ import {
|
||||
} from './magic-notes/magic-note-analyzer'
|
||||
import { AgentEventBuffer } from './agent-event-buffer'
|
||||
import { withImageConversationContext } from './agent/image-conversation-context'
|
||||
import type { ImageGenerationService } from './agent/image-generation-service'
|
||||
import { imageOperationTargetSchema } from '../shared/image-operation-ipc'
|
||||
import {
|
||||
ExecutionSpaceResolver,
|
||||
REMOTE_EXECUTION_SPACE_UNAVAILABLE
|
||||
@@ -1334,7 +1336,8 @@ export function registerIpcHandlers(
|
||||
'getHostConnectionState' | 'onHostConnectionStateChange'
|
||||
>,
|
||||
terminalSessionManager?: TerminalSessionManager,
|
||||
localToolEnvironmentService?: LocalToolEnvironmentService
|
||||
localToolEnvironmentService?: LocalToolEnvironmentService,
|
||||
imageGenerationService?: ImageGenerationService
|
||||
): () => Promise<void> {
|
||||
type ActiveRequestLease = {
|
||||
controller: AbortController
|
||||
@@ -4018,6 +4021,34 @@ export function registerIpcHandlers(
|
||||
const attachedRequest = contextManager.enrichRequest(
|
||||
parsedRequest
|
||||
)
|
||||
let uploadedImageSourceIds: string[] = []
|
||||
if (imageGenerationService && parsedRequest.currentUserMessageId && parsedRequest.currentAssistantMessageId) {
|
||||
const conversation = assistantDatabase.getConversation(parsedRequest.conversationId)
|
||||
const now = Date.now()
|
||||
assistantDatabase.saveLocalConversations([{
|
||||
header: {
|
||||
id: conversation.id, projectId: conversation.projectId, title: conversation.title,
|
||||
updatedAt: now, workMode: normalizedWorkMode, runtimeSelection: parsedRequest.runtimeSelection,
|
||||
knowledgeLibraryIds: conversation.knowledgeLibraryIds,
|
||||
knowledgeRetrievalMode: conversation.knowledgeRetrievalMode,
|
||||
contextMetrics: conversation.contextMetrics, contextCompressionState: conversation.contextCompressionState,
|
||||
branch: conversation.branch
|
||||
},
|
||||
messages: [
|
||||
...(conversation.messages.some(message => message.id === parsedRequest.currentUserMessageId) ? [] : [{
|
||||
id: parsedRequest.currentUserMessageId, role: 'user' as const,
|
||||
content: parsedRequest.prompt, createdAt: now, state: 'complete' as const
|
||||
}]),
|
||||
...(conversation.messages.some(message => message.id === parsedRequest.currentAssistantMessageId) ? [] : [{
|
||||
id: parsedRequest.currentAssistantMessageId, role: 'assistant' as const,
|
||||
content: '', createdAt: now, state: 'streaming' as const
|
||||
}])
|
||||
]
|
||||
}])
|
||||
if (attachedRequest.images?.length) {
|
||||
uploadedImageSourceIds = imageGenerationService.persistUploads({ conversationId: parsedRequest.conversationId, messageId: parsedRequest.currentUserMessageId }, attachedRequest.images)
|
||||
}
|
||||
}
|
||||
const enrichedRequest = imageGeneration
|
||||
? withImageConversationContext(
|
||||
attachedRequest,
|
||||
@@ -4045,7 +4076,7 @@ export function registerIpcHandlers(
|
||||
!agentRuntimeSelected
|
||||
? capabilityService.getWebSearchCapabilityStatus?.()
|
||||
: undefined,
|
||||
configAccess !== 'none' && !enrichedRequest.projectId
|
||||
(configAccess !== 'none' && !enrichedRequest.projectId) || (imageGenerationService && !imageGeneration && enrichedRequest.images?.length)
|
||||
? settingsStore.getResolvedSettings()
|
||||
: undefined,
|
||||
selectedRuntimeTarget
|
||||
@@ -4077,6 +4108,15 @@ export function registerIpcHandlers(
|
||||
: undefined
|
||||
: resolvedRuntimeSettings?.workspacePath
|
||||
const controller = new AbortController()
|
||||
const imageToolBinding = !imageGeneration && imageGenerationService && enrichedRequest.currentAssistantMessageId
|
||||
? imageGenerationService.bind({
|
||||
conversationId: enrichedRequest.conversationId,
|
||||
messageId: enrichedRequest.currentAssistantMessageId,
|
||||
requestId: enrichedRequest.requestId,
|
||||
workMode: normalizedWorkMode
|
||||
}, () => normalizeInteractiveWorkMode(assistantDatabase.getConversation(enrichedRequest.conversationId).workMode))
|
||||
: undefined
|
||||
const imageToolAvailable = Boolean(await imageToolBinding?.describe())
|
||||
const scopedCapability = await grantScopedDataCapability({
|
||||
gateway: knowledgeGateway,
|
||||
browserControl: browserControl as BrowserCapabilityControl | undefined,
|
||||
@@ -4101,6 +4141,7 @@ export function registerIpcHandlers(
|
||||
})
|
||||
const knowledgeCapabilityToken = scopedCapability.token
|
||||
const availableTools = [
|
||||
...(imageToolAvailable ? ['generate_image'] : []),
|
||||
...(webSearchEnabled ? ['web_search', 'web_fetch'] : []),
|
||||
...(!agentRuntimeSelected && !imageGeneration
|
||||
? ['workspace_rg', 'workspace_read_text', 'output_read', 'subagent_delegate']
|
||||
@@ -4142,6 +4183,28 @@ export function registerIpcHandlers(
|
||||
if (scopedCapability.browserTabId) {
|
||||
request.browserTabId = scopedCapability.browserTabId
|
||||
}
|
||||
if (imageToolBinding) {
|
||||
request.imageToolBinding = imageToolBinding
|
||||
const imageReferenceIds = [...uploadedImageSourceIds, ...(request.imageContextArtifactIds ?? [])]
|
||||
if (imageReferenceIds.length) {
|
||||
request.trustedInstructions = [request.trustedInstructions,
|
||||
`Image references for this request: ${JSON.stringify(imageReferenceIds)}. These are references, not visual observations; use explicit sourceArtifactIds when editing.`].filter(Boolean).join('\n')
|
||||
}
|
||||
if (request.images?.length && resolvedRuntimeSettings) {
|
||||
const effective = request.runtimeSelection
|
||||
? applyRuntimeSelection(resolvedRuntimeSettings, request.runtimeSelection).settings
|
||||
: resolvedRuntimeSettings
|
||||
const supportsImages = selectedRuntimeTarget === 'model' ? effective.supportsImageInput
|
||||
: selectedRuntimeTarget === 'opencode' ? effective.opencodeModelProfile?.supportsImageInput
|
||||
: selectedRuntimeTarget === 'continue' ? effective.continueModelProfile?.supportsImageInput
|
||||
: effective.deepseekHarnessModelProfile?.supportsImageInput
|
||||
if (supportsImages !== true) {
|
||||
request.images = undefined
|
||||
request.trustedInstructions = [request.trustedInstructions,
|
||||
'The uploaded images were saved as conversation image references for the image tool. Your chat model cannot view them. Do not claim visual observations; use the user description to edit them.'].filter(Boolean).join('\n')
|
||||
}
|
||||
}
|
||||
}
|
||||
const managedSshExecution =
|
||||
agentRuntimeSelected &&
|
||||
configExecutionSpace?.kind === 'ssh'
|
||||
@@ -6762,6 +6825,7 @@ export function registerIpcHandlers(
|
||||
(event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
const conversationId = assistantIdSchema.parse(input)
|
||||
imageGenerationService?.cancelConversation(conversationId)
|
||||
const queuedItems =
|
||||
assistantDatabase.listConversationQueueItems(conversationId)
|
||||
for (const item of queuedItems) {
|
||||
@@ -6789,6 +6853,24 @@ export function registerIpcHandlers(
|
||||
}
|
||||
)
|
||||
|
||||
registerHandler(ipcChannels.imageOperationCancel, (event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
const target = imageOperationTargetSchema.parse(input)
|
||||
if (!imageGenerationService) throw new Error('Image service is unavailable')
|
||||
return imageGenerationService.cancel(target.conversationId, target.operationId)
|
||||
})
|
||||
registerHandler(ipcChannels.imageOperationRegenerate, (event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
const target = imageOperationTargetSchema.parse(input)
|
||||
if (!imageGenerationService) throw new Error('Image service is unavailable')
|
||||
const previous = imageGenerationService.getOperation(target.conversationId, target.operationId)
|
||||
const currentWorkMode = (): 'ask' | 'execute' => normalizeInteractiveWorkMode(assistantDatabase.getConversation(target.conversationId).workMode)
|
||||
return imageGenerationService.regenerate({
|
||||
conversationId: target.conversationId, messageId: previous.messageId,
|
||||
requestId: randomUUID(), workMode: currentWorkMode()
|
||||
}, target.operationId, currentWorkMode)
|
||||
})
|
||||
|
||||
registerHandler(
|
||||
ipcChannels.conversationQueueList,
|
||||
(event, input: unknown) => {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import type { ImageToolBinding } from '../agent/image-tool-binding'
|
||||
import type { RuntimeProtocolBinaryChannel } from './protocol-remote-runtime-channel'
|
||||
import {
|
||||
decodeRemoteImageToolMessage, encodeRemoteImageToolMessage,
|
||||
remoteImageToolCallSchema, remoteImageToolReplySchema
|
||||
} from '../../shared/remote-image-tool-contracts'
|
||||
|
||||
export class MainImageToolSession {
|
||||
private readonly wait = new AbortController()
|
||||
private readonly unsubscribe: () => void
|
||||
private readonly removeAbort: () => void
|
||||
|
||||
constructor(
|
||||
private readonly channel: RuntimeProtocolBinaryChannel,
|
||||
private readonly binding: ImageToolBinding,
|
||||
signal?: AbortSignal
|
||||
) {
|
||||
this.unsubscribe = channel.onClose(() => this.close())
|
||||
const abort = (): void => this.close()
|
||||
signal?.addEventListener('abort', abort, { once: true })
|
||||
this.removeAbort = () => signal?.removeEventListener('abort', abort)
|
||||
if (signal?.aborted) this.close()
|
||||
void this.receive().catch(() => this.close())
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.wait.signal.aborted) return
|
||||
// This aborts binding.call's wait, never the Main-owned image operation.
|
||||
this.wait.abort()
|
||||
this.removeAbort()
|
||||
this.unsubscribe()
|
||||
this.channel.close()
|
||||
}
|
||||
|
||||
private async receive(): Promise<void> {
|
||||
while (!this.wait.signal.aborted) {
|
||||
const frame = await this.channel.receive(this.wait.signal)
|
||||
const call = remoteImageToolCallSchema.parse(decodeRemoteImageToolMessage(frame.payload))
|
||||
await frame.consume()
|
||||
void this.dispatch(call).catch(() => this.close())
|
||||
}
|
||||
}
|
||||
|
||||
private async dispatch(call: ReturnType<typeof remoteImageToolCallSchema.parse>): Promise<void> {
|
||||
let reply: ReturnType<typeof remoteImageToolReplySchema.parse>
|
||||
try {
|
||||
this.wait.signal.throwIfAborted()
|
||||
if (this.binding.context.workMode !== 'execute') throw new Error('Image tools are unavailable in Ask mode')
|
||||
const result = await this.binding.call(call.input, call.callId, this.wait.signal)
|
||||
reply = remoteImageToolReplySchema.parse({ callId: call.callId, result })
|
||||
} catch (error) {
|
||||
reply = { callId: call.callId, error: (error instanceof Error ? error.message : 'Image tool failed').slice(0, 4_000) }
|
||||
}
|
||||
if (!this.wait.signal.aborted) await this.channel.send(encodeRemoteImageToolMessage(reply), this.wait.signal)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { ImageToolBinding } from '../agent/image-tool-binding'
|
||||
import {
|
||||
AGENT_PROTOCOL_LIMITS,
|
||||
ChannelProtocolError,
|
||||
@@ -489,6 +490,46 @@ async function openChannel(fixture = connectionFixture()) {
|
||||
}
|
||||
|
||||
describe('ProtocolRemoteRuntimeChannel', () => {
|
||||
it('refreshes image binding descriptions per prompt, hides Ask, and closes failed preparations', async () => {
|
||||
const fixture = await openChannel()
|
||||
const describe = vi.fn(async () => 'First image catalog')
|
||||
const binding: ImageToolBinding = {
|
||||
context: { conversationId: 'conversation', messageId: 'message', requestId: 'operation-1', workMode: 'execute' },
|
||||
describe, call: vi.fn()
|
||||
}
|
||||
const prepare = (requestId: string, workMode: 'ask' | 'execute' = 'execute') => ({
|
||||
bindingId: 'binding-1', operationId: requestId, requestId, workMode,
|
||||
controllerId: 'controller-1', controllerGeneration: 1, connectionGeneration: fixture.channel.generation,
|
||||
channelEpoch: fixture.channel.channelEpoch, hostId: 'host-1', hostRevision: 1, hostKeyGeneration: 1,
|
||||
workspaceIdentity: 'workspace-1', agentInstallationId: 'agent-1', runtimeId: 'opencode',
|
||||
runtimeBundleDigest, runtimeAdapterDigest: digest('d'), deadlineAt: new Date(Date.now() + 10000).toISOString(),
|
||||
budget: { maximumInputBytes: 1024, maximumOutputBytes: 1024 }, promptSequence: 0
|
||||
})
|
||||
fixture.client.responder = (method, params) => {
|
||||
if (method !== 'runtime/preparePrompt') return defaultResponse(method, params)
|
||||
const input = params as ReturnType<typeof prepare> & { imageTool?: { description: string } }
|
||||
return { bindingId: input.bindingId, operationId: input.operationId, requestId: input.requestId,
|
||||
workMode: input.workMode, deadlineAt: input.deadlineAt, acceptedAt: new Date().toISOString(),
|
||||
...(input.imageTool ? { imageToolUrl: 'http://127.0.0.1:1234/image' } : {}) }
|
||||
}
|
||||
try {
|
||||
const wait = new AbortController()
|
||||
const first = await fixture.channel.preparePrompt(prepare('operation-1'), binding, wait.signal)
|
||||
fixture.channel.setRecoveryBoundary(first.deadlineAt, wait.signal)
|
||||
expect(fixture.client.blobBinaries[0]!.closed).toBe(false)
|
||||
describe.mockResolvedValue('Refreshed image catalog')
|
||||
await fixture.channel.preparePrompt(prepare('operation-2'), binding, wait.signal)
|
||||
expect(fixture.client.blobBinaries[0]!.closed).toBe(true)
|
||||
expect(fixture.client.requests.at(-1)!.params).toMatchObject({ imageTool: { description: 'Refreshed image catalog' } })
|
||||
await fixture.channel.preparePrompt(prepare('operation-3', 'ask'), binding, wait.signal)
|
||||
expect(describe).toHaveBeenCalledTimes(2)
|
||||
expect(fixture.client.blobBinaries[1]!.closed).toBe(true)
|
||||
expect(fixture.client.requests.at(-1)!.params).not.toHaveProperty('imageTool', expect.anything())
|
||||
fixture.client.responder = () => { throw new Error('preparation response lost') }
|
||||
await expect(fixture.channel.preparePrompt(prepare('operation-4'), binding, wait.signal)).rejects.toThrow()
|
||||
expect(fixture.client.blobBinaries[2]!.closed).toBe(true)
|
||||
} finally { await fixture.channel.close() }
|
||||
})
|
||||
it('sends question replies on the authenticated control channel and rejects foreign bindings', async () => {
|
||||
const fixture = await openChannel()
|
||||
const response = { bindingId: 'binding-1', operationId: 'operation-1', questionId: 'question-1', answers: [] }
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { z } from 'zod'
|
||||
import type { ImageToolBinding } from '../agent/image-tool-binding'
|
||||
import { MainImageToolSession } from './main-image-tool-session'
|
||||
import { remoteQuestionResponseSchema, type RemoteQuestionResponse } from '../../shared/remote-question-contracts'
|
||||
import {
|
||||
AGENT_PROTOCOL_LIMITS,
|
||||
@@ -584,38 +586,51 @@ export class ProtocolRemoteRuntimeChannel
|
||||
}
|
||||
|
||||
async preparePrompt(
|
||||
preparation: z.input<typeof remotePromptOperationPreparationSchema>
|
||||
preparation: z.input<typeof remotePromptOperationPreparationSchema>,
|
||||
imageToolBinding?: ImageToolBinding,
|
||||
waitSignal?: AbortSignal
|
||||
): Promise<
|
||||
z.infer<typeof remotePromptOperationAcceptanceSchema>
|
||||
> {
|
||||
const parsed = remotePromptOperationPreparationSchema.parse(
|
||||
preparation
|
||||
)
|
||||
this.#assertPreparationIdentity(parsed)
|
||||
const raw = await this.#request(
|
||||
'runtime/preparePrompt',
|
||||
parsed
|
||||
)
|
||||
const acceptance = parseRemoteResult(
|
||||
remotePromptOperationAcceptanceSchema,
|
||||
raw,
|
||||
'Remote Runtime prompt acceptance is invalid'
|
||||
)
|
||||
this.#imageTool?.close()
|
||||
this.#imageTool = undefined
|
||||
const description = preparation.workMode === 'execute' ? await imageToolBinding?.describe() : undefined
|
||||
waitSignal?.throwIfAborted()
|
||||
let imageTool: z.infer<typeof remotePromptOperationPreparationSchema>['imageTool']
|
||||
if (description && imageToolBinding) {
|
||||
this.#assertCurrent()
|
||||
const binary = this.#state.client.allocateBinaryChannel?.({ kind: 'blob' })
|
||||
if (!binary) throw new Error('Remote image tool transport is unavailable')
|
||||
imageTool = { channelId: binary.channelId, channelEpoch: binary.channelEpoch, description }
|
||||
this.#imageTool = new MainImageToolSession(binary, imageToolBinding, waitSignal)
|
||||
}
|
||||
try {
|
||||
assertRemotePromptAcceptanceMatchesPreparation(
|
||||
parsed,
|
||||
acceptance
|
||||
const parsed = remotePromptOperationPreparationSchema.parse({ ...preparation, imageTool })
|
||||
this.#assertPreparationIdentity(parsed)
|
||||
const raw = await this.#request('runtime/preparePrompt', parsed)
|
||||
const acceptance = parseRemoteResult(
|
||||
remotePromptOperationAcceptanceSchema, raw,
|
||||
'Remote Runtime prompt acceptance is invalid'
|
||||
)
|
||||
} catch {
|
||||
throw new ProtocolRemoteRuntimeChannelError(
|
||||
'Remote Runtime prompt acceptance identity is invalid',
|
||||
'binding-mismatch'
|
||||
)
|
||||
try {
|
||||
assertRemotePromptAcceptanceMatchesPreparation(parsed, acceptance)
|
||||
if (Boolean(imageTool) !== Boolean(acceptance.imageToolUrl)) throw new Error('Image tool registration mismatch')
|
||||
} catch {
|
||||
throw new ProtocolRemoteRuntimeChannelError(
|
||||
'Remote Runtime prompt acceptance identity is invalid', 'binding-mismatch'
|
||||
)
|
||||
}
|
||||
this.#assertCurrent()
|
||||
return acceptance
|
||||
} catch (error) {
|
||||
this.#imageTool?.close()
|
||||
this.#imageTool = undefined
|
||||
throw error
|
||||
}
|
||||
this.#assertCurrent()
|
||||
return acceptance
|
||||
}
|
||||
|
||||
#imageTool?: MainImageToolSession
|
||||
|
||||
async startOwnedPrompt(
|
||||
request: z.input<typeof remoteOwnedPromptStartRequestSchema>
|
||||
): Promise<z.infer<typeof remoteOwnedPromptStartResultSchema>> {
|
||||
@@ -754,7 +769,8 @@ export class ProtocolRemoteRuntimeChannel
|
||||
'protocol'
|
||||
)
|
||||
}
|
||||
this.clearRecoveryBoundary()
|
||||
this.#removeRecoveryAbort?.()
|
||||
this.#removeRecoveryAbort = undefined
|
||||
this.#recoveryDeadlineAt = deadline
|
||||
this.#recoverySignal = signal
|
||||
const abort = (): void => {
|
||||
@@ -774,6 +790,8 @@ export class ProtocolRemoteRuntimeChannel
|
||||
}
|
||||
|
||||
clearRecoveryBoundary(): void {
|
||||
this.#imageTool?.close()
|
||||
this.#imageTool = undefined
|
||||
this.#removeRecoveryAbort?.()
|
||||
this.#removeRecoveryAbort = undefined
|
||||
this.#recoverySignal = undefined
|
||||
@@ -1628,6 +1646,7 @@ export class ProtocolRemoteRuntimeChannel
|
||||
this.#state.capabilityGeneration
|
||||
)
|
||||
this.#closing = true
|
||||
this.#imageTool?.close()
|
||||
this.clearRecoveryBoundary()
|
||||
this.#lifetime.abort(
|
||||
new DOMException('Remote Runtime channel closed', 'AbortError')
|
||||
|
||||
@@ -80,6 +80,57 @@ afterEach(async () => {
|
||||
})
|
||||
|
||||
describe('RuntimeSettingsStore', () => {
|
||||
it('round-trips conversation image settings and retains unavailable defaults without resolving credentials', async () => {
|
||||
const { store, filePath } = await createStore()
|
||||
const initial = await store.getPublicSettings()
|
||||
expect(initial.modelProfiles[0]?.allowConversationInvocation).toBe(false)
|
||||
expect(initial.defaultImageModelProfileId).toBeUndefined()
|
||||
const imageId = '00000000-0000-4000-8000-000000000099'
|
||||
const text = {
|
||||
id: initial.modelProfiles[0]!.id, name: 'Text',
|
||||
baseUrl: 'https://example.com', modelName: 'text',
|
||||
protocol: 'openai-chat-completions' as const,
|
||||
authentication: 'api-key' as const,
|
||||
imageGenerationQuality: 'auto' as const,
|
||||
apiKey: { action: 'keep' as const }
|
||||
}
|
||||
const image = {
|
||||
...text, id: imageId, name: 'Image model',
|
||||
protocol: 'openai-images-generations' as const,
|
||||
allowConversationInvocation: true,
|
||||
apiKey: { action: 'replace' as const, value: 'image-secret' }
|
||||
}
|
||||
const input = settings({ modelProfiles: [text, image], defaultModelProfileId: text.id, defaultImageModelProfileId: imageId })
|
||||
await store.update(runtimeSettingsInputSchema.parse(input))
|
||||
const reloaded = new RuntimeSettingsStore(filePath, cipher, {})
|
||||
expect((await reloaded.getPublicSettings()).modelProfiles[1]).toMatchObject({ allowConversationInvocation: true, apiKeyConfigured: true })
|
||||
expect((await reloaded.getResolvedSettings()).defaultImageModelProfileId).toBe(imageId)
|
||||
expect((await reloaded.getResolvedSettings()).modelProfiles[1]).toMatchObject({ allowConversationInvocation: true, apiKey: 'image-secret' })
|
||||
expect((await reloaded.getPublicSettings()).configured?.modelProfiles[1]?.allowConversationInvocation).toBe(true)
|
||||
const decrypt = vi.fn(() => { throw new Error('Credential unavailable') })
|
||||
const credentialFree = new RuntimeSettingsStore(filePath, { ...cipher, decrypt }, {})
|
||||
expect(await credentialFree.getSavedModelProfiles()).toEqual([
|
||||
{ id: text.id, name: text.name, protocol: text.protocol, allowConversationInvocation: false },
|
||||
{ id: imageId, name: image.name, protocol: image.protocol, allowConversationInvocation: true }
|
||||
])
|
||||
expect(decrypt).not.toHaveBeenCalled()
|
||||
await reloaded.update(settings())
|
||||
expect((await reloaded.getSavedModelProfiles())[1]?.allowConversationInvocation).toBe(true)
|
||||
expect((await reloaded.getPublicSettings()).defaultImageModelProfileId).toBe(imageId)
|
||||
const disabled = await reloaded.update({ ...input, modelProfiles: [text, { ...image, allowConversationInvocation: false, apiKey: { action: 'keep' } }] })
|
||||
expect(disabled.defaultImageModelProfileId).toBe(imageId)
|
||||
expect(disabled.modelProfiles[1]).toMatchObject({ allowConversationInvocation: false, apiKeyConfigured: true })
|
||||
const removed = await reloaded.update({ ...input, modelProfiles: [text] })
|
||||
expect(removed.defaultImageModelProfileId).toBe(imageId)
|
||||
expect((await new RuntimeSettingsStore(filePath, cipher, {}).getPublicSettings()).defaultImageModelProfileId).toBe(imageId)
|
||||
expect((await reloaded.update({ ...input, modelProfiles: [text], defaultImageModelProfileId: null })).defaultImageModelProfileId).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects malformed conversation image settings', () => {
|
||||
expect(runtimeSettingsInputSchema.safeParse(settings({ defaultImageModelProfileId: 'not-a-uuid' })).success).toBe(false)
|
||||
expect(runtimeSettingsInputSchema.safeParse({ ...settings(), modelProfiles: [{ id: '00000000-0000-4000-8000-000000000001', name: 'Image', baseUrl: 'https://example.com', modelName: 'image', protocol: 'openai-images-generations', authentication: 'none', imageGenerationQuality: 'auto', apiKey: { action: 'keep' }, allowConversationInvocation: 'true' }] }).success).toBe(false)
|
||||
})
|
||||
|
||||
it('defaults to automatic Execute tool authorization and preserves an explicitly saved deny policy', async () => {
|
||||
const { store, filePath } = await createStore()
|
||||
await expect(store.getPublicSettings()).resolves.toMatchObject({ toolApproval: 'always' })
|
||||
|
||||
@@ -292,12 +292,14 @@ const storedSettingsSchema = version20StoredSettingsSchema
|
||||
modelProfiles: z
|
||||
.array(
|
||||
currentStoredModelProfileSchema.extend({
|
||||
allowConversationInvocation: z.boolean().optional(),
|
||||
requestHeaders: modelRequestHeadersSchema,
|
||||
requestBody: modelRequestBodySchema
|
||||
})
|
||||
)
|
||||
.min(1)
|
||||
.max(20)
|
||||
.max(20),
|
||||
defaultImageModelProfileId: z.string().uuid().nullable().optional()
|
||||
})
|
||||
|
||||
type StoredSettings = z.infer<typeof storedSettingsSchema>
|
||||
@@ -424,6 +426,7 @@ export type ResolvedRuntimeSettings = {
|
||||
apiKey?: string
|
||||
modelProfiles: ResolvedModelProfile[]
|
||||
defaultModelProfileId: string
|
||||
defaultImageModelProfileId?: string | null
|
||||
opencodeModelProfile?: ResolvedModelProfile
|
||||
continueModelProfile?: ResolvedModelProfile
|
||||
deepseekHarnessModelProfile?: ResolvedModelProfile
|
||||
@@ -458,6 +461,7 @@ export type RuntimePolicySettings = Pick<
|
||||
>
|
||||
|
||||
export type ResolvedModelProfile = {
|
||||
allowConversationInvocation?: boolean
|
||||
id: string
|
||||
name: string
|
||||
baseUrl: string
|
||||
@@ -1599,6 +1603,7 @@ export class RuntimeSettingsStore {
|
||||
protocol: effective.protocol,
|
||||
authentication: effective.authentication,
|
||||
supportsImageInput: effective.supportsImageInput,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: effective.contextWindowTokens,
|
||||
maximumOutputTokens: effective.maximumOutputTokens,
|
||||
imageGenerationQuality: effective.imageGenerationQuality,
|
||||
@@ -1619,6 +1624,7 @@ export class RuntimeSettingsStore {
|
||||
protocol: profile.protocol,
|
||||
authentication: profile.authentication,
|
||||
supportsImageInput: profile.supportsImageInput,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: profile.contextWindowTokens,
|
||||
maximumOutputTokens: profile.maximumOutputTokens,
|
||||
imageGenerationQuality: profile.imageGenerationQuality,
|
||||
@@ -1704,6 +1710,7 @@ export class RuntimeSettingsStore {
|
||||
protocol: resolved.protocol,
|
||||
authentication: resolved.authentication,
|
||||
supportsImageInput: resolved.supportsImageInput,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: resolved.contextWindowTokens,
|
||||
maximumOutputTokens: resolved.maximumOutputTokens,
|
||||
imageGenerationQuality:
|
||||
@@ -1728,6 +1735,7 @@ export class RuntimeSettingsStore {
|
||||
protocol: profile.protocol,
|
||||
authentication: profile.authentication,
|
||||
supportsImageInput: profile.supportsImageInput,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: profile.contextWindowTokens,
|
||||
maximumOutputTokens: profile.maximumOutputTokens,
|
||||
imageGenerationQuality:
|
||||
@@ -1860,6 +1868,7 @@ export class RuntimeSettingsStore {
|
||||
credentialSource: effective.credentialSource,
|
||||
modelProfiles,
|
||||
defaultModelProfileId: settings.defaultModelProfileId,
|
||||
defaultImageModelProfileId: settings.defaultImageModelProfileId,
|
||||
opencodeModelSource: agent.opencodeBaseUrl
|
||||
? { kind: 'platform' }
|
||||
: settings.opencodeModelSource,
|
||||
@@ -1891,6 +1900,18 @@ export class RuntimeSettingsStore {
|
||||
return this.toPublicSettings(await this.load())
|
||||
}
|
||||
|
||||
async getSavedModelProfiles(): Promise<Array<Pick<
|
||||
RuntimeSettings['modelProfiles'][number],
|
||||
'id' | 'name' | 'protocol' | 'allowConversationInvocation'
|
||||
>>> {
|
||||
return (await this.load()).modelProfiles.map((profile) => ({
|
||||
id: profile.id,
|
||||
name: profile.name,
|
||||
protocol: profile.protocol,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false
|
||||
}))
|
||||
}
|
||||
|
||||
captureRollback(): Promise<RuntimeSettingsRollback> {
|
||||
let result: RuntimeSettingsRollback | undefined
|
||||
const operation = this.updateQueue.then(async () => {
|
||||
@@ -1990,6 +2011,7 @@ export class RuntimeSettingsStore {
|
||||
modelProfiles,
|
||||
defaultModelProfileId: settings.defaultModelProfileId,
|
||||
opencodeModelProfile,
|
||||
defaultImageModelProfileId: settings.defaultImageModelProfileId,
|
||||
continueModelProfile,
|
||||
deepseekHarnessModelProfile,
|
||||
...agent,
|
||||
@@ -2128,6 +2150,7 @@ export class RuntimeSettingsStore {
|
||||
protocol: input.modelProtocol,
|
||||
authentication: input.modelAuthentication,
|
||||
supportsImageInput: profile.supportsImageInput,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: profile.contextWindowTokens,
|
||||
maximumOutputTokens: profile.maximumOutputTokens,
|
||||
imageGenerationQuality: input.imageGenerationQuality,
|
||||
@@ -2143,6 +2166,7 @@ export class RuntimeSettingsStore {
|
||||
protocol: profile.protocol,
|
||||
authentication: profile.authentication,
|
||||
supportsImageInput: profile.supportsImageInput,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: profile.contextWindowTokens,
|
||||
maximumOutputTokens: profile.maximumOutputTokens,
|
||||
imageGenerationQuality: profile.imageGenerationQuality,
|
||||
@@ -2197,6 +2221,9 @@ export class RuntimeSettingsStore {
|
||||
protocol: profile.protocol,
|
||||
authentication: profile.authentication,
|
||||
supportsImageInput: profile.supportsImageInput ?? false,
|
||||
allowConversationInvocation:
|
||||
profile.allowConversationInvocation ??
|
||||
existing?.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: profile.contextWindowTokens,
|
||||
maximumOutputTokens: profile.maximumOutputTokens,
|
||||
imageGenerationQuality: profile.imageGenerationQuality,
|
||||
@@ -2491,6 +2518,9 @@ export class RuntimeSettingsStore {
|
||||
modelProfiles,
|
||||
defaultModelProfileId,
|
||||
opencodeModelSource,
|
||||
defaultImageModelProfileId: input.defaultImageModelProfileId === undefined
|
||||
? current.defaultImageModelProfileId
|
||||
: input.defaultImageModelProfileId,
|
||||
continueModelSource,
|
||||
deepseekHarnessModelSource:
|
||||
requestedDeepSeekHarnessSource,
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { DesktopApi } from '../shared/contracts'
|
||||
import { ipcChannels } from '../shared/ipc-channels'
|
||||
|
||||
const electron = vi.hoisted(() => ({
|
||||
contextBridge: { exposeInMainWorld: vi.fn() },
|
||||
ipcRenderer: { invoke: vi.fn(), on: vi.fn(), removeListener: vi.fn() },
|
||||
webUtils: {}
|
||||
}))
|
||||
vi.mock('electron', () => electron)
|
||||
|
||||
describe('image operation preload', () => {
|
||||
it('forwards explicit actions and delivers operation data without Electron events', async () => {
|
||||
await import('./index')
|
||||
const api = (electron.contextBridge.exposeInMainWorld.mock.calls[0]![1] as DesktopApi).conversations.imageOperations
|
||||
const target = { conversationId: 'conversation', operationId: 'operation' }
|
||||
await api.cancel(target)
|
||||
await api.regenerate(target)
|
||||
expect(electron.ipcRenderer.invoke.mock.calls).toEqual([
|
||||
[ipcChannels.imageOperationCancel, target], [ipcChannels.imageOperationRegenerate, target]
|
||||
])
|
||||
const listener = vi.fn()
|
||||
const unsubscribe = api.onChanged(listener)
|
||||
const [channel, handler] = electron.ipcRenderer.on.mock.calls[0]!
|
||||
const operation = { ...target, state: 'completed' }
|
||||
handler({ sender: 'private' }, operation)
|
||||
expect(listener).toHaveBeenCalledExactlyOnceWith(operation)
|
||||
unsubscribe()
|
||||
expect(electron.ipcRenderer.removeListener).toHaveBeenCalledExactlyOnceWith(channel, handler)
|
||||
})
|
||||
})
|
||||
@@ -1057,6 +1057,15 @@ const desktopApi: DesktopApi = {
|
||||
}
|
||||
},
|
||||
conversations: {
|
||||
imageOperations: {
|
||||
cancel: (input) => ipcRenderer.invoke(ipcChannels.imageOperationCancel, input),
|
||||
regenerate: (input) => ipcRenderer.invoke(ipcChannels.imageOperationRegenerate, input),
|
||||
onChanged: (listener) => {
|
||||
const handler = (_event: Electron.IpcRendererEvent, operation: import('../shared/image-generation-contracts').ImageOperation): void => listener(operation)
|
||||
ipcRenderer.on(ipcChannels.imageOperationChanged, handler)
|
||||
return () => ipcRenderer.removeListener(ipcChannels.imageOperationChanged, handler)
|
||||
}
|
||||
},
|
||||
list: () =>
|
||||
ipcRenderer.invoke(
|
||||
ipcChannels.conversationsList
|
||||
|
||||
@@ -483,6 +483,11 @@ const api: DesktopApi = {
|
||||
},
|
||||
},
|
||||
conversations: {
|
||||
imageOperations: {
|
||||
cancel: vi.fn(),
|
||||
regenerate: vi.fn(),
|
||||
onChanged: vi.fn(() => () => undefined),
|
||||
},
|
||||
list: vi.fn(async () => []),
|
||||
replace: vi.fn(async () => {}),
|
||||
saveLocal: vi.fn(async () => {}),
|
||||
@@ -8926,6 +8931,19 @@ describe("App", () => {
|
||||
expect(mode).toHaveAccessibleName("工作模式：Ask · 只读问答");
|
||||
});
|
||||
|
||||
it('opens model settings from no-tool image guidance without losing the request draft', async () => {
|
||||
vi.mocked(api.agent.getStatus).mockResolvedValue({ id: 'model', label: 'Legacy chat', available: true, supportsToolExecution: false, capability: 'chat', detail: 'Ready' });
|
||||
render(<App />);
|
||||
expect(await screen.findByText('当前聊天模型无法自动调用图片工具，可使用已有直连图片工作流。')).toBeVisible();
|
||||
fireEvent.change(screen.getByLabelText('向 GoodBuddy 提问'), { target: { value: 'Keep my image request' } });
|
||||
fireEvent.click(screen.getByRole('button', { name: '前往图片模型设置' }));
|
||||
expect(await screen.findByRole('tab', { name: '模型连接' })).toHaveAttribute('aria-selected', 'true');
|
||||
fireEvent.click(screen.getByRole('button', { name: '关闭设置' }));
|
||||
expect(screen.getByLabelText('向 GoodBuddy 提问')).toHaveValue('Keep my image request');
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(api.conversations.imageOperations.regenerate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows a direct model to submit Execute with GoodBuddy approvals", async () => {
|
||||
vi.mocked(api.agent.getStatus).mockResolvedValue({
|
||||
id: "model",
|
||||
@@ -10290,6 +10308,93 @@ describe("App", () => {
|
||||
anchorClick.mockRestore();
|
||||
});
|
||||
|
||||
it("updates the original image message after leaving its conversation and queues history editing across Runtime switches", async () => {
|
||||
const conversationId = crypto.randomUUID();
|
||||
const messageId = crypto.randomUUID();
|
||||
const otherId = crypto.randomUUID();
|
||||
const operation: import('../../shared/image-generation-contracts').ImageOperation = {
|
||||
id: crypto.randomUUID(), conversationId, messageId, requestId: crypto.randomUUID(), callId: 'image-call',
|
||||
modelProfileId: crypto.randomUUID(), modelName: 'image-model', modelProfileName: '图片连接',
|
||||
input: { intent: 'create', prompt: 'Blue circle', sourceArtifactIds: [] }, state: 'running', artifactIds: [], createdAt: 1, updatedAt: 1,
|
||||
};
|
||||
const secondOperation = { ...operation, id: crypto.randomUUID(), modelProfileName: 'Second image connection' };
|
||||
let imageListener: ((value: typeof operation) => void) | undefined;
|
||||
vi.mocked(api.conversations.imageOperations.onChanged).mockImplementationOnce(listener => { imageListener = listener; return () => { imageListener = undefined; }; });
|
||||
vi.mocked(api.conversations.list).mockResolvedValueOnce([
|
||||
{ id: conversationId, projectId, title: '图片原会话', updatedAt: 2, workMode: 'execute', messages: [
|
||||
{ id: messageId, role: 'assistant', content: '图片稍后保存', createdAt: 1, state: 'complete', imageOperations: [operation, secondOperation] },
|
||||
] },
|
||||
{ id: otherId, projectId, title: '另一会话', updatedAt: 1, messages: [
|
||||
{ id: crypto.randomUUID(), role: 'assistant', content: '另一会话正文', createdAt: 1, state: 'complete' },
|
||||
] },
|
||||
]);
|
||||
const artifact = { id: crypto.randomUUID(), kind: 'image' as const, title: 'Blue circle', mimeType: 'image/png',
|
||||
content: 'data:image/png;base64,iVBORw0KGgo=', byteSize: 8, createdAt: '2026-09-15T00:00:00Z', updatedAt: '2026-09-15T00:00:00Z' };
|
||||
vi.mocked(api.artifacts.get).mockResolvedValueOnce(artifact);
|
||||
render(<App />);
|
||||
expect(await screen.findByText('图片稍后保存')).toBeVisible();
|
||||
fireEvent.click(screen.getByRole('button', { name: /^另一会话/u }));
|
||||
expect(await screen.findByText('另一会话正文')).toBeVisible();
|
||||
act(() => imageListener?.({ ...operation, state: 'completed', artifactIds: [artifact.id], updatedAt: 3 }));
|
||||
expect(screen.queryByRole('img', { name: 'Blue circle' })).not.toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole('button', { name: /^图片原会话/u }));
|
||||
expect(await screen.findByRole('img', { name: 'Blue circle' })).toBeVisible();
|
||||
expect(screen.getByText(/图片已保存/u)).toBeVisible();
|
||||
expect(screen.getAllByRole('region', { name: '图片生成' }).map(card => within(card).getByRole('status').textContent)).toEqual([
|
||||
'图片已保存 · 图片连接', '正在生成图片 · Second image connection',
|
||||
]);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
fireEvent.click(screen.getByRole('button', { name: '继续修改' }));
|
||||
fireEvent.click(screen.getByRole('button', { name: /sonnet-5/u }));
|
||||
fireEvent.click(screen.getByRole('menuitemradio', { name: /^OpenCode · 默认模型/u }));
|
||||
await waitFor(() => expect(api.agent.getStatus).toHaveBeenLastCalledWith({ provider: 'opencode' }));
|
||||
fireEvent.change(screen.getByLabelText('向 GoodBuddy 提问'), { target: { value: 'Turn the circle red' } });
|
||||
fireEvent.click(screen.getByLabelText('发送'));
|
||||
await waitFor(() => expect(run).toHaveBeenCalledOnce());
|
||||
expect(run.mock.calls[0]![0]).toMatchObject({ runtimeSelection: { provider: 'opencode' }, imageContextArtifactIds: [artifact.id] });
|
||||
expect(api.conversationQueue.enqueueUser).toHaveBeenCalledWith(expect.objectContaining({ imageContextArtifactIds: [artifact.id] }));
|
||||
expect(api.context.addPastedImage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('recovers failed image editing through settings and composer attachments without resubmitting the operation', async () => {
|
||||
const conversationId = crypto.randomUUID();
|
||||
const operation: import('../../shared/image-generation-contracts').ImageOperation = {
|
||||
id: crypto.randomUUID(), conversationId, messageId: crypto.randomUUID(), requestId: crypto.randomUUID(), callId: 'failed-edit',
|
||||
modelProfileId: crypto.randomUUID(), modelName: 'original-image-model', modelProfileName: 'Original image connection',
|
||||
input: { intent: 'edit', prompt: 'Change the background', sourceArtifactIds: [crypto.randomUUID()] },
|
||||
state: 'failed', error: 'Original source unavailable', artifactIds: [], createdAt: 1, updatedAt: 1,
|
||||
};
|
||||
vi.mocked(api.conversations.list).mockResolvedValueOnce([{ id: conversationId, projectId, title: 'Failed image editing', updatedAt: 2, workMode: 'execute', messages: [
|
||||
{ id: operation.messageId, role: 'assistant', content: '', createdAt: 1, state: 'complete', imageOperations: [operation] },
|
||||
] }]);
|
||||
const source = { id: crypto.randomUUID(), name: 'replacement.png', size: 64, preview: 'Replacement image', kind: 'image' as const, contentUrl: 'data:image/png;base64,iVBORw0KGgo=' };
|
||||
vi.mocked(api.context.selectFiles).mockResolvedValueOnce([source]);
|
||||
render(<App />);
|
||||
fireEvent.click(await screen.findByRole('button', { name: '重新选择素材' }));
|
||||
const composer = screen.getByLabelText('向 GoodBuddy 提问');
|
||||
expect(composer).toHaveValue('使用图片模型“Original image connection”：Change the background');
|
||||
expect(composer).toHaveFocus();
|
||||
expect(await within(composer.closest<HTMLElement>('.composer')!).findByText('replacement.png')).toBeVisible();
|
||||
expect(api.context.selectFiles).toHaveBeenCalledOnce();
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(api.conversations.imageOperations.regenerate).not.toHaveBeenCalled();
|
||||
fireEvent.change(composer, { target: { value: 'My edited image request' } });
|
||||
vi.mocked(api.context.selectFiles).mockResolvedValueOnce([]);
|
||||
fireEvent.click(screen.getByRole('button', { name: '重新选择素材' }));
|
||||
await waitFor(() => expect(api.context.selectFiles).toHaveBeenCalledTimes(2));
|
||||
expect(composer).toHaveValue('My edited image request');
|
||||
fireEvent.click(screen.getByRole('button', { name: '重新选择模型' }));
|
||||
expect(await screen.findByRole('tab', { name: '模型连接' })).toHaveAttribute('aria-selected', 'true');
|
||||
fireEvent.click(screen.getByRole('button', { name: '关闭设置' }));
|
||||
expect(screen.getByLabelText('向 GoodBuddy 提问')).toHaveValue('My edited image request');
|
||||
expect(screen.getByText('Original source unavailable')).toBeVisible();
|
||||
expect(api.conversations.imageOperations.regenerate).not.toHaveBeenCalled();
|
||||
fireEvent.click(screen.getByLabelText('发送'));
|
||||
await waitFor(() => expect(run).toHaveBeenCalledOnce());
|
||||
expect(run.mock.calls[0]![0]).toMatchObject({ conversationId, prompt: 'My edited image request', contextIds: [source.id] });
|
||||
expect(run.mock.calls[0]![0].imageContextArtifactIds).toBeUndefined();
|
||||
});
|
||||
|
||||
it("can dispatch a request to the parallel expert team", async () => {
|
||||
render(<App />);
|
||||
|
||||
|
||||
+113
-11
@@ -1,3 +1,5 @@
|
||||
import { ImageCapabilityNotice } from "./ImageCapabilityNotice";
|
||||
import type { ImageOperation } from "../../shared/image-generation-contracts";
|
||||
import {
|
||||
ArrowDown,
|
||||
Bot,
|
||||
@@ -803,6 +805,9 @@ type ChatScrollSnapshot = {
|
||||
};
|
||||
|
||||
function ChatHistoryPane({
|
||||
onOpenImageModelSettings,
|
||||
onReselectImageSources,
|
||||
onEditImage,
|
||||
active,
|
||||
artifactById,
|
||||
conversationHtmlRenderingEnabled,
|
||||
@@ -824,6 +829,9 @@ function ChatHistoryPane({
|
||||
taskStrip,
|
||||
visibleMessageCount,
|
||||
}: {
|
||||
onOpenImageModelSettings: () => void;
|
||||
onReselectImageSources: (operation: ImageOperation) => void;
|
||||
onEditImage: (artifact: AssistantArtifact) => void;
|
||||
active: boolean;
|
||||
artifactById: ReadonlyMap<string, AssistantArtifact>;
|
||||
conversationHtmlRenderingEnabled: boolean;
|
||||
@@ -1107,6 +1115,9 @@ function ChatHistoryPane({
|
||||
</div>
|
||||
)}
|
||||
<ChatTimeline
|
||||
onOpenImageModelSettings={onOpenImageModelSettings}
|
||||
onReselectImageSources={onReselectImageSources}
|
||||
onEditImage={onEditImage}
|
||||
artifactById={artifactById}
|
||||
conversationId={conversation.id}
|
||||
hiddenMessageCount={hiddenMessageCount}
|
||||
@@ -1344,6 +1355,8 @@ function toConversationMessage(message: Message): ConversationMessage {
|
||||
sourceReferences: message.sourceReferences,
|
||||
knowledgeRetrieval: message.knowledgeRetrieval,
|
||||
artifactIds: message.artifactIds,
|
||||
imageOperations: message.imageOperations,
|
||||
imageSourceArtifactIds: message.imageSourceArtifactIds,
|
||||
imageContextNotice: message.imageContextNotice,
|
||||
task: message.task,
|
||||
attachments: message.attachments,
|
||||
@@ -1477,15 +1490,26 @@ function mergePersistedConversations(
|
||||
conversation.messages.map((message) => message.id),
|
||||
);
|
||||
const messages = [
|
||||
...conversation.messages.map((message) => {
|
||||
const localMessage = localMessageById.get(message.id);
|
||||
...conversation.messages.map((persistedMessage) => {
|
||||
const localMessage = localMessageById.get(persistedMessage.id);
|
||||
const operations = new Map(persistedMessage.imageOperations?.map(operation => [operation.id, operation]));
|
||||
for (const operation of localMessage?.imageOperations ?? []) {
|
||||
if (operation.updatedAt > (operations.get(operation.id)?.updatedAt ?? -1)) operations.set(operation.id, operation);
|
||||
}
|
||||
const imageMetadata = {
|
||||
imageOperations: operations.size ? [...operations.values()] : undefined,
|
||||
imageSourceArtifactIds: persistedMessage.imageSourceArtifactIds ?? localMessage?.imageSourceArtifactIds,
|
||||
artifactIds: [...new Set([...(persistedMessage.artifactIds ?? []),
|
||||
...[...operations.values()].flatMap(operation => operation.artifactIds)])].slice(-8),
|
||||
};
|
||||
const message = { ...persistedMessage, ...imageMetadata };
|
||||
if (!conversation.activeRequest && local.activeRequest?.messageId === message.id) {
|
||||
return { ...message, pendingQuestions: undefined };
|
||||
}
|
||||
if (!localIsNewer) {
|
||||
// Pending prompts are live-only and are omitted from persisted messages.
|
||||
if (localMessage?.state === "streaming" && message.state === "streaming") {
|
||||
return { ...message, approval: localMessage.approval, pendingQuestions: localMessage.pendingQuestions };
|
||||
return { ...localMessage, ...imageMetadata };
|
||||
}
|
||||
return message;
|
||||
}
|
||||
@@ -1495,7 +1519,8 @@ function mergePersistedConversations(
|
||||
) {
|
||||
return message;
|
||||
}
|
||||
return localMessage;
|
||||
return { ...localMessage, ...imageMetadata,
|
||||
artifactIds: [...new Set([...(localMessage.artifactIds ?? []), ...(message.artifactIds ?? [])])].slice(-8) };
|
||||
}),
|
||||
...local.messages.filter((message) => !serverMessageIds.has(message.id)),
|
||||
];
|
||||
@@ -2505,6 +2530,8 @@ function App(): React.JSX.Element {
|
||||
Record<string, ContextAttachment[]>
|
||||
>({});
|
||||
const attachments = attachmentsByConversation[activeId] ?? [];
|
||||
const [imageReferencesByConversation, setImageReferencesByConversation] = useState<Record<string, AssistantArtifact[]>>({});
|
||||
const imageReferences = imageReferencesByConversation[activeId] ?? [];
|
||||
const attachmentsRef = useRef(new Map<string, ContextAttachment[]>());
|
||||
const updateAttachments = useCallback(
|
||||
(
|
||||
@@ -2618,6 +2645,7 @@ function App(): React.JSX.Element {
|
||||
);
|
||||
const hydratingArtifactIds = useRef(new Set<string>());
|
||||
const inputRef = useRef<HTMLTextAreaElement>(null);
|
||||
const attachmentButtonRef = useRef<HTMLButtonElement>(null);
|
||||
const [chatScrollSnapshots, setChatScrollSnapshots] = useState<
|
||||
Record<string, ChatScrollSnapshot>
|
||||
>({});
|
||||
@@ -6017,7 +6045,7 @@ function App(): React.JSX.Element {
|
||||
const missingIds = [
|
||||
...new Set(
|
||||
(activeConversation?.messages ?? []).flatMap(
|
||||
(message) => message.artifactIds ?? [],
|
||||
(message) => [...(message.artifactIds ?? []), ...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? [])],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -6123,11 +6151,25 @@ function App(): React.JSX.Element {
|
||||
);
|
||||
const removeAgentListener =
|
||||
window.goodbuddy.agent.onEvent(handleAgentEvent);
|
||||
const removeImageListener = window.goodbuddy.conversations.imageOperations.onChanged((operation) => {
|
||||
setConversations(current => current.map(conversation => conversation.id !== operation.conversationId ? conversation : {
|
||||
...conversation,
|
||||
updatedAt: Math.max(conversation.updatedAt, operation.updatedAt),
|
||||
messages: conversation.messages.map(message => message.id !== operation.messageId ? message : {
|
||||
...message,
|
||||
imageOperations: message.imageOperations?.some(item => item.id === operation.id)
|
||||
? message.imageOperations.map(item => item.id === operation.id ? operation : item)
|
||||
: [...(message.imageOperations ?? []), operation],
|
||||
artifactIds: [...new Set([...(message.artifactIds ?? []), ...operation.artifactIds])].slice(-8),
|
||||
}),
|
||||
}));
|
||||
});
|
||||
const removeOpenSettingsListener = window.goodbuddy.app.onOpenSettings(() =>
|
||||
setView("settings"),
|
||||
);
|
||||
return () => {
|
||||
removeAgentListener();
|
||||
removeImageListener();
|
||||
removeOpenSettingsListener();
|
||||
};
|
||||
}, [handleAgentEvent, setView]);
|
||||
@@ -7103,6 +7145,7 @@ function App(): React.JSX.Element {
|
||||
includeMemoryContext: !command,
|
||||
prompt,
|
||||
attachments: attachmentSnapshot,
|
||||
imageContextArtifactIds: imageReferences.map(artifact => artifact.id),
|
||||
knowledgeLibraryIds: knowledgeLibraryIdsSnapshot,
|
||||
knowledgeRetrievalMode: knowledgeRetrievalModeSnapshot,
|
||||
};
|
||||
@@ -7119,6 +7162,7 @@ function App(): React.JSX.Element {
|
||||
setRuntimeMenuOpen(false);
|
||||
setInput("");
|
||||
updateAttachments([]);
|
||||
setImageReferencesByConversation(current => ({ ...current, [conversationId]: [] }));
|
||||
if (command) {
|
||||
setSelectedRuntimeCommand("");
|
||||
}
|
||||
@@ -7236,6 +7280,22 @@ function App(): React.JSX.Element {
|
||||
),
|
||||
);
|
||||
try {
|
||||
const saveOrigin = conversationPersistenceQueueRef.current.then(async () => {
|
||||
const origin = {
|
||||
...conversationSnapshot,
|
||||
title: conversationSnapshot.title === "新对话" ? prompt.slice(0, 24) : conversationSnapshot.title,
|
||||
updatedAt: assistantMessage.createdAt,
|
||||
workMode: workModeSnapshot,
|
||||
messages: [...historySnapshot, userMessage, assistantMessage],
|
||||
};
|
||||
await window.goodbuddy.conversations.saveLocal([{
|
||||
header: toLocalConversationHeader(origin),
|
||||
messages: [toConversationMessage(userMessage), toConversationMessage(assistantMessage)],
|
||||
}]);
|
||||
persistedLocalConversationsRef.current.set(conversationId, origin);
|
||||
});
|
||||
conversationPersistenceQueueRef.current = saveOrigin.catch(() => undefined);
|
||||
await saveOrigin;
|
||||
await window.goodbuddy.agent.run({
|
||||
requestId,
|
||||
conversationId,
|
||||
@@ -7254,9 +7314,9 @@ function App(): React.JSX.Element {
|
||||
knowledgeLibraryIds: knowledgeLibraryIdsSnapshot,
|
||||
knowledgeRetrievalMode: knowledgeRetrievalModeSnapshot,
|
||||
contextIds: attachmentSnapshot.map((attachment) => attachment.id),
|
||||
...(runtimeSelectionSnapshot.provider === "model"
|
||||
? {
|
||||
imageContextArtifactIds: [...historySnapshot]
|
||||
imageContextArtifactIds: queuedInput?.imageContextArtifactIds?.length
|
||||
? queuedInput.imageContextArtifactIds
|
||||
: attachmentSnapshot.some(attachment => attachment.kind === 'image') ? undefined : [...historySnapshot]
|
||||
.reverse()
|
||||
.find(
|
||||
(message) =>
|
||||
@@ -7264,8 +7324,6 @@ function App(): React.JSX.Element {
|
||||
message.state === "complete" &&
|
||||
message.artifactIds?.length,
|
||||
)?.artifactIds,
|
||||
}
|
||||
: {}),
|
||||
contextCompressionState: conversationSnapshot.contextCompressionState,
|
||||
history: retainedHistorySnapshot.map((message) => ({
|
||||
role: message.role,
|
||||
@@ -7588,6 +7646,30 @@ function App(): React.JSX.Element {
|
||||
}
|
||||
};
|
||||
|
||||
const editImage = useCallback((artifact: AssistantArtifact): void => {
|
||||
const conversationId = activeConversationIdRef.current;
|
||||
setContextError(undefined);
|
||||
setImageReferencesByConversation(current => ({ ...current,
|
||||
[conversationId]: [...(current[conversationId] ?? []).filter(item => item.id !== artifact.id), artifact].slice(-8),
|
||||
}));
|
||||
inputRef.current?.focus();
|
||||
}, []);
|
||||
|
||||
const openImageModelSettings = useCallback((): void => {
|
||||
setSettingsInitialCategory("model");
|
||||
setView("settings");
|
||||
}, [setView]);
|
||||
|
||||
const reselectImageSources = useCallback((operation: ImageOperation): void => {
|
||||
if (operation.conversationId !== activeId) return;
|
||||
setInput(current => current || t("chat.images.recoveryPrompt", {
|
||||
model: operation.modelProfileName ?? operation.modelName,
|
||||
prompt: operation.input.prompt,
|
||||
}));
|
||||
inputRef.current?.focus();
|
||||
attachmentButtonRef.current?.click();
|
||||
}, [activeId, setInput, t]);
|
||||
|
||||
const selectContextFiles = async (paths?: string[]): Promise<void> => {
|
||||
if (selectingContextFilesRef.current) {
|
||||
return;
|
||||
@@ -9229,6 +9311,9 @@ function App(): React.JSX.Element {
|
||||
locale={locale}
|
||||
onCopyMessage={copyMessage}
|
||||
onDownloadImage={downloadImage}
|
||||
onEditImage={editImage}
|
||||
onOpenImageModelSettings={openImageModelSettings}
|
||||
onReselectImageSources={reselectImageSources}
|
||||
onOpenCitationContext={openCitationContext}
|
||||
onOpenCitationSource={openCitationSource}
|
||||
onOpenImage={openImageViewer}
|
||||
@@ -9343,12 +9428,20 @@ function App(): React.JSX.Element {
|
||||
running={conversationExecutionRunning}
|
||||
/>
|
||||
<div className="composer">
|
||||
<ImageCapabilityNotice
|
||||
runtime={runtime}
|
||||
workMode={effectiveWorkMode}
|
||||
hasCallableImageModels={runtimeSettings?.modelProfiles.some(profile =>
|
||||
profile.protocol === "openai-images-generations" && profile.allowConversationInvocation === true
|
||||
) ?? false}
|
||||
onOpenModelSettings={openImageModelSettings}
|
||||
/>
|
||||
{composerOptionSummary && (
|
||||
<div className="composer__option-summary" aria-label={t("composer.settings")}>
|
||||
{composerOptionSummary}
|
||||
</div>
|
||||
)}
|
||||
{(attachments.length > 0 ||
|
||||
{(attachments.length > 0 || imageReferences.length > 0 ||
|
||||
selectingContextFiles) && (
|
||||
<div
|
||||
aria-busy={selectingContextFiles}
|
||||
@@ -9360,6 +9453,14 @@ function App(): React.JSX.Element {
|
||||
aria-invalid={contextError ? true : undefined}
|
||||
className="context-list"
|
||||
>
|
||||
{imageReferences.map(artifact => <div className="context-chip" key={artifact.id}>
|
||||
{artifact.content && <img alt="" className="context-chip__thumbnail" src={artifact.content} />}
|
||||
<span><strong>{artifact.title}</strong><small>{t('chat.images.edit')}</small></span>
|
||||
<button type="button" aria-label={t('composer.removeAttachment', { name: artifact.title })}
|
||||
onClick={() => setImageReferencesByConversation(current => ({ ...current,
|
||||
[activeId]: (current[activeId] ?? []).filter(item => item.id !== artifact.id),
|
||||
}))}>×</button>
|
||||
</div>)}
|
||||
{attachments.map((attachment) => (
|
||||
<div
|
||||
className="context-chip"
|
||||
@@ -9558,6 +9659,7 @@ function App(): React.JSX.Element {
|
||||
}
|
||||
aria-invalid={contextError ? true : undefined}
|
||||
disabled={selectingContextFiles}
|
||||
ref={attachmentButtonRef}
|
||||
onClick={() => void selectContextFiles()}
|
||||
title={t("composer.addAttachment")}
|
||||
type="button"
|
||||
|
||||
@@ -11,9 +11,11 @@ type CapabilitySettingsTab = 'skills' | 'mcp' | 'tools'
|
||||
|
||||
export function CapabilitiesAndToolsSettingsSection({
|
||||
magicNotesEnabled,
|
||||
onOpenImageModelSettings,
|
||||
onNotify
|
||||
}: {
|
||||
magicNotesEnabled: boolean
|
||||
onOpenImageModelSettings: () => void
|
||||
onNotify?: (notification: AppNotificationInput) => void
|
||||
}): React.JSX.Element {
|
||||
const { t } = useTranslation('settings')
|
||||
@@ -76,7 +78,10 @@ export function CapabilitiesAndToolsSettingsSection({
|
||||
role="tabpanel"
|
||||
>
|
||||
{mcpVisited && (
|
||||
<McpSettingsSection magicNotesEnabled={magicNotesEnabled} />
|
||||
<McpSettingsSection
|
||||
magicNotesEnabled={magicNotesEnabled}
|
||||
onOpenImageModelSettings={onOpenImageModelSettings}
|
||||
/>
|
||||
)}
|
||||
</section>
|
||||
<section
|
||||
|
||||
@@ -15,8 +15,10 @@ import {
|
||||
UserRound,
|
||||
XCircle
|
||||
} from 'lucide-react'
|
||||
import { memo, useEffect, useRef, useState } from 'react'
|
||||
import { memo, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { ImageOperationStatus } from './ImageOperationStatus'
|
||||
import type { ImageOperation } from '../../shared/image-generation-contracts'
|
||||
import type {
|
||||
ApprovalDecision,
|
||||
AgentEvent,
|
||||
@@ -75,6 +77,8 @@ export type Message = {
|
||||
sourceReferences?: KnowledgeSearchReference[]
|
||||
knowledgeRetrieval?: KnowledgeRetrievalStatus
|
||||
artifactIds?: string[]
|
||||
imageOperations?: ConversationMessage['imageOperations']
|
||||
imageSourceArtifactIds?: string[]
|
||||
imageContextNotice?: ConversationMessage['imageContextNotice']
|
||||
task?: ConversationMessage['task']
|
||||
attachments?: ConversationAttachment[]
|
||||
@@ -488,6 +492,9 @@ function SubagentStatusList({
|
||||
}
|
||||
|
||||
type ChatMessageRowProps = {
|
||||
onOpenImageModelSettings?: () => void
|
||||
onReselectImageSources?: (operation: ImageOperation) => void
|
||||
onEditImage?: (artifact: AssistantArtifact) => void
|
||||
artifactById: ReadonlyMap<string, AssistantArtifact>
|
||||
canRetry: boolean
|
||||
conversationId: string
|
||||
@@ -522,6 +529,9 @@ type ChatMessageRowProps = {
|
||||
}
|
||||
|
||||
function ChatMessageRowView({
|
||||
onOpenImageModelSettings,
|
||||
onReselectImageSources,
|
||||
onEditImage,
|
||||
artifactById,
|
||||
canRetry,
|
||||
conversationId,
|
||||
@@ -541,6 +551,10 @@ function ChatMessageRowView({
|
||||
retryContent
|
||||
}: ChatMessageRowProps): React.JSX.Element {
|
||||
const { t } = useTranslation('app')
|
||||
const imageArtifactIds = useMemo(() => [...new Set([
|
||||
...(message.artifactIds ?? []),
|
||||
...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? [])
|
||||
])], [message.artifactIds, message.imageOperations])
|
||||
const compressionMarkers =
|
||||
message.contextCompressions ??
|
||||
(message.contextCompression ? [message.contextCompression] : [])
|
||||
@@ -799,7 +813,8 @@ function ChatMessageRowView({
|
||||
subagents={unorderedSubagents}
|
||||
/>
|
||||
)}
|
||||
{message.artifactIds?.map((artifactId) => {
|
||||
{message.imageOperations?.map(operation => <ImageOperationStatus key={operation.id} operation={operation} onOpenImageModelSettings={onOpenImageModelSettings} onReselectImageSources={onReselectImageSources} />)}
|
||||
{imageArtifactIds.map((artifactId) => {
|
||||
const candidate = artifactById.get(artifactId)
|
||||
const artifact =
|
||||
candidate?.kind === 'image' &&
|
||||
@@ -833,6 +848,7 @@ function ChatMessageRowView({
|
||||
</button>
|
||||
<figcaption>{artifact.title}</figcaption>
|
||||
<div className="message-image-actions">
|
||||
{onEditImage && <button type="button" onClick={() => onEditImage(artifact)}>{t('chat.images.edit')}</button>}
|
||||
<button
|
||||
onClick={(event) =>
|
||||
onOpenImage(
|
||||
@@ -1254,6 +1270,9 @@ function ChatMessageRowView({
|
||||
export const ChatMessageRow = memo(ChatMessageRowView)
|
||||
|
||||
type ChatTimelineProps = {
|
||||
onOpenImageModelSettings?: () => void
|
||||
onReselectImageSources?: (operation: ImageOperation) => void
|
||||
onEditImage?: (artifact: AssistantArtifact) => void
|
||||
artifactById: ReadonlyMap<string, AssistantArtifact>
|
||||
conversationId: string
|
||||
hiddenMessageCount: number
|
||||
@@ -1291,6 +1310,9 @@ type ChatTimelineProps = {
|
||||
}
|
||||
|
||||
export const ChatTimeline = memo(function ChatTimeline({
|
||||
onOpenImageModelSettings,
|
||||
onReselectImageSources,
|
||||
onEditImage,
|
||||
artifactById,
|
||||
conversationId,
|
||||
hiddenMessageCount,
|
||||
@@ -1331,6 +1353,9 @@ export const ChatTimeline = memo(function ChatTimeline({
|
||||
const messageIndex = messageStartIndex + visibleMessageIndex
|
||||
return (
|
||||
<ChatMessageRow
|
||||
onOpenImageModelSettings={onOpenImageModelSettings}
|
||||
onReselectImageSources={onReselectImageSources}
|
||||
onEditImage={onEditImage}
|
||||
artifactById={artifactById}
|
||||
canRetry={
|
||||
message.state === 'error' &&
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { cleanup, fireEvent, render, screen } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ImageCapabilityNotice } from './ImageCapabilityNotice'
|
||||
import type { AgentRuntimeStatus } from '../../shared/contracts'
|
||||
|
||||
afterEach(cleanup)
|
||||
const runtime: AgentRuntimeStatus = { id: 'model', label: 'Chat', available: true, capability: 'chat', supportsToolExecution: false, detail: 'Ready' }
|
||||
|
||||
describe('conversation image capability guidance', () => {
|
||||
it('explains the existing direct image workflow and links to configuration without selecting a model', () => {
|
||||
const onOpenModelSettings = vi.fn()
|
||||
const view = render(<ImageCapabilityNotice runtime={runtime} workMode="ask" hasCallableImageModels={false} onOpenModelSettings={onOpenModelSettings} />)
|
||||
expect(screen.getByRole('status')).toHaveTextContent('当前聊天模型无法自动调用图片工具，可使用已有直连图片工作流。')
|
||||
expect(screen.getByRole('status')).toHaveTextContent('在 Runtime 选择器中选择图片模型')
|
||||
fireEvent.click(screen.getByRole('button', { name: '前往图片模型设置' }))
|
||||
expect(onOpenModelSettings).toHaveBeenCalledOnce()
|
||||
view.rerender(<ImageCapabilityNotice runtime={{ ...runtime, supportsToolExecution: true }} workMode="ask" hasCallableImageModels onOpenModelSettings={onOpenModelSettings} />)
|
||||
expect(screen.getByRole('status')).toHaveTextContent('生成或编辑图片需要 Execute 模式')
|
||||
expect(screen.queryByRole('button')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it.each([
|
||||
undefined,
|
||||
{ ...runtime, available: false },
|
||||
{ ...runtime, capability: 'image-generation' as const },
|
||||
{ ...runtime, supportsToolExecution: true },
|
||||
])('does not claim missing tool capability for loading, unavailable, image, or tool-capable runtimes: %j', current => {
|
||||
const { container } = render(<ImageCapabilityNotice runtime={current} workMode="execute" hasCallableImageModels onOpenModelSettings={vi.fn()} />)
|
||||
expect(container).toBeEmptyDOMElement()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,28 @@
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { AgentRuntimeStatus } from '../../shared/contracts'
|
||||
|
||||
export function ImageCapabilityNotice({ runtime, workMode, hasCallableImageModels, onOpenModelSettings }: {
|
||||
runtime?: AgentRuntimeStatus
|
||||
workMode: 'ask' | 'execute'
|
||||
hasCallableImageModels: boolean
|
||||
onOpenModelSettings: () => void
|
||||
}): React.JSX.Element | null {
|
||||
const { t } = useTranslation('app')
|
||||
if (!runtime?.available || runtime.capability === 'image-generation') return null
|
||||
const noTools = !runtime.supportsToolExecution
|
||||
if (!noTools && (!hasCallableImageModels || workMode !== 'ask')) return null
|
||||
return (
|
||||
<div className="composer__option-summary" role="status">
|
||||
<span>{t(noTools ? 'chat.images.noToolsNotice' : 'chat.images.askNotice')}</span>
|
||||
{noTools && (
|
||||
<>
|
||||
{' '}
|
||||
<span>{t('chat.images.directWorkflowNotice')}</span>
|
||||
<div className="message-image-actions">
|
||||
<button type="button" onClick={onOpenModelSettings}>{t('chat.images.openModelSettings')}</button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ImageOperationStatus } from './ImageOperationStatus'
|
||||
import type { ImageOperation } from '../../shared/image-generation-contracts'
|
||||
|
||||
afterEach(() => { cleanup(); vi.unstubAllGlobals() })
|
||||
|
||||
describe('image operation controls', () => {
|
||||
it('offers explicit failed-edit recovery without changing regeneration inputs', async () => {
|
||||
const regenerate = vi.fn(async () => undefined)
|
||||
Object.defineProperty(window, 'goodbuddy', { configurable: true, value: { conversations: { imageOperations: { regenerate } } } })
|
||||
const operation: ImageOperation = {
|
||||
id: crypto.randomUUID(), conversationId: crypto.randomUUID(), messageId: crypto.randomUUID(),
|
||||
requestId: crypto.randomUUID(), callId: 'edit-call', modelProfileId: crypto.randomUUID(), modelName: 'original-model',
|
||||
input: { intent: 'edit', prompt: 'Change the background', sourceArtifactIds: [crypto.randomUUID()] },
|
||||
state: 'failed', error: 'Source unavailable', artifactIds: [], createdAt: 1, updatedAt: 1
|
||||
}
|
||||
const onOpenImageModelSettings = vi.fn()
|
||||
const onReselectImageSources = vi.fn()
|
||||
const view = render(<ImageOperationStatus operation={operation} onOpenImageModelSettings={onOpenImageModelSettings} onReselectImageSources={onReselectImageSources} />)
|
||||
fireEvent.click(screen.getByRole('button', { name: '重新选择模型' }))
|
||||
fireEvent.click(screen.getByRole('button', { name: '重新选择素材' }))
|
||||
expect(onOpenImageModelSettings).toHaveBeenCalledOnce()
|
||||
expect(onReselectImageSources).toHaveBeenCalledExactlyOnceWith(operation)
|
||||
expect(regenerate).not.toHaveBeenCalled()
|
||||
fireEvent.click(screen.getByRole('button', { name: '重新生成' }))
|
||||
await waitFor(() => expect(regenerate).toHaveBeenCalledExactlyOnceWith({ conversationId: operation.conversationId, operationId: operation.id }))
|
||||
view.rerender(<ImageOperationStatus operation={{ ...operation, state: 'running' }} onOpenImageModelSettings={onOpenImageModelSettings} onReselectImageSources={onReselectImageSources} />)
|
||||
expect(screen.queryByRole('button', { name: '重新选择模型' })).not.toBeInTheDocument()
|
||||
expect(screen.queryByRole('button', { name: '重新选择素材' })).not.toBeInTheDocument()
|
||||
view.rerender(<ImageOperationStatus operation={{ ...operation, input: { ...operation.input, intent: 'create', sourceArtifactIds: [] } }} onOpenImageModelSettings={onOpenImageModelSettings} onReselectImageSources={onReselectImageSources} />)
|
||||
expect(screen.getByRole('button', { name: '重新选择模型' })).toBeInTheDocument()
|
||||
expect(screen.queryByRole('button', { name: '重新选择素材' })).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('cancels independently and offers an explicit new request with local error recovery', async () => {
|
||||
const cancel = vi.fn(async () => undefined)
|
||||
const regenerate = vi.fn(async () => { throw new Error('Selected image model is unavailable') })
|
||||
Object.defineProperty(window, 'goodbuddy', { configurable: true, value: { conversations: { imageOperations: { cancel, regenerate } } } })
|
||||
const operation: ImageOperation = { id: crypto.randomUUID(), conversationId: crypto.randomUUID(), messageId: crypto.randomUUID(),
|
||||
requestId: crypto.randomUUID(), callId: 'image-call', modelProfileId: crypto.randomUUID(), modelName: 'image-model',
|
||||
input: { intent: 'create', prompt: 'Blue circle', sourceArtifactIds: [] }, state: 'running', artifactIds: [], createdAt: 1, updatedAt: 1 }
|
||||
const view = render(<ImageOperationStatus operation={operation} />)
|
||||
fireEvent.click(screen.getByRole('button', { name: '取消生成' }))
|
||||
await waitFor(() => expect(cancel).toHaveBeenCalledExactlyOnceWith({ conversationId: operation.conversationId, operationId: operation.id }))
|
||||
view.rerender(<ImageOperationStatus operation={{ ...operation, state: 'unconfirmed' }} />)
|
||||
expect(screen.getByText('将发起新的模型请求。')).toBeInTheDocument()
|
||||
fireEvent.click(await screen.findByRole('button', { name: '重新生成' }))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent('Selected image model is unavailable')
|
||||
expect(screen.getByText('Blue circle')).toBeInTheDocument()
|
||||
expect(screen.getByRole('button', { name: '重新生成' })).toBeEnabled()
|
||||
expect(screen.getByRole('button', { name: '重新生成' })).toHaveAccessibleDescription('Selected image model is unavailable')
|
||||
expect(regenerate).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('distinguishes pending cancellation, stopped waiting, and a saved late result', () => {
|
||||
const operation: ImageOperation = {
|
||||
id: crypto.randomUUID(), conversationId: crypto.randomUUID(), messageId: crypto.randomUUID(),
|
||||
requestId: crypto.randomUUID(), callId: 'image-call', modelProfileId: crypto.randomUUID(), modelName: 'image-model',
|
||||
input: { intent: 'create', prompt: 'Blue circle', sourceArtifactIds: [] },
|
||||
state: 'cancelling', cancellationRequested: true, artifactIds: [], createdAt: 1, updatedAt: 1
|
||||
}
|
||||
const view = render(<ImageOperationStatus operation={operation} />)
|
||||
expect(screen.getByText('已请求取消。提供商可能继续生成并计费。')).toBeVisible()
|
||||
expect(screen.queryByText(/已停止等待/)).not.toBeInTheDocument()
|
||||
expect(screen.getByRole('button', { name: '取消生成' })).toBeDisabled()
|
||||
view.rerender(<ImageOperationStatus operation={{ ...operation, state: 'stopped' }} />)
|
||||
expect(screen.getByText('已停止等待。提供商可能继续生成并计费，晚到的结果会保存在这里。')).toBeVisible()
|
||||
view.rerender(<ImageOperationStatus operation={{ ...operation, state: 'completed', artifactIds: [crypto.randomUUID()] }} />)
|
||||
expect(screen.getByText('取消未生效，已生成并保存图片。')).toBeVisible()
|
||||
expect(screen.queryByText(/已停止等待/)).not.toBeInTheDocument()
|
||||
expect(screen.getByRole('button', { name: '重新生成' })).toBeEnabled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,62 @@
|
||||
import { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { ImageOperation } from '../../shared/image-generation-contracts'
|
||||
import { displayErrorMessage } from './error-message'
|
||||
|
||||
export function ImageOperationStatus({ operation, onOpenImageModelSettings, onReselectImageSources }: {
|
||||
operation: ImageOperation
|
||||
onOpenImageModelSettings?: () => void
|
||||
onReselectImageSources?: (operation: ImageOperation) => void
|
||||
}): React.JSX.Element {
|
||||
const { t } = useTranslation('app')
|
||||
const [pending, setPending] = useState(false)
|
||||
const [error, setError] = useState<string>()
|
||||
const active = ['running', 'saving', 'cancelling'].includes(operation.state)
|
||||
const needsRecovery = !active && (operation.state === 'failed' || Boolean(error))
|
||||
const act = async (): Promise<void> => {
|
||||
setPending(true)
|
||||
setError(undefined)
|
||||
try {
|
||||
const api = window.goodbuddy.conversations.imageOperations
|
||||
await api[active ? 'cancel' : 'regenerate']({ conversationId: operation.conversationId, operationId: operation.id })
|
||||
} catch (reason) {
|
||||
setError(displayErrorMessage(reason, t('chat.images.states.failed')))
|
||||
} finally {
|
||||
setPending(false)
|
||||
}
|
||||
}
|
||||
return <section className="message-retrieval-status" aria-label={t('chat.images.operation')}>
|
||||
<div>
|
||||
<strong role="status">{t(`chat.images.states.${operation.state}`)} · {operation.modelProfileName ?? operation.modelName}</strong>
|
||||
{operation.cancellationRequested && <p>{t(
|
||||
operation.state === 'completed'
|
||||
? 'chat.images.cancelTooLateNotice'
|
||||
: active ? 'chat.images.cancellingNotice' : 'chat.images.cancelNotice'
|
||||
)}</p>}
|
||||
{operation.state === 'unconfirmed' && <p>{t('chat.images.unconfirmedNotice')}</p>}
|
||||
{operation.error && <p>{operation.error}</p>}
|
||||
<details><summary>{t('chat.images.details')}</summary><div>{operation.input.prompt}</div>
|
||||
<small>{operation.modelName} · {operation.input.quality ?? 'auto'}</small>
|
||||
{operation.input.sourceArtifactIds.length > 0 && <div>{t('chat.images.sources', { count: operation.input.sourceArtifactIds.length })}</div>}
|
||||
</details>
|
||||
<div className="message-image-actions">
|
||||
<button type="button" aria-describedby={error ? `image-operation-error-${operation.id}` : undefined} disabled={pending || operation.state === 'cancelling'} onClick={() => void act()}>
|
||||
{t(active ? 'chat.images.cancel' : 'chat.images.regenerate')}
|
||||
</button>
|
||||
{!active && <small>{t('chat.images.regenerateNotice')}</small>}
|
||||
{needsRecovery && onOpenImageModelSettings && (
|
||||
<button type="button" disabled={pending} onClick={onOpenImageModelSettings}>
|
||||
{t('chat.images.reselectModel')}
|
||||
</button>
|
||||
)}
|
||||
{needsRecovery && operation.input.intent === 'edit' && onReselectImageSources && (
|
||||
<button type="button" disabled={pending} onClick={() => onReselectImageSources(operation)}>
|
||||
{t('chat.images.reselectSources')}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
{needsRecovery && <small>{t('chat.images.recoveryNotice')}</small>}
|
||||
{error && <p id={`image-operation-error-${operation.id}`} role="alert">{error}</p>}
|
||||
</div>
|
||||
</section>
|
||||
}
|
||||
@@ -96,8 +96,10 @@ function editorFromServer(server: McpServerSummary): McpEditor {
|
||||
}
|
||||
|
||||
export function McpSettingsSection({
|
||||
onOpenImageModelSettings,
|
||||
magicNotesEnabled = false
|
||||
}: {
|
||||
onOpenImageModelSettings: () => void
|
||||
magicNotesEnabled?: boolean
|
||||
}): React.JSX.Element {
|
||||
const { t } = useTranslation('integrations')
|
||||
@@ -574,11 +576,49 @@ export function McpSettingsSection({
|
||||
</div>
|
||||
<small>
|
||||
{t('mcp.profiles.count', {
|
||||
count: builtinMcpServers.length
|
||||
count: builtinMcpServers.length + 1
|
||||
})}
|
||||
</small>
|
||||
</div>
|
||||
<div className="mcp-server-list">
|
||||
<article
|
||||
className="capability-card builtin-mcp-card"
|
||||
aria-label={t('conversationImages.title')}
|
||||
>
|
||||
<div className="capability-card__header">
|
||||
<div>
|
||||
<strong>{t('conversationImages.title')}</strong>
|
||||
<small>{t('conversationImages.managed')}</small>
|
||||
</div>
|
||||
</div>
|
||||
<p id="image-generation-management">
|
||||
{t('conversationImages.description')}
|
||||
</p>
|
||||
<p>{t('conversationImages.runtimeScope')}</p>
|
||||
<div className="runtime-assignments">
|
||||
<small>{t('mcp.builtin.assignedTo')}</small>
|
||||
{configurableMcpTargets.map((target) => (
|
||||
<label key={target}>
|
||||
<input
|
||||
aria-describedby="image-generation-management"
|
||||
checked={
|
||||
snapshot?.imageGeneration?.assignments.includes(target) ?? false
|
||||
}
|
||||
disabled
|
||||
type="checkbox"
|
||||
/>
|
||||
{runtimeLabels[target]}
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
<button
|
||||
className="secondary-button"
|
||||
onClick={onOpenImageModelSettings}
|
||||
type="button"
|
||||
>
|
||||
{t('conversationImages.openSettings')}
|
||||
</button>
|
||||
</article>
|
||||
{builtinMcpServers.map((server) => {
|
||||
const expansionId = `builtin:${server.id}`
|
||||
const expanded = expandedItemIds.has(expansionId)
|
||||
|
||||
@@ -4790,6 +4790,11 @@ describe('SettingsPanel runtime files', () => {
|
||||
target: { value: 'openai-images-generations' }
|
||||
})
|
||||
const qualitySelect = screen.getByLabelText('图片质量 默认模型')
|
||||
const invocationSwitch = screen.getByRole('switch', { name: '允许 AI 在会话中调用' })
|
||||
expect(invocationSwitch).not.toBeChecked()
|
||||
expect(screen.queryByRole('radio', { name: '设为默认生图模型' })).not.toBeInTheDocument()
|
||||
fireEvent.click(invocationSwitch)
|
||||
fireEvent.click(screen.getByRole('radio', { name: '设为默认生图模型' }))
|
||||
expect(qualitySelect).toHaveValue('auto')
|
||||
fireEvent.change(qualitySelect, {
|
||||
target: { value: 'high' }
|
||||
@@ -4807,10 +4812,12 @@ describe('SettingsPanel runtime files', () => {
|
||||
modelProfiles: [
|
||||
expect.objectContaining({
|
||||
protocol: 'openai-images-generations',
|
||||
allowConversationInvocation: true,
|
||||
imageGenerationQuality: 'high'
|
||||
})
|
||||
],
|
||||
imageGenerationQuality: 'high'
|
||||
imageGenerationQuality: 'high',
|
||||
defaultImageModelProfileId: modelProfileId
|
||||
})
|
||||
)
|
||||
)
|
||||
@@ -5179,6 +5186,77 @@ describe('SettingsPanel runtime files', () => {
|
||||
expect(screen.queryByText('智能心跳')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it.each([false, true])('shows immutable image assignments and source navigation when enabled=%s', async (enabled) => {
|
||||
const snapshot = {
|
||||
...capabilitySnapshot,
|
||||
imageGeneration: {
|
||||
modelProfiles: enabled ? [{ id: modelProfileId, name: 'Image' }] : [],
|
||||
assignments: enabled ? ['model', 'opencode', 'continue', 'deepseek-harness'] : []
|
||||
}
|
||||
}
|
||||
getCapabilitySnapshot.mockResolvedValueOnce(snapshot).mockResolvedValueOnce(snapshot)
|
||||
render(<SettingsPanel {...heartbeatSettingsProps} open onClearLocalData={vi.fn(async () => {})} onClose={vi.fn()} onSaved={vi.fn()} />)
|
||||
fireEvent.click(screen.getByRole('tab', { name: '能力与工具' }))
|
||||
await screen.findByText('文档写作')
|
||||
fireEvent.click(screen.getByRole('tab', { name: 'MCP' }))
|
||||
const card = screen.getByRole('article', { name: '图片生成与编辑' })
|
||||
await waitFor(() => {
|
||||
const checkboxes = within(card).getAllByRole('checkbox')
|
||||
expect(checkboxes).toHaveLength(4)
|
||||
for (const checkbox of checkboxes) {
|
||||
expect(checkbox).toBeDisabled()
|
||||
if (enabled) expect(checkbox).toBeChecked()
|
||||
else expect(checkbox).not.toBeChecked()
|
||||
}
|
||||
})
|
||||
expect(within(card).getByText('内置 · 自动管理')).toBeInTheDocument()
|
||||
expect(within(card).queryByRole('switch')).not.toBeInTheDocument()
|
||||
expect(within(card).queryByText(/暂不支持/)).not.toBeInTheDocument()
|
||||
for (const checkbox of within(card).getAllByRole('checkbox')) {
|
||||
expect(checkbox).toHaveAccessibleDescription('由图片模型的「允许 AI 在会话中调用」设置自动管理。')
|
||||
fireEvent.click(checkbox)
|
||||
fireEvent.click(checkbox.closest('label')!)
|
||||
fireEvent.keyDown(checkbox, { key: ' ' })
|
||||
}
|
||||
expect(setBuiltinMcpServerAssignments).not.toHaveBeenCalled()
|
||||
expect(setBuiltinMcpServerEnabled).not.toHaveBeenCalled()
|
||||
expect(saveMcpServer).not.toHaveBeenCalled()
|
||||
const navigation = within(card).getByRole('button', { name: '前往图片模型设置' })
|
||||
navigation.focus()
|
||||
expect(navigation).toHaveFocus()
|
||||
fireEvent.click(navigation)
|
||||
expect(screen.getByRole('tab', { name: '模型连接' })).toHaveAttribute('aria-selected', 'true')
|
||||
expect(await screen.findByRole('button', { name: '添加自定义' })).toBeEnabled()
|
||||
})
|
||||
|
||||
it('retains a disabled or deleted image default and keeps drafts on save failure', async () => {
|
||||
const imageId = '00000000-0000-4000-8000-000000000099'
|
||||
getRuntime.mockResolvedValueOnce({
|
||||
...runtimeSettings,
|
||||
defaultImageModelProfileId: imageId,
|
||||
modelProfiles: [runtimeSettings.modelProfiles[0]!, {
|
||||
...runtimeSettings.modelProfiles[0]!, id: imageId, name: 'Image',
|
||||
protocol: 'openai-images-generations', allowConversationInvocation: true
|
||||
}]
|
||||
})
|
||||
render(<SettingsPanel {...heartbeatSettingsProps} open onClearLocalData={vi.fn(async () => {})} onClose={vi.fn()} onSaved={vi.fn()} />)
|
||||
fireEvent.click(screen.getByRole('tab', { name: '模型连接' }))
|
||||
fireEvent.click(await screen.findByRole('button', { name: '编辑模型连接 Image' }))
|
||||
fireEvent.click(screen.getByRole('switch', { name: '允许 AI 在会话中调用' }))
|
||||
expect(screen.getByText('默认生图模型不可用，请开启该模型或明确选择其他图片模型。')).toBeInTheDocument()
|
||||
updateRuntime.mockRejectedValueOnce(new Error('Image settings save failed'))
|
||||
fireEvent.click(screen.getByRole('button', { name: '保存设置' }))
|
||||
await screen.findByText('Image settings save failed')
|
||||
expect(screen.getByRole('switch', { name: '允许 AI 在会话中调用' })).not.toBeChecked()
|
||||
expect(updateRuntime).toHaveBeenLastCalledWith(expect.objectContaining({ defaultImageModelProfileId: imageId }))
|
||||
fireEvent.click(screen.getByRole('button', { name: '删除模型连接 Image' }))
|
||||
fireEvent.click(screen.getByRole('button', { name: '保存设置' }))
|
||||
await waitFor(() => expect(updateRuntime).toHaveBeenLastCalledWith(expect.objectContaining({
|
||||
defaultImageModelProfileId: imageId,
|
||||
modelProfiles: [expect.objectContaining({ id: modelProfileId })]
|
||||
})))
|
||||
})
|
||||
|
||||
it('groups Skills and MCP under Capabilities and tools', async () => {
|
||||
render(
|
||||
<SettingsPanel
|
||||
|
||||
@@ -279,6 +279,7 @@ function toModelProfileDrafts(
|
||||
({ requestHeaders, requestBody, ...profile }) => ({
|
||||
...profile,
|
||||
supportsImageInput: profile.supportsImageInput ?? false,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
maximumOutputTokens:
|
||||
profile.maximumOutputTokens ??
|
||||
defaultAnthropicMaximumOutputTokens,
|
||||
@@ -356,6 +357,7 @@ function hydrateRuntimeSettings(
|
||||
modelProfiles: (value: ModelProfileDraft[]) => void
|
||||
selectedModelProfileId: (value: RuntimeDraftSelection) => void
|
||||
defaultModelProfileId: (value: string) => void
|
||||
defaultImageModelProfileId: (value: string | null) => void
|
||||
opencodeModelSource: (value: RuntimeModelSource) => void
|
||||
continueModelSource: (value: RuntimeModelSource) => void
|
||||
deepseekHarnessModelSource: (value: RuntimeModelSource) => void
|
||||
@@ -404,6 +406,7 @@ function hydrateRuntimeSettings(
|
||||
: fallbackProfileId
|
||||
)
|
||||
setters.defaultModelProfileId(value.defaultModelProfileId)
|
||||
setters.defaultImageModelProfileId(value.defaultImageModelProfileId ?? null)
|
||||
setters.opencodeModelSource(configured.opencodeModelSource)
|
||||
setters.continueModelSource(configured.continueModelSource)
|
||||
setters.deepseekHarnessModelSource(
|
||||
@@ -671,6 +674,8 @@ export function SettingsPanel({
|
||||
const [selectedModelProfileId, setSelectedModelProfileId] =
|
||||
useState('')
|
||||
const [defaultModelProfileId, setDefaultModelProfileId] = useState('')
|
||||
const [defaultImageModelProfileId, setDefaultImageModelProfileId] =
|
||||
useState<string | null>(null)
|
||||
const [opencodeModelSource, setOpencodeModelSource] =
|
||||
useState<RuntimeModelSource>({ kind: 'platform' })
|
||||
const [continueModelSource, setContinueModelSource] =
|
||||
@@ -844,6 +849,7 @@ export function SettingsPanel({
|
||||
modelProfiles: setModelProfiles,
|
||||
selectedModelProfileId: setSelectedModelProfileId,
|
||||
defaultModelProfileId: setDefaultModelProfileId,
|
||||
defaultImageModelProfileId: setDefaultImageModelProfileId,
|
||||
opencodeModelSource: setOpencodeModelSource,
|
||||
continueModelSource: setContinueModelSource,
|
||||
deepseekHarnessModelSource: setDeepseekHarnessModelSource,
|
||||
@@ -905,6 +911,7 @@ export function SettingsPanel({
|
||||
provider,
|
||||
modelProfiles,
|
||||
defaultModelProfileId,
|
||||
defaultImageModelProfileId,
|
||||
opencodeModelSource,
|
||||
continueModelSource,
|
||||
deepseekHarnessModelSource,
|
||||
@@ -936,6 +943,7 @@ export function SettingsPanel({
|
||||
provider: settings.provider,
|
||||
modelProfiles: toModelProfileDrafts(settings),
|
||||
defaultModelProfileId: settings.defaultModelProfileId,
|
||||
defaultImageModelProfileId: settings.defaultImageModelProfileId ?? null,
|
||||
opencodeModelSource:
|
||||
savedConfiguredSettings?.opencodeModelSource,
|
||||
continueModelSource:
|
||||
@@ -1420,6 +1428,7 @@ export function SettingsPanel({
|
||||
protocol: profile.protocol,
|
||||
authentication: profile.authentication,
|
||||
supportsImageInput: profile.supportsImageInput,
|
||||
allowConversationInvocation: profile.allowConversationInvocation ?? false,
|
||||
contextWindowTokens: profile.contextWindowTokens,
|
||||
maximumOutputTokens: profile.maximumOutputTokens,
|
||||
imageGenerationQuality: profile.imageGenerationQuality,
|
||||
@@ -1506,6 +1515,7 @@ export function SettingsPanel({
|
||||
apiKey: defaultProfileInput.apiKey,
|
||||
modelProfiles: profileInputs,
|
||||
defaultModelProfileId: defaultProfile.id,
|
||||
defaultImageModelProfileId,
|
||||
opencodeModelSource,
|
||||
continueModelSource,
|
||||
deepseekHarnessModelSource,
|
||||
@@ -1802,6 +1812,7 @@ export function SettingsPanel({
|
||||
protocol: 'openai-chat-completions',
|
||||
authentication: defaultRuntimeSettings.modelAuthentication,
|
||||
supportsImageInput: defaultRuntimeSettings.supportsImageInput,
|
||||
allowConversationInvocation: false,
|
||||
maximumOutputTokens: defaultAnthropicMaximumOutputTokens,
|
||||
imageGenerationQuality:
|
||||
defaultRuntimeSettings.imageGenerationQuality,
|
||||
@@ -2929,6 +2940,15 @@ export function SettingsPanel({
|
||||
{t('actions.addCustom')}
|
||||
</button>
|
||||
</div>
|
||||
{defaultImageModelProfileId && !modelProfiles.some((profile) =>
|
||||
profile.id === defaultImageModelProfileId &&
|
||||
profile.protocol === 'openai-images-generations' &&
|
||||
profile.allowConversationInvocation
|
||||
) && (
|
||||
<p className="settings-warning" role="status">
|
||||
{t('conversationImages.unavailable')}
|
||||
</p>
|
||||
)}
|
||||
<div className="model-connection-manager">
|
||||
<aside
|
||||
aria-label={t('model.profile.listAriaLabel')}
|
||||
@@ -2963,6 +2983,12 @@ export function SettingsPanel({
|
||||
{defaultModelProfileId === profile.id && (
|
||||
<span>{t('model.profile.defaultBadge')}</span>
|
||||
)}
|
||||
{profile.protocol === 'openai-images-generations' && profile.allowConversationInvocation && (
|
||||
<span>{t('conversationImages.callable')}</span>
|
||||
)}
|
||||
{defaultImageModelProfileId === profile.id && (
|
||||
<span>{t('conversationImages.defaultBadge')}</span>
|
||||
)}
|
||||
{profile.protocol ===
|
||||
'openai-images-generations' && (
|
||||
<span>{t('model.profile.imageBadge')}</span>
|
||||
@@ -3247,6 +3273,36 @@ export function SettingsPanel({
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{profile.protocol === 'openai-images-generations' && (
|
||||
<div className="field">
|
||||
<label className="toggle-row">
|
||||
<input
|
||||
checked={profile.allowConversationInvocation ?? false}
|
||||
onChange={(event) => updateModelProfile(profile.id, {
|
||||
allowConversationInvocation: event.target.checked
|
||||
})}
|
||||
aria-describedby={`conversation-images-help-${profile.id}`}
|
||||
role="switch"
|
||||
type="checkbox"
|
||||
/>
|
||||
<span>{t('conversationImages.allow')}</span>
|
||||
</label>
|
||||
<small id={`conversation-images-help-${profile.id}`}>
|
||||
{t('conversationImages.description')}
|
||||
</small>
|
||||
{profile.allowConversationInvocation && (
|
||||
<label className="check-field">
|
||||
<input
|
||||
checked={defaultImageModelProfileId === profile.id}
|
||||
name="default-image-model-profile"
|
||||
onChange={() => setDefaultImageModelProfileId(profile.id)}
|
||||
type="radio"
|
||||
/>
|
||||
<span>{t('conversationImages.setDefault')}</span>
|
||||
</label>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{isAgentRuntimeModelProtocol(profile.protocol) && (
|
||||
<>
|
||||
<div className="field">
|
||||
@@ -4081,6 +4137,17 @@ export function SettingsPanel({
|
||||
)}
|
||||
{activeTab === 'capabilities' && (
|
||||
<CapabilitiesAndToolsSettingsSection
|
||||
onOpenImageModelSettings={() => {
|
||||
if (requestTabChange('model')) {
|
||||
setModelType('llm')
|
||||
const imageProfile = modelProfiles.find(
|
||||
(profile) => profile.protocol === 'openai-images-generations'
|
||||
)
|
||||
if (imageProfile) {
|
||||
setSelectedModelProfileId(imageProfile.id)
|
||||
}
|
||||
}
|
||||
}}
|
||||
magicNotesEnabled={magicNotesEnabled}
|
||||
onNotify={onNotify}
|
||||
/>
|
||||
|
||||
@@ -92,6 +92,18 @@ describe('WorkspacePrimitives', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('wraps long model names, prompts, and errors inside shared message status cards', () => {
|
||||
expect(stylesheet).toMatch(
|
||||
/\.message-retrieval-status\s*\{[^}]*overflow-wrap:\s*anywhere;/u
|
||||
)
|
||||
expect(stylesheet).toMatch(
|
||||
/\.message-retrieval-status > div\s*\{[^}]*min-width:\s*0;/u
|
||||
)
|
||||
expect(stylesheet).toMatch(
|
||||
/\.message-image-actions\s*\{[^}]*flex-wrap:\s*wrap;/u
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps shared switch rows aligned without standalone dividers', () => {
|
||||
const toggleRowStyles = stylesheet.match(
|
||||
/\.toggle-row\s*\{(?<rules>[^}]*)\}/u
|
||||
|
||||
@@ -267,6 +267,26 @@ export const app = {
|
||||
},
|
||||
exportSpeaker: '{{speaker}}:\n{{content}}',
|
||||
images: {
|
||||
operation: 'Image generation',
|
||||
edit: 'Continue editing',
|
||||
cancel: 'Cancel generation',
|
||||
regenerate: 'Generate again',
|
||||
regenerateNotice: 'Starts a new model request.',
|
||||
reselectModel: 'Choose another model',
|
||||
reselectSources: 'Choose source images',
|
||||
recoveryNotice: 'Generate again keeps the original model and sources. To change the model, name it explicitly in a new composer request. To change sources, upload attachments or choose Continue editing on a previous image.',
|
||||
recoveryPrompt: 'Use image model "{{model}}": {{prompt}}',
|
||||
noToolsNotice: 'This chat model cannot call image tools automatically. You can use the existing direct image workflow.',
|
||||
directWorkflowNotice: 'Choose an image model in the Runtime selector and send your request. If none is configured, add an image connection in model settings.',
|
||||
askNotice: 'Generating or editing images requires Execute mode. Change it using the work mode selector.',
|
||||
openModelSettings: 'Open image model settings',
|
||||
cancelNotice: 'Waiting was stopped. The provider may still generate and charge for the image; a late result will be saved here.',
|
||||
cancellingNotice: 'Cancellation was requested. The provider may still generate and charge for the image.',
|
||||
cancelTooLateNotice: 'Cancellation did not take effect. The generated image has been saved.',
|
||||
unconfirmedNotice: 'The result could not be confirmed. Check before starting another request.',
|
||||
details: 'Request details',
|
||||
sources: '{{count}} source images',
|
||||
states: { running: 'Generating image', saving: 'Saving image', completed: 'Image saved', cancelling: 'Cancelling generation', stopped: 'Stopped waiting', failed: 'Image generation failed', unconfirmed: 'Result unconfirmed' },
|
||||
contextNotice: {
|
||||
'reference-unavailable': 'Some previous images could not be loaded. Only available images and text context were used.',
|
||||
'editing-unavailable': 'The provider does not support image editing. This image was generated from text without the reference images.'
|
||||
|
||||
@@ -2,6 +2,13 @@ import type { TranslationShape } from '../../resource-types'
|
||||
import type { integrations as chineseIntegrations } from '../zh-CN/integrations'
|
||||
|
||||
export const integrations = {
|
||||
conversationImages: {
|
||||
title: 'Image generation and editing',
|
||||
managed: 'Builtin · Automatically managed',
|
||||
description: 'Automatically managed by the image models’ “Allow AI to invoke in conversations” setting.',
|
||||
runtimeScope: 'Assignments include applicable remote Runtimes through GoodBuddy Agent. Generation requires Execute and a chat model with tool calling; assignments do not indicate runtime readiness.',
|
||||
openSettings: 'Go to image model settings'
|
||||
},
|
||||
channels: {
|
||||
tabs: {
|
||||
weixin: 'WeChat ClawBot',
|
||||
|
||||
@@ -2,6 +2,14 @@ import type { TranslationShape } from '../../resource-types'
|
||||
import type { settings as chineseSettings } from '../zh-CN/settings'
|
||||
|
||||
export const settings = {
|
||||
conversationImages: {
|
||||
allow: 'Allow AI to invoke in conversations',
|
||||
description: 'When enabled, assistants with tool calling can use this model to generate and edit images. Editing support depends on the image protocol. Results are saved in the originating conversation.',
|
||||
callable: 'Conversation enabled',
|
||||
defaultBadge: 'Default image model',
|
||||
setDefault: 'Set as default image model',
|
||||
unavailable: 'The default image model is unavailable. Enable it or explicitly choose another image model.'
|
||||
},
|
||||
modelDownloadSources: {
|
||||
modelscope: 'ModelScope',
|
||||
'hugging-face': 'Hugging Face'
|
||||
|
||||
@@ -252,6 +252,26 @@ export const app = {
|
||||
},
|
||||
exportSpeaker: '{{speaker}}：\n{{content}}',
|
||||
images: {
|
||||
operation: '图片生成',
|
||||
edit: '继续修改',
|
||||
cancel: '取消生成',
|
||||
regenerate: '重新生成',
|
||||
regenerateNotice: '将发起新的模型请求。',
|
||||
reselectModel: '重新选择模型',
|
||||
reselectSources: '重新选择素材',
|
||||
recoveryNotice: '重新生成仍使用原模型与素材。更换模型后，请在输入区明确指定模型并重新发送；更换素材可上传附件，或在历史图片上点击“继续修改”。',
|
||||
recoveryPrompt: '使用图片模型“{{model}}”：{{prompt}}',
|
||||
noToolsNotice: '当前聊天模型无法自动调用图片工具，可使用已有直连图片工作流。',
|
||||
directWorkflowNotice: '在 Runtime 选择器中选择图片模型后发送请求；尚未配置时，请前往模型设置添加图片连接。',
|
||||
askNotice: '生成或编辑图片需要 Execute 模式，请通过工作模式选择器切换。',
|
||||
openModelSettings: '前往图片模型设置',
|
||||
cancelNotice: '已停止等待。提供商可能继续生成并计费，晚到的结果会保存在这里。',
|
||||
cancellingNotice: '已请求取消。提供商可能继续生成并计费。',
|
||||
cancelTooLateNotice: '取消未生效，已生成并保存图片。',
|
||||
unconfirmedNotice: '暂时无法确认结果，请在重新发起请求前核对。',
|
||||
details: '请求详情',
|
||||
sources: '{{count}} 张来源图片',
|
||||
states: { running: '正在生成图片', saving: '正在保存图片', completed: '图片已保存', cancelling: '正在取消生成', stopped: '已停止等待', failed: '图片生成失败', unconfirmed: '结果未确认' },
|
||||
contextNotice: {
|
||||
'reference-unavailable': '部分历史图片无法读取，本次仅使用可用的图片与文字上下文。',
|
||||
'editing-unavailable': '上游不支持图片编辑，本次按文字要求生成，未使用参考图片。'
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
export const integrations = {
|
||||
conversationImages: {
|
||||
title: '图片生成与编辑',
|
||||
managed: '内置 · 自动管理',
|
||||
description: '由图片模型的「允许 AI 在会话中调用」设置自动管理。',
|
||||
runtimeScope: '分配包含 GoodBuddy Agent 远程路径上的适用 Runtime。生成需要 Execute 模式及支持工具调用的聊天模型；勾选不代表 Runtime 已就绪。',
|
||||
openSettings: '前往图片模型设置'
|
||||
},
|
||||
channels: {
|
||||
tabs: {
|
||||
weixin: '微信 ClawBot',
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
export const settings = {
|
||||
conversationImages: {
|
||||
allow: '允许 AI 在会话中调用',
|
||||
description: '开启后，支持工具调用的聊天助手可使用此模型生成和编辑图片，编辑能力取决于图片协议。结果保存在发起请求的会话中。',
|
||||
callable: '可在会话中调用',
|
||||
defaultBadge: '默认生图模型',
|
||||
setDefault: '设为默认生图模型',
|
||||
unavailable: '默认生图模型不可用，请开启该模型或明确选择其他图片模型。'
|
||||
},
|
||||
modelDownloadSources: {
|
||||
modelscope: 'ModelScope',
|
||||
'hugging-face': 'Hugging Face'
|
||||
|
||||
@@ -1020,6 +1020,7 @@
|
||||
|
||||
.message-retrieval-status {
|
||||
display: flex;
|
||||
overflow-wrap: anywhere;
|
||||
align-items: flex-start;
|
||||
gap: var(--space-2);
|
||||
margin-top: var(--space-3);
|
||||
@@ -5749,6 +5750,7 @@ button > svg {
|
||||
|
||||
.message-image-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
@@ -419,6 +419,8 @@ export const conversationMessageSchema = z
|
||||
.strict()
|
||||
.optional(),
|
||||
artifactIds: z.array(assistantIdSchema).max(8).optional(),
|
||||
imageOperations: z.array(imageOperationSchema).optional(),
|
||||
imageSourceArtifactIds: z.array(assistantIdSchema).max(8).optional(),
|
||||
imageContextNotice: z
|
||||
.enum(['reference-unavailable', 'editing-unavailable'])
|
||||
.optional(),
|
||||
@@ -1032,3 +1034,4 @@ export type AssistantExpert = z.output<typeof expertCreateSchema> & {
|
||||
createdAt: string
|
||||
updatedAt: string
|
||||
}
|
||||
import { imageOperationSchema } from './image-generation-contracts'
|
||||
|
||||
@@ -11,12 +11,12 @@ describe('built-in model tool catalog', () => {
|
||||
expect(new Set(names).size).toBe(names.length)
|
||||
})
|
||||
|
||||
it('places every direct-model group outside the shared browser capability', () => {
|
||||
it('places direct-model groups outside the shared browser and managed image capabilities', () => {
|
||||
const groupedNames = builtinModelToolGroups.flatMap((group) =>
|
||||
group.tools.map((tool) => tool.name)
|
||||
)
|
||||
const directModelTools = builtinModelTools.filter(
|
||||
(tool) => tool.group !== 'browser'
|
||||
(tool) => tool.group !== 'browser' && tool.group !== 'image'
|
||||
)
|
||||
|
||||
expect(groupedNames).toHaveLength(directModelTools.length)
|
||||
@@ -24,6 +24,10 @@ describe('built-in model tool catalog', () => {
|
||||
expect(groupedNames).toEqual(
|
||||
expect.arrayContaining(directModelTools.map((tool) => tool.name))
|
||||
)
|
||||
expect(groupedNames).not.toContain('generate_image')
|
||||
expect(builtinModelTools.find((tool) => tool.name === 'generate_image')).toMatchObject({
|
||||
group: 'image', access: 'write'
|
||||
})
|
||||
|
||||
for (const group of builtinModelToolGroups) {
|
||||
expect(group.tools.every((tool) => tool.group === group.id)).toBe(true)
|
||||
|
||||
@@ -3,10 +3,17 @@ export type BuiltinModelToolSummary = {
|
||||
displayName: string
|
||||
description: string
|
||||
access: 'read' | 'write'
|
||||
group: 'filesystem' | 'browser' | 'web' | 'programming'
|
||||
group: 'filesystem' | 'browser' | 'web' | 'programming' | 'image'
|
||||
}
|
||||
|
||||
export const builtinModelTools = [
|
||||
{
|
||||
name: 'generate_image',
|
||||
displayName: 'Generate or edit image',
|
||||
description: 'Generate or edit an image with an enabled image model and save it in this conversation.',
|
||||
access: 'write',
|
||||
group: 'image'
|
||||
},
|
||||
{
|
||||
name: 'workspace_rg',
|
||||
displayName: '搜索工作区',
|
||||
|
||||
@@ -368,6 +368,16 @@ export type WebSearchCapability = z.infer<
|
||||
export const capabilitySnapshotSchema = z
|
||||
.object({
|
||||
skills: z.array(skillSummarySchema).max(256),
|
||||
imageGeneration: z
|
||||
.object({
|
||||
modelProfiles: z.array(z.object({
|
||||
id: z.string().uuid(),
|
||||
name: z.string()
|
||||
}).strict()).max(20),
|
||||
assignments: capabilityAssignmentsSchema
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
builtinMcpServers: z
|
||||
.array(builtinMcpServerStateSummarySchema)
|
||||
.max(4)
|
||||
|
||||
@@ -385,6 +385,7 @@ export const conversationQueueUserInputSchema = z
|
||||
includeMemoryContext: z.boolean().default(true),
|
||||
prompt: z.string().trim().min(1).max(100_000),
|
||||
attachments: z.array(conversationAttachmentSchema).max(8).default([]),
|
||||
imageContextArtifactIds: z.array(assistantIdSchema).max(8).optional(),
|
||||
knowledgeLibraryIds: z
|
||||
.array(z.string().uuid())
|
||||
.max(20)
|
||||
@@ -625,6 +626,7 @@ const modelProfileInputSchema = z
|
||||
protocol: modelProtocolSchema,
|
||||
authentication: modelAuthenticationSchema,
|
||||
supportsImageInput: z.boolean().optional(),
|
||||
allowConversationInvocation: z.boolean().optional(),
|
||||
contextWindowTokens: z
|
||||
.number()
|
||||
.int()
|
||||
@@ -740,6 +742,7 @@ export const runtimeSettingsInputSchema = z
|
||||
apiKey: modelApiKeyUpdateSchema,
|
||||
modelProfiles: z.array(modelProfileInputSchema).min(1).max(20).optional(),
|
||||
defaultModelProfileId: modelProfileIdSchema.optional(),
|
||||
defaultImageModelProfileId: modelProfileIdSchema.nullable().optional(),
|
||||
opencodeModelSource: runtimeModelSourceSchema.optional(),
|
||||
continueModelSource: runtimeModelSourceSchema.optional(),
|
||||
deepseekHarnessModelSource: runtimeModelSourceSchema
|
||||
@@ -1019,6 +1022,7 @@ export type ModelConnectionSettings = {
|
||||
protocol: ModelProtocol
|
||||
authentication: ModelAuthentication
|
||||
supportsImageInput?: boolean
|
||||
allowConversationInvocation?: boolean
|
||||
contextWindowTokens?: number
|
||||
maximumOutputTokens?: number
|
||||
imageGenerationQuality: ImageGenerationQuality
|
||||
@@ -1111,6 +1115,7 @@ export type RuntimeSettings = {
|
||||
credentialSource: 'none' | 'encrypted' | 'environment' | 'unreadable'
|
||||
modelProfiles: ModelConnectionSettings[]
|
||||
defaultModelProfileId: string
|
||||
defaultImageModelProfileId?: string | null
|
||||
opencodeModelSource: RuntimeModelSource
|
||||
continueModelSource: RuntimeModelSource
|
||||
deepseekHarnessModelSource?: RuntimeModelSource
|
||||
@@ -2047,6 +2052,7 @@ export type DesktopApi = {
|
||||
}
|
||||
}
|
||||
conversations: {
|
||||
imageOperations: import('./image-operation-ipc').ImageOperationsApi
|
||||
list: () => Promise<ConversationSnapshot[]>
|
||||
replace: (conversations: ConversationSnapshot[]) => Promise<void>
|
||||
saveLocal: (batch: LocalConversationSaveBatch) => Promise<void>
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const imageToolInputSchema = z.object({
|
||||
prompt: z.string().trim().min(1).max(32_000),
|
||||
intent: z.enum(['create', 'edit']),
|
||||
modelProfileId: z.string().uuid().optional(),
|
||||
sourceArtifactIds: z.array(z.string().uuid()).max(8).default([]),
|
||||
quality: z.enum(['auto', 'low', 'medium', 'high']).optional()
|
||||
}).strict().superRefine((input, context) => {
|
||||
if ((input.intent === 'edit') !== (input.sourceArtifactIds.length > 0)) {
|
||||
context.addIssue({ code: 'custom', message: 'Editing requires source images; creating must not include source images.' })
|
||||
}
|
||||
})
|
||||
export type ImageToolInput = z.infer<typeof imageToolInputSchema>
|
||||
|
||||
export const imageOperationSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
conversationId: z.string(),
|
||||
messageId: z.string(),
|
||||
requestId: z.string(),
|
||||
callId: z.string(),
|
||||
modelProfileId: z.string().uuid(),
|
||||
modelName: z.string(),
|
||||
modelProfileName: z.string().optional(),
|
||||
input: imageToolInputSchema,
|
||||
state: z.enum(['running', 'saving', 'completed', 'cancelling', 'stopped', 'failed', 'unconfirmed']),
|
||||
createdAt: z.number(),
|
||||
updatedAt: z.number(),
|
||||
artifactIds: z.array(z.string().uuid()).default([]),
|
||||
error: z.string().optional(),
|
||||
cancellationRequested: z.boolean().optional()
|
||||
}).strict()
|
||||
export type ImageOperation = z.infer<typeof imageOperationSchema>
|
||||
|
||||
/** Bound by Main/Agent transport, never accepted as model tool arguments. */
|
||||
export type ImageRequestContext = {
|
||||
conversationId: string
|
||||
messageId: string
|
||||
requestId: string
|
||||
workMode: 'ask' | 'execute'
|
||||
}
|
||||
|
||||
export const imageToolName = 'generate_image' as const
|
||||
export const imageToolDescriptionLimit = 16_000
|
||||
export const imageToolDescription = 'Generate or edit an image using a configured image model. Use intent=create for a new unrelated image, or intent=edit with explicit sourceArtifactIds. Never invent profile or artifact IDs. Clarify ambiguous references. A failed edit must not be replaced by generation. Results contain operation and artifact references, not image bytes. Do not retry a paid request automatically. A running operation continues after this chat stops; inspect its conversation card for completion.'
|
||||
@@ -0,0 +1,14 @@
|
||||
import { z } from 'zod'
|
||||
import type { ImageOperation } from './image-generation-contracts'
|
||||
|
||||
export const imageOperationTargetSchema = z.object({
|
||||
conversationId: z.string().min(1).max(128),
|
||||
operationId: z.string().uuid()
|
||||
}).strict()
|
||||
|
||||
export type ImageOperationTarget = z.infer<typeof imageOperationTargetSchema>
|
||||
export type ImageOperationsApi = {
|
||||
cancel(input: ImageOperationTarget): Promise<ImageOperation>
|
||||
regenerate(input: ImageOperationTarget): Promise<ImageOperation>
|
||||
onChanged(listener: (operation: ImageOperation) => void): () => void
|
||||
}
|
||||
@@ -190,6 +190,9 @@ export const ipcChannels = {
|
||||
conversationsBranchLocal: 'conversations:branch-local',
|
||||
conversationsDeleteLocal: 'conversations:delete-local',
|
||||
conversationsChanged: 'conversations:changed',
|
||||
imageOperationCancel: 'image-operation:cancel',
|
||||
imageOperationRegenerate: 'image-operation:regenerate',
|
||||
imageOperationChanged: 'image-operation:changed',
|
||||
conversationQueueList: 'conversation-queue:list',
|
||||
conversationQueueEnqueueUser: 'conversation-queue:enqueue-user',
|
||||
conversationQueueRemove: 'conversation-queue:remove',
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { z } from 'zod'
|
||||
import { remoteImageToolSchema } from './remote-image-tool-contracts'
|
||||
import { workspaceManagementActionSchema } from './workspace-management-contracts'
|
||||
import {
|
||||
acpFrameDirectionSchema,
|
||||
@@ -164,6 +165,7 @@ const remotePromptOperationIdentityFields = {
|
||||
runtimeAdapterDigest: sha256DigestSchema,
|
||||
modelBridge: remotePromptModelBridgeSchema.optional(),
|
||||
modelProfile: agentPromptModelProfileSchema.optional(),
|
||||
imageTool: remoteImageToolSchema.optional(),
|
||||
promptSequence: z
|
||||
.number()
|
||||
.int()
|
||||
@@ -350,6 +352,7 @@ export type RemoteSemanticTranscriptEvent = z.infer<
|
||||
|
||||
export const remotePromptOperationAcceptanceSchema = z
|
||||
.object({
|
||||
imageToolUrl: z.string().url().optional(),
|
||||
bindingId: agentIdentifierSchema,
|
||||
operationId: agentIdentifierSchema,
|
||||
requestId: agentIdentifierSchema,
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { z } from 'zod'
|
||||
import { agentIdentifierSchema, positiveAgentSequenceSchema } from './agent-protocol/contracts'
|
||||
import { imageOperationSchema, imageToolDescriptionLimit, imageToolInputSchema } from './image-generation-contracts'
|
||||
|
||||
export const remoteImageToolSchema = z.object({
|
||||
channelId: agentIdentifierSchema,
|
||||
channelEpoch: positiveAgentSequenceSchema,
|
||||
description: z.string().min(1).max(imageToolDescriptionLimit)
|
||||
}).strict()
|
||||
|
||||
export const remoteImageToolCallSchema = z.object({
|
||||
callId: z.string().uuid(),
|
||||
input: imageToolInputSchema
|
||||
}).strict()
|
||||
|
||||
export const remoteImageToolReplySchema = z.object({
|
||||
callId: z.string().uuid(),
|
||||
result: imageOperationSchema.optional(),
|
||||
error: z.string().max(4_000).optional()
|
||||
}).strict()
|
||||
|
||||
// References and operation metadata only. Image bytes never cross this channel.
|
||||
export const remoteImageToolMaximumBytes = 256 * 1024
|
||||
|
||||
export function encodeRemoteImageToolMessage(value: unknown): Uint8Array {
|
||||
const bytes = new TextEncoder().encode(JSON.stringify(value))
|
||||
if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large')
|
||||
return bytes
|
||||
}
|
||||
|
||||
export function decodeRemoteImageToolMessage(bytes: Uint8Array): unknown {
|
||||
if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large')
|
||||
return JSON.parse(new TextDecoder().decode(bytes)) as unknown
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
// Used by Main and the Agent daemon, never by Renderer contracts.
|
||||
export function remoteImageToolMcpName(bindingId: string): string {
|
||||
return `goodbuddy_image_${createHash('sha256').update(bindingId).digest('hex').slice(0, 24)}`
|
||||
}
|
||||
Reference in New Issue
Block a user