feat: generate and edit conversation images across runtimes
Add model-controlled image invocation and read-only runtime assignments. Route image tools through Main so accepted operations retain their originating conversation when chat runtimes change, with saved references, cancellation, and manual recovery. Keep Node hashing outside browser-shared contracts to restore production builds. Validation includes the production build, typecheck, lint, focused regressions, real Host transport checks, and two successful real image requests. Full UI-driven runtime switching acceptance remains documented as pending. Release note: 开启图片模型的会话调用后,聊天助手可调用图片生成与编辑工具,Runtime 分配自动管理。图片成果保存在原会话,可继续修改,并独立显示生成状态和取消入口。
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
import type { ImageToolBinding } from '../agent/image-tool-binding'
|
||||
import type { RuntimeProtocolBinaryChannel } from './protocol-remote-runtime-channel'
|
||||
import {
|
||||
decodeRemoteImageToolMessage, encodeRemoteImageToolMessage,
|
||||
remoteImageToolCallSchema, remoteImageToolReplySchema
|
||||
} from '../../shared/remote-image-tool-contracts'
|
||||
|
||||
export class MainImageToolSession {
|
||||
private readonly wait = new AbortController()
|
||||
private readonly unsubscribe: () => void
|
||||
private readonly removeAbort: () => void
|
||||
|
||||
constructor(
|
||||
private readonly channel: RuntimeProtocolBinaryChannel,
|
||||
private readonly binding: ImageToolBinding,
|
||||
signal?: AbortSignal
|
||||
) {
|
||||
this.unsubscribe = channel.onClose(() => this.close())
|
||||
const abort = (): void => this.close()
|
||||
signal?.addEventListener('abort', abort, { once: true })
|
||||
this.removeAbort = () => signal?.removeEventListener('abort', abort)
|
||||
if (signal?.aborted) this.close()
|
||||
void this.receive().catch(() => this.close())
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.wait.signal.aborted) return
|
||||
// This aborts binding.call's wait, never the Main-owned image operation.
|
||||
this.wait.abort()
|
||||
this.removeAbort()
|
||||
this.unsubscribe()
|
||||
this.channel.close()
|
||||
}
|
||||
|
||||
private async receive(): Promise<void> {
|
||||
while (!this.wait.signal.aborted) {
|
||||
const frame = await this.channel.receive(this.wait.signal)
|
||||
const call = remoteImageToolCallSchema.parse(decodeRemoteImageToolMessage(frame.payload))
|
||||
await frame.consume()
|
||||
void this.dispatch(call).catch(() => this.close())
|
||||
}
|
||||
}
|
||||
|
||||
private async dispatch(call: ReturnType<typeof remoteImageToolCallSchema.parse>): Promise<void> {
|
||||
let reply: ReturnType<typeof remoteImageToolReplySchema.parse>
|
||||
try {
|
||||
this.wait.signal.throwIfAborted()
|
||||
if (this.binding.context.workMode !== 'execute') throw new Error('Image tools are unavailable in Ask mode')
|
||||
const result = await this.binding.call(call.input, call.callId, this.wait.signal)
|
||||
reply = remoteImageToolReplySchema.parse({ callId: call.callId, result })
|
||||
} catch (error) {
|
||||
reply = { callId: call.callId, error: (error instanceof Error ? error.message : 'Image tool failed').slice(0, 4_000) }
|
||||
}
|
||||
if (!this.wait.signal.aborted) await this.channel.send(encodeRemoteImageToolMessage(reply), this.wait.signal)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { ImageToolBinding } from '../agent/image-tool-binding'
|
||||
import {
|
||||
AGENT_PROTOCOL_LIMITS,
|
||||
ChannelProtocolError,
|
||||
@@ -489,6 +490,46 @@ async function openChannel(fixture = connectionFixture()) {
|
||||
}
|
||||
|
||||
describe('ProtocolRemoteRuntimeChannel', () => {
|
||||
it('refreshes image binding descriptions per prompt, hides Ask, and closes failed preparations', async () => {
|
||||
const fixture = await openChannel()
|
||||
const describe = vi.fn(async () => 'First image catalog')
|
||||
const binding: ImageToolBinding = {
|
||||
context: { conversationId: 'conversation', messageId: 'message', requestId: 'operation-1', workMode: 'execute' },
|
||||
describe, call: vi.fn()
|
||||
}
|
||||
const prepare = (requestId: string, workMode: 'ask' | 'execute' = 'execute') => ({
|
||||
bindingId: 'binding-1', operationId: requestId, requestId, workMode,
|
||||
controllerId: 'controller-1', controllerGeneration: 1, connectionGeneration: fixture.channel.generation,
|
||||
channelEpoch: fixture.channel.channelEpoch, hostId: 'host-1', hostRevision: 1, hostKeyGeneration: 1,
|
||||
workspaceIdentity: 'workspace-1', agentInstallationId: 'agent-1', runtimeId: 'opencode',
|
||||
runtimeBundleDigest, runtimeAdapterDigest: digest('d'), deadlineAt: new Date(Date.now() + 10000).toISOString(),
|
||||
budget: { maximumInputBytes: 1024, maximumOutputBytes: 1024 }, promptSequence: 0
|
||||
})
|
||||
fixture.client.responder = (method, params) => {
|
||||
if (method !== 'runtime/preparePrompt') return defaultResponse(method, params)
|
||||
const input = params as ReturnType<typeof prepare> & { imageTool?: { description: string } }
|
||||
return { bindingId: input.bindingId, operationId: input.operationId, requestId: input.requestId,
|
||||
workMode: input.workMode, deadlineAt: input.deadlineAt, acceptedAt: new Date().toISOString(),
|
||||
...(input.imageTool ? { imageToolUrl: 'http://127.0.0.1:1234/image' } : {}) }
|
||||
}
|
||||
try {
|
||||
const wait = new AbortController()
|
||||
const first = await fixture.channel.preparePrompt(prepare('operation-1'), binding, wait.signal)
|
||||
fixture.channel.setRecoveryBoundary(first.deadlineAt, wait.signal)
|
||||
expect(fixture.client.blobBinaries[0]!.closed).toBe(false)
|
||||
describe.mockResolvedValue('Refreshed image catalog')
|
||||
await fixture.channel.preparePrompt(prepare('operation-2'), binding, wait.signal)
|
||||
expect(fixture.client.blobBinaries[0]!.closed).toBe(true)
|
||||
expect(fixture.client.requests.at(-1)!.params).toMatchObject({ imageTool: { description: 'Refreshed image catalog' } })
|
||||
await fixture.channel.preparePrompt(prepare('operation-3', 'ask'), binding, wait.signal)
|
||||
expect(describe).toHaveBeenCalledTimes(2)
|
||||
expect(fixture.client.blobBinaries[1]!.closed).toBe(true)
|
||||
expect(fixture.client.requests.at(-1)!.params).not.toHaveProperty('imageTool', expect.anything())
|
||||
fixture.client.responder = () => { throw new Error('preparation response lost') }
|
||||
await expect(fixture.channel.preparePrompt(prepare('operation-4'), binding, wait.signal)).rejects.toThrow()
|
||||
expect(fixture.client.blobBinaries[2]!.closed).toBe(true)
|
||||
} finally { await fixture.channel.close() }
|
||||
})
|
||||
it('sends question replies on the authenticated control channel and rejects foreign bindings', async () => {
|
||||
const fixture = await openChannel()
|
||||
const response = { bindingId: 'binding-1', operationId: 'operation-1', questionId: 'question-1', answers: [] }
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { z } from 'zod'
|
||||
import type { ImageToolBinding } from '../agent/image-tool-binding'
|
||||
import { MainImageToolSession } from './main-image-tool-session'
|
||||
import { remoteQuestionResponseSchema, type RemoteQuestionResponse } from '../../shared/remote-question-contracts'
|
||||
import {
|
||||
AGENT_PROTOCOL_LIMITS,
|
||||
@@ -584,38 +586,51 @@ export class ProtocolRemoteRuntimeChannel
|
||||
}
|
||||
|
||||
async preparePrompt(
|
||||
preparation: z.input<typeof remotePromptOperationPreparationSchema>
|
||||
preparation: z.input<typeof remotePromptOperationPreparationSchema>,
|
||||
imageToolBinding?: ImageToolBinding,
|
||||
waitSignal?: AbortSignal
|
||||
): Promise<
|
||||
z.infer<typeof remotePromptOperationAcceptanceSchema>
|
||||
> {
|
||||
const parsed = remotePromptOperationPreparationSchema.parse(
|
||||
preparation
|
||||
)
|
||||
this.#assertPreparationIdentity(parsed)
|
||||
const raw = await this.#request(
|
||||
'runtime/preparePrompt',
|
||||
parsed
|
||||
)
|
||||
const acceptance = parseRemoteResult(
|
||||
remotePromptOperationAcceptanceSchema,
|
||||
raw,
|
||||
'Remote Runtime prompt acceptance is invalid'
|
||||
)
|
||||
this.#imageTool?.close()
|
||||
this.#imageTool = undefined
|
||||
const description = preparation.workMode === 'execute' ? await imageToolBinding?.describe() : undefined
|
||||
waitSignal?.throwIfAborted()
|
||||
let imageTool: z.infer<typeof remotePromptOperationPreparationSchema>['imageTool']
|
||||
if (description && imageToolBinding) {
|
||||
this.#assertCurrent()
|
||||
const binary = this.#state.client.allocateBinaryChannel?.({ kind: 'blob' })
|
||||
if (!binary) throw new Error('Remote image tool transport is unavailable')
|
||||
imageTool = { channelId: binary.channelId, channelEpoch: binary.channelEpoch, description }
|
||||
this.#imageTool = new MainImageToolSession(binary, imageToolBinding, waitSignal)
|
||||
}
|
||||
try {
|
||||
assertRemotePromptAcceptanceMatchesPreparation(
|
||||
parsed,
|
||||
acceptance
|
||||
const parsed = remotePromptOperationPreparationSchema.parse({ ...preparation, imageTool })
|
||||
this.#assertPreparationIdentity(parsed)
|
||||
const raw = await this.#request('runtime/preparePrompt', parsed)
|
||||
const acceptance = parseRemoteResult(
|
||||
remotePromptOperationAcceptanceSchema, raw,
|
||||
'Remote Runtime prompt acceptance is invalid'
|
||||
)
|
||||
} catch {
|
||||
throw new ProtocolRemoteRuntimeChannelError(
|
||||
'Remote Runtime prompt acceptance identity is invalid',
|
||||
'binding-mismatch'
|
||||
)
|
||||
try {
|
||||
assertRemotePromptAcceptanceMatchesPreparation(parsed, acceptance)
|
||||
if (Boolean(imageTool) !== Boolean(acceptance.imageToolUrl)) throw new Error('Image tool registration mismatch')
|
||||
} catch {
|
||||
throw new ProtocolRemoteRuntimeChannelError(
|
||||
'Remote Runtime prompt acceptance identity is invalid', 'binding-mismatch'
|
||||
)
|
||||
}
|
||||
this.#assertCurrent()
|
||||
return acceptance
|
||||
} catch (error) {
|
||||
this.#imageTool?.close()
|
||||
this.#imageTool = undefined
|
||||
throw error
|
||||
}
|
||||
this.#assertCurrent()
|
||||
return acceptance
|
||||
}
|
||||
|
||||
#imageTool?: MainImageToolSession
|
||||
|
||||
async startOwnedPrompt(
|
||||
request: z.input<typeof remoteOwnedPromptStartRequestSchema>
|
||||
): Promise<z.infer<typeof remoteOwnedPromptStartResultSchema>> {
|
||||
@@ -754,7 +769,8 @@ export class ProtocolRemoteRuntimeChannel
|
||||
'protocol'
|
||||
)
|
||||
}
|
||||
this.clearRecoveryBoundary()
|
||||
this.#removeRecoveryAbort?.()
|
||||
this.#removeRecoveryAbort = undefined
|
||||
this.#recoveryDeadlineAt = deadline
|
||||
this.#recoverySignal = signal
|
||||
const abort = (): void => {
|
||||
@@ -774,6 +790,8 @@ export class ProtocolRemoteRuntimeChannel
|
||||
}
|
||||
|
||||
clearRecoveryBoundary(): void {
|
||||
this.#imageTool?.close()
|
||||
this.#imageTool = undefined
|
||||
this.#removeRecoveryAbort?.()
|
||||
this.#removeRecoveryAbort = undefined
|
||||
this.#recoverySignal = undefined
|
||||
@@ -1628,6 +1646,7 @@ export class ProtocolRemoteRuntimeChannel
|
||||
this.#state.capabilityGeneration
|
||||
)
|
||||
this.#closing = true
|
||||
this.#imageTool?.close()
|
||||
this.clearRecoveryBoundary()
|
||||
this.#lifetime.abort(
|
||||
new DOMException('Remote Runtime channel closed', 'AbortError')
|
||||
|
||||
Reference in New Issue
Block a user