feat: generate and edit conversation images across runtimes

Add model-controlled image invocation and read-only runtime assignments. Route image tools through Main so accepted operations retain their originating conversation when chat runtimes change, with saved references, cancellation, and manual recovery.

Keep Node hashing outside browser-shared contracts to restore production builds. Validation includes the production build, typecheck, lint, focused regressions, real Host transport checks, and two successful real image requests. Full UI-driven runtime switching acceptance remains documented as pending.

Release note: 开启图片模型的会话调用后,聊天助手可调用图片生成与编辑工具,Runtime 分配自动管理。图片成果保存在原会话,可继续修改,并独立显示生成状态和取消入口。
This commit is contained in:
mesalogo
2026-09-15 15:51:52 +08:00
parent e5aa9ed688
commit f8d07419d2
78 changed files with 2948 additions and 330 deletions
+3
View File
@@ -419,6 +419,8 @@ export const conversationMessageSchema = z
.strict()
.optional(),
artifactIds: z.array(assistantIdSchema).max(8).optional(),
imageOperations: z.array(imageOperationSchema).optional(),
imageSourceArtifactIds: z.array(assistantIdSchema).max(8).optional(),
imageContextNotice: z
.enum(['reference-unavailable', 'editing-unavailable'])
.optional(),
@@ -1032,3 +1034,4 @@ export type AssistantExpert = z.output<typeof expertCreateSchema> & {
createdAt: string
updatedAt: string
}
import { imageOperationSchema } from './image-generation-contracts'
+6 -2
View File
@@ -11,12 +11,12 @@ describe('built-in model tool catalog', () => {
expect(new Set(names).size).toBe(names.length)
})
it('places every direct-model group outside the shared browser capability', () => {
it('places direct-model groups outside the shared browser and managed image capabilities', () => {
const groupedNames = builtinModelToolGroups.flatMap((group) =>
group.tools.map((tool) => tool.name)
)
const directModelTools = builtinModelTools.filter(
(tool) => tool.group !== 'browser'
(tool) => tool.group !== 'browser' && tool.group !== 'image'
)
expect(groupedNames).toHaveLength(directModelTools.length)
@@ -24,6 +24,10 @@ describe('built-in model tool catalog', () => {
expect(groupedNames).toEqual(
expect.arrayContaining(directModelTools.map((tool) => tool.name))
)
expect(groupedNames).not.toContain('generate_image')
expect(builtinModelTools.find((tool) => tool.name === 'generate_image')).toMatchObject({
group: 'image', access: 'write'
})
for (const group of builtinModelToolGroups) {
expect(group.tools.every((tool) => tool.group === group.id)).toBe(true)
+8 -1
View File
@@ -3,10 +3,17 @@ export type BuiltinModelToolSummary = {
displayName: string
description: string
access: 'read' | 'write'
group: 'filesystem' | 'browser' | 'web' | 'programming'
group: 'filesystem' | 'browser' | 'web' | 'programming' | 'image'
}
export const builtinModelTools = [
{
name: 'generate_image',
displayName: 'Generate or edit image',
description: 'Generate or edit an image with an enabled image model and save it in this conversation.',
access: 'write',
group: 'image'
},
{
name: 'workspace_rg',
displayName: '搜索工作区',
+10
View File
@@ -368,6 +368,16 @@ export type WebSearchCapability = z.infer<
export const capabilitySnapshotSchema = z
.object({
skills: z.array(skillSummarySchema).max(256),
imageGeneration: z
.object({
modelProfiles: z.array(z.object({
id: z.string().uuid(),
name: z.string()
}).strict()).max(20),
assignments: capabilityAssignmentsSchema
})
.strict()
.optional(),
builtinMcpServers: z
.array(builtinMcpServerStateSummarySchema)
.max(4)
+6
View File
@@ -385,6 +385,7 @@ export const conversationQueueUserInputSchema = z
includeMemoryContext: z.boolean().default(true),
prompt: z.string().trim().min(1).max(100_000),
attachments: z.array(conversationAttachmentSchema).max(8).default([]),
imageContextArtifactIds: z.array(assistantIdSchema).max(8).optional(),
knowledgeLibraryIds: z
.array(z.string().uuid())
.max(20)
@@ -625,6 +626,7 @@ const modelProfileInputSchema = z
protocol: modelProtocolSchema,
authentication: modelAuthenticationSchema,
supportsImageInput: z.boolean().optional(),
allowConversationInvocation: z.boolean().optional(),
contextWindowTokens: z
.number()
.int()
@@ -740,6 +742,7 @@ export const runtimeSettingsInputSchema = z
apiKey: modelApiKeyUpdateSchema,
modelProfiles: z.array(modelProfileInputSchema).min(1).max(20).optional(),
defaultModelProfileId: modelProfileIdSchema.optional(),
defaultImageModelProfileId: modelProfileIdSchema.nullable().optional(),
opencodeModelSource: runtimeModelSourceSchema.optional(),
continueModelSource: runtimeModelSourceSchema.optional(),
deepseekHarnessModelSource: runtimeModelSourceSchema
@@ -1019,6 +1022,7 @@ export type ModelConnectionSettings = {
protocol: ModelProtocol
authentication: ModelAuthentication
supportsImageInput?: boolean
allowConversationInvocation?: boolean
contextWindowTokens?: number
maximumOutputTokens?: number
imageGenerationQuality: ImageGenerationQuality
@@ -1111,6 +1115,7 @@ export type RuntimeSettings = {
credentialSource: 'none' | 'encrypted' | 'environment' | 'unreadable'
modelProfiles: ModelConnectionSettings[]
defaultModelProfileId: string
defaultImageModelProfileId?: string | null
opencodeModelSource: RuntimeModelSource
continueModelSource: RuntimeModelSource
deepseekHarnessModelSource?: RuntimeModelSource
@@ -2047,6 +2052,7 @@ export type DesktopApi = {
}
}
conversations: {
imageOperations: import('./image-operation-ipc').ImageOperationsApi
list: () => Promise<ConversationSnapshot[]>
replace: (conversations: ConversationSnapshot[]) => Promise<void>
saveLocal: (batch: LocalConversationSaveBatch) => Promise<void>
+45
View File
@@ -0,0 +1,45 @@
import { z } from 'zod'
export const imageToolInputSchema = z.object({
prompt: z.string().trim().min(1).max(32_000),
intent: z.enum(['create', 'edit']),
modelProfileId: z.string().uuid().optional(),
sourceArtifactIds: z.array(z.string().uuid()).max(8).default([]),
quality: z.enum(['auto', 'low', 'medium', 'high']).optional()
}).strict().superRefine((input, context) => {
if ((input.intent === 'edit') !== (input.sourceArtifactIds.length > 0)) {
context.addIssue({ code: 'custom', message: 'Editing requires source images; creating must not include source images.' })
}
})
export type ImageToolInput = z.infer<typeof imageToolInputSchema>
export const imageOperationSchema = z.object({
id: z.string().uuid(),
conversationId: z.string(),
messageId: z.string(),
requestId: z.string(),
callId: z.string(),
modelProfileId: z.string().uuid(),
modelName: z.string(),
modelProfileName: z.string().optional(),
input: imageToolInputSchema,
state: z.enum(['running', 'saving', 'completed', 'cancelling', 'stopped', 'failed', 'unconfirmed']),
createdAt: z.number(),
updatedAt: z.number(),
artifactIds: z.array(z.string().uuid()).default([]),
error: z.string().optional(),
cancellationRequested: z.boolean().optional()
}).strict()
export type ImageOperation = z.infer<typeof imageOperationSchema>
/** Bound by Main/Agent transport, never accepted as model tool arguments. */
export type ImageRequestContext = {
conversationId: string
messageId: string
requestId: string
workMode: 'ask' | 'execute'
}
export const imageToolName = 'generate_image' as const
export const imageToolDescriptionLimit = 16_000
export const imageToolDescription = 'Generate or edit an image using a configured image model. Use intent=create for a new unrelated image, or intent=edit with explicit sourceArtifactIds. Never invent profile or artifact IDs. Clarify ambiguous references. A failed edit must not be replaced by generation. Results contain operation and artifact references, not image bytes. Do not retry a paid request automatically. A running operation continues after this chat stops; inspect its conversation card for completion.'
+14
View File
@@ -0,0 +1,14 @@
import { z } from 'zod'
import type { ImageOperation } from './image-generation-contracts'
export const imageOperationTargetSchema = z.object({
conversationId: z.string().min(1).max(128),
operationId: z.string().uuid()
}).strict()
export type ImageOperationTarget = z.infer<typeof imageOperationTargetSchema>
export type ImageOperationsApi = {
cancel(input: ImageOperationTarget): Promise<ImageOperation>
regenerate(input: ImageOperationTarget): Promise<ImageOperation>
onChanged(listener: (operation: ImageOperation) => void): () => void
}
+3
View File
@@ -190,6 +190,9 @@ export const ipcChannels = {
conversationsBranchLocal: 'conversations:branch-local',
conversationsDeleteLocal: 'conversations:delete-local',
conversationsChanged: 'conversations:changed',
imageOperationCancel: 'image-operation:cancel',
imageOperationRegenerate: 'image-operation:regenerate',
imageOperationChanged: 'image-operation:changed',
conversationQueueList: 'conversation-queue:list',
conversationQueueEnqueueUser: 'conversation-queue:enqueue-user',
conversationQueueRemove: 'conversation-queue:remove',
+3
View File
@@ -1,4 +1,5 @@
import { z } from 'zod'
import { remoteImageToolSchema } from './remote-image-tool-contracts'
import { workspaceManagementActionSchema } from './workspace-management-contracts'
import {
acpFrameDirectionSchema,
@@ -164,6 +165,7 @@ const remotePromptOperationIdentityFields = {
runtimeAdapterDigest: sha256DigestSchema,
modelBridge: remotePromptModelBridgeSchema.optional(),
modelProfile: agentPromptModelProfileSchema.optional(),
imageTool: remoteImageToolSchema.optional(),
promptSequence: z
.number()
.int()
@@ -350,6 +352,7 @@ export type RemoteSemanticTranscriptEvent = z.infer<
export const remotePromptOperationAcceptanceSchema = z
.object({
imageToolUrl: z.string().url().optional(),
bindingId: agentIdentifierSchema,
operationId: agentIdentifierSchema,
requestId: agentIdentifierSchema,
+34
View File
@@ -0,0 +1,34 @@
import { z } from 'zod'
import { agentIdentifierSchema, positiveAgentSequenceSchema } from './agent-protocol/contracts'
import { imageOperationSchema, imageToolDescriptionLimit, imageToolInputSchema } from './image-generation-contracts'
export const remoteImageToolSchema = z.object({
channelId: agentIdentifierSchema,
channelEpoch: positiveAgentSequenceSchema,
description: z.string().min(1).max(imageToolDescriptionLimit)
}).strict()
export const remoteImageToolCallSchema = z.object({
callId: z.string().uuid(),
input: imageToolInputSchema
}).strict()
export const remoteImageToolReplySchema = z.object({
callId: z.string().uuid(),
result: imageOperationSchema.optional(),
error: z.string().max(4_000).optional()
}).strict()
// References and operation metadata only. Image bytes never cross this channel.
export const remoteImageToolMaximumBytes = 256 * 1024
export function encodeRemoteImageToolMessage(value: unknown): Uint8Array {
const bytes = new TextEncoder().encode(JSON.stringify(value))
if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large')
return bytes
}
export function decodeRemoteImageToolMessage(bytes: Uint8Array): unknown {
if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large')
return JSON.parse(new TextDecoder().decode(bytes)) as unknown
}
+6
View File
@@ -0,0 +1,6 @@
import { createHash } from 'node:crypto'
// Used by Main and the Agent daemon, never by Renderer contracts.
export function remoteImageToolMcpName(bindingId: string): string {
return `goodbuddy_image_${createHash('sha256').update(bindingId).digest('hex').slice(0, 24)}`
}