feat: generate and edit conversation images across runtimes
Add model-controlled image invocation and read-only runtime assignments. Route image tools through Main so accepted operations retain their originating conversation when chat runtimes change, with saved references, cancellation, and manual recovery. Keep Node hashing outside browser-shared contracts to restore production builds. Validation includes the production build, typecheck, lint, focused regressions, real Host transport checks, and two successful real image requests. Full UI-driven runtime switching acceptance remains documented as pending. Release note: 开启图片模型的会话调用后,聊天助手可调用图片生成与编辑工具,Runtime 分配自动管理。图片成果保存在原会话,可继续修改,并独立显示生成状态和取消入口。
This commit is contained in:
@@ -419,6 +419,8 @@ export const conversationMessageSchema = z
|
||||
.strict()
|
||||
.optional(),
|
||||
artifactIds: z.array(assistantIdSchema).max(8).optional(),
|
||||
imageOperations: z.array(imageOperationSchema).optional(),
|
||||
imageSourceArtifactIds: z.array(assistantIdSchema).max(8).optional(),
|
||||
imageContextNotice: z
|
||||
.enum(['reference-unavailable', 'editing-unavailable'])
|
||||
.optional(),
|
||||
@@ -1032,3 +1034,4 @@ export type AssistantExpert = z.output<typeof expertCreateSchema> & {
|
||||
createdAt: string
|
||||
updatedAt: string
|
||||
}
|
||||
import { imageOperationSchema } from './image-generation-contracts'
|
||||
|
||||
@@ -11,12 +11,12 @@ describe('built-in model tool catalog', () => {
|
||||
expect(new Set(names).size).toBe(names.length)
|
||||
})
|
||||
|
||||
it('places every direct-model group outside the shared browser capability', () => {
|
||||
it('places direct-model groups outside the shared browser and managed image capabilities', () => {
|
||||
const groupedNames = builtinModelToolGroups.flatMap((group) =>
|
||||
group.tools.map((tool) => tool.name)
|
||||
)
|
||||
const directModelTools = builtinModelTools.filter(
|
||||
(tool) => tool.group !== 'browser'
|
||||
(tool) => tool.group !== 'browser' && tool.group !== 'image'
|
||||
)
|
||||
|
||||
expect(groupedNames).toHaveLength(directModelTools.length)
|
||||
@@ -24,6 +24,10 @@ describe('built-in model tool catalog', () => {
|
||||
expect(groupedNames).toEqual(
|
||||
expect.arrayContaining(directModelTools.map((tool) => tool.name))
|
||||
)
|
||||
expect(groupedNames).not.toContain('generate_image')
|
||||
expect(builtinModelTools.find((tool) => tool.name === 'generate_image')).toMatchObject({
|
||||
group: 'image', access: 'write'
|
||||
})
|
||||
|
||||
for (const group of builtinModelToolGroups) {
|
||||
expect(group.tools.every((tool) => tool.group === group.id)).toBe(true)
|
||||
|
||||
@@ -3,10 +3,17 @@ export type BuiltinModelToolSummary = {
|
||||
displayName: string
|
||||
description: string
|
||||
access: 'read' | 'write'
|
||||
group: 'filesystem' | 'browser' | 'web' | 'programming'
|
||||
group: 'filesystem' | 'browser' | 'web' | 'programming' | 'image'
|
||||
}
|
||||
|
||||
export const builtinModelTools = [
|
||||
{
|
||||
name: 'generate_image',
|
||||
displayName: 'Generate or edit image',
|
||||
description: 'Generate or edit an image with an enabled image model and save it in this conversation.',
|
||||
access: 'write',
|
||||
group: 'image'
|
||||
},
|
||||
{
|
||||
name: 'workspace_rg',
|
||||
displayName: '搜索工作区',
|
||||
|
||||
@@ -368,6 +368,16 @@ export type WebSearchCapability = z.infer<
|
||||
export const capabilitySnapshotSchema = z
|
||||
.object({
|
||||
skills: z.array(skillSummarySchema).max(256),
|
||||
imageGeneration: z
|
||||
.object({
|
||||
modelProfiles: z.array(z.object({
|
||||
id: z.string().uuid(),
|
||||
name: z.string()
|
||||
}).strict()).max(20),
|
||||
assignments: capabilityAssignmentsSchema
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
builtinMcpServers: z
|
||||
.array(builtinMcpServerStateSummarySchema)
|
||||
.max(4)
|
||||
|
||||
@@ -385,6 +385,7 @@ export const conversationQueueUserInputSchema = z
|
||||
includeMemoryContext: z.boolean().default(true),
|
||||
prompt: z.string().trim().min(1).max(100_000),
|
||||
attachments: z.array(conversationAttachmentSchema).max(8).default([]),
|
||||
imageContextArtifactIds: z.array(assistantIdSchema).max(8).optional(),
|
||||
knowledgeLibraryIds: z
|
||||
.array(z.string().uuid())
|
||||
.max(20)
|
||||
@@ -625,6 +626,7 @@ const modelProfileInputSchema = z
|
||||
protocol: modelProtocolSchema,
|
||||
authentication: modelAuthenticationSchema,
|
||||
supportsImageInput: z.boolean().optional(),
|
||||
allowConversationInvocation: z.boolean().optional(),
|
||||
contextWindowTokens: z
|
||||
.number()
|
||||
.int()
|
||||
@@ -740,6 +742,7 @@ export const runtimeSettingsInputSchema = z
|
||||
apiKey: modelApiKeyUpdateSchema,
|
||||
modelProfiles: z.array(modelProfileInputSchema).min(1).max(20).optional(),
|
||||
defaultModelProfileId: modelProfileIdSchema.optional(),
|
||||
defaultImageModelProfileId: modelProfileIdSchema.nullable().optional(),
|
||||
opencodeModelSource: runtimeModelSourceSchema.optional(),
|
||||
continueModelSource: runtimeModelSourceSchema.optional(),
|
||||
deepseekHarnessModelSource: runtimeModelSourceSchema
|
||||
@@ -1019,6 +1022,7 @@ export type ModelConnectionSettings = {
|
||||
protocol: ModelProtocol
|
||||
authentication: ModelAuthentication
|
||||
supportsImageInput?: boolean
|
||||
allowConversationInvocation?: boolean
|
||||
contextWindowTokens?: number
|
||||
maximumOutputTokens?: number
|
||||
imageGenerationQuality: ImageGenerationQuality
|
||||
@@ -1111,6 +1115,7 @@ export type RuntimeSettings = {
|
||||
credentialSource: 'none' | 'encrypted' | 'environment' | 'unreadable'
|
||||
modelProfiles: ModelConnectionSettings[]
|
||||
defaultModelProfileId: string
|
||||
defaultImageModelProfileId?: string | null
|
||||
opencodeModelSource: RuntimeModelSource
|
||||
continueModelSource: RuntimeModelSource
|
||||
deepseekHarnessModelSource?: RuntimeModelSource
|
||||
@@ -2047,6 +2052,7 @@ export type DesktopApi = {
|
||||
}
|
||||
}
|
||||
conversations: {
|
||||
imageOperations: import('./image-operation-ipc').ImageOperationsApi
|
||||
list: () => Promise<ConversationSnapshot[]>
|
||||
replace: (conversations: ConversationSnapshot[]) => Promise<void>
|
||||
saveLocal: (batch: LocalConversationSaveBatch) => Promise<void>
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const imageToolInputSchema = z.object({
|
||||
prompt: z.string().trim().min(1).max(32_000),
|
||||
intent: z.enum(['create', 'edit']),
|
||||
modelProfileId: z.string().uuid().optional(),
|
||||
sourceArtifactIds: z.array(z.string().uuid()).max(8).default([]),
|
||||
quality: z.enum(['auto', 'low', 'medium', 'high']).optional()
|
||||
}).strict().superRefine((input, context) => {
|
||||
if ((input.intent === 'edit') !== (input.sourceArtifactIds.length > 0)) {
|
||||
context.addIssue({ code: 'custom', message: 'Editing requires source images; creating must not include source images.' })
|
||||
}
|
||||
})
|
||||
export type ImageToolInput = z.infer<typeof imageToolInputSchema>
|
||||
|
||||
export const imageOperationSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
conversationId: z.string(),
|
||||
messageId: z.string(),
|
||||
requestId: z.string(),
|
||||
callId: z.string(),
|
||||
modelProfileId: z.string().uuid(),
|
||||
modelName: z.string(),
|
||||
modelProfileName: z.string().optional(),
|
||||
input: imageToolInputSchema,
|
||||
state: z.enum(['running', 'saving', 'completed', 'cancelling', 'stopped', 'failed', 'unconfirmed']),
|
||||
createdAt: z.number(),
|
||||
updatedAt: z.number(),
|
||||
artifactIds: z.array(z.string().uuid()).default([]),
|
||||
error: z.string().optional(),
|
||||
cancellationRequested: z.boolean().optional()
|
||||
}).strict()
|
||||
export type ImageOperation = z.infer<typeof imageOperationSchema>
|
||||
|
||||
/** Bound by Main/Agent transport, never accepted as model tool arguments. */
|
||||
export type ImageRequestContext = {
|
||||
conversationId: string
|
||||
messageId: string
|
||||
requestId: string
|
||||
workMode: 'ask' | 'execute'
|
||||
}
|
||||
|
||||
export const imageToolName = 'generate_image' as const
|
||||
export const imageToolDescriptionLimit = 16_000
|
||||
export const imageToolDescription = 'Generate or edit an image using a configured image model. Use intent=create for a new unrelated image, or intent=edit with explicit sourceArtifactIds. Never invent profile or artifact IDs. Clarify ambiguous references. A failed edit must not be replaced by generation. Results contain operation and artifact references, not image bytes. Do not retry a paid request automatically. A running operation continues after this chat stops; inspect its conversation card for completion.'
|
||||
@@ -0,0 +1,14 @@
|
||||
import { z } from 'zod'
|
||||
import type { ImageOperation } from './image-generation-contracts'
|
||||
|
||||
export const imageOperationTargetSchema = z.object({
|
||||
conversationId: z.string().min(1).max(128),
|
||||
operationId: z.string().uuid()
|
||||
}).strict()
|
||||
|
||||
export type ImageOperationTarget = z.infer<typeof imageOperationTargetSchema>
|
||||
export type ImageOperationsApi = {
|
||||
cancel(input: ImageOperationTarget): Promise<ImageOperation>
|
||||
regenerate(input: ImageOperationTarget): Promise<ImageOperation>
|
||||
onChanged(listener: (operation: ImageOperation) => void): () => void
|
||||
}
|
||||
@@ -190,6 +190,9 @@ export const ipcChannels = {
|
||||
conversationsBranchLocal: 'conversations:branch-local',
|
||||
conversationsDeleteLocal: 'conversations:delete-local',
|
||||
conversationsChanged: 'conversations:changed',
|
||||
imageOperationCancel: 'image-operation:cancel',
|
||||
imageOperationRegenerate: 'image-operation:regenerate',
|
||||
imageOperationChanged: 'image-operation:changed',
|
||||
conversationQueueList: 'conversation-queue:list',
|
||||
conversationQueueEnqueueUser: 'conversation-queue:enqueue-user',
|
||||
conversationQueueRemove: 'conversation-queue:remove',
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { z } from 'zod'
|
||||
import { remoteImageToolSchema } from './remote-image-tool-contracts'
|
||||
import { workspaceManagementActionSchema } from './workspace-management-contracts'
|
||||
import {
|
||||
acpFrameDirectionSchema,
|
||||
@@ -164,6 +165,7 @@ const remotePromptOperationIdentityFields = {
|
||||
runtimeAdapterDigest: sha256DigestSchema,
|
||||
modelBridge: remotePromptModelBridgeSchema.optional(),
|
||||
modelProfile: agentPromptModelProfileSchema.optional(),
|
||||
imageTool: remoteImageToolSchema.optional(),
|
||||
promptSequence: z
|
||||
.number()
|
||||
.int()
|
||||
@@ -350,6 +352,7 @@ export type RemoteSemanticTranscriptEvent = z.infer<
|
||||
|
||||
export const remotePromptOperationAcceptanceSchema = z
|
||||
.object({
|
||||
imageToolUrl: z.string().url().optional(),
|
||||
bindingId: agentIdentifierSchema,
|
||||
operationId: agentIdentifierSchema,
|
||||
requestId: agentIdentifierSchema,
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { z } from 'zod'
|
||||
import { agentIdentifierSchema, positiveAgentSequenceSchema } from './agent-protocol/contracts'
|
||||
import { imageOperationSchema, imageToolDescriptionLimit, imageToolInputSchema } from './image-generation-contracts'
|
||||
|
||||
export const remoteImageToolSchema = z.object({
|
||||
channelId: agentIdentifierSchema,
|
||||
channelEpoch: positiveAgentSequenceSchema,
|
||||
description: z.string().min(1).max(imageToolDescriptionLimit)
|
||||
}).strict()
|
||||
|
||||
export const remoteImageToolCallSchema = z.object({
|
||||
callId: z.string().uuid(),
|
||||
input: imageToolInputSchema
|
||||
}).strict()
|
||||
|
||||
export const remoteImageToolReplySchema = z.object({
|
||||
callId: z.string().uuid(),
|
||||
result: imageOperationSchema.optional(),
|
||||
error: z.string().max(4_000).optional()
|
||||
}).strict()
|
||||
|
||||
// References and operation metadata only. Image bytes never cross this channel.
|
||||
export const remoteImageToolMaximumBytes = 256 * 1024
|
||||
|
||||
export function encodeRemoteImageToolMessage(value: unknown): Uint8Array {
|
||||
const bytes = new TextEncoder().encode(JSON.stringify(value))
|
||||
if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large')
|
||||
return bytes
|
||||
}
|
||||
|
||||
export function decodeRemoteImageToolMessage(bytes: Uint8Array): unknown {
|
||||
if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large')
|
||||
return JSON.parse(new TextDecoder().decode(bytes)) as unknown
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
// Used by Main and the Agent daemon, never by Renderer contracts.
|
||||
export function remoteImageToolMcpName(bindingId: string): string {
|
||||
return `goodbuddy_image_${createHash('sha256').update(bindingId).digest('hex').slice(0, 24)}`
|
||||
}
|
||||
Reference in New Issue
Block a user