fix: support reference image editing

src/main/agent/model-runtime.test.ts

@@ -5493,6 +5493,75 @@ describe('ModelAgentRuntime', () => {
     expect(events.at(-1)).toMatchObject({ type: 'done' })
   })
 
+  it('edits reference images through the multipart image edits endpoint', async () => {
+    const source = Buffer.from([
+      0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+      0x00
+    ])
+    const fetcher = vi.fn<typeof fetch>(async () =>
+      Response.json({ data: [{ b64_json: source.toString('base64') }] })
+    )
+    const runtime = new ModelAgentRuntime({
+      apiKey: 'test-key',
+      baseUrl: 'https://bigtoken.ai/v1',
+      model: 'gpt-image-2',
+      protocol: 'openai-images-generations',
+      authentication: 'api-key',
+      imageGenerationQuality: 'medium',
+      requestHeaders: { 'x-image-route': 'internal' },
+      requestBody: { background: 'transparent' },
+      fetcher
+    })
+    const events = []
+
+    for await (const event of runtime.run(
+      {
+        requestId: crypto.randomUUID(),
+        conversationId: crypto.randomUUID(),
+        prompt: '把背景改成白色',
+        images: [
+          {
+            name: 'source.png',
+            mediaType: 'image/png',
+            data: source.toString('base64')
+          }
+        ]
+      },
+      new AbortController().signal
+    )) {
+      events.push(event)
+    }
+
+    const [input, init] = fetcher.mock.calls[0] ?? []
+    expect(input?.toString()).toBe(
+      'https://bigtoken.ai/v1/images/edits'
+    )
+    expect(new Headers(init?.headers).get('authorization')).toBe(
+      'Bearer test-key'
+    )
+    expect(new Headers(init?.headers).get('content-type')).toBeNull()
+    expect(new Headers(init?.headers).get('x-image-route')).toBe(
+      'internal'
+    )
+    const form = init?.body as FormData
+    expect(form.get('model')).toBe('gpt-image-2')
+    expect(form.get('prompt')).toBe('把背景改成白色')
+    expect(form.get('n')).toBe('1')
+    expect(form.get('quality')).toBe('medium')
+    expect(form.get('response_format')).toBe('b64_json')
+    expect(form.get('background')).toBe('transparent')
+    const uploaded = form.get('image')
+    expect(uploaded).toBeInstanceOf(Blob)
+    expect((uploaded as Blob).type).toBe('image/png')
+    expect(Buffer.from(await (uploaded as Blob).arrayBuffer())).toEqual(
+      source
+    )
+    expect(events).toContainEqual(
+      expect.objectContaining({ type: 'generated-image' })
+    )
+    expect(events.at(-1)).toMatchObject({ type: 'done' })
+  })
+
   it('rejects remote image URLs instead of fetching provider output', async () => {
     const runtime = new ModelAgentRuntime({
       apiKey: 'test-key',

src/main/agent/model-runtime.ts

@@ -40,6 +40,7 @@ import {
 } from './model-tool-provider'
 import {
   createOpenAIChatCompletionsUrl,
+  createOpenAIImagesEditsUrl,
   createOpenAIImagesGenerationsUrl,
   createOpenAIResponsesUrl
 } from './openai-endpoint'
@@ -1745,12 +1746,10 @@ export class ModelAgentRuntime implements AgentRuntime {
       : createOpenAIChatCompletionsUrl(this.options.baseUrl)
   }
 
-  private getHeaders(): Record<string, string> {
+  private getHeaders(json = true): Record<string, string> {
     const headers = mergeModelRequestHeaders(
       this.options.requestHeaders ?? {},
-      {
-        'content-type': 'application/json'
-      }
+      json ? { 'content-type': 'application/json' } : {}
     )
     if (
       this.options.authentication === 'api-key' &&
@@ -1789,6 +1788,32 @@ export class ModelAgentRuntime implements AgentRuntime {
     })
   }
 
+  private createImageEditRequest(
+    prompt: string,
+    images: NonNullable<AgentExecutionRequest['images']>
+  ): FormData {
+    const form = new FormData()
+    const fields = this.createImageGenerationRequest(prompt)
+    for (const [name, value] of Object.entries(fields)) {
+      form.append(
+        name,
+        typeof value === 'string' ? value : JSON.stringify(value)
+      )
+    }
+    const imageField = images.length === 1 ? 'image' : 'image[]'
+    images.forEach((image, index) => {
+      const extension = image.mediaType === 'image/png' ? 'png' : 'jpg'
+      form.append(
+        imageField,
+        new Blob([Buffer.from(image.data, 'base64')], {
+          type: image.mediaType
+        }),
+        `reference-${index + 1}.${extension}`
+      )
+    })
+    return form
+  }
+
   private async fetchWithTimeout(
     input: URL,
     init: RequestInit,
@@ -2620,23 +2645,26 @@ export class ModelAgentRuntime implements AgentRuntime {
     request: AgentExecutionRequest,
     signal: AbortSignal
   ): AsyncGenerator<RuntimeEvent, void, void> {
-    if (request.images?.length) {
-      throw new Error('当前图像生成接口暂不支持参考图或图片编辑')
-    }
+    const images = request.images ?? []
     yield {
       requestId: request.requestId,
       type: 'status',
-      message: `${this.options.model} 正在生成图片`
+      message: `${this.options.model} 正在${images.length > 0 ? '编辑' : '生成'}图片`
     }
-    const imageRequest = this.createImageGenerationRequest(
-      request.prompt
-    )
+    const body =
+      images.length > 0
+        ? this.createImageEditRequest(request.prompt, images)
+        : JSON.stringify(
+            this.createImageGenerationRequest(request.prompt)
+          )
     const modelRequest = await this.fetchWithTimeout(
-      this.getEndpoint(),
+      images.length > 0
+        ? createOpenAIImagesEditsUrl(this.options.baseUrl)
+        : this.getEndpoint(),
       {
         method: 'POST',
-        headers: this.getHeaders(),
-        body: JSON.stringify(imageRequest)
+        headers: this.getHeaders(images.length === 0),
+        body
       },
       signal
     )

src/main/agent/openai-endpoint.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'
 import {
   createOpenAIApiBaseUrl,
   createOpenAIChatCompletionsUrl,
+  createOpenAIImagesEditsUrl,
   createOpenAIImagesGenerationsUrl,
   createOpenAIResponsesUrl
 } from './openai-endpoint'
@@ -26,6 +27,9 @@ describe('OpenAI endpoint normalization', () => {
     expect(createOpenAIImagesGenerationsUrl(baseUrl).toString()).toBe(
       'http://192.168.1.50:8000/openai/v1/images/generations'
     )
+    expect(createOpenAIImagesEditsUrl(baseUrl).toString()).toBe(
+      'http://192.168.1.50:8000/openai/v1/images/edits'
+    )
   })
 
   it('preserves a gateway query on base and request URLs', () => {

src/main/agent/openai-endpoint.ts

@@ -31,3 +31,7 @@ export function createOpenAIResponsesUrl(baseUrl: string): URL {
 export function createOpenAIImagesGenerationsUrl(baseUrl: string): URL {
   return createOpenAIRequestUrl(baseUrl, '/images/generations')
 }
+
+export function createOpenAIImagesEditsUrl(baseUrl: string): URL {
+  return createOpenAIRequestUrl(baseUrl, '/images/edits')
+}