diff --git a/src/main/agent/model-runtime.test.ts b/src/main/agent/model-runtime.test.ts index ef1f8fef..2eb36d2c 100644 --- a/src/main/agent/model-runtime.test.ts +++ b/src/main/agent/model-runtime.test.ts @@ -5493,6 +5493,75 @@ describe('ModelAgentRuntime', () => { expect(events.at(-1)).toMatchObject({ type: 'done' }) }) + it('edits reference images through the multipart image edits endpoint', async () => { + const source = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + 0x00 + ]) + const fetcher = vi.fn(async () => + Response.json({ data: [{ b64_json: source.toString('base64') }] }) + ) + const runtime = new ModelAgentRuntime({ + apiKey: 'test-key', + baseUrl: 'https://bigtoken.ai/v1', + model: 'gpt-image-2', + protocol: 'openai-images-generations', + authentication: 'api-key', + imageGenerationQuality: 'medium', + requestHeaders: { 'x-image-route': 'internal' }, + requestBody: { background: 'transparent' }, + fetcher + }) + const events = [] + + for await (const event of runtime.run( + { + requestId: crypto.randomUUID(), + conversationId: crypto.randomUUID(), + prompt: '把背景改成白色', + images: [ + { + name: 'source.png', + mediaType: 'image/png', + data: source.toString('base64') + } + ] + }, + new AbortController().signal + )) { + events.push(event) + } + + const [input, init] = fetcher.mock.calls[0] ?? [] + expect(input?.toString()).toBe( + 'https://bigtoken.ai/v1/images/edits' + ) + expect(new Headers(init?.headers).get('authorization')).toBe( + 'Bearer test-key' + ) + expect(new Headers(init?.headers).get('content-type')).toBeNull() + expect(new Headers(init?.headers).get('x-image-route')).toBe( + 'internal' + ) + const form = init?.body as FormData + expect(form.get('model')).toBe('gpt-image-2') + expect(form.get('prompt')).toBe('把背景改成白色') + expect(form.get('n')).toBe('1') + expect(form.get('quality')).toBe('medium') + expect(form.get('response_format')).toBe('b64_json') + expect(form.get('background')).toBe('transparent') + const uploaded = form.get('image') + expect(uploaded).toBeInstanceOf(Blob) + expect((uploaded as Blob).type).toBe('image/png') + expect(Buffer.from(await (uploaded as Blob).arrayBuffer())).toEqual( + source + ) + expect(events).toContainEqual( + expect.objectContaining({ type: 'generated-image' }) + ) + expect(events.at(-1)).toMatchObject({ type: 'done' }) + }) + it('rejects remote image URLs instead of fetching provider output', async () => { const runtime = new ModelAgentRuntime({ apiKey: 'test-key', diff --git a/src/main/agent/model-runtime.ts b/src/main/agent/model-runtime.ts index 0ce4733c..c3be5225 100644 --- a/src/main/agent/model-runtime.ts +++ b/src/main/agent/model-runtime.ts @@ -40,6 +40,7 @@ import { } from './model-tool-provider' import { createOpenAIChatCompletionsUrl, + createOpenAIImagesEditsUrl, createOpenAIImagesGenerationsUrl, createOpenAIResponsesUrl } from './openai-endpoint' @@ -1745,12 +1746,10 @@ export class ModelAgentRuntime implements AgentRuntime { : createOpenAIChatCompletionsUrl(this.options.baseUrl) } - private getHeaders(): Record { + private getHeaders(json = true): Record { const headers = mergeModelRequestHeaders( this.options.requestHeaders ?? {}, - { - 'content-type': 'application/json' - } + json ? { 'content-type': 'application/json' } : {} ) if ( this.options.authentication === 'api-key' && @@ -1789,6 +1788,32 @@ export class ModelAgentRuntime implements AgentRuntime { }) } + private createImageEditRequest( + prompt: string, + images: NonNullable + ): FormData { + const form = new FormData() + const fields = this.createImageGenerationRequest(prompt) + for (const [name, value] of Object.entries(fields)) { + form.append( + name, + typeof value === 'string' ? value : JSON.stringify(value) + ) + } + const imageField = images.length === 1 ? 'image' : 'image[]' + images.forEach((image, index) => { + const extension = image.mediaType === 'image/png' ? 'png' : 'jpg' + form.append( + imageField, + new Blob([Buffer.from(image.data, 'base64')], { + type: image.mediaType + }), + `reference-${index + 1}.${extension}` + ) + }) + return form + } + private async fetchWithTimeout( input: URL, init: RequestInit, @@ -2620,23 +2645,26 @@ export class ModelAgentRuntime implements AgentRuntime { request: AgentExecutionRequest, signal: AbortSignal ): AsyncGenerator { - if (request.images?.length) { - throw new Error('当前图像生成接口暂不支持参考图或图片编辑') - } + const images = request.images ?? [] yield { requestId: request.requestId, type: 'status', - message: `${this.options.model} 正在生成图片` + message: `${this.options.model} 正在${images.length > 0 ? '编辑' : '生成'}图片` } - const imageRequest = this.createImageGenerationRequest( - request.prompt - ) + const body = + images.length > 0 + ? this.createImageEditRequest(request.prompt, images) + : JSON.stringify( + this.createImageGenerationRequest(request.prompt) + ) const modelRequest = await this.fetchWithTimeout( - this.getEndpoint(), + images.length > 0 + ? createOpenAIImagesEditsUrl(this.options.baseUrl) + : this.getEndpoint(), { method: 'POST', - headers: this.getHeaders(), - body: JSON.stringify(imageRequest) + headers: this.getHeaders(images.length === 0), + body }, signal ) diff --git a/src/main/agent/openai-endpoint.test.ts b/src/main/agent/openai-endpoint.test.ts index 47913b18..95904c05 100644 --- a/src/main/agent/openai-endpoint.test.ts +++ b/src/main/agent/openai-endpoint.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { createOpenAIApiBaseUrl, createOpenAIChatCompletionsUrl, + createOpenAIImagesEditsUrl, createOpenAIImagesGenerationsUrl, createOpenAIResponsesUrl } from './openai-endpoint' @@ -26,6 +27,9 @@ describe('OpenAI endpoint normalization', () => { expect(createOpenAIImagesGenerationsUrl(baseUrl).toString()).toBe( 'http://192.168.1.50:8000/openai/v1/images/generations' ) + expect(createOpenAIImagesEditsUrl(baseUrl).toString()).toBe( + 'http://192.168.1.50:8000/openai/v1/images/edits' + ) }) it('preserves a gateway query on base and request URLs', () => { diff --git a/src/main/agent/openai-endpoint.ts b/src/main/agent/openai-endpoint.ts index 0b19d78c..615e1d56 100644 --- a/src/main/agent/openai-endpoint.ts +++ b/src/main/agent/openai-endpoint.ts @@ -31,3 +31,7 @@ export function createOpenAIResponsesUrl(baseUrl: string): URL { export function createOpenAIImagesGenerationsUrl(baseUrl: string): URL { return createOpenAIRequestUrl(baseUrl, '/images/generations') } + +export function createOpenAIImagesEditsUrl(baseUrl: string): URL { + return createOpenAIRequestUrl(baseUrl, '/images/edits') +}