feat: add model-aware DSH image input
DeepSeek Harness previously treated every selected model as text-only. It now carries the model profile's image capability through Main, the Utility Host, ACP, and Pi-AI, rejects unsupported images before model invocation, and validates supported JPEG/PNG data in a bounded process-local attachment store. The Composer keeps universal controls on its first row, places OpenCode and Continue controls on a dedicated responsive row, and anchors context compaction without shifting the footer. Runtime supervision ownership for future Subagents and Jobs is documented for the right sidebar. Release note: DeepSeek Harness 现可按所选模型连接的能力安全接收 JPEG/PNG 图片;Composer 同时将 Runtime 专属选择器移到独立一行,并固定上下文压缩入口。
This commit is contained in:
@@ -103,7 +103,7 @@ describe('createAgentRuntime model compatibility', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('creates DeepSeek Harness with a compatible HTTPS gateway profile', async () => {
|
||||
it('forwards a compatible gateway profile to DeepSeek Harness', async () => {
|
||||
const profile = {
|
||||
id: '00000000-0000-4000-8000-000000000006',
|
||||
name: 'OpenAI-compatible gateway',
|
||||
@@ -111,9 +111,13 @@ describe('createAgentRuntime model compatibility', () => {
|
||||
modelName: 'qwen-plus',
|
||||
protocol: 'openai-chat-completions' as const,
|
||||
authentication: 'api-key' as const,
|
||||
supportsImageInput: true,
|
||||
imageGenerationQuality: 'auto' as const,
|
||||
apiKey: 'gateway-key'
|
||||
}
|
||||
const deepseekHarnessLauncher = vi
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('stop after launch options'))
|
||||
const runtime = createAgentRuntime(
|
||||
process.cwd(),
|
||||
settings({
|
||||
@@ -122,10 +126,16 @@ describe('createAgentRuntime model compatibility', () => {
|
||||
defaultModelProfileId: profile.id,
|
||||
deepseekHarnessModelProfile: profile
|
||||
}),
|
||||
{ deepseekHarnessLauncher: vi.fn() }
|
||||
{ deepseekHarnessLauncher }
|
||||
)
|
||||
|
||||
expect(runtime.runtimeId).toBe('deepseek-harness')
|
||||
await expect(runtime.getStatus()).resolves.toMatchObject({
|
||||
available: false
|
||||
})
|
||||
expect(deepseekHarnessLauncher).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ supportsImageInput: true })
|
||||
)
|
||||
await runtime.dispose()
|
||||
})
|
||||
|
||||
|
||||
@@ -171,6 +171,7 @@ export function createAgentRuntime(
|
||||
defaultWorkspace: workspace,
|
||||
baseUrl: profile.baseUrl,
|
||||
model: profile.modelName,
|
||||
supportsImageInput: profile.supportsImageInput === true,
|
||||
launch: capabilities.deepseekHarnessLauncher,
|
||||
credentialRefs: {
|
||||
GOODBUDDY_HARNESS_MODEL_API_KEY: profile.apiKey
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { createCanvas } from '@napi-rs/canvas'
|
||||
import {
|
||||
CallId,
|
||||
type GenerateOptions,
|
||||
@@ -31,8 +32,8 @@ import {
|
||||
} from './deepseek-harness-runtime'
|
||||
import { GOODBUDDY_HARNESS_MAX_STEP_TOKENS } from './goodbuddy-harness-control-plane'
|
||||
import { DshNpmExtensionInstaller } from './dsh-extension-marketplace'
|
||||
import { DEEPSEEK_HARNESS_MAX_FRAME_BYTES } from './deepseek-harness-control-protocol'
|
||||
|
||||
const MAX_FRAME_BYTES = 1024 * 1024
|
||||
const CREDENTIAL_REF = 'GOODBUDDY_HARNESS_MODEL_API_KEY'
|
||||
const SKILL_CALL_ID = 'e2e-skill-call'
|
||||
const MCP_CALL_ID = 'e2e-mcp-call'
|
||||
@@ -327,6 +328,7 @@ function createInProcessLaunch(
|
||||
api: 'openai-completions',
|
||||
provider: 'goodbuddy',
|
||||
model: options.model,
|
||||
supportsImageInput: options.supportsImageInput,
|
||||
harnessVersion: '0.1.0-rc.6',
|
||||
credentialRefs: options.credentialRefs,
|
||||
skillPackages: options.skillPackages,
|
||||
@@ -334,7 +336,7 @@ function createInProcessLaunch(
|
||||
stream: createBoundedNdJsonStream(
|
||||
hostToClient.writable,
|
||||
clientToHost.readable,
|
||||
MAX_FRAME_BYTES
|
||||
DEEPSEEK_HARNESS_MAX_FRAME_BYTES
|
||||
)
|
||||
})
|
||||
hosts.push(host)
|
||||
@@ -378,6 +380,88 @@ function createInProcessLaunch(
|
||||
}
|
||||
|
||||
describe('DeepSeek Harness real ACP control-plane E2E', () => {
|
||||
it('delivers bounded inline images to an image-capable Harness model', async () => {
|
||||
const root = await realpath(
|
||||
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-acp-image-'))
|
||||
)
|
||||
const workspace = join(root, 'workspace')
|
||||
const dshHome = join(root, 'dsh-home')
|
||||
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
||||
let observedRequest: GenerateOptions | undefined
|
||||
const inProcess = createInProcessLaunch(dshHome, {
|
||||
stream(options) {
|
||||
observedRequest = options
|
||||
return textResponse('Image received.')
|
||||
}
|
||||
})
|
||||
const runtime = new DeepSeekHarnessRuntime({
|
||||
defaultWorkspace: workspace,
|
||||
baseUrl: 'https://api.deepseek.com',
|
||||
model: 'vision-test',
|
||||
supportsImageInput: true,
|
||||
launch: (options) => inProcess.launch(options),
|
||||
credentialRefs: {
|
||||
[CREDENTIAL_REF]: 'unused-in-memory-model-credential'
|
||||
},
|
||||
initializationTimeoutMs: 20_000,
|
||||
promptTimeoutMs: 20_000,
|
||||
shutdownTimeoutMs: 5_000
|
||||
})
|
||||
const png = createCanvas(1, 1).toBuffer('image/png')
|
||||
|
||||
try {
|
||||
const events = await collect(
|
||||
runtime.run(
|
||||
{
|
||||
requestId: 'request-acp-image',
|
||||
conversationId: 'acp-image',
|
||||
prompt: 'Describe this image.',
|
||||
workMode: 'ask',
|
||||
images: [
|
||||
{
|
||||
name: 'reference.png',
|
||||
mediaType: 'image/png',
|
||||
data: png.toString('base64')
|
||||
}
|
||||
]
|
||||
},
|
||||
new AbortController().signal
|
||||
)
|
||||
)
|
||||
const image = observedRequest?.messages
|
||||
.flatMap((message) => message.content)
|
||||
.find(
|
||||
(
|
||||
block
|
||||
): block is Extract<
|
||||
GenerateOptions['messages'][number]['content'][number],
|
||||
{ type: 'image' }
|
||||
> => block.type === 'image'
|
||||
)
|
||||
|
||||
expect(events).toContainEqual(
|
||||
expect.objectContaining({ type: 'done' })
|
||||
)
|
||||
expect(image?.attachment).toMatchObject({
|
||||
mediaType: 'image/png',
|
||||
bytes: png.byteLength,
|
||||
width: 1,
|
||||
height: 1
|
||||
})
|
||||
const stored =
|
||||
await inProcess.hosts[0]!.context.attachments.readImage(
|
||||
image!.attachment
|
||||
)
|
||||
expect(Buffer.from(stored.data).equals(png)).toBe(true)
|
||||
} finally {
|
||||
await runtime.dispose()
|
||||
await Promise.allSettled(
|
||||
inProcess.hosts.map((host) => host.dispose())
|
||||
)
|
||||
await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it(
|
||||
'coalesces micro reasoning deltas without losing content and caps each model step',
|
||||
async () => {
|
||||
|
||||
@@ -8,10 +8,12 @@ import {
|
||||
|
||||
export const DEEPSEEK_HARNESS_CONTROL_PROTOCOL =
|
||||
'goodbuddy.deepseek-harness.control'
|
||||
export const DEEPSEEK_HARNESS_CONTROL_VERSION = 1
|
||||
export const DEEPSEEK_HARNESS_CONTROL_VERSION = 2
|
||||
export const DEEPSEEK_HARNESS_HOST_VERSION = '0.1.0-rc.6'
|
||||
export const DEEPSEEK_HARNESS_CREDENTIAL_REF =
|
||||
'GOODBUDDY_HARNESS_MODEL_API_KEY'
|
||||
export const DEEPSEEK_HARNESS_MAX_FRAME_BYTES =
|
||||
8 * 1024 * 1024
|
||||
|
||||
const skillPackageSchema = z
|
||||
.object({
|
||||
@@ -43,13 +45,14 @@ export const controlledHarnessHostConfigSchema = z
|
||||
api: z.literal('openai-completions'),
|
||||
provider: z.literal('goodbuddy'),
|
||||
model: z.string().min(1).max(128),
|
||||
supportsImageInput: z.boolean(),
|
||||
harnessVersion: z.literal(DEEPSEEK_HARNESS_HOST_VERSION),
|
||||
credentialRefs: z
|
||||
.tuple([z.literal(DEEPSEEK_HARNESS_CREDENTIAL_REF)])
|
||||
.readonly(),
|
||||
skillPackages: z.array(skillPackageSchema).max(64),
|
||||
extensionPackages: z.array(extensionPackageSchema).max(64),
|
||||
maxFrameBytes: z.literal(1024 * 1024)
|
||||
maxFrameBytes: z.literal(DEEPSEEK_HARNESS_MAX_FRAME_BYTES)
|
||||
})
|
||||
.strict()
|
||||
|
||||
|
||||
@@ -45,6 +45,8 @@ function setup(
|
||||
promptTimeoutMs?: number
|
||||
maxEventCharacters?: number
|
||||
maxRequestOutputCharacters?: number
|
||||
supportsImageInput?: boolean
|
||||
advertisedImageInput?: boolean
|
||||
} = {}
|
||||
) {
|
||||
const exit = deferred<{
|
||||
@@ -95,7 +97,13 @@ function setup(
|
||||
if (method === 'initialize') {
|
||||
return {
|
||||
protocolVersion: 1,
|
||||
agentCapabilities: {}
|
||||
agentCapabilities: {
|
||||
promptCapabilities: {
|
||||
image:
|
||||
options.advertisedImageInput ??
|
||||
(options.supportsImageInput === true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (method === 'session/new') {
|
||||
@@ -212,6 +220,7 @@ function setup(
|
||||
defaultWorkspace: 'C:\\workspace',
|
||||
baseUrl: 'https://api.deepseek.com',
|
||||
model: 'deepseek-test',
|
||||
supportsImageInput: options.supportsImageInput,
|
||||
launch,
|
||||
loadAcpSdk: async () => sdk,
|
||||
initializationTimeoutMs: 100,
|
||||
@@ -417,6 +426,86 @@ describe('DeepSeekHarnessRuntime', () => {
|
||||
).toBeInstanceOf(RequestError)
|
||||
})
|
||||
|
||||
it('rejects images before launch when the selected model is text-only', async () => {
|
||||
const harness = setup()
|
||||
|
||||
await expect(
|
||||
collect(
|
||||
harness.runtime.run(
|
||||
{
|
||||
...request('text-only-image'),
|
||||
images: [
|
||||
{
|
||||
name: 'reference.png',
|
||||
mediaType: 'image/png',
|
||||
data: 'aW1hZ2U='
|
||||
}
|
||||
]
|
||||
},
|
||||
new AbortController().signal
|
||||
)
|
||||
)
|
||||
).rejects.toThrow('未启用图像输入')
|
||||
expect(harness.launch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('forwards inline images when the selected model supports them', async () => {
|
||||
const harness = setup({ supportsImageInput: true })
|
||||
const running = collect(
|
||||
harness.runtime.run(
|
||||
{
|
||||
...request('vision'),
|
||||
images: [
|
||||
{
|
||||
name: 'reference.png',
|
||||
mediaType: 'image/png',
|
||||
data: 'aW1hZ2U='
|
||||
}
|
||||
]
|
||||
},
|
||||
new AbortController().signal
|
||||
)
|
||||
)
|
||||
await vi.waitFor(() =>
|
||||
expect(harness.promptGates).toHaveLength(1)
|
||||
)
|
||||
|
||||
expect(harness.launch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ supportsImageInput: true })
|
||||
)
|
||||
expect(
|
||||
harness.requests.find(
|
||||
(entry) => entry.method === 'session/prompt'
|
||||
)?.params
|
||||
).toMatchObject({
|
||||
prompt: [
|
||||
{ type: 'text', text: 'hello' },
|
||||
{
|
||||
type: 'image',
|
||||
mimeType: 'image/png',
|
||||
data: 'aW1hZ2U='
|
||||
}
|
||||
]
|
||||
})
|
||||
harness.promptGates[0]!.resolve({ stopReason: 'end_turn' })
|
||||
await expect(running).resolves.toContainEqual(
|
||||
expect.objectContaining({ type: 'done' })
|
||||
)
|
||||
})
|
||||
|
||||
it('fails closed when Host image capability disagrees with the model', async () => {
|
||||
const harness = setup({
|
||||
supportsImageInput: true,
|
||||
advertisedImageInput: false
|
||||
})
|
||||
|
||||
await expect(harness.runtime.getStatus()).resolves.toMatchObject({
|
||||
available: false,
|
||||
detail: expect.stringContaining('图片能力')
|
||||
})
|
||||
expect(harness.child.terminate).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses ACP stdio, maps conversations to sessions, and streams text', async () => {
|
||||
const harness = setup()
|
||||
const first = collect(
|
||||
@@ -456,6 +545,7 @@ describe('DeepSeekHarnessRuntime', () => {
|
||||
signal: expect.any(AbortSignal),
|
||||
baseUrl: 'https://api.deepseek.com',
|
||||
model: 'deepseek-test',
|
||||
supportsImageInput: false,
|
||||
credentialRefs: [],
|
||||
skillPackages: [],
|
||||
extensionPackages: []
|
||||
|
||||
@@ -171,6 +171,7 @@ export type DeepSeekHarnessLaunchOptions = {
|
||||
signal: AbortSignal
|
||||
baseUrl: string
|
||||
model: string
|
||||
supportsImageInput: boolean
|
||||
credentialRefs: readonly string[]
|
||||
skillPackages: readonly RuntimeSkillPackage[]
|
||||
extensionPackages: readonly ControlledHarnessExtensionPackage[]
|
||||
@@ -180,6 +181,7 @@ export type DeepSeekHarnessRuntimeOptions = {
|
||||
defaultWorkspace: string
|
||||
baseUrl: string
|
||||
model: string
|
||||
supportsImageInput?: boolean
|
||||
launch: (
|
||||
options: DeepSeekHarnessLaunchOptions
|
||||
) => Promise<DeepSeekHarnessChild>
|
||||
@@ -812,6 +814,8 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
signal: launchController.signal,
|
||||
baseUrl: this.options.baseUrl,
|
||||
model: this.options.model,
|
||||
supportsImageInput:
|
||||
this.options.supportsImageInput === true,
|
||||
credentialRefs: Object.keys(
|
||||
this.options.credentialRefs ?? {}
|
||||
),
|
||||
@@ -1022,7 +1026,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
() =>
|
||||
this.fail(new Error('DeepSeek Harness ACP 连接异常关闭'))
|
||||
)
|
||||
await withTimeout(
|
||||
const initialization = await withTimeout(
|
||||
stateWithoutCapabilities.agent.initialize({
|
||||
protocolVersion: sdk.PROTOCOL_VERSION,
|
||||
clientCapabilities: {},
|
||||
@@ -1034,6 +1038,26 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
this.initializationTimeoutMs,
|
||||
'ACP 初始化'
|
||||
)
|
||||
const advertisedImageInput =
|
||||
Boolean(
|
||||
initialization &&
|
||||
typeof initialization === 'object' &&
|
||||
(
|
||||
initialization as {
|
||||
agentCapabilities?: {
|
||||
promptCapabilities?: { image?: unknown }
|
||||
}
|
||||
}
|
||||
).agentCapabilities?.promptCapabilities?.image === true
|
||||
)
|
||||
if (
|
||||
advertisedImageInput !==
|
||||
(this.options.supportsImageInput === true)
|
||||
) {
|
||||
throw new Error(
|
||||
'DeepSeek Harness Host 图片能力与所选模型连接不一致'
|
||||
)
|
||||
}
|
||||
const capabilities = this.parseCapabilities(
|
||||
await withTimeout(
|
||||
stateWithoutCapabilities.agent.extMethod(
|
||||
@@ -1408,8 +1432,11 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
authorize?: RuntimeAuthorizer
|
||||
): AsyncGenerator<RuntimeEvent, void, void> {
|
||||
signal.throwIfAborted()
|
||||
if (request.images?.length) {
|
||||
throw new Error('DeepSeek Harness Runtime 暂不支持图像输入')
|
||||
if (
|
||||
request.images?.length &&
|
||||
this.options.supportsImageInput !== true
|
||||
) {
|
||||
throw new Error('当前 DeepSeek Harness 模型连接未启用图像输入')
|
||||
}
|
||||
const release = await this.acquireConversation(
|
||||
request.conversationId,
|
||||
@@ -1477,7 +1504,12 @@ export class DeepSeekHarnessRuntime implements AgentRuntime {
|
||||
{
|
||||
type: 'text',
|
||||
text: flattenPrompt(request)
|
||||
}
|
||||
},
|
||||
...(request.images ?? []).map((image) => ({
|
||||
type: 'image' as const,
|
||||
data: image.data,
|
||||
mimeType: image.mediaType
|
||||
}))
|
||||
]
|
||||
}),
|
||||
this.promptTimeoutMs,
|
||||
|
||||
@@ -52,6 +52,7 @@ async function fixture() {
|
||||
signal: new AbortController().signal,
|
||||
baseUrl: 'https://gateway.example/openai/v1',
|
||||
model: 'qwen-plus',
|
||||
supportsImageInput: false,
|
||||
credentialRefs: [DEEPSEEK_HARNESS_CREDENTIAL_REF],
|
||||
skillPackages: [],
|
||||
extensionPackages: []
|
||||
@@ -103,6 +104,7 @@ describe('DeepSeek Harness utility launcher', () => {
|
||||
config: {
|
||||
baseUrl: 'https://gateway.example/openai/v1',
|
||||
model: 'qwen-plus',
|
||||
supportsImageInput: false,
|
||||
credentialRefs: [DEEPSEEK_HARNESS_CREDENTIAL_REF]
|
||||
}
|
||||
})
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
DEEPSEEK_HARNESS_CONTROL_VERSION,
|
||||
DEEPSEEK_HARNESS_CREDENTIAL_REF,
|
||||
DEEPSEEK_HARNESS_HOST_VERSION,
|
||||
DEEPSEEK_HARNESS_MAX_FRAME_BYTES,
|
||||
parseHarnessControlMessage,
|
||||
type DeepSeekHarnessControlMessage as HarnessControlMessage
|
||||
} from './deepseek-harness-control-protocol'
|
||||
@@ -24,6 +25,7 @@ export {
|
||||
DEEPSEEK_HARNESS_CONTROL_VERSION,
|
||||
DEEPSEEK_HARNESS_CREDENTIAL_REF,
|
||||
DEEPSEEK_HARNESS_HOST_VERSION,
|
||||
DEEPSEEK_HARNESS_MAX_FRAME_BYTES,
|
||||
parseHarnessControlMessage
|
||||
} from './deepseek-harness-control-protocol'
|
||||
export type {
|
||||
@@ -274,11 +276,12 @@ export function createDeepSeekHarnessUtilityLauncher(
|
||||
api: 'openai-completions',
|
||||
provider: 'goodbuddy',
|
||||
model: options.model,
|
||||
supportsImageInput: options.supportsImageInput,
|
||||
harnessVersion: DEEPSEEK_HARNESS_HOST_VERSION,
|
||||
credentialRefs: [DEEPSEEK_HARNESS_CREDENTIAL_REF],
|
||||
skillPackages: canonicalSkillPackages,
|
||||
extensionPackages: canonicalExtensionPackages,
|
||||
maxFrameBytes: 1024 * 1024
|
||||
maxFrameBytes: DEEPSEEK_HARNESS_MAX_FRAME_BYTES
|
||||
})
|
||||
utility.postMessage({
|
||||
protocol: DEEPSEEK_HARNESS_CONTROL_PROTOCOL,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
import {
|
||||
DEEPSEEK_HARNESS_BYTE_PROTOCOL,
|
||||
DEEPSEEK_HARNESS_BYTE_PROTOCOL_VERSION,
|
||||
@@ -7,6 +9,10 @@ import {
|
||||
createDeepSeekHarnessUtilityChild,
|
||||
type DeepSeekHarnessParentPortLike
|
||||
} from './deepseek-harness-utility-transport'
|
||||
import {
|
||||
DEEPSEEK_HARNESS_CONTROL_PROTOCOL,
|
||||
DEEPSEEK_HARNESS_CONTROL_VERSION
|
||||
} from './deepseek-harness-control-protocol'
|
||||
|
||||
type Listener = (value: unknown) => void
|
||||
|
||||
@@ -120,13 +126,27 @@ function setup() {
|
||||
const tick = () => new Promise<void>((resolve) => queueMicrotask(resolve))
|
||||
|
||||
describe('DeepSeek Harness utility byte transport', () => {
|
||||
it('keeps the Electron smoke protocol versions aligned', () => {
|
||||
const smokeSource = readFileSync(
|
||||
resolve('build/deepseek-harness-utility-smoke.cjs'),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
expect(smokeSource).toContain(
|
||||
`const controlVersion = ${DEEPSEEK_HARNESS_CONTROL_VERSION}`
|
||||
)
|
||||
expect(smokeSource).toContain(
|
||||
`const byteProtocolVersion = ${DEEPSEEK_HARNESS_BYTE_PROTOCOL_VERSION}`
|
||||
)
|
||||
})
|
||||
|
||||
it('ignores trusted control-plane messages that share the UtilityProcess port', async () => {
|
||||
const { child, hostPort, utility } = setup()
|
||||
await tick()
|
||||
utility.kill.mockClear()
|
||||
utility.emitMessage({
|
||||
protocol: 'goodbuddy.deepseek-harness.control',
|
||||
version: 1,
|
||||
protocol: DEEPSEEK_HARNESS_CONTROL_PROTOCOL,
|
||||
version: DEEPSEEK_HARNESS_CONTROL_VERSION,
|
||||
type: 'ready',
|
||||
failedExtensionIds: []
|
||||
})
|
||||
@@ -153,8 +173,8 @@ describe('DeepSeek Harness utility byte transport', () => {
|
||||
const { child, utility } = setup()
|
||||
const reader = child.stdout.getReader()
|
||||
utility.emitMessage({
|
||||
protocol: 'goodbuddy.deepseek-harness.control',
|
||||
version: 1,
|
||||
protocol: DEEPSEEK_HARNESS_CONTROL_PROTOCOL,
|
||||
version: DEEPSEEK_HARNESS_CONTROL_VERSION,
|
||||
type: 'ready',
|
||||
failedExtensionIds: [],
|
||||
unexpected: true
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
// @vitest-environment node
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import { createCanvas } from '@napi-rs/canvas'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { GoodBuddyHarnessAttachmentStore } from './goodbuddy-harness-attachment-store'
|
||||
|
||||
const canvas = createCanvas(1, 1)
|
||||
const transparentPng = canvas.toBuffer('image/png')
|
||||
const jpeg = canvas.toBuffer('image/jpeg')
|
||||
const secondPng = createCanvas(2, 1).toBuffer('image/png')
|
||||
|
||||
describe('GoodBuddy Harness attachment store', () => {
|
||||
it('decodes, stores, verifies, and releases inline images', async () => {
|
||||
const store = new GoodBuddyHarnessAttachmentStore(new Context())
|
||||
const input = {
|
||||
data: transparentPng,
|
||||
mediaType: 'image/png' as const,
|
||||
name: '..\\screenshots\\reference.png'
|
||||
}
|
||||
|
||||
const first = await store.saveImage(input)
|
||||
const second = await store.saveImage(input)
|
||||
|
||||
expect(first).toEqual(second)
|
||||
expect(first).toMatchObject({
|
||||
mediaType: 'image/png',
|
||||
bytes: transparentPng.byteLength,
|
||||
width: 1,
|
||||
height: 1,
|
||||
name: 'reference.png'
|
||||
})
|
||||
const stored = await store.readImage(first)
|
||||
expect(stored.ref).toBe(first)
|
||||
expect(Buffer.from(stored.data).equals(transparentPng)).toBe(true)
|
||||
|
||||
store.releaseImage(first)
|
||||
await expect(store.readImage(first)).resolves.toBeDefined()
|
||||
store.releaseImage(second)
|
||||
await expect(store.readImage(first)).rejects.toMatchObject({
|
||||
code: 'NOT_FOUND'
|
||||
})
|
||||
|
||||
const jpegRef = await store.saveImage({
|
||||
data: jpeg,
|
||||
mediaType: 'image/jpeg'
|
||||
})
|
||||
expect(jpegRef).toMatchObject({
|
||||
mediaType: 'image/jpeg',
|
||||
bytes: jpeg.byteLength,
|
||||
width: 1,
|
||||
height: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects mismatched, malformed, and over-capacity images', async () => {
|
||||
const store = new GoodBuddyHarnessAttachmentStore(new Context(), {
|
||||
maxStoredImages: 1
|
||||
})
|
||||
|
||||
await expect(
|
||||
store.saveImage({
|
||||
data: transparentPng,
|
||||
mediaType: 'image/jpeg'
|
||||
})
|
||||
).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
await expect(
|
||||
store.saveImage({
|
||||
data: Buffer.from([
|
||||
0x89, 0x50, 0x4e, 0x47,
|
||||
0x0d, 0x0a, 0x1a, 0x0a
|
||||
]),
|
||||
mediaType: 'image/png'
|
||||
})
|
||||
).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
const corruptPng = Buffer.from(transparentPng)
|
||||
corruptPng[corruptPng.length - 8] =
|
||||
(corruptPng[corruptPng.length - 8] ?? 0) ^ 1
|
||||
await expect(
|
||||
store.saveImage({
|
||||
data: corruptPng,
|
||||
mediaType: 'image/png'
|
||||
})
|
||||
).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
|
||||
await store.saveImage({
|
||||
data: transparentPng,
|
||||
mediaType: 'image/png'
|
||||
})
|
||||
await expect(
|
||||
store.saveImage({
|
||||
data: secondPng,
|
||||
mediaType: 'image/png'
|
||||
})
|
||||
).rejects.toMatchObject({ code: 'STORAGE_LIMIT' })
|
||||
})
|
||||
|
||||
it('does not retain a partial batch when capacity is exceeded', async () => {
|
||||
const store = new GoodBuddyHarnessAttachmentStore(new Context(), {
|
||||
maxStoredImages: 1
|
||||
})
|
||||
|
||||
await expect(
|
||||
store.saveImages([
|
||||
{ data: transparentPng, mediaType: 'image/png' },
|
||||
{ data: jpeg, mediaType: 'image/jpeg' }
|
||||
])
|
||||
).rejects.toMatchObject({ code: 'STORAGE_LIMIT' })
|
||||
await expect(
|
||||
store.saveImage({
|
||||
data: jpeg,
|
||||
mediaType: 'image/jpeg'
|
||||
})
|
||||
).resolves.toMatchObject({ mediaType: 'image/jpeg' })
|
||||
})
|
||||
|
||||
it('bounds aggregate decoded pixels before retaining a batch', async () => {
|
||||
const store = new GoodBuddyHarnessAttachmentStore(new Context(), {
|
||||
maxBatchImagePixels: 1
|
||||
})
|
||||
const input = {
|
||||
data: transparentPng,
|
||||
mediaType: 'image/png' as const
|
||||
}
|
||||
|
||||
await expect(
|
||||
store.saveImages([input, input])
|
||||
).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
await expect(store.saveImage(input)).resolves.toMatchObject({
|
||||
width: 1,
|
||||
height: 1
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,449 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { basename } from 'node:path'
|
||||
import { crc32 } from 'node:zlib'
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import {
|
||||
AttachmentError,
|
||||
AttachmentId,
|
||||
AttachmentStore,
|
||||
type ImageAttachmentLimits,
|
||||
type ImageAttachmentRef,
|
||||
type ImageMediaType,
|
||||
type SaveImageAttachment,
|
||||
type StoredImageAttachment
|
||||
} from '@deepseek-ai/dsh-attachment'
|
||||
|
||||
const DEFAULT_MAX_STORE_BYTES = 32 * 1024 * 1024
|
||||
const DEFAULT_MAX_STORED_IMAGES = 256
|
||||
const DEFAULT_MAX_BATCH_IMAGE_PIXELS = 32_000_000
|
||||
|
||||
export const GOODBUDDY_HARNESS_IMAGE_LIMITS: ImageAttachmentLimits =
|
||||
Object.freeze({
|
||||
maxImageBytes: 1024 * 1024,
|
||||
maxImagesPerMessage: 8,
|
||||
maxMessageImageBytes: 2 * 1024 * 1024,
|
||||
maxImagePixels: 16_000_000,
|
||||
mediaTypes: Object.freeze([
|
||||
'image/png',
|
||||
'image/jpeg'
|
||||
] satisfies ImageMediaType[])
|
||||
})
|
||||
|
||||
type StoredImage = {
|
||||
ref: ImageAttachmentRef
|
||||
data: Buffer
|
||||
references: number
|
||||
}
|
||||
|
||||
type InspectedImage = {
|
||||
data: Buffer
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
export type GoodBuddyHarnessAttachmentStoreConfig = {
|
||||
maxStoreBytes?: number
|
||||
maxStoredImages?: number
|
||||
maxBatchImagePixels?: number
|
||||
}
|
||||
|
||||
function invalidImage(message: string, cause?: unknown): AttachmentError {
|
||||
return new AttachmentError(message, 'INVALID_IMAGE', {
|
||||
...(cause === undefined ? {} : { cause })
|
||||
})
|
||||
}
|
||||
|
||||
function safeImageName(name: string | undefined): string | undefined {
|
||||
if (!name) {
|
||||
return undefined
|
||||
}
|
||||
const safe = basename(name.replaceAll('\\', '/'))
|
||||
.replace(/\p{Cc}/gu, '_')
|
||||
.trim()
|
||||
.slice(0, 200)
|
||||
return safe || undefined
|
||||
}
|
||||
|
||||
function matchesSignature(
|
||||
data: Buffer,
|
||||
mediaType: ImageMediaType
|
||||
): boolean {
|
||||
if (mediaType === 'image/png') {
|
||||
return (
|
||||
data.length >= 8 &&
|
||||
data.subarray(0, 8).equals(
|
||||
Buffer.from([
|
||||
0x89, 0x50, 0x4e, 0x47,
|
||||
0x0d, 0x0a, 0x1a, 0x0a
|
||||
])
|
||||
)
|
||||
)
|
||||
}
|
||||
return (
|
||||
data.length >= 4 &&
|
||||
data[0] === 0xff &&
|
||||
data[1] === 0xd8 &&
|
||||
data.at(-2) === 0xff &&
|
||||
data.at(-1) === 0xd9
|
||||
)
|
||||
}
|
||||
|
||||
function pngDimensions(data: Buffer): {
|
||||
width: number
|
||||
height: number
|
||||
} | undefined {
|
||||
let offset = 8
|
||||
let chunks = 0
|
||||
let width: number | undefined
|
||||
let height: number | undefined
|
||||
let sawImageData = false
|
||||
while (offset + 12 <= data.length && chunks < 256) {
|
||||
chunks += 1
|
||||
const length = data.readUInt32BE(offset)
|
||||
const typeStart = offset + 4
|
||||
const dataStart = typeStart + 4
|
||||
const dataEnd = dataStart + length
|
||||
const chunkEnd = dataEnd + 4
|
||||
if (dataEnd < dataStart || chunkEnd > data.length) {
|
||||
return undefined
|
||||
}
|
||||
const typeBytes = data.subarray(typeStart, dataStart)
|
||||
const type = typeBytes.toString('ascii')
|
||||
if (!/^[A-Za-z]{4}$/u.test(type)) {
|
||||
return undefined
|
||||
}
|
||||
if (
|
||||
crc32(data.subarray(typeStart, dataEnd)) !==
|
||||
data.readUInt32BE(dataEnd)
|
||||
) {
|
||||
return undefined
|
||||
}
|
||||
if (chunks === 1) {
|
||||
if (type !== 'IHDR' || length !== 13) {
|
||||
return undefined
|
||||
}
|
||||
width = data.readUInt32BE(dataStart)
|
||||
height = data.readUInt32BE(dataStart + 4)
|
||||
} else if (type === 'IHDR') {
|
||||
return undefined
|
||||
}
|
||||
if (type === 'IDAT') {
|
||||
sawImageData = true
|
||||
}
|
||||
if (type === 'IEND') {
|
||||
return (
|
||||
length === 0 &&
|
||||
chunkEnd === data.length &&
|
||||
sawImageData &&
|
||||
width !== undefined &&
|
||||
height !== undefined
|
||||
)
|
||||
? { width, height }
|
||||
: undefined
|
||||
}
|
||||
offset = chunkEnd
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function jpegDimensions(data: Buffer): {
|
||||
width: number
|
||||
height: number
|
||||
} | undefined {
|
||||
let offset = 2
|
||||
while (offset + 4 <= data.length - 2) {
|
||||
if (data[offset] !== 0xff) {
|
||||
return undefined
|
||||
}
|
||||
while (data[offset] === 0xff) {
|
||||
offset += 1
|
||||
}
|
||||
const marker = data[offset]
|
||||
offset += 1
|
||||
if (marker === undefined || marker === 0x00 || marker === 0xd9) {
|
||||
return undefined
|
||||
}
|
||||
if (marker === 0xda) {
|
||||
return undefined
|
||||
}
|
||||
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
|
||||
continue
|
||||
}
|
||||
if (offset + 2 > data.length - 2) {
|
||||
return undefined
|
||||
}
|
||||
const length = data.readUInt16BE(offset)
|
||||
if (length < 2 || offset + length > data.length - 2) {
|
||||
return undefined
|
||||
}
|
||||
const isStartOfFrame =
|
||||
marker >= 0xc0 &&
|
||||
marker <= 0xcf &&
|
||||
marker !== 0xc4 &&
|
||||
marker !== 0xc8 &&
|
||||
marker !== 0xcc
|
||||
if (isStartOfFrame) {
|
||||
if (length < 7) {
|
||||
return undefined
|
||||
}
|
||||
return {
|
||||
height: data.readUInt16BE(offset + 3),
|
||||
width: data.readUInt16BE(offset + 5)
|
||||
}
|
||||
}
|
||||
offset += length
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
async function inspectImage(
|
||||
input: SaveImageAttachment,
|
||||
limits: ImageAttachmentLimits
|
||||
): Promise<InspectedImage> {
|
||||
if (!limits.mediaTypes.includes(input.mediaType)) {
|
||||
throw invalidImage('Image media type is not supported')
|
||||
}
|
||||
if (
|
||||
input.data.byteLength === 0 ||
|
||||
input.data.byteLength > limits.maxImageBytes
|
||||
) {
|
||||
throw invalidImage('Image exceeds the per-image byte limit')
|
||||
}
|
||||
const data = Buffer.from(input.data)
|
||||
if (!matchesSignature(data, input.mediaType)) {
|
||||
throw invalidImage('Image media type does not match its bytes')
|
||||
}
|
||||
const encodedDimensions =
|
||||
input.mediaType === 'image/png'
|
||||
? pngDimensions(data)
|
||||
: jpegDimensions(data)
|
||||
if (!encodedDimensions) {
|
||||
throw invalidImage('Image container is malformed')
|
||||
}
|
||||
if (
|
||||
encodedDimensions.width < 1 ||
|
||||
encodedDimensions.height < 1 ||
|
||||
encodedDimensions.width * encodedDimensions.height >
|
||||
limits.maxImagePixels
|
||||
) {
|
||||
throw invalidImage('Image dimensions exceed the pixel limit')
|
||||
}
|
||||
let width: number
|
||||
let height: number
|
||||
let loadImage: typeof import('@napi-rs/canvas')['loadImage']
|
||||
try {
|
||||
const canvas = await import('@napi-rs/canvas')
|
||||
loadImage = canvas.loadImage
|
||||
} catch (error) {
|
||||
throw new AttachmentError(
|
||||
'Harness image decoder is unavailable',
|
||||
'DECODER_UNAVAILABLE',
|
||||
{ cause: error }
|
||||
)
|
||||
}
|
||||
try {
|
||||
const image = await loadImage(data)
|
||||
width = image.naturalWidth || image.width
|
||||
height = image.naturalHeight || image.height
|
||||
} catch (error) {
|
||||
throw invalidImage('Image bytes could not be decoded', error)
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(width) ||
|
||||
!Number.isSafeInteger(height) ||
|
||||
width < 1 ||
|
||||
height < 1 ||
|
||||
width * height > limits.maxImagePixels ||
|
||||
width !== encodedDimensions.width ||
|
||||
height !== encodedDimensions.height
|
||||
) {
|
||||
throw invalidImage('Image dimensions exceed the pixel limit')
|
||||
}
|
||||
return { data, width, height }
|
||||
}
|
||||
|
||||
/**
|
||||
* Process-local attachment storage for the non-persistent Harness sessions.
|
||||
* Images are fully decoded before an immutable content-addressed reference is
|
||||
* published. The store is bounded independently of per-message admission.
|
||||
*/
|
||||
export class GoodBuddyHarnessAttachmentStore extends AttachmentStore {
|
||||
readonly imageLimits = GOODBUDDY_HARNESS_IMAGE_LIMITS
|
||||
private readonly images = new Map<string, StoredImage>()
|
||||
private readonly maxBatchImagePixels: number
|
||||
private readonly maxStoreBytes: number
|
||||
private readonly maxStoredImages: number
|
||||
private storedBytes = 0
|
||||
|
||||
constructor(
|
||||
ctx: Context,
|
||||
config: GoodBuddyHarnessAttachmentStoreConfig = {}
|
||||
) {
|
||||
super(ctx)
|
||||
this.maxStoreBytes =
|
||||
config.maxStoreBytes ?? DEFAULT_MAX_STORE_BYTES
|
||||
this.maxStoredImages =
|
||||
config.maxStoredImages ?? DEFAULT_MAX_STORED_IMAGES
|
||||
this.maxBatchImagePixels =
|
||||
config.maxBatchImagePixels ??
|
||||
DEFAULT_MAX_BATCH_IMAGE_PIXELS
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxStoreBytes) ||
|
||||
this.maxStoreBytes < this.imageLimits.maxImageBytes ||
|
||||
!Number.isSafeInteger(this.maxStoredImages) ||
|
||||
this.maxStoredImages < 1 ||
|
||||
!Number.isSafeInteger(this.maxBatchImagePixels) ||
|
||||
this.maxBatchImagePixels < 1
|
||||
) {
|
||||
throw new TypeError(
|
||||
'GoodBuddy Harness attachment-store limits are invalid'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async validateImage(input: SaveImageAttachment): Promise<void> {
|
||||
await inspectImage(input, this.imageLimits)
|
||||
}
|
||||
|
||||
async saveImage(
|
||||
input: SaveImageAttachment
|
||||
): Promise<ImageAttachmentRef> {
|
||||
return (await this.saveImages([input]))[0]!
|
||||
}
|
||||
|
||||
async saveImages(
|
||||
inputs: readonly SaveImageAttachment[]
|
||||
): Promise<ImageAttachmentRef[]> {
|
||||
const inspectedByContent = new Map<string, InspectedImage>()
|
||||
const candidates: Array<{
|
||||
input: SaveImageAttachment
|
||||
inspected: InspectedImage
|
||||
attachmentId: ImageAttachmentRef['attachmentId']
|
||||
}> = []
|
||||
let batchPixels = 0
|
||||
for (const input of inputs) {
|
||||
if (
|
||||
!this.imageLimits.mediaTypes.includes(input.mediaType) ||
|
||||
input.data.byteLength === 0 ||
|
||||
input.data.byteLength > this.imageLimits.maxImageBytes
|
||||
) {
|
||||
throw invalidImage('Image exceeds the attachment limits')
|
||||
}
|
||||
const digest = createHash('sha256')
|
||||
.update(input.data)
|
||||
.digest('hex')
|
||||
const contentKey = `${input.mediaType}:${digest}`
|
||||
let inspected = inspectedByContent.get(contentKey)
|
||||
if (!inspected) {
|
||||
inspected = await inspectImage(input, this.imageLimits)
|
||||
inspectedByContent.set(contentKey, inspected)
|
||||
}
|
||||
batchPixels += inspected.width * inspected.height
|
||||
if (batchPixels > this.maxBatchImagePixels) {
|
||||
throw invalidImage('Images exceed the batch pixel limit')
|
||||
}
|
||||
candidates.push({
|
||||
input,
|
||||
inspected,
|
||||
attachmentId: AttachmentId(`sha256:${digest}`)
|
||||
})
|
||||
}
|
||||
const additions = new Map<
|
||||
ImageAttachmentRef['attachmentId'],
|
||||
InspectedImage
|
||||
>()
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
!this.images.has(candidate.attachmentId) &&
|
||||
!additions.has(candidate.attachmentId)
|
||||
) {
|
||||
additions.set(candidate.attachmentId, candidate.inspected)
|
||||
}
|
||||
}
|
||||
const additionalBytes = [...additions.values()].reduce(
|
||||
(total, inspected) => total + inspected.data.byteLength,
|
||||
0
|
||||
)
|
||||
if (
|
||||
this.images.size + additions.size > this.maxStoredImages ||
|
||||
this.storedBytes + additionalBytes > this.maxStoreBytes
|
||||
) {
|
||||
throw new AttachmentError(
|
||||
'Harness attachment store is full',
|
||||
'STORAGE_LIMIT'
|
||||
)
|
||||
}
|
||||
return candidates.map(({ input, inspected, attachmentId }) => {
|
||||
const existing = this.images.get(attachmentId)
|
||||
if (existing) {
|
||||
existing.references += 1
|
||||
return existing.ref
|
||||
}
|
||||
const name = safeImageName(input.name)
|
||||
const ref = Object.freeze({
|
||||
attachmentId,
|
||||
mediaType: input.mediaType,
|
||||
bytes: inspected.data.byteLength,
|
||||
width: inspected.width,
|
||||
height: inspected.height,
|
||||
...(name ? { name } : {})
|
||||
})
|
||||
this.images.set(attachmentId, {
|
||||
ref,
|
||||
data: inspected.data,
|
||||
references: 1
|
||||
})
|
||||
this.storedBytes += inspected.data.byteLength
|
||||
return ref
|
||||
})
|
||||
}
|
||||
|
||||
async readImage(
|
||||
ref: ImageAttachmentRef,
|
||||
signal?: AbortSignal
|
||||
): Promise<StoredImageAttachment> {
|
||||
signal?.throwIfAborted()
|
||||
const stored = this.images.get(ref.attachmentId)
|
||||
if (!stored) {
|
||||
throw new AttachmentError(
|
||||
'Harness image attachment was not found',
|
||||
'NOT_FOUND'
|
||||
)
|
||||
}
|
||||
if (
|
||||
stored.ref.attachmentId !== ref.attachmentId ||
|
||||
stored.ref.mediaType !== ref.mediaType ||
|
||||
stored.ref.bytes !== ref.bytes ||
|
||||
stored.ref.width !== ref.width ||
|
||||
stored.ref.height !== ref.height ||
|
||||
stored.ref.name !== ref.name
|
||||
) {
|
||||
throw new AttachmentError(
|
||||
'Harness image attachment failed integrity validation',
|
||||
'INTEGRITY'
|
||||
)
|
||||
}
|
||||
return {
|
||||
ref: stored.ref,
|
||||
data: Uint8Array.from(stored.data)
|
||||
}
|
||||
}
|
||||
|
||||
releaseImage(ref: ImageAttachmentRef): void {
|
||||
const stored = this.images.get(ref.attachmentId)
|
||||
if (!stored) {
|
||||
return
|
||||
}
|
||||
stored.references -= 1
|
||||
if (stored.references > 0) {
|
||||
return
|
||||
}
|
||||
this.images.delete(ref.attachmentId)
|
||||
this.storedBytes -= stored.data.byteLength
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.images.clear()
|
||||
this.storedBytes = 0
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import { createCanvas } from '@napi-rs/canvas'
|
||||
import type { Stream } from '@agentclientprotocol/sdk'
|
||||
import { resolve } from 'node:path'
|
||||
import {
|
||||
@@ -9,17 +10,24 @@ import {
|
||||
GoodBuddyHarnessControlPlane,
|
||||
createBoundedAcpStream
|
||||
} from './goodbuddy-harness-control-plane'
|
||||
import { GoodBuddyHarnessAttachmentStore } from './goodbuddy-harness-attachment-store'
|
||||
|
||||
function controlPlane() {
|
||||
return new GoodBuddyHarnessControlPlane({} as Context, {
|
||||
provider: 'goodbuddy',
|
||||
model: 'deepseek-test',
|
||||
workspace: resolve('workspace'),
|
||||
harnessVersion: '0.1.0-rc.6',
|
||||
execution: { mode: 'host' },
|
||||
credentialRefs: ['GOODBUDDY_API_KEY'],
|
||||
skills: []
|
||||
})
|
||||
return new GoodBuddyHarnessControlPlane(
|
||||
{
|
||||
on: vi.fn(),
|
||||
get: vi.fn()
|
||||
} as unknown as Context,
|
||||
{
|
||||
provider: 'goodbuddy',
|
||||
model: 'deepseek-test',
|
||||
workspace: resolve('workspace'),
|
||||
harnessVersion: '0.1.0-rc.6',
|
||||
execution: { mode: 'host' },
|
||||
credentialRefs: ['GOODBUDDY_API_KEY'],
|
||||
skills: []
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function stubAgentContext() {
|
||||
@@ -123,6 +131,127 @@ function stubAgentContext() {
|
||||
}
|
||||
|
||||
describe('GoodBuddy Harness internal control plane', () => {
|
||||
it('advertises and stores only model-enabled inline image prompts', async () => {
|
||||
const textOnly = controlPlane() as unknown as {
|
||||
createAgentApi(): {
|
||||
initialize(): Promise<{
|
||||
agentCapabilities: {
|
||||
promptCapabilities: { image: boolean }
|
||||
}
|
||||
}>
|
||||
}
|
||||
storePromptImages(
|
||||
prompt: Array<Record<string, unknown>>
|
||||
): Promise<unknown[]>
|
||||
}
|
||||
await expect(textOnly.createAgentApi().initialize()).resolves.toMatchObject({
|
||||
agentCapabilities: {
|
||||
promptCapabilities: { image: false }
|
||||
}
|
||||
})
|
||||
await expect(
|
||||
textOnly.storePromptImages([
|
||||
{
|
||||
type: 'image',
|
||||
mimeType: 'image/png',
|
||||
data: 'aW1hZ2U='
|
||||
}
|
||||
])
|
||||
).rejects.toThrow('does not accept image input')
|
||||
|
||||
const storeContext = new Context()
|
||||
const store = new GoodBuddyHarnessAttachmentStore(storeContext)
|
||||
const ctx = {
|
||||
on: vi.fn(),
|
||||
get: vi.fn((name: string) =>
|
||||
name === 'attachments' ? store : undefined
|
||||
)
|
||||
} as unknown as Context
|
||||
const subject = new GoodBuddyHarnessControlPlane(ctx, {
|
||||
provider: 'goodbuddy',
|
||||
model: 'vision-test',
|
||||
supportsImageInput: true,
|
||||
workspace: resolve('workspace'),
|
||||
harnessVersion: '0.1.0-rc.6',
|
||||
execution: { mode: 'host' },
|
||||
credentialRefs: ['GOODBUDDY_API_KEY'],
|
||||
skills: []
|
||||
}) as unknown as {
|
||||
createAgentApi(): {
|
||||
initialize(): Promise<{
|
||||
agentCapabilities: {
|
||||
promptCapabilities: { image: boolean }
|
||||
}
|
||||
}>
|
||||
}
|
||||
storePromptImages(
|
||||
prompt: Array<Record<string, unknown>>
|
||||
): Promise<
|
||||
Array<
|
||||
Parameters<GoodBuddyHarnessAttachmentStore['readImage']>[0]
|
||||
>
|
||||
>
|
||||
releaseAttachments(
|
||||
refs: Array<
|
||||
Parameters<GoodBuddyHarnessAttachmentStore['readImage']>[0]
|
||||
>
|
||||
): void
|
||||
}
|
||||
const png = createCanvas(1, 1).toBuffer('image/png')
|
||||
|
||||
await expect(subject.createAgentApi().initialize()).resolves.toMatchObject({
|
||||
agentCapabilities: {
|
||||
promptCapabilities: { image: true }
|
||||
}
|
||||
})
|
||||
const refs = await subject.storePromptImages([
|
||||
{ type: 'text', text: 'describe this image' },
|
||||
{
|
||||
type: 'image',
|
||||
mimeType: 'image/png',
|
||||
data: png.toString('base64')
|
||||
}
|
||||
])
|
||||
expect(refs).toHaveLength(1)
|
||||
await expect(store.readImage(refs[0]!)).resolves.toMatchObject({
|
||||
ref: expect.objectContaining({
|
||||
mediaType: 'image/png',
|
||||
width: 1,
|
||||
height: 1
|
||||
})
|
||||
})
|
||||
|
||||
subject.releaseAttachments(refs)
|
||||
await expect(store.readImage(refs[0]!)).rejects.toMatchObject({
|
||||
code: 'NOT_FOUND'
|
||||
})
|
||||
await expect(
|
||||
subject.storePromptImages([
|
||||
{
|
||||
type: 'image',
|
||||
mimeType: 'image/png',
|
||||
data: png.toString('base64'),
|
||||
uri: 'https://example.com/reference.png'
|
||||
}
|
||||
])
|
||||
).rejects.toThrow('invalid inline image')
|
||||
const saveImages = vi.spyOn(store, 'saveImages')
|
||||
const largeInlineData = Buffer.alloc(800 * 1024).toString(
|
||||
'base64'
|
||||
)
|
||||
await expect(
|
||||
subject.storePromptImages(
|
||||
Array.from({ length: 3 }, () => ({
|
||||
type: 'image' as const,
|
||||
mimeType: 'image/png',
|
||||
data: largeInlineData
|
||||
}))
|
||||
)
|
||||
).rejects.toThrow('invalid inline image')
|
||||
expect(saveImages).not.toHaveBeenCalled()
|
||||
await storeContext.fiber.dispose()
|
||||
})
|
||||
|
||||
it('requires a versioned handshake before privileged extensions', async () => {
|
||||
const subject = controlPlane()
|
||||
|
||||
|
||||
@@ -6,10 +6,16 @@ import {
|
||||
RequestError,
|
||||
type Agent,
|
||||
type AgentSideConnection as AcpAgentConnection,
|
||||
type ContentBlock as AcpContentBlock,
|
||||
type Stream
|
||||
} from '@agentclientprotocol/sdk'
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import type { AgentHandle } from '@deepseek-ai/dsh-agent'
|
||||
import {
|
||||
AttachmentError,
|
||||
type ImageAttachmentRef,
|
||||
type SaveImageAttachment
|
||||
} from '@deepseek-ai/dsh-attachment'
|
||||
import {
|
||||
CredentialProvider,
|
||||
type CredentialInfo,
|
||||
@@ -19,6 +25,7 @@ import {
|
||||
import {
|
||||
createUserMessage,
|
||||
errorChain,
|
||||
type ContentBlock as HarnessContentBlock,
|
||||
type TokenUsage
|
||||
} from '@deepseek-ai/dsh-llm'
|
||||
import {
|
||||
@@ -39,6 +46,7 @@ import {
|
||||
GOODBUDDY_TOOLS_CALL,
|
||||
GOODBUDDY_TOOLS_LIST
|
||||
} from './deepseek-harness-protocol'
|
||||
import { GoodBuddyHarnessAttachmentStore } from './goodbuddy-harness-attachment-store'
|
||||
export {
|
||||
GOODBUDDY_CONTROL_PROTOCOL_VERSION,
|
||||
GOODBUDDY_CREDENTIAL,
|
||||
@@ -94,6 +102,7 @@ export type GoodBuddyHarnessCapabilities = {
|
||||
export type GoodBuddyHarnessControlConfig = {
|
||||
provider: string
|
||||
model: string
|
||||
supportsImageInput?: boolean
|
||||
workspace: string
|
||||
harnessVersion: string
|
||||
execution: GoodBuddyHarnessCapabilities['execution']
|
||||
@@ -117,6 +126,7 @@ type Preparation = {
|
||||
|
||||
type OwnedSession = {
|
||||
handle: AgentHandle
|
||||
attachmentRefs: ImageAttachmentRef[]
|
||||
preparation?: Preparation
|
||||
proxyTools: Map<
|
||||
string,
|
||||
@@ -375,24 +385,27 @@ function boundedJson(value: unknown): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
function promptText(
|
||||
prompt: readonly { type: string; text?: string }[]
|
||||
): string {
|
||||
if (
|
||||
prompt.some(
|
||||
(block) =>
|
||||
block.type !== 'text' &&
|
||||
block.type !== 'resource_link'
|
||||
)
|
||||
) {
|
||||
throw RequestError.invalidParams(
|
||||
undefined,
|
||||
'only text and resource_link prompt content is supported'
|
||||
)
|
||||
function promptText(prompt: readonly AcpContentBlock[]): string {
|
||||
const text: string[] = []
|
||||
for (const block of prompt) {
|
||||
switch (block.type) {
|
||||
case 'text':
|
||||
text.push(block.text)
|
||||
break
|
||||
case 'image':
|
||||
case 'resource_link':
|
||||
break
|
||||
case 'audio':
|
||||
case 'resource':
|
||||
throw RequestError.invalidParams(
|
||||
undefined,
|
||||
`unsupported ACP prompt content: ${block.type}`
|
||||
)
|
||||
default:
|
||||
block satisfies never
|
||||
}
|
||||
}
|
||||
return prompt
|
||||
.map((block) => (block.type === 'text' ? block.text ?? '' : ''))
|
||||
.join('')
|
||||
return text.join('')
|
||||
}
|
||||
|
||||
function turnReason(event: SessionEvent): string | undefined {
|
||||
@@ -957,6 +970,118 @@ export class GoodBuddyHarnessControlPlane {
|
||||
}
|
||||
}
|
||||
|
||||
private attachmentStore(): GoodBuddyHarnessAttachmentStore {
|
||||
const store = this.ctx.get('attachments')
|
||||
if (!(store instanceof GoodBuddyHarnessAttachmentStore)) {
|
||||
throw RequestError.internalError(
|
||||
undefined,
|
||||
'Harness image attachment service is unavailable'
|
||||
)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
private releaseAttachments(refs: readonly ImageAttachmentRef[]): void {
|
||||
if (refs.length === 0) {
|
||||
return
|
||||
}
|
||||
const store = this.attachmentStore()
|
||||
for (const ref of refs) {
|
||||
store.releaseImage(ref)
|
||||
}
|
||||
}
|
||||
|
||||
private async storePromptImages(
|
||||
prompt: readonly AcpContentBlock[]
|
||||
): Promise<ImageAttachmentRef[]> {
|
||||
const imageBlocks = prompt.filter(
|
||||
(
|
||||
block
|
||||
): block is Extract<AcpContentBlock, { type: 'image' }> =>
|
||||
block.type === 'image'
|
||||
)
|
||||
if (imageBlocks.length === 0) {
|
||||
return []
|
||||
}
|
||||
if (!this.config.supportsImageInput) {
|
||||
throw RequestError.invalidParams(
|
||||
undefined,
|
||||
'the selected model does not accept image input'
|
||||
)
|
||||
}
|
||||
const store = this.attachmentStore()
|
||||
if (imageBlocks.length > store.imageLimits.maxImagesPerMessage) {
|
||||
throw RequestError.invalidParams(
|
||||
undefined,
|
||||
'too many images in one prompt'
|
||||
)
|
||||
}
|
||||
const inputs: SaveImageAttachment[] = []
|
||||
let totalBytes = 0
|
||||
for (const block of imageBlocks) {
|
||||
if (
|
||||
block.uri != null ||
|
||||
!store.imageLimits.mediaTypes.includes(
|
||||
block.mimeType as SaveImageAttachment['mediaType']
|
||||
) ||
|
||||
block.data.length === 0 ||
|
||||
block.data.length % 4 !== 0 ||
|
||||
block.data.length >
|
||||
Math.ceil(store.imageLimits.maxImageBytes / 3) * 4 ||
|
||||
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(
|
||||
block.data
|
||||
)
|
||||
) {
|
||||
throw RequestError.invalidParams(
|
||||
undefined,
|
||||
'invalid inline image prompt content'
|
||||
)
|
||||
}
|
||||
const data = Buffer.from(block.data, 'base64')
|
||||
totalBytes += data.byteLength
|
||||
if (
|
||||
data.byteLength === 0 ||
|
||||
data.byteLength > store.imageLimits.maxImageBytes ||
|
||||
totalBytes > store.imageLimits.maxMessageImageBytes ||
|
||||
data.toString('base64') !== block.data
|
||||
) {
|
||||
throw RequestError.invalidParams(
|
||||
undefined,
|
||||
'invalid inline image prompt content'
|
||||
)
|
||||
}
|
||||
inputs.push({
|
||||
data,
|
||||
mediaType: block.mimeType as SaveImageAttachment['mediaType']
|
||||
})
|
||||
}
|
||||
try {
|
||||
return await store.saveImages(inputs)
|
||||
} catch (error) {
|
||||
if (error instanceof AttachmentError) {
|
||||
if (error.code === 'INVALID_IMAGE') {
|
||||
throw RequestError.invalidParams(
|
||||
undefined,
|
||||
'invalid inline image prompt content'
|
||||
)
|
||||
}
|
||||
if (
|
||||
error.code === 'STORAGE_LIMIT' ||
|
||||
error.code === 'DECODER_UNAVAILABLE'
|
||||
) {
|
||||
throw RequestError.internalError(
|
||||
undefined,
|
||||
'Harness image attachment service is unavailable'
|
||||
)
|
||||
}
|
||||
}
|
||||
throw RequestError.internalError(
|
||||
undefined,
|
||||
'Harness image attachment storage failed'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private createAgentApi(): Agent {
|
||||
this.observeSessions()
|
||||
return {
|
||||
@@ -968,7 +1093,7 @@ export class GoodBuddyHarnessControlPlane {
|
||||
},
|
||||
agentCapabilities: {
|
||||
promptCapabilities: {
|
||||
image: false,
|
||||
image: this.config.supportsImageInput === true,
|
||||
audio: false,
|
||||
embeddedContext: false
|
||||
},
|
||||
@@ -1050,6 +1175,7 @@ export class GoodBuddyHarnessControlPlane {
|
||||
}
|
||||
this.sessions.set(sessionId, {
|
||||
handle,
|
||||
attachmentRefs: [],
|
||||
proxyTools: new Map(),
|
||||
askToolDefinitions
|
||||
})
|
||||
@@ -1091,10 +1217,26 @@ export class GoodBuddyHarnessControlPlane {
|
||||
'empty prompt'
|
||||
)
|
||||
}
|
||||
const message = createUserMessage({
|
||||
content: [{ type: 'text', text }],
|
||||
source: { kind: 'user' }
|
||||
})
|
||||
const attachmentRefs = await this.storePromptImages(
|
||||
params.prompt
|
||||
)
|
||||
let message: ReturnType<typeof createUserMessage>
|
||||
try {
|
||||
const content: HarnessContentBlock[] = [
|
||||
{ type: 'text', text },
|
||||
...attachmentRefs.map((attachment) => ({
|
||||
type: 'image' as const,
|
||||
attachment
|
||||
}))
|
||||
]
|
||||
message = createUserMessage({
|
||||
content,
|
||||
source: { kind: 'user' }
|
||||
})
|
||||
} catch (error) {
|
||||
this.releaseAttachments(attachmentRefs)
|
||||
throw error
|
||||
}
|
||||
const stopReason = await new Promise<string>(
|
||||
(resolve, reject) => {
|
||||
record.inflight = {
|
||||
@@ -1110,9 +1252,11 @@ export class GoodBuddyHarnessControlPlane {
|
||||
record.handle.agent.followup(message)
|
||||
} catch (error) {
|
||||
record.inflight = undefined
|
||||
this.releaseAttachments(attachmentRefs)
|
||||
reject(error)
|
||||
return
|
||||
}
|
||||
record.attachmentRefs.push(...attachmentRefs)
|
||||
void record.handle.agent.whenIdle().then(() => {
|
||||
const current = record.inflight
|
||||
if (current?.messageId !== message.id) {
|
||||
@@ -1255,7 +1399,11 @@ export class GoodBuddyHarnessControlPlane {
|
||||
record.inflight.resolve('cancelled')
|
||||
record.inflight = undefined
|
||||
}
|
||||
await record.handle.dispose()
|
||||
try {
|
||||
await record.handle.dispose()
|
||||
} finally {
|
||||
this.releaseAttachments(record.attachmentRefs)
|
||||
}
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
@@ -1273,6 +1421,9 @@ export class GoodBuddyHarnessControlPlane {
|
||||
await Promise.allSettled(
|
||||
sessions.map(([, record]) => record.handle.dispose())
|
||||
)
|
||||
for (const [, record] of sessions) {
|
||||
this.releaseAttachments(record.attachmentRefs)
|
||||
}
|
||||
})()
|
||||
await this.disposing
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
CreateAgentOptions
|
||||
} from '@deepseek-ai/dsh-agent'
|
||||
import { GOODBUDDY_HARNESS_MAX_STEP_TOKENS } from './agent/goodbuddy-harness-control-plane'
|
||||
import { GoodBuddyHarnessAttachmentStore } from './agent/goodbuddy-harness-attachment-store'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { basename, join } from 'node:path'
|
||||
|
||||
@@ -80,6 +81,7 @@ describe('controlled DeepSeek Harness host', () => {
|
||||
api: 'openai-completions',
|
||||
provider: 'goodbuddy',
|
||||
model: 'qwen-plus',
|
||||
supportsImageInput: true,
|
||||
harnessVersion: '0.1.0-rc.6',
|
||||
credentialRefs: ['GOODBUDDY_API_KEY'],
|
||||
skillPackages: [],
|
||||
@@ -91,6 +93,9 @@ describe('controlled DeepSeek Harness host', () => {
|
||||
|
||||
expect(host.context.fs.sandboxMode).toBeUndefined()
|
||||
expect(host.context.shell.sandboxMode).toBeUndefined()
|
||||
expect(host.context.get('attachments')).toBeInstanceOf(
|
||||
GoodBuddyHarnessAttachmentStore
|
||||
)
|
||||
expect(
|
||||
host.context.shell.resolve({
|
||||
command: 'echo goodbuddy-host-execution'
|
||||
|
||||
@@ -19,6 +19,9 @@ import * as ToolBash from '@deepseek-ai/dsh-tool-bash'
|
||||
import * as ToolFs from '@deepseek-ai/dsh-tool-fs'
|
||||
import * as ToolPwsh from '@deepseek-ai/dsh-tool-pwsh'
|
||||
import * as ShellEnv from '@deepseek-ai/dsh-shell-env'
|
||||
import {
|
||||
GoodBuddyHarnessAttachmentStore
|
||||
} from './agent/goodbuddy-harness-attachment-store'
|
||||
import {
|
||||
GoodBuddyCredentialProvider,
|
||||
GoodBuddyHarnessControlPlane,
|
||||
@@ -31,8 +34,8 @@ import {
|
||||
} from './agent/deepseek-harness-extension-loader'
|
||||
import type { Stream } from '@agentclientprotocol/sdk'
|
||||
import { isDeepSeekHarnessCompatibleBaseUrl } from '../shared/deepseek-harness-compatibility'
|
||||
import { DEEPSEEK_HARNESS_MAX_FRAME_BYTES } from './agent/deepseek-harness-control-protocol'
|
||||
|
||||
const DEFAULT_MAX_FRAME_BYTES = 1024 * 1024
|
||||
const MAX_DIAGNOSTIC_BYTES = 64 * 1024
|
||||
|
||||
export type ControlledHarnessHostConfig = Omit<
|
||||
@@ -204,9 +207,10 @@ async function loadControlledSkills(
|
||||
* Boots a fixed, programmatic Cordis graph. It never imports app-boot, a
|
||||
* profile loader, settings-file, local credentials, persistence, telemetry,
|
||||
* web, HMR, marketplace discovery, direct MCP clients, jobs, subagents,
|
||||
* hooks, or workflow packages. The control plane registers Main-selected
|
||||
* Skill snapshots, Main-mediated MCP tool proxies, and explicitly enabled
|
||||
* extension entrypoints.
|
||||
* hooks, or workflow packages. When the selected model declares image input,
|
||||
* the graph adds only a bounded process-local attachment store. The control
|
||||
* plane registers Main-selected Skill snapshots, Main-mediated MCP tool
|
||||
* proxies, and explicitly enabled extension entrypoints.
|
||||
*/
|
||||
export async function startControlledDeepSeekHarnessHost(
|
||||
input: ControlledHarnessHostConfig
|
||||
@@ -219,6 +223,9 @@ export async function startControlledDeepSeekHarnessHost(
|
||||
const specs: PluginSpec[] = [
|
||||
{ plugin: LlmRuntime },
|
||||
{ plugin: SessionStore },
|
||||
...(config.supportsImageInput
|
||||
? [{ plugin: GoodBuddyHarnessAttachmentStore }]
|
||||
: []),
|
||||
{ plugin: SkillRegistry },
|
||||
{
|
||||
plugin: SystemPrompt,
|
||||
@@ -242,7 +249,14 @@ export async function startControlledDeepSeekHarnessHost(
|
||||
apiKeyEnv: config.credentialRefs[0],
|
||||
api: config.api,
|
||||
baseURL: config.baseUrl,
|
||||
models: [{ id: config.model, input: ['text'] }]
|
||||
models: [
|
||||
{
|
||||
id: config.model,
|
||||
input: config.supportsImageInput
|
||||
? ['text', 'image']
|
||||
: ['text']
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -310,13 +324,22 @@ export async function startControlledDeepSeekHarnessHost(
|
||||
'Controlled Harness credential provider failed to start'
|
||||
)
|
||||
}
|
||||
const attachmentStore = ctx.get('attachments')
|
||||
if (
|
||||
config.supportsImageInput &&
|
||||
!(attachmentStore instanceof GoodBuddyHarnessAttachmentStore)
|
||||
) {
|
||||
throw new Error(
|
||||
'Controlled Harness attachment store failed to start'
|
||||
)
|
||||
}
|
||||
startupCode = 'HOST_CONTROL_PLANE_FAILED'
|
||||
const rawStream =
|
||||
config.stream ??
|
||||
createBoundedNdJsonStream(
|
||||
stdoutStream(),
|
||||
stdinStream(),
|
||||
config.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES
|
||||
config.maxFrameBytes ?? DEEPSEEK_HARNESS_MAX_FRAME_BYTES
|
||||
)
|
||||
const controlPlane = new GoodBuddyHarnessControlPlane(ctx, {
|
||||
...config,
|
||||
@@ -325,7 +348,7 @@ export async function startControlledDeepSeekHarnessHost(
|
||||
execution: { mode: 'host' },
|
||||
stream: createBoundedAcpStream(
|
||||
rawStream,
|
||||
config.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES
|
||||
config.maxFrameBytes ?? DEEPSEEK_HARNESS_MAX_FRAME_BYTES
|
||||
)
|
||||
})
|
||||
controlPlane.bindCredentialProvider(credentialProvider)
|
||||
@@ -337,6 +360,9 @@ export async function startControlledDeepSeekHarnessHost(
|
||||
extensionFailures: extensions.failures,
|
||||
async dispose() {
|
||||
await controlPlane.dispose()
|
||||
if (attachmentStore instanceof GoodBuddyHarnessAttachmentStore) {
|
||||
attachmentStore.clear()
|
||||
}
|
||||
await ctx.fiber.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user