mesalogo
66f77b83c2
Expose saved decisions, timelines and source evidence through read-only Story Graph tools across Model, Harness and local/remote OpenCode and Continue. Supervisor enablement and runtime assignments govern discovery and calls; reads do not invoke models or modify stored evidence. Remote use requires matching Desktop and Agent changes. Make Continue discussion show the complete editable message with a separated destination and shared action layout. Sending uses the visible draft without reattaching removed context. Validation: existing focused MCP, runtime, controlled Linux Host, discussion component and Electron layout checks plus typechecks; staged diff check passed. Cross-scope fact reuse, historical reconstruction and full package acceptance remain deferred. Release note: Agents can consult saved supervisor decisions and their sources when continuing work. Continue discussion now lets you review and edit the complete message before sending.
1621 lines
56 KiB
TypeScript
1621 lines
56 KiB
TypeScript
import {
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
realpath,
|
|
rm,
|
|
writeFile
|
|
} from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join, resolve } from 'node:path'
|
|
import { describe, expect, it, vi } from 'vitest'
|
|
import { createCanvas } from '@napi-rs/canvas'
|
|
import {
|
|
ToolCallId,
|
|
type GenerateOptions,
|
|
type StreamChunk
|
|
} from '@deepseek-ai/dsh-llm'
|
|
import type { RuntimeEvent } from './runtime'
|
|
import {
|
|
ModelToolProvider,
|
|
type ModelToolCallContext
|
|
} from './model-tool-provider'
|
|
import type { ResolvedMcpServer } from '../capabilities/capability-service'
|
|
import {
|
|
createBoundedNdJsonStream,
|
|
startControlledDeepSeekHarnessHost,
|
|
type ControlledHarnessHost
|
|
} from '../deepseek-harness-host'
|
|
import {
|
|
DeepSeekHarnessRuntime,
|
|
type DeepSeekHarnessChild,
|
|
type DeepSeekHarnessLaunchOptions
|
|
} from './deepseek-harness-runtime'
|
|
import { DshNpmExtensionInstaller } from './dsh-extension-marketplace'
|
|
import { DEEPSEEK_HARNESS_MAX_FRAME_BYTES } from './deepseek-harness-control-protocol'
|
|
import { createOpenAIChatCompletionsUrl } from './openai-endpoint'
|
|
import { createModelRequestProbe } from '../../../tests/support/model-request-probe'
|
|
import { ExecutionSpaceResolver } from '../execution-space/execution-space-resolver'
|
|
import { SelectedRuntimeManager } from './selected-runtime-manager'
|
|
import { LocalRuntimeRegistry } from './local-runtime-registry'
|
|
import { createAgentRuntime } from './create-runtime'
|
|
import { defaultRuntimeSettings } from '../../shared/contracts'
|
|
import type { ResolvedRuntimeSettings } from '../runtime-settings-store'
|
|
import { AssistantDatabase } from '../assistant/assistant-database'
|
|
import { ImageGenerationService } from './image-generation-service'
|
|
import { KnowledgeMcpGateway } from './knowledge-mcp-gateway'
|
|
import type { KnowledgeService } from '../knowledge/knowledge-service'
|
|
|
|
vi.mock('electron', () => ({ nativeImage: {} }))
|
|
|
|
it('generates and edits durable images through the controlled Harness Main proxy and refreshes it in Ask', async () => {
|
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-images-')))
|
|
const database = new AssistantDatabase(join(root, 'assistant.sqlite'))
|
|
database.initialize(root)
|
|
const conversationId = crypto.randomUUID()
|
|
const profileId = crypto.randomUUID()
|
|
const fetcher = vi.fn<typeof fetch>(async () => Response.json({ data: [{ b64_json: 'iVBORw0KGgo=' }] }))
|
|
const service = new ImageGenerationService({ database, fetcher, getSettings: async () => ({ defaultImageModelProfileId: profileId, modelProfiles: [{
|
|
id: profileId, name: 'Image fixture', modelName: 'fixture', protocol: 'openai-images-generations',
|
|
allowConversationInvocation: true, baseUrl: 'https://image.test/v1', authentication: 'none'
|
|
}] }) })
|
|
let input: Record<string, unknown> = { intent: 'create', prompt: 'Blue circle' }
|
|
const inProcess = createInProcessLaunch(root, {
|
|
stream(options) {
|
|
const prompt = latestUserText(options)
|
|
const tool = options.tools?.find(tool => tool.name === 'generate_image')
|
|
if (prompt === 'ask-image') {
|
|
expect(tool).toBeUndefined()
|
|
return textResponse('Ask is read-only')
|
|
}
|
|
expect(tool).toBeDefined()
|
|
expect(tool?.description).toContain(profileId)
|
|
if (!toolResultText(options, prompt)) return toolCall(prompt, 'generate_image', input)
|
|
expect(toolResultText(options, prompt)).toContain('completed')
|
|
return textResponse('IMAGE_SAVED')
|
|
}
|
|
})
|
|
const runtime = new DeepSeekHarnessRuntime({ defaultWorkspace: root, baseUrl: 'https://chat.test/v1', model: 'fixture',
|
|
launch: inProcess.launch, credentialRefs: { [CREDENTIAL_REF]: 'fixture' }, toolProvider: new ModelToolProvider(root),
|
|
initializationTimeoutMs: 20_000, promptTimeoutMs: 20_000, shutdownTimeoutMs: 5_000 })
|
|
try {
|
|
for (const intent of ['create', 'edit'] as const) {
|
|
const messageId = crypto.randomUUID()
|
|
const requestId = crypto.randomUUID()
|
|
database.saveLocalConversations([{ header: { id: conversationId, title: 'Images', updatedAt: Date.now() },
|
|
messages: [{ id: messageId, role: 'assistant', content: '', createdAt: Date.now(), state: 'complete' }] }])
|
|
const events = await collect(runtime.run({ requestId, conversationId, prompt: `${intent}-${requestId}`, workMode: 'execute',
|
|
imageToolBinding: service.bind({ conversationId, messageId, requestId, workMode: 'execute' }) }, new AbortController().signal, async () => 'once'))
|
|
expect(events.at(-1)?.type).toBe('done')
|
|
const operation = database.getConversation(conversationId).messages.find(message => message.id === messageId)!.imageOperations![0]!
|
|
expect(operation.state).toBe('completed')
|
|
input = { intent: 'edit', prompt: 'Turn red', sourceArtifactIds: operation.artifactIds }
|
|
}
|
|
await collect(runtime.run({ requestId: crypto.randomUUID(), conversationId, prompt: 'ask-image', workMode: 'ask' }, new AbortController().signal))
|
|
expect(fetcher).toHaveBeenCalledTimes(2)
|
|
expect(String(fetcher.mock.calls[1]![0])).toMatch(/images\/edits$/u)
|
|
} finally {
|
|
await runtime.dispose()
|
|
await service.dispose()
|
|
database.close()
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 60_000)
|
|
|
|
const CREDENTIAL_REF = 'GOODBUDDY_HARNESS_MODEL_API_KEY'
|
|
|
|
it('reads Story Graph through the controlled Harness ACP Main proxy in Ask and Execute, then removes it when disabled', async () => {
|
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-graph-')))
|
|
const db = new AssistantDatabase(':memory:'); db.initialize(root)
|
|
let enabled = true
|
|
const projectId = db.listProjects()[0]!.id
|
|
const gateway = new KnowledgeMcpGateway({} as KnowledgeService, { storyGraphService: {
|
|
available: async () => enabled, read: db.readStoryGraph.bind(db)
|
|
} })
|
|
const signal = new AbortController().signal
|
|
const token = gateway.grant('harness-graph', [], signal, 'none', undefined, undefined, undefined, undefined, undefined,
|
|
{ projectId, runtimeTarget: 'deepseek-harness' })!
|
|
const inProcess = createInProcessLaunch(root, { stream(options) {
|
|
const prompt = latestUserText(options)
|
|
const tool = options.tools?.find(tool => tool.name === 'story_graph_search')
|
|
if (!enabled) { expect(tool).toBeUndefined(); return textResponse('DISABLED') }
|
|
expect(tool).toBeDefined()
|
|
if (!toolResultText(options, prompt)) return toolCall(prompt, 'story_graph_search', { query: 'Decision' })
|
|
expect(toolResultText(options, prompt)).toContain(projectId)
|
|
return textResponse('GRAPH_READ')
|
|
} })
|
|
const provider = new ModelToolProvider(root, [], undefined, gateway)
|
|
const runtime = new DeepSeekHarnessRuntime({ defaultWorkspace: root, baseUrl: 'https://chat.test/v1', model: 'fixture',
|
|
launch: inProcess.launch, credentialRefs: { [CREDENTIAL_REF]: 'fixture' }, toolProvider: provider,
|
|
initializationTimeoutMs: 20_000, promptTimeoutMs: 20_000, shutdownTimeoutMs: 5_000 })
|
|
try {
|
|
for (const mode of ['ask', 'execute', 'disabled'] as const) {
|
|
enabled = mode !== 'disabled'
|
|
const events = await collect(runtime.run({ requestId: crypto.randomUUID(), conversationId: 'graph', prompt: mode,
|
|
workMode: mode === 'execute' ? 'execute' : 'ask', knowledgeCapabilityToken: token }, signal))
|
|
expect(events.at(-1)?.type).toBe('done')
|
|
expect(JSON.stringify(events)).toContain(enabled ? 'GRAPH_READ' : 'DISABLED')
|
|
}
|
|
} finally { await runtime.dispose(); await provider.dispose(); await gateway.dispose(); db.close(); await rm(root, { recursive: true, force: true }) }
|
|
}, 60_000)
|
|
const SKILL_CALL_ID = 'e2e-skill-call'
|
|
const MCP_CALL_ID = 'e2e-mcp-call'
|
|
const ASK_MCP_CALL_ID = 'e2e-ask-mcp-call'
|
|
const MICRO_DELTA_COUNT = 30_000
|
|
const liveModelEnabled =
|
|
process.env.GOODBUDDY_DSH_MODEL_E2E === '1'
|
|
const liveApiKey = process.env.GOODBUDDY_DSH_API_KEY ?? ''
|
|
const liveBaseUrl =
|
|
process.env.GOODBUDDY_DSH_BASE_URL ?? 'https://api.deepseek.com'
|
|
const liveModel =
|
|
process.env.GOODBUDDY_DSH_MODEL ?? 'deepseek-chat'
|
|
|
|
function deferred<T>() {
|
|
let resolvePromise!: (value: T) => void
|
|
const promise = new Promise<T>((resolve) => {
|
|
resolvePromise = resolve
|
|
})
|
|
return { promise, resolve: resolvePromise }
|
|
}
|
|
|
|
function toolResultText(
|
|
options: GenerateOptions,
|
|
callId: string
|
|
): string | undefined {
|
|
for (const message of options.messages) {
|
|
if (message.role === 'tool' && message.toolCallId === callId) {
|
|
return message.content
|
|
.filter(
|
|
(
|
|
content
|
|
): content is Extract<
|
|
(typeof message.content)[number],
|
|
{ type: 'text' }
|
|
> => content.type === 'text'
|
|
)
|
|
.map((content) => content.text)
|
|
.join('\n')
|
|
}
|
|
}
|
|
return undefined
|
|
}
|
|
|
|
function latestUserText(options: GenerateOptions): string {
|
|
return options.messages
|
|
.filter(
|
|
(message) =>
|
|
message.role === 'user' &&
|
|
message.source?.kind === 'user'
|
|
)
|
|
.flatMap((message) =>
|
|
message.content
|
|
.filter(
|
|
(
|
|
content
|
|
): content is Extract<
|
|
(typeof message.content)[number],
|
|
{ type: 'text' }
|
|
> => content.type === 'text'
|
|
)
|
|
.map((content) => content.text)
|
|
)
|
|
.at(-1) ?? ''
|
|
}
|
|
|
|
async function* toolCall(
|
|
callId: string,
|
|
name: string,
|
|
argumentsValue: Record<string, unknown>
|
|
): AsyncGenerator<StreamChunk> {
|
|
const id = ToolCallId(callId)
|
|
const argumentsText = JSON.stringify(argumentsValue)
|
|
yield {
|
|
type: 'block-start',
|
|
index: 0,
|
|
blockType: 'tool-call'
|
|
}
|
|
yield {
|
|
type: 'tool-call-delta',
|
|
index: 0,
|
|
id,
|
|
name,
|
|
argumentsDelta: argumentsText
|
|
}
|
|
yield {
|
|
type: 'block-end',
|
|
index: 0,
|
|
block: {
|
|
type: 'tool-call',
|
|
id,
|
|
name,
|
|
arguments: argumentsText
|
|
}
|
|
}
|
|
yield {
|
|
type: 'usage',
|
|
usage: {
|
|
inputTokens: 10,
|
|
outputTokens: 5,
|
|
cacheReadTokens: 0,
|
|
cacheWriteTokens: 0
|
|
}
|
|
}
|
|
yield {
|
|
type: 'finish',
|
|
reason: { kind: 'tool-calls' }
|
|
}
|
|
}
|
|
|
|
async function* textResponse(
|
|
text: string
|
|
): AsyncGenerator<StreamChunk> {
|
|
yield {
|
|
type: 'block-start',
|
|
index: 0,
|
|
blockType: 'text'
|
|
}
|
|
yield {
|
|
type: 'text-delta',
|
|
index: 0,
|
|
text
|
|
}
|
|
yield {
|
|
type: 'block-end',
|
|
index: 0,
|
|
block: { type: 'text', text }
|
|
}
|
|
yield {
|
|
type: 'usage',
|
|
usage: {
|
|
inputTokens: 20,
|
|
outputTokens: 8,
|
|
cacheReadTokens: 0,
|
|
cacheWriteTokens: 0
|
|
}
|
|
}
|
|
yield {
|
|
type: 'finish',
|
|
reason: { kind: 'stop' }
|
|
}
|
|
}
|
|
|
|
async function* microDeltaResponse(): AsyncGenerator<StreamChunk> {
|
|
yield {
|
|
type: 'block-start',
|
|
index: 0,
|
|
blockType: 'reasoning'
|
|
}
|
|
for (let index = 0; index < MICRO_DELTA_COUNT; index += 1) {
|
|
yield {
|
|
type: 'reasoning-delta',
|
|
index: 0,
|
|
text: String(index % 10)
|
|
}
|
|
}
|
|
yield {
|
|
type: 'block-end',
|
|
index: 0,
|
|
block: {
|
|
type: 'reasoning',
|
|
text: Array.from(
|
|
{ length: MICRO_DELTA_COUNT },
|
|
(_value, index) => String(index % 10)
|
|
).join('')
|
|
}
|
|
}
|
|
yield {
|
|
type: 'usage',
|
|
usage: {
|
|
inputTokens: 20,
|
|
outputTokens: 8_000,
|
|
cacheReadTokens: 0,
|
|
cacheWriteTokens: 0
|
|
}
|
|
}
|
|
yield {
|
|
type: 'finish',
|
|
reason: { kind: 'stop' }
|
|
}
|
|
}
|
|
|
|
class FakeGameModel {
|
|
mcpToolName?: string
|
|
skillResult?: string
|
|
blueprint?: Record<string, unknown>
|
|
askToolResult?: string
|
|
executeToolNames: string[] = []
|
|
askToolNames: string[] = []
|
|
|
|
stream(options: GenerateOptions): AsyncIterable<StreamChunk> {
|
|
const prompt = latestUserText(options)
|
|
const toolNames = options.tools?.map((tool) => tool.name) ?? []
|
|
|
|
if (prompt.includes('ASK_BOUNDARY_PROBE')) {
|
|
this.askToolNames = toolNames
|
|
const result = toolResultText(options, ASK_MCP_CALL_ID)
|
|
if (!result) {
|
|
if (!this.mcpToolName) {
|
|
throw new Error('Fake model has no prior MCP tool identity')
|
|
}
|
|
return toolCall(ASK_MCP_CALL_ID, this.mcpToolName, {
|
|
theme: 'neon-ruins',
|
|
seed: 'ask-must-not-execute',
|
|
targetCount: 5
|
|
})
|
|
}
|
|
this.askToolResult = result
|
|
return textResponse('Ask mode MCP proxy unavailable as required.')
|
|
}
|
|
|
|
this.executeToolNames = toolNames
|
|
const skillResult = toolResultText(options, SKILL_CALL_ID)
|
|
if (!skillResult) {
|
|
return toolCall(SKILL_CALL_ID, 'skill', {
|
|
name: 'web-3d-game'
|
|
})
|
|
}
|
|
this.skillResult = skillResult
|
|
|
|
const blueprintResult = toolResultText(options, MCP_CALL_ID)
|
|
if (!blueprintResult) {
|
|
const mcpTool = options.tools?.find((tool) =>
|
|
tool.name.endsWith('_create_game_blueprint')
|
|
)
|
|
if (!mcpTool) {
|
|
throw new Error(
|
|
'Main-mediated 3D blueprint MCP tool was not exposed'
|
|
)
|
|
}
|
|
this.mcpToolName = mcpTool.name
|
|
return toolCall(MCP_CALL_ID, mcpTool.name, {
|
|
theme: 'neon-ruins',
|
|
seed: 'goodbuddy-0.9.0',
|
|
targetCount: 5
|
|
})
|
|
}
|
|
this.blueprint = JSON.parse(
|
|
blueprintResult
|
|
) as Record<string, unknown>
|
|
return textResponse(
|
|
'Loaded the Web 3D Game Skill and the approved Prism Relay blueprint.'
|
|
)
|
|
}
|
|
}
|
|
|
|
type HarnessModel = {
|
|
stream(options: GenerateOptions): AsyncIterable<StreamChunk>
|
|
}
|
|
|
|
async function collect(
|
|
stream: AsyncGenerator<RuntimeEvent, void, void>
|
|
): Promise<RuntimeEvent[]> {
|
|
const events: RuntimeEvent[] = []
|
|
for await (const event of stream) {
|
|
events.push(event)
|
|
}
|
|
return events
|
|
}
|
|
|
|
function createInProcessLaunch(
|
|
dshHome: string,
|
|
model?: HarnessModel,
|
|
observeStream?: (options: GenerateOptions) => void
|
|
): {
|
|
launch(
|
|
options: DeepSeekHarnessLaunchOptions
|
|
): Promise<DeepSeekHarnessChild>
|
|
hosts: ControlledHarnessHost[]
|
|
} {
|
|
const hosts: ControlledHarnessHost[] = []
|
|
return {
|
|
hosts,
|
|
async launch(options) {
|
|
const clientToHost =
|
|
new TransformStream<Uint8Array, Uint8Array>()
|
|
const hostToClient =
|
|
new TransformStream<Uint8Array, Uint8Array>()
|
|
const exited = deferred<{
|
|
exitCode: number | null
|
|
signal?: string | null
|
|
}>()
|
|
const host = await startControlledDeepSeekHarnessHost({
|
|
workspace: options.cwd,
|
|
dshHome,
|
|
baseUrl: options.baseUrl,
|
|
api: 'openai-completions',
|
|
provider: 'goodbuddy',
|
|
model: options.model,
|
|
supportsImageInput: options.supportsImageInput,
|
|
requestHeaders: options.requestHeaders,
|
|
harnessVersion: '0.1.7-rc.2',
|
|
credentialRefs: options.credentialRefs,
|
|
skillPackages: options.skillPackages,
|
|
extensionPackages: options.extensionPackages,
|
|
stream: createBoundedNdJsonStream(
|
|
hostToClient.writable,
|
|
clientToHost.readable,
|
|
DEEPSEEK_HARNESS_MAX_FRAME_BYTES
|
|
)
|
|
})
|
|
hosts.push(host)
|
|
if (observeStream) {
|
|
host.context.on(
|
|
'llm/stream',
|
|
(request, next) => {
|
|
observeStream(request)
|
|
return next()
|
|
},
|
|
{ global: true, prepend: true }
|
|
)
|
|
}
|
|
if (model) {
|
|
host.context.on(
|
|
'llm/stream',
|
|
(request) => model.stream(request),
|
|
{ global: true, prepend: true }
|
|
)
|
|
}
|
|
let terminated = false
|
|
return {
|
|
stdin: clientToHost.writable,
|
|
stdout: hostToClient.readable,
|
|
exited: exited.promise,
|
|
async terminate() {
|
|
if (terminated) {
|
|
return
|
|
}
|
|
terminated = true
|
|
await host.dispose().catch(() => undefined)
|
|
await Promise.allSettled([
|
|
clientToHost.writable.close(),
|
|
hostToClient.writable.close()
|
|
])
|
|
exited.resolve({ exitCode: 0 })
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
describe('DeepSeek Harness real ACP control-plane E2E', () => {
|
|
it.each([2, 10])('shares one production-composed Host across %i projects with independent native file and shell tools', async (projectCount) => {
|
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-reuse-')))
|
|
const directories = Array.from(
|
|
{ length: projectCount }, (_, index) => join(root, `project-${index}`)
|
|
)
|
|
await Promise.all(directories.map(async (directory, index) => {
|
|
await mkdir(directory)
|
|
await writeFile(join(directory, 'marker.txt'), `PROJECT_${index}`)
|
|
}))
|
|
const inProcess = createInProcessLaunch(root, {
|
|
stream(options) {
|
|
const id = latestUserText(options)
|
|
const read = toolResultText(options, `${id}-read`)
|
|
if (read === undefined) return toolCall(`${id}-read`, 'read', { file_path: 'marker.txt' })
|
|
const write = toolResultText(options, `${id}-write`)
|
|
if (write === undefined) return toolCall(`${id}-write`, 'write', {
|
|
file_path: `${id}.txt`, content: read
|
|
})
|
|
const shell = toolResultText(options, `${id}-shell`)
|
|
if (shell === undefined) return toolCall(`${id}-shell`, process.platform === 'win32' ? 'pwsh' : 'bash', {
|
|
command: process.platform === 'win32' ? 'Get-Content marker.txt' : 'cat marker.txt',
|
|
description: 'Read the session workspace fixture'
|
|
})
|
|
return textResponse(shell)
|
|
}
|
|
})
|
|
const launch = vi.fn(inProcess.launch)
|
|
const registry = new LocalRuntimeRegistry()
|
|
const resolver = new ExecutionSpaceResolver()
|
|
const profile = {
|
|
id: '00000000-0000-4000-8000-000000000009',
|
|
name: 'Reuse fixture', baseUrl: 'http://127.0.0.1:9',
|
|
modelName: 'reuse', protocol: 'openai-chat-completions' as const,
|
|
authentication: 'api-key' as const, apiKey: 'unused-deterministic-model'
|
|
}
|
|
const settings = {
|
|
...defaultRuntimeSettings, provider: 'deepseek-harness',
|
|
modelProfiles: [profile], deepseekHarnessModelProfile: profile
|
|
} as ResolvedRuntimeSettings
|
|
const manager = new SelectedRuntimeManager(async (_selection, space) =>
|
|
createAgentRuntime(root, settings, {
|
|
executionSpace: space, localRuntimeRegistry: registry,
|
|
deepseekHarnessLauncher: launch
|
|
}), 1, 1, undefined, undefined, (id) => registry.releaseConversation(id))
|
|
try {
|
|
const selection = { provider: 'deepseek-harness' as const }
|
|
for (let wave = 0; wave < 2; wave++) {
|
|
const outputs = await Promise.all(directories.flatMap((directory, project) =>
|
|
[0, 1].map(async (conversation) => {
|
|
const runtime = await manager.getRuntime(selection, resolver.resolveLocal(directory))
|
|
const id = `p${project}-c${conversation}-w${wave}`
|
|
const events = await collect(runtime.run({
|
|
requestId: id, conversationId: `p${project}-c${conversation}`,
|
|
prompt: id, workMode: 'execute'
|
|
}, AbortSignal.timeout(20_000)))
|
|
expect(events.at(-1)?.type).toBe('done')
|
|
expect(await readFile(join(directory, `${id}.txt`), 'utf8')).toContain(`PROJECT_${project}`)
|
|
expect(events.filter((event) => event.type === 'text').map((event) => event.delta).join(''))
|
|
.toContain(`PROJECT_${project}`)
|
|
return events
|
|
})
|
|
))
|
|
expect(outputs).toHaveLength(projectCount * 2)
|
|
expect(launch).toHaveBeenCalledOnce()
|
|
}
|
|
await manager.releaseConversation('p0-c0')
|
|
expect(launch).toHaveBeenCalledOnce()
|
|
expect(inProcess.hosts).toHaveLength(1)
|
|
} finally {
|
|
await manager.dispose()
|
|
await registry.dispose()
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 60_000)
|
|
|
|
it.skipIf(process.platform !== 'win32')('reuses a working runtime for mixed-separator and canonical Windows paths', async () => {
|
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-path-')))
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'home')
|
|
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
|
const canonical = await realpath(workspace)
|
|
const mixed = `${root.replaceAll('\\', '/')}\\workspace`
|
|
const resolver = new ExecutionSpaceResolver()
|
|
const inProcess = createInProcessLaunch(dshHome, {
|
|
stream: () => textResponse('PATH_OK')
|
|
})
|
|
const launch = vi.fn(inProcess.launch)
|
|
const manager = new SelectedRuntimeManager(async (_selection, space) => {
|
|
if (space?.kind !== 'local') throw new Error('local workspace required')
|
|
return new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: space.rootPath, baseUrl: 'https://api.deepseek.com',
|
|
model: 'path-test', launch,
|
|
credentialRefs: { [CREDENTIAL_REF]: 'unused-in-memory-model-credential' },
|
|
initializationTimeoutMs: 20_000, promptTimeoutMs: 20_000, shutdownTimeoutMs: 5_000
|
|
})
|
|
})
|
|
try {
|
|
expect(mixed).not.toBe(canonical)
|
|
expect(await realpath(mixed)).toBe(canonical)
|
|
const first = await manager.getRuntime({ provider: 'deepseek-harness' }, resolver.resolveLocal(mixed))
|
|
for (const [index, spelling] of [mixed, canonical].entries()) {
|
|
const runtime = await manager.getRuntime({ provider: 'deepseek-harness' }, resolver.resolveLocal(spelling))
|
|
expect(runtime).toBe(first)
|
|
const events = await collect(runtime.run({
|
|
requestId: `path-${index}`, conversationId: `path-${index}`,
|
|
workMode: 'ask', prompt: 'Return PATH_OK.'
|
|
}, AbortSignal.timeout(20_000)))
|
|
expect(events.at(-1)).toMatchObject({ type: 'done' })
|
|
expect(events.filter(event => event.type === 'text').map(event => event.delta).join('')).toBe('PATH_OK')
|
|
}
|
|
expect(launch).toHaveBeenCalledOnce()
|
|
expect(launch.mock.calls[0]?.[0].cwd).toBe(canonical)
|
|
} finally {
|
|
await manager.dispose()
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
it('keeps native Skills, plugin inventory, MCP calls and images scoped across shared projects', async () => {
|
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-shared-capabilities-')))
|
|
const directories = [join(root, 'one'), join(root, 'two')]
|
|
await Promise.all(directories.map(directory => mkdir(directory)))
|
|
const plugin = join(root, 'inventory-plugin.mjs')
|
|
await writeFile(plugin, [
|
|
"export const name = 'shared-inventory'",
|
|
"export const inject = ['skills']",
|
|
'export function apply(ctx) {',
|
|
"ctx.skills.register({ name: 'shared-plugin-skill', description: 'Shared plugin fixture', content: '# Shared plugin', source: 'custom' })",
|
|
'}'
|
|
].join('\n'))
|
|
const models = [new FakeGameModel(), new FakeGameModel()]
|
|
const images = [createCanvas(1, 1).toBuffer('image/png'), createCanvas(2, 1).toBuffer('image/png')]
|
|
const seenImages = new Map<number, GenerateOptions['messages'][number]['content']>()
|
|
const inProcess = createInProcessLaunch(root, {
|
|
stream(options) {
|
|
const project = latestUserText(options).includes('PROJECT_1') ? 1 : 0
|
|
seenImages.set(project, options.messages.flatMap(message => message.content).filter(block => block.type === 'image'))
|
|
return models[project]!.stream(options)
|
|
}
|
|
})
|
|
const launch = vi.fn(inProcess.launch)
|
|
const calls = vi.spyOn(ModelToolProvider.prototype, 'callTool')
|
|
const registry = new LocalRuntimeRegistry()
|
|
const profile = {
|
|
id: '00000000-0000-4000-8000-000000000010',
|
|
name: 'Capabilities fixture', baseUrl: 'http://127.0.0.1:9',
|
|
modelName: 'capabilities', protocol: 'openai-chat-completions' as const,
|
|
authentication: 'api-key' as const, apiKey: 'unused-deterministic-model',
|
|
supportsImageInput: true
|
|
}
|
|
const resolver = new ExecutionSpaceResolver()
|
|
const runtimes = directories.map(directory => createAgentRuntime(root, {
|
|
...defaultRuntimeSettings, provider: 'deepseek-harness',
|
|
modelProfiles: [profile], deepseekHarnessModelProfile: profile
|
|
} as ResolvedRuntimeSettings, {
|
|
executionSpace: resolver.resolveLocal(directory), localRuntimeRegistry: registry,
|
|
deepseekHarnessLauncher: launch,
|
|
skillPackages: [{ id: 'web-3d-game', directory: resolve('tests/fixtures/web-3d-game-skill') }],
|
|
deepseekHarnessExtensions: [{ id: 'shared-inventory', entrypoint: plugin, configuration: {} }],
|
|
mcpServers: [{
|
|
id: 'fbf42200-4e60-48d0-b5f2-e816db38ac54', name: 'Shared blueprint', description: '',
|
|
enabled: true, allowDynamicTools: false, assignments: ['deepseek-harness'],
|
|
secretConfigured: false, transport: 'stdio',
|
|
command: process.versions.electron ? 'node' : process.execPath,
|
|
args: [resolve('tests/fixtures/web-3d-game-mcp.mjs')]
|
|
}]
|
|
}))
|
|
try {
|
|
for (const mode of ['execute', 'ask'] as const) {
|
|
await Promise.all(runtimes.map(async (runtime, project) => {
|
|
const events = await collect(runtime.run({
|
|
requestId: `capabilities-${project}-${mode}`, conversationId: `capabilities-${project}`,
|
|
workMode: mode,
|
|
prompt: `${mode === 'ask' ? 'ASK_BOUNDARY_PROBE' : 'Use Skill and MCP'} PROJECT_${project}`,
|
|
...(mode === 'execute' ? { images: [{
|
|
name: `project-${project}.png`, mediaType: 'image/png', data: images[project]!.toString('base64')
|
|
}] } : {})
|
|
}, AbortSignal.timeout(30_000), async () => 'once'))
|
|
expect(events.at(-1)?.type).toBe('done')
|
|
const model = models[project]!
|
|
if (mode === 'execute') {
|
|
expect(model.skillResult).toContain('window.__GOODBUDDY_GAME__')
|
|
expect(model.blueprint).toMatchObject({ title: 'Prism Relay' })
|
|
const seen = seenImages.get(project)!
|
|
expect(seen).toHaveLength(1)
|
|
const image = seen[0]!
|
|
if (image.type !== 'image') throw new Error('Expected image block')
|
|
expect(image.attachment).toMatchObject({ width: project + 1, height: 1 })
|
|
const stored = await inProcess.hosts[0]!.context.attachments.readImage(image.attachment)
|
|
expect(Buffer.from(stored.data).equals(images[project]!)).toBe(true)
|
|
} else {
|
|
expect(model.askToolNames).not.toContain(model.mcpToolName)
|
|
expect(model.askToolResult).toContain('Ask 模式不允许执行非只读工具')
|
|
}
|
|
expect(await runtime.getNativeSnapshot?.()).toMatchObject({
|
|
skills: expect.arrayContaining([expect.objectContaining({ id: 'shared-plugin-skill', source: 'plugin' })])
|
|
})
|
|
}))
|
|
expect(launch).toHaveBeenCalledOnce()
|
|
expect(calls).toHaveBeenCalledTimes(2)
|
|
}
|
|
expect(new Set(calls.mock.contexts).size).toBe(2)
|
|
expect(calls.mock.calls.map(call => call[3]?.conversationId).sort())
|
|
.toEqual(['capabilities-0', 'capabilities-1'])
|
|
await runtimes[0]!.releaseConversation?.('capabilities-0')
|
|
expect(await runtimes[1]!.getStatus()).toMatchObject({ available: true })
|
|
expect(launch).toHaveBeenCalledOnce()
|
|
} finally {
|
|
calls.mockRestore()
|
|
await Promise.all(runtimes.map(runtime => runtime.dispose()))
|
|
await registry.dispose()
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 60_000)
|
|
|
|
it('writes outside the Workspace in Execute without approval and rejects the same tool in Ask', async () => {
|
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-directory-')))
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'home')
|
|
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
|
const externalFile = join(root, 'external.txt')
|
|
const results: string[] = []
|
|
const inProcess = createInProcessLaunch(dshHome, {
|
|
stream(options) {
|
|
const result = toolResultText(options, 'external-write')
|
|
if (result !== undefined) {
|
|
results.push(result)
|
|
return textResponse('DIRECTORY_TEST_DONE')
|
|
}
|
|
return toolCall('external-write', 'write', {
|
|
file_path: externalFile, content: latestUserText(options)
|
|
})
|
|
}
|
|
})
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace, baseUrl: 'https://api.deepseek.com',
|
|
model: 'directory-test', launch: options => inProcess.launch(options),
|
|
credentialRefs: { [CREDENTIAL_REF]: 'unused-in-memory-model-credential' },
|
|
initializationTimeoutMs: 20_000, promptTimeoutMs: 20_000, shutdownTimeoutMs: 5_000
|
|
})
|
|
const authorize = vi.fn(async () => 'deny' as const)
|
|
try {
|
|
for (const mode of ['execute', 'ask'] as const) {
|
|
const events = await collect(runtime.run({
|
|
requestId: `directory-${mode}`, conversationId: `directory-${mode}`,
|
|
workMode: mode, prompt: mode === 'execute' ? 'EXECUTE_WRITTEN' : 'ASK_MUST_NOT_WRITE'
|
|
}, AbortSignal.timeout(20_000), authorize))
|
|
expect(events.at(-1)).toMatchObject({ type: 'done' })
|
|
expect(await readFile(externalFile, 'utf8')).toBe('EXECUTE_WRITTEN')
|
|
}
|
|
expect(results).toHaveLength(2)
|
|
expect(authorize).not.toHaveBeenCalled()
|
|
} finally {
|
|
await runtime.dispose()
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
it('delivers bounded inline images to an image-capable Harness model', async () => {
|
|
const root = await realpath(
|
|
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-acp-image-'))
|
|
)
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'dsh-home')
|
|
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
|
let observedRequest: GenerateOptions | undefined
|
|
const inProcess = createInProcessLaunch(dshHome, {
|
|
stream(options) {
|
|
observedRequest = options
|
|
return textResponse('Image received.')
|
|
}
|
|
})
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace,
|
|
baseUrl: 'https://api.deepseek.com',
|
|
model: 'vision-test',
|
|
supportsImageInput: true,
|
|
launch: (options) => inProcess.launch(options),
|
|
credentialRefs: {
|
|
[CREDENTIAL_REF]: 'unused-in-memory-model-credential'
|
|
},
|
|
initializationTimeoutMs: 20_000,
|
|
promptTimeoutMs: 20_000,
|
|
shutdownTimeoutMs: 5_000
|
|
})
|
|
const png = createCanvas(1, 1).toBuffer('image/png')
|
|
|
|
try {
|
|
const events = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-acp-image',
|
|
conversationId: 'acp-image',
|
|
prompt: 'Describe this image.',
|
|
workMode: 'ask',
|
|
images: [
|
|
{
|
|
name: 'reference.png',
|
|
mediaType: 'image/png',
|
|
data: png.toString('base64')
|
|
}
|
|
]
|
|
},
|
|
new AbortController().signal
|
|
)
|
|
)
|
|
const image = observedRequest?.messages
|
|
.flatMap((message) => message.content)
|
|
.find(
|
|
(
|
|
block
|
|
): block is Extract<
|
|
GenerateOptions['messages'][number]['content'][number],
|
|
{ type: 'image' }
|
|
> => block.type === 'image'
|
|
)
|
|
|
|
expect(events).toContainEqual(
|
|
expect.objectContaining({ type: 'done' })
|
|
)
|
|
expect(image?.attachment).toMatchObject({
|
|
mediaType: 'image/png',
|
|
bytes: png.byteLength,
|
|
width: 1,
|
|
height: 1
|
|
})
|
|
const stored =
|
|
await inProcess.hosts[0]!.context.attachments.readImage(
|
|
image!.attachment
|
|
)
|
|
expect(Buffer.from(stored.data).equals(png)).toBe(true)
|
|
} finally {
|
|
await runtime.dispose()
|
|
await Promise.allSettled(
|
|
inProcess.hosts.map((host) => host.dispose())
|
|
)
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
it(
|
|
'coalesces micro reasoning deltas without losing content or overriding model output tokens',
|
|
async () => {
|
|
const root = await realpath(
|
|
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-acp-deltas-'))
|
|
)
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'dsh-home')
|
|
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
|
let observedRequest: GenerateOptions | undefined
|
|
const inProcess = createInProcessLaunch(dshHome, {
|
|
stream(options) {
|
|
observedRequest = options
|
|
return microDeltaResponse()
|
|
}
|
|
})
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace,
|
|
baseUrl: 'https://api.deepseek.com',
|
|
model: 'deepseek-test',
|
|
launch: (options) => inProcess.launch(options),
|
|
credentialRefs: {
|
|
[CREDENTIAL_REF]: 'unused-in-memory-model-credential'
|
|
},
|
|
initializationTimeoutMs: 20_000,
|
|
promptTimeoutMs: 20_000,
|
|
shutdownTimeoutMs: 5_000
|
|
})
|
|
|
|
try {
|
|
const events = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-acp-deltas',
|
|
conversationId: 'acp-deltas',
|
|
prompt: 'Return the deterministic reasoning stream.',
|
|
workMode: 'execute'
|
|
},
|
|
new AbortController().signal
|
|
)
|
|
)
|
|
const reasoning = events.filter(
|
|
(
|
|
event
|
|
): event is Extract<
|
|
RuntimeEvent,
|
|
{ type: 'reasoning' }
|
|
> => event.type === 'reasoning'
|
|
)
|
|
|
|
expect(observedRequest?.maxTokens).toBeUndefined()
|
|
expect(
|
|
observedRequest?.messages
|
|
.filter((message) => message.role === 'system')
|
|
.flatMap((message) => message.content)
|
|
.filter((block) => block.type === 'text')
|
|
.map((block) => block.text)
|
|
.join('\n')
|
|
).toContain('act through the available tools')
|
|
expect(reasoning).toHaveLength(8)
|
|
expect(
|
|
reasoning.map((event) => event.delta).join('')
|
|
).toBe(
|
|
Array.from(
|
|
{ length: MICRO_DELTA_COUNT },
|
|
(_value, index) => String(index % 10)
|
|
).join('')
|
|
)
|
|
expect(events.at(-1)).toMatchObject({ type: 'done' })
|
|
} finally {
|
|
await runtime.dispose()
|
|
await Promise.allSettled(
|
|
inProcess.hosts.map((host) => host.dispose())
|
|
)
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
},
|
|
30_000
|
|
)
|
|
|
|
it(
|
|
'rejects the ACP prompt with a bounded model turn error',
|
|
async () => {
|
|
const root = await realpath(
|
|
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-acp-error-'))
|
|
)
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'dsh-home')
|
|
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
|
const inProcess = createInProcessLaunch(dshHome, {
|
|
stream() {
|
|
throw new Error('synthetic model turn failed')
|
|
}
|
|
})
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace,
|
|
baseUrl: 'https://api.deepseek.com',
|
|
model: 'deepseek-test',
|
|
launch: (options) => inProcess.launch(options),
|
|
credentialRefs: {
|
|
[CREDENTIAL_REF]: 'unused-in-memory-model-credential'
|
|
},
|
|
initializationTimeoutMs: 20_000,
|
|
promptTimeoutMs: 2_000,
|
|
shutdownTimeoutMs: 5_000
|
|
})
|
|
|
|
try {
|
|
await expect(
|
|
collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-acp-error',
|
|
conversationId: 'acp-error',
|
|
prompt: 'Trigger the synthetic model failure.',
|
|
workMode: 'ask'
|
|
},
|
|
new AbortController().signal
|
|
)
|
|
)
|
|
).rejects.toThrow('synthetic model turn failed')
|
|
} finally {
|
|
await runtime.dispose()
|
|
await Promise.allSettled(
|
|
inProcess.hosts.map((host) => host.dispose())
|
|
)
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
},
|
|
30_000
|
|
)
|
|
|
|
it(
|
|
'loads a native Skill, calls an approved real MCP, forwards events, and removes MCP in Ask',
|
|
async () => {
|
|
const root = await realpath(
|
|
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-acp-e2e-'))
|
|
)
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'dsh-home')
|
|
await Promise.all([
|
|
mkdir(workspace),
|
|
mkdir(dshHome)
|
|
])
|
|
const inventoryPlugin = join(
|
|
root,
|
|
'native-inventory-plugin.mjs'
|
|
)
|
|
await writeFile(
|
|
inventoryPlugin,
|
|
[
|
|
"export const name = 'native-inventory-plugin'",
|
|
"export const inject = ['skills']",
|
|
'export function apply(ctx) {',
|
|
' ctx.skills.register({',
|
|
" name: 'plugin-native-skill',",
|
|
" description: 'Skill contributed by a Host plugin.',",
|
|
" content: '# Plugin native skill',",
|
|
" source: 'custom'",
|
|
' })',
|
|
'}'
|
|
].join('\n'),
|
|
'utf8'
|
|
)
|
|
const provider = new ModelToolProvider(workspace, [
|
|
{
|
|
id: 'fbf42200-4e60-48d0-b5f2-e816db38ac54',
|
|
name: 'Local 3D Game Blueprint',
|
|
description: 'Deterministic integration fixture',
|
|
enabled: true,
|
|
allowDynamicTools: false,
|
|
assignments: ['deepseek-harness'],
|
|
secretConfigured: false,
|
|
transport: 'stdio',
|
|
command: process.versions.electron ? 'node' : process.execPath,
|
|
args: [
|
|
resolve(
|
|
'tests',
|
|
'fixtures',
|
|
'web-3d-game-mcp.mjs'
|
|
)
|
|
]
|
|
} satisfies ResolvedMcpServer
|
|
])
|
|
const callTool = vi.spyOn(provider, 'callTool')
|
|
const listTools = vi.spyOn(provider, 'listTools')
|
|
const fakeModel = new FakeGameModel()
|
|
const inProcess = createInProcessLaunch(dshHome, fakeModel)
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace,
|
|
baseUrl: 'https://api.deepseek.com',
|
|
model: 'deepseek-test',
|
|
launch: (options) => inProcess.launch(options),
|
|
credentialRefs: {
|
|
[CREDENTIAL_REF]: 'unused-in-memory-model-credential'
|
|
},
|
|
skillPackages: [
|
|
{
|
|
id: 'web-3d-game',
|
|
directory: resolve(
|
|
'tests',
|
|
'fixtures',
|
|
'web-3d-game-skill'
|
|
)
|
|
}
|
|
],
|
|
extensionPackages: [
|
|
{
|
|
id: 'native-inventory-plugin',
|
|
entrypoint: inventoryPlugin,
|
|
configuration: {}
|
|
}
|
|
],
|
|
toolProvider: provider,
|
|
initializationTimeoutMs: 20_000,
|
|
promptTimeoutMs: 20_000,
|
|
shutdownTimeoutMs: 5_000
|
|
})
|
|
const authorize = vi.fn(
|
|
async (
|
|
request: Parameters<
|
|
NonNullable<
|
|
Parameters<DeepSeekHarnessRuntime['run']>[2]
|
|
>
|
|
>[0]
|
|
) =>
|
|
request.scopeKey.startsWith('model:mcp:')
|
|
? ('once' as const)
|
|
: ('deny' as const)
|
|
)
|
|
|
|
try {
|
|
await runtime.testConnection()
|
|
expect(inProcess.hosts[0]?.extensionFailures).toEqual([])
|
|
await expect(runtime.getNativeSnapshot()).resolves.toMatchObject({
|
|
provider: 'deepseek-harness',
|
|
available: true,
|
|
inventoryStatus: 'available',
|
|
toolsSupported: true,
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
id: 'read',
|
|
kind: 'read',
|
|
source: 'runtime',
|
|
ask: 'allowed',
|
|
execute: 'allowed'
|
|
}),
|
|
expect.objectContaining({
|
|
id: 'edit',
|
|
kind: 'write',
|
|
source: 'runtime',
|
|
ask: 'blocked',
|
|
execute: 'allowed'
|
|
})
|
|
]),
|
|
skills: [
|
|
{
|
|
id: 'plugin-native-skill',
|
|
name: 'plugin-native-skill',
|
|
description: 'Skill contributed by a Host plugin.',
|
|
source: 'plugin'
|
|
}
|
|
],
|
|
mcpServers: [],
|
|
agents: [],
|
|
commands: [],
|
|
lsp: [],
|
|
formatters: [],
|
|
prompts: [],
|
|
resources: [],
|
|
resourcesSupported: false,
|
|
context: {
|
|
strategy: 'unsupported',
|
|
manualCompact: false
|
|
}
|
|
})
|
|
const executeEvents = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-acp-execute',
|
|
conversationId: 'acp-e2e',
|
|
prompt:
|
|
'Use the Web 3D Game Skill and assigned blueprint MCP.',
|
|
workMode: 'execute'
|
|
},
|
|
new AbortController().signal,
|
|
authorize
|
|
)
|
|
)
|
|
|
|
expect(fakeModel.executeToolNames).toContain('skill')
|
|
expect(fakeModel.mcpToolName).toMatch(
|
|
/_create_game_blueprint$/u
|
|
)
|
|
expect(fakeModel.skillResult).toContain(
|
|
'window.__GOODBUDDY_GAME__'
|
|
)
|
|
expect(fakeModel.blueprint).toMatchObject({
|
|
title: 'Prism Relay',
|
|
objective: { targetCount: 5 },
|
|
acceptance: {
|
|
testSurface: 'window.__GOODBUDDY_GAME__'
|
|
}
|
|
})
|
|
expect(authorize).toHaveBeenCalledOnce()
|
|
expect(callTool).toHaveBeenCalledWith(
|
|
fakeModel.mcpToolName,
|
|
{
|
|
theme: 'neon-ruins',
|
|
seed: 'goodbuddy-0.9.0',
|
|
targetCount: 5
|
|
},
|
|
expect.any(AbortSignal),
|
|
{
|
|
conversationId: 'acp-e2e',
|
|
workMode: 'execute',
|
|
knowledgeCapabilityToken: undefined
|
|
} satisfies ModelToolCallContext
|
|
)
|
|
expect(executeEvents).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
callId: SKILL_CALL_ID,
|
|
name: 'skill',
|
|
state: 'pending'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
callId: SKILL_CALL_ID,
|
|
state: 'completed'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
callId: MCP_CALL_ID,
|
|
name: fakeModel.mcpToolName,
|
|
state: 'pending'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
callId: MCP_CALL_ID,
|
|
state: 'completed'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'text',
|
|
delta: expect.stringContaining('Prism Relay')
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'model-usage',
|
|
runtime: 'deepseek-harness'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'done',
|
|
sessionId: expect.any(String)
|
|
})
|
|
])
|
|
)
|
|
expect(
|
|
executeEvents.filter(
|
|
(event) =>
|
|
event.type === 'tool' &&
|
|
event.state === 'running'
|
|
)
|
|
).toHaveLength(0)
|
|
|
|
const callsBeforeAsk = callTool.mock.calls.length
|
|
const listsBeforeAsk = listTools.mock.calls.length
|
|
const approvalsBeforeAsk = authorize.mock.calls.length
|
|
const askEvents = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-acp-ask',
|
|
conversationId: 'acp-e2e',
|
|
prompt:
|
|
'ASK_BOUNDARY_PROBE: attempt the previous MCP tool.',
|
|
workMode: 'ask'
|
|
},
|
|
new AbortController().signal,
|
|
authorize
|
|
)
|
|
)
|
|
|
|
expect(fakeModel.askToolNames).not.toContain(
|
|
fakeModel.mcpToolName
|
|
)
|
|
expect(fakeModel.askToolResult).toContain(
|
|
'Ask 模式不允许执行非只读工具'
|
|
)
|
|
expect(callTool).toHaveBeenCalledTimes(callsBeforeAsk)
|
|
expect(listTools).toHaveBeenCalledTimes(listsBeforeAsk + 1)
|
|
expect(listTools).toHaveBeenLastCalledWith(
|
|
{
|
|
conversationId: 'acp-e2e',
|
|
workMode: 'ask',
|
|
knowledgeCapabilityToken: undefined
|
|
},
|
|
expect.any(AbortSignal)
|
|
)
|
|
expect(authorize).toHaveBeenCalledTimes(approvalsBeforeAsk)
|
|
expect(askEvents).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
callId: ASK_MCP_CALL_ID,
|
|
name: fakeModel.mcpToolName,
|
|
state: 'pending'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
callId: ASK_MCP_CALL_ID,
|
|
state: 'failed'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'text',
|
|
delta: expect.stringContaining(
|
|
'MCP proxy unavailable'
|
|
)
|
|
}),
|
|
expect.objectContaining({ type: 'done' })
|
|
])
|
|
)
|
|
} finally {
|
|
await runtime.dispose()
|
|
await Promise.allSettled(
|
|
inProcess.hosts.map((host) => host.dispose())
|
|
)
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
},
|
|
60_000
|
|
)
|
|
|
|
it.runIf(liveModelEnabled)(
|
|
'forwards a custom model header through the real DeepSeek Harness provider',
|
|
async () => {
|
|
if (!liveApiKey) {
|
|
throw new Error(
|
|
'GOODBUDDY_DSH_API_KEY is required for live DSH model E2E'
|
|
)
|
|
}
|
|
const root = await realpath(
|
|
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-header-model-'))
|
|
)
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'dsh-home')
|
|
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
|
const probe = await createModelRequestProbe({
|
|
upstreamUrl: createOpenAIChatCompletionsUrl(liveBaseUrl),
|
|
headerName: 'x-goodbuddy-e2e'
|
|
})
|
|
const inProcess = createInProcessLaunch(dshHome)
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace,
|
|
baseUrl: probe.baseUrl,
|
|
model: liveModel,
|
|
requestHeaders: {
|
|
'x-goodbuddy-e2e': 'harness-header-present'
|
|
},
|
|
launch: (options) => inProcess.launch(options),
|
|
credentialRefs: {
|
|
[CREDENTIAL_REF]: liveApiKey
|
|
},
|
|
initializationTimeoutMs: 20_000,
|
|
promptTimeoutMs: 120_000,
|
|
shutdownTimeoutMs: 5_000
|
|
})
|
|
|
|
try {
|
|
const events = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-live-custom-header',
|
|
conversationId: 'live-custom-header',
|
|
prompt:
|
|
'Reply with exactly DSH_CUSTOM_HEADER_E2E_OK and nothing else.',
|
|
workMode: 'ask'
|
|
},
|
|
new AbortController().signal
|
|
)
|
|
)
|
|
expect(
|
|
events
|
|
.flatMap((event) =>
|
|
event.type === 'text' ? [event.delta] : []
|
|
)
|
|
.join('')
|
|
).toContain('DSH_CUSTOM_HEADER_E2E_OK')
|
|
expect(probe.observations).toEqual([
|
|
{
|
|
headerValue: 'harness-header-present',
|
|
bodyFields: {}
|
|
}
|
|
])
|
|
} finally {
|
|
await runtime.dispose()
|
|
await Promise.allSettled(
|
|
inProcess.hosts.map((host) => host.dispose())
|
|
)
|
|
await probe.close()
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
},
|
|
180_000
|
|
)
|
|
|
|
it.runIf(liveModelEnabled)(
|
|
'lets a real model use Main-brokered Web Search and Fetch in Ask',
|
|
async () => {
|
|
if (!liveApiKey) {
|
|
throw new Error(
|
|
'GOODBUDDY_DSH_API_KEY is required for live DSH model E2E'
|
|
)
|
|
}
|
|
const root = await realpath(
|
|
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-web-model-'))
|
|
)
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'dsh-home')
|
|
await Promise.all([mkdir(workspace), mkdir(dshHome)])
|
|
const observedRequests: GenerateOptions[] = []
|
|
const inProcess = createInProcessLaunch(
|
|
dshHome,
|
|
undefined,
|
|
(options) => observedRequests.push(options)
|
|
)
|
|
const toolProvider = new ModelToolProvider(
|
|
workspace,
|
|
[],
|
|
undefined,
|
|
undefined,
|
|
true
|
|
)
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace,
|
|
baseUrl: liveBaseUrl,
|
|
model: liveModel,
|
|
launch: (options) => inProcess.launch(options),
|
|
credentialRefs: {
|
|
[CREDENTIAL_REF]: liveApiKey
|
|
},
|
|
toolProvider,
|
|
initializationTimeoutMs: 20_000,
|
|
promptTimeoutMs: 120_000,
|
|
shutdownTimeoutMs: 5_000
|
|
})
|
|
|
|
try {
|
|
const events = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-live-web-search',
|
|
conversationId: 'live-web-search',
|
|
prompt:
|
|
'DSH_WEB_TOOLS_PROBE: First call web_search exactly once with query "GoodBuddy GitHub desktop assistant" and numResults 2. Then call web_fetch exactly once with urls ["https://example.com/"] and maxCharacters 1000. Do not call another tool. After both results, reply with DSH_WEB_TOOLS_E2E_OK.',
|
|
workMode: 'ask'
|
|
},
|
|
new AbortController().signal
|
|
)
|
|
)
|
|
expect(
|
|
observedRequests.flatMap(
|
|
(options) =>
|
|
options.tools?.map((tool) => tool.name) ?? []
|
|
)
|
|
).toEqual(
|
|
expect.arrayContaining(['web_search', 'web_fetch'])
|
|
)
|
|
expect(events).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
name: 'web_search',
|
|
state: 'completed'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
name: 'web_fetch',
|
|
state: 'completed'
|
|
})
|
|
])
|
|
)
|
|
expect(
|
|
events
|
|
.flatMap((event) =>
|
|
event.type === 'text' ? [event.delta] : []
|
|
)
|
|
.join('')
|
|
).toContain('DSH_WEB_TOOLS_E2E_OK')
|
|
} finally {
|
|
await runtime.dispose()
|
|
await toolProvider.dispose()
|
|
await Promise.allSettled(
|
|
inProcess.hosts.map((host) => host.dispose())
|
|
)
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
},
|
|
180_000
|
|
)
|
|
|
|
it.runIf(liveModelEnabled)(
|
|
'rejects a real npm plugin in Ask and lets a real model call it in Execute',
|
|
async () => {
|
|
if (!liveApiKey) {
|
|
throw new Error(
|
|
'GOODBUDDY_DSH_API_KEY is required for live DSH model E2E'
|
|
)
|
|
}
|
|
const root = await realpath(
|
|
await mkdtemp(join(tmpdir(), 'goodbuddy-harness-plugin-model-'))
|
|
)
|
|
const workspace = join(root, 'workspace')
|
|
const dshHome = join(root, 'dsh-home')
|
|
const installation = join(root, 'extension')
|
|
await Promise.all([
|
|
mkdir(workspace),
|
|
mkdir(dshHome),
|
|
mkdir(installation)
|
|
])
|
|
let installer: DshNpmExtensionInstaller | undefined
|
|
try {
|
|
const entry = {
|
|
id: 'dsh-plugin-greet-live',
|
|
package: {
|
|
name: 'dsh-plugin-greet',
|
|
version: '0.2.0'
|
|
},
|
|
displayName: 'dsh-plugin-greet',
|
|
description: 'Reviewed minimal live DSH plugin fixture.'
|
|
}
|
|
const npmCliPath = process.env.GOODBUDDY_DSH_NPM_CLI
|
|
? resolve(process.env.GOODBUDDY_DSH_NPM_CLI)
|
|
: resolve(
|
|
'node_modules',
|
|
'npm',
|
|
'bin',
|
|
'npm-cli.js'
|
|
)
|
|
const nodeExecutablePath =
|
|
process.env.GOODBUDDY_DSH_NODE_EXECUTABLE
|
|
? resolve(
|
|
process.env.GOODBUDDY_DSH_NODE_EXECUTABLE
|
|
)
|
|
: undefined
|
|
installer = new DshNpmExtensionInstaller({
|
|
dshHome,
|
|
npmCliPath,
|
|
...(nodeExecutablePath ? { nodeExecutablePath } : {})
|
|
})
|
|
const installed = await installer.install({
|
|
entry,
|
|
destinationDirectory: installation
|
|
})
|
|
const observedRequests: GenerateOptions[] = []
|
|
const inProcess = createInProcessLaunch(
|
|
dshHome,
|
|
undefined,
|
|
(options) => observedRequests.push(options)
|
|
)
|
|
const runtime = new DeepSeekHarnessRuntime({
|
|
defaultWorkspace: workspace,
|
|
baseUrl: liveBaseUrl,
|
|
model: liveModel,
|
|
launch: (options) => inProcess.launch(options),
|
|
credentialRefs: {
|
|
[CREDENTIAL_REF]: liveApiKey
|
|
},
|
|
extensionPackages: [
|
|
{
|
|
id: entry.id,
|
|
entrypoint: join(
|
|
installation,
|
|
...installed.entrypoint.split('/')
|
|
),
|
|
configuration: {}
|
|
}
|
|
],
|
|
initializationTimeoutMs: 20_000,
|
|
promptTimeoutMs: 120_000,
|
|
shutdownTimeoutMs: 5_000
|
|
})
|
|
|
|
try {
|
|
const askEvents = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-live-plugin-ask',
|
|
conversationId: 'live-plugin-ask',
|
|
prompt:
|
|
'DSH_ASK_PLUGIN_PROBE: attempt to call greet exactly once with name GoodBuddyAsk. The runtime must reject it. After the tool result, reply with DSH_ASK_PLUGIN_BLOCKED.',
|
|
workMode: 'ask'
|
|
},
|
|
new AbortController().signal
|
|
)
|
|
)
|
|
const askRequests = observedRequests.filter((options) =>
|
|
latestUserText(options).includes(
|
|
'DSH_ASK_PLUGIN_PROBE'
|
|
)
|
|
)
|
|
expect(askRequests.length).toBeGreaterThan(0)
|
|
expect(
|
|
askRequests.flatMap(
|
|
(options) =>
|
|
options.tools?.map((tool) => tool.name) ?? []
|
|
)
|
|
).toContain('greet')
|
|
expect(askEvents).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
name: 'greet',
|
|
state: 'pending'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
state: 'failed',
|
|
output: expect.stringContaining(
|
|
'Ask 模式不允许执行非只读工具'
|
|
)
|
|
}),
|
|
expect.objectContaining({ type: 'done' })
|
|
])
|
|
)
|
|
expect(
|
|
askEvents.some(
|
|
(event) =>
|
|
event.type === 'tool' &&
|
|
event.state === 'completed'
|
|
)
|
|
).toBe(false)
|
|
expect(
|
|
askEvents
|
|
.flatMap((event) =>
|
|
event.type === 'text' ? [event.delta] : []
|
|
)
|
|
.join('')
|
|
).toContain('DSH_ASK_PLUGIN_BLOCKED')
|
|
|
|
const executeEvents = await collect(
|
|
runtime.run(
|
|
{
|
|
requestId: 'request-live-plugin-execute',
|
|
conversationId: 'live-plugin-execute',
|
|
prompt:
|
|
'DSH_EXECUTE_PLUGIN_PROBE: call greet exactly once with name GoodBuddyLive. After its result, reply with DSH_EXECUTE_PLUGIN_OK and the exact greeting.',
|
|
workMode: 'execute'
|
|
},
|
|
new AbortController().signal
|
|
)
|
|
)
|
|
const executeRequests = observedRequests.filter((options) =>
|
|
latestUserText(options).includes(
|
|
'DSH_EXECUTE_PLUGIN_PROBE'
|
|
)
|
|
)
|
|
expect(executeRequests.length).toBeGreaterThan(0)
|
|
expect(
|
|
executeRequests.some((options) =>
|
|
options.tools?.some((tool) => tool.name === 'greet')
|
|
)
|
|
).toBe(true)
|
|
expect(executeEvents).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
name: 'greet',
|
|
state: 'pending'
|
|
}),
|
|
expect.objectContaining({
|
|
type: 'tool',
|
|
state: 'completed',
|
|
output: expect.stringContaining(
|
|
'Hello, GoodBuddyLive!'
|
|
)
|
|
}),
|
|
expect.objectContaining({ type: 'done' })
|
|
])
|
|
)
|
|
expect(
|
|
executeEvents
|
|
.flatMap((event) =>
|
|
event.type === 'text' ? [event.delta] : []
|
|
)
|
|
.join('')
|
|
).toContain('DSH_EXECUTE_PLUGIN_OK')
|
|
} finally {
|
|
await Promise.allSettled([
|
|
runtime.dispose(),
|
|
...inProcess.hosts.map((host) => host.dispose())
|
|
])
|
|
}
|
|
} finally {
|
|
await installer?.dispose().catch(() => undefined)
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
},
|
|
180_000
|
|
)
|
|
})
|