root/goodbuddy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve runtime visibility and browser interaction

Browse Source
This commit is contained in:
lofyer
2026-08-07 17:53:14 +08:00
parent 2c715e5e81
commit 53d18e2b06
36 changed files with 1680 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/agent/approval-summary.test.ts
+52 -3
View File
@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest'
import {
boundedToolDetail,
safeToolArgumentSummary,
safeToolErrorDetail
} from './approval-summary'
@@ -30,8 +31,29 @@ describe('safeToolArgumentSummary', () => {
})
})
describe('boundedToolDetail', () => {
it('preserves conversation details verbatim while bounding output', () => {
expect(
boundedToolDetail(
{
command: 'npm test',
token: 'secret-token',
output: 'Authorization: Bearer inline-secret'
},
1_000
)
).toBe(
'{\n "command": "npm test",\n "token": "secret-token",\n "output": "Authorization: Bearer inline-secret"\n}'
)
expect(
boundedToolDetail(' exact output\r\n', 1_000)
).toBe(' exact output\r\n')
expect(boundedToolDetail('x'.repeat(100), 20)).toHaveLength(20)
})
})
describe('safeToolErrorDetail', () => {
it('extracts nested runtime errors while redacting secrets', () => {
it('extracts nested runtime errors without rewriting their contents', () => {
expect(
safeToolErrorDetail([
{
@@ -39,14 +61,14 @@ describe('safeToolErrorDetail', () => {
'exit code 1\nAuthorization: Bearer secret-token'
}
])
).toBe('exit code 1\nAuthorization: [REDACTED]')
).toBe('exit code 1\nAuthorization: Bearer secret-token')
expect(
safeToolErrorDetail({
message:
'{"token":"json-secret","authorization":"Basic abc123"}'
})
).toBe(
'{"token":"[REDACTED]","authorization":"[REDACTED]"}'
'{"token":"json-secret","authorization":"Basic abc123"}'
)
})
@@ -66,4 +88,31 @@ describe('safeToolErrorDetail', () => {
})
).toBeUndefined()
})
it('includes nested fetch causes and network diagnostics', () => {
const cause = Object.assign(
new Error('connect ECONNREFUSED 127.0.0.1:11434'),
{
code: 'ECONNREFUSED',
errno: -4078,
syscall: 'connect',
address: '127.0.0.1',
port: 11434
}
)
const error = new TypeError('fetch failed', { cause })
expect(safeToolErrorDetail(error)).toBe(
[
'fetch failed',
'cause:',
'connect ECONNREFUSED 127.0.0.1:11434',
'code: ECONNREFUSED',
'errno: -4078',
'syscall: connect',
'address: 127.0.0.1',
'port: 11434'
].join('\n')
)
})
})
src/main/agent/approval-summary.ts
+74 -25
View File
@@ -8,6 +8,9 @@ function redactValue(
if (depth > 8) {
return '[TRUNCATED]'
}
if (typeof value === 'string') {
return redactSensitiveText(value)
}
if (!value || typeof value !== 'object') {
return value
}
@@ -64,6 +67,32 @@ export function safeToolErrorDetail(
let remaining = maximum
const seen = new WeakSet<object>()
const append = (value: string): void => {
const text = [...value]
.filter((character) => {
const code = character.charCodeAt(0)
return (
code === 9 ||
code === 10 ||
code === 13 ||
(code > 31 && code !== 127)
)
})
.join('')
.trim()
if (!text || remaining <= 0) {
return
}
const separator = parts.length > 0 ? '\n' : ''
const available = Math.max(0, remaining - separator.length)
if (available === 0) {
return
}
const bounded = text.slice(0, available)
parts.push(`${separator}${bounded}`)
remaining -= separator.length + bounded.length
}
const collect = (candidate: unknown, depth = 0): void => {
if (remaining <= 0 || depth > 4 || candidate === undefined) {
return
@@ -73,30 +102,7 @@ export function safeToolErrorDetail(
0,
Math.min(candidate.length, remaining * 4)
)
const text = redactSensitiveText(
[...boundedCandidate]
.filter((character) => {
const code = character.charCodeAt(0)
return (
code === 9 ||
code === 10 ||
code === 13 ||
(code > 31 && code !== 127)
)
})
.join('')
).trim()
if (!text) {
return
}
const separator = parts.length > 0 ? '\n' : ''
const available = Math.max(0, remaining - separator.length)
if (available === 0) {
return
}
const bounded = text.slice(0, available)
parts.push(`${separator}${bounded}`)
remaining -= separator.length + bounded.length
append(boundedCandidate)
return
}
if (!candidate || typeof candidate !== 'object') {
@@ -113,10 +119,33 @@ export function safeToolErrorDetail(
return
}
const record = candidate as Record<string, unknown>
collect(record.message, depth + 1)
for (const key of [
'code',
'errno',
'syscall',
'hostname',
'address',
'port',
'status',
'statusCode'
]) {
const metadata = record[key]
if (
typeof metadata === 'string' ||
typeof metadata === 'number'
) {
append(`${key}: ${metadata}`)
}
}
if (record.cause !== undefined) {
append('cause:')
collect(record.cause, depth + 1)
}
for (const key of [
'content',
'message',
'error',
'errors',
'stderr',
'detail',
'data'
@@ -152,3 +181,23 @@ export function safeToolArgumentSummary(
redactValue(toolArguments, new WeakSet())
).slice(0, maximum)
}
export function boundedToolDetail(
value: unknown,
maximum: number
): string | undefined {
if (!Number.isSafeInteger(maximum) || maximum < 1 || value === undefined) {
return undefined
}
let text: string | undefined
if (typeof value === 'string') {
text = value
} else {
try {
text = JSON.stringify(value, null, 2)
} catch {
return undefined
}
}
return text ? text.slice(0, maximum) : undefined
}
src/main/agent/continue-host-adapter.test.ts
+18 -6
View File
@@ -876,7 +876,7 @@ describe('ContinueHostAdapter', () => {
name: 'Bash',
state: 'failed',
error:
'PowerShell parser failed Authorization: [REDACTED]'
'PowerShell parser failed Authorization: Bearer secret-token'
}
]
})
@@ -940,7 +940,9 @@ describe('ContinueHostAdapter', () => {
type: 'tool',
callId: 'call-1',
name: 'Bash',
state: 'running'
state: 'running',
input:
'{"command":"npm test","token":"secret-token"}'
}
]
})
@@ -973,7 +975,9 @@ describe('ContinueHostAdapter', () => {
type: 'tool',
callId: 'call-1',
name: 'Bash',
state: 'completed'
state: 'completed',
output:
'Tests passed\nAuthorization: Bearer secret-token'
},
{ type: 'text', delta: 'TOOLS_OK' }
]
@@ -1012,7 +1016,11 @@ describe('ContinueHostAdapter', () => {
{
callId: 'call-1',
name: 'Bash',
state: 'completed'
state: 'completed',
input:
'{"command":"npm test","token":"secret-token"}',
output:
'Tests passed\nAuthorization: Bearer secret-token'
}
]
})
@@ -1023,7 +1031,9 @@ describe('ContinueHostAdapter', () => {
tool: {
callId: 'call-1',
name: 'Bash',
state: 'running'
state: 'running',
input:
'{"command":"npm test","token":"secret-token"}'
}
},
{
@@ -1031,7 +1041,9 @@ describe('ContinueHostAdapter', () => {
tool: {
callId: 'call-1',
name: 'Bash',
state: 'completed'
state: 'completed',
output:
'Tests passed\nAuthorization: Bearer secret-token'
}
},
{ type: 'text', delta: 'TOOLS_OK' }
src/main/agent/continue-host-adapter.ts
+36 -5
View File
@@ -33,7 +33,7 @@ import {
import { createAnthropicApiBaseUrl } from './anthropic-endpoint'
import { createOpenAIApiBaseUrl } from './openai-endpoint'
import {
redactSensitiveText,
boundedToolDetail,
safeToolErrorDetail
} from './approval-summary'
@@ -88,6 +88,8 @@ const continueHostStreamEventSchema = z.discriminatedUnion('type', [
callId: z.string().min(1).max(256),
name: z.string().min(1).max(200),
state: z.enum(['running', 'completed', 'failed']),
input: z.string().max(4_000).optional(),
output: z.string().max(16_000).optional(),
error: z.string().max(1_000).optional()
})
.strict()
@@ -142,6 +144,8 @@ export type ContinueHostTool = {
callId: string
name: string
state: 'pending' | 'running' | 'completed' | 'failed'
input?: string
output?: string
error?: string
}
@@ -383,7 +387,7 @@ function parseContinueFailure(text: string): string | undefined {
: record.message
const detail =
typeof message === 'string' && message.trim()
? `：${redactSensitiveText(message.trim()).slice(0, 500)}`
? `：${message.trim().slice(0, 500)}`
: ''
return `Continue 模型请求失败${detail}`
} catch {
@@ -465,10 +469,23 @@ function extractContinueTools(
normalizedState === 'failed'
? normalizeContinueToolError(state.output)
: undefined
const input =
toolFunction && typeof toolFunction === 'object'
? boundedToolDetail(
(toolFunction as Record<string, unknown>).arguments,
4_000
)
: undefined
const output =
normalizedState === 'completed'
? boundedToolDetail(state.output, 16_000)
: undefined
tools.set(callId, {
callId,
name: name.trim().slice(0, 200),
state: normalizedState,
...(input ? { input } : {}),
...(output ? { output } : {}),
...(error ? { error } : {})
})
}
@@ -482,7 +499,14 @@ function mergeContinueTools(
): ContinueHostTool[] {
const tools = new Map(current.map((tool) => [tool.callId, tool]))
for (const tool of updates) {
tools.set(tool.callId, tool)
const previous = tools.get(tool.callId)
tools.set(tool.callId, {
...previous,
...tool,
input: tool.input ?? previous?.input,
output: tool.output ?? previous?.output,
error: tool.error ?? previous?.error
})
}
return [...tools.values()]
}
@@ -661,7 +685,7 @@ export class ContinueHostAdapter {
patched = replaceExactly(
patched,
streamCallbacksMarker,
'a={onContent:u=>{u&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"text",delta:u})},onContentComplete:u=>{},onToolStart:(u,l,c)=>{c&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"tool",callId:c,name:u,state:"running"})},onToolResult:(u,l,c,d)=>{d&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"tool",callId:d,name:l,state:c==="done"?"completed":"failed"})},onToolError:(u,l,c)=>{c&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"tool",callId:c,name:l??"unknown",state:"failed",error:String(u).slice(0,1e3)})},onToolPermissionRequest:'
'a={onContent:u=>{u&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"text",delta:u})},onContentComplete:u=>{},onToolStart:(u,l,c)=>{c&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"tool",callId:c,name:u,state:"running",input:(()=>{try{return JSON.stringify(l).slice(0,4e3)}catch{return"[无法序列化]"}})()})},onToolResult:(u,l,c,d)=>{d&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"tool",callId:d,name:l,state:c==="done"?"completed":"failed",output:String(u).slice(0,16e3)})},onToolError:(u,l,c)=>{c&&e.goodbuddyEvents.length<5e3&&e.goodbuddyEvents.push({type:"tool",callId:c,name:l??"unknown",state:"failed",error:String(u).slice(0,1e3)})},onToolPermissionRequest:'
)
patched = replaceExactly(
patched,
@@ -1112,6 +1136,12 @@ export class ContinueHostAdapter {
callId: event.callId,
name: event.name,
state: event.state,
...(event.input
? { input: boundedToolDetail(event.input, 4_000) }
: {}),
...(event.output
? { output: boundedToolDetail(event.output, 16_000) }
: {}),
...(event.error
? { error: normalizeContinueToolError(event.error) }
: {})
@@ -1143,7 +1173,8 @@ export class ContinueHostAdapter {
{
callId: pendingCallId,
name: pending.toolName,
state: 'pending'
state: 'pending',
input: boundedToolDetail(pending.toolArgs, 4_000)
}
]
}
src/main/agent/continue-runtime.test.ts
+10 -2
View File
@@ -384,7 +384,13 @@ describe('ContinueAgentRuntime', () => {
mocks.runHost.mockResolvedValue({
text: 'Continue response',
tools: [
{ callId: 'call-1', name: 'Bash', state: 'completed' },
{
callId: 'call-1',
name: 'Bash',
state: 'completed',
input: '{"command":"npm test"}',
output: 'Tests passed'
},
{ callId: 'call-2', name: 'Write', state: 'completed' }
]
})
@@ -396,7 +402,9 @@ describe('ContinueAgentRuntime', () => {
type: 'tool',
name: 'Bash',
state: 'completed',
summary: 'Continue 工具：Bash'
summary: 'Continue 工具：Bash',
input: '{"command":"npm test"}',
output: 'Tests passed'
}),
expect.objectContaining({
type: 'tool',
src/main/agent/continue-runtime.ts
+2
View File
@@ -72,6 +72,8 @@ function toContinueToolEvent(
? 'failed'
: tool.state,
summary: `Continue 工具：${tool.name}`,
...(tool.input ? { input: tool.input } : {}),
...(tool.output ? { output: tool.output } : {}),
...(tool.error ? { error: tool.error } : {})
}
}
src/main/agent/model-runtime.test.ts
+12 -3
View File
@@ -286,7 +286,7 @@ describe('ModelAgentRuntime', () => {
await expect(consume()).rejects.toThrow('意外中断')
})
it('redacts credentials from provider error messages', async () => {
it('preserves bounded provider error messages', async () => {
const runtime = new ModelAgentRuntime({
apiKey: 'test-key',
baseUrl: 'https://bigtoken.ai',
@@ -319,7 +319,7 @@ describe('ModelAgentRuntime', () => {
}
await expect(consume()).rejects.toThrow(
'upstream failed Authorization: [REDACTED]'
'upstream failed Authorization: Bearer secret-token'
)
})
@@ -702,6 +702,15 @@ describe('ModelAgentRuntime', () => {
.filter((event) => event.type === 'tool')
.map((event) => event.state)
).toEqual(['pending', 'running', 'completed'])
expect(events).toContainEqual(
expect.objectContaining({
type: 'tool',
state: 'completed',
input: '{\n "path": "README.md"\n}',
output:
'tool result\n\n[图片结果 1：image/png]'
})
)
expect(events).toContainEqual(
expect.objectContaining({
type: 'text',
@@ -1693,7 +1702,7 @@ describe('ModelAgentRuntime', () => {
'x-request-id': 'image-request-502'
},
expected:
'upstream unavailable Authorization: [REDACTED]（HTTP 502，请求 ID image-request-502）'
'upstream unavailable Authorization: Bearer secret-token（HTTP 502，请求 ID image-request-502）'
},
{
body: '<html>Bad Gateway</html>',
src/main/agent/model-runtime.ts
+46 -17
View File
@@ -31,8 +31,8 @@ import type {
RuntimeModelUsageEvent
} from './runtime'
import {
redactSensitiveText,
safeToolArgumentSummary
boundedToolDetail,
safeToolErrorDetail
} from './approval-summary'
type ConversationMessage = {
@@ -121,7 +121,7 @@ function getErrorMessage(value: unknown): string | undefined {
}
const error = 'error' in value ? value.error : undefined
if (typeof error === 'string') {
return redactSensitiveText(error).slice(0, 1_000)
return error.slice(0, 1_000)
}
if (
error &&
@@ -129,13 +129,13 @@ function getErrorMessage(value: unknown): string | undefined {
'message' in error &&
typeof error.message === 'string'
) {
return redactSensitiveText(error.message).slice(0, 1_000)
return error.message.slice(0, 1_000)
}
if (
'message' in value &&
typeof value.message === 'string'
) {
return redactSensitiveText(value.message).slice(0, 1_000)
return value.message.slice(0, 1_000)
}
return undefined
}
@@ -624,14 +624,29 @@ function getChatToolResultText(parts: ModelToolResultPart[]): string {
.join('\n\n')
}
function getToolResultPreview(parts: ModelToolResultPart[]): string {
let imageNumber = 0
return parts
.map((part) => {
if (part.type === 'text') {
return part.text
}
imageNumber += 1
return `[图片结果 ${imageNumber}：${part.mimeType}]`
})
.filter(Boolean)
.join('\n\n')
.slice(0, 16_000)
}
function createRecoverableToolErrorResult(
error: RecoverableModelToolError
): ModelToolResult {
const text = JSON.stringify({
ok: false,
recoverable: true,
error: redactSensitiveText(error.message).slice(0, 1_000),
nextAction: redactSensitiveText(error.nextAction).slice(0, 1_000)
error: error.message.slice(0, 1_000),
nextAction: error.nextAction.slice(0, 1_000)
})
return {
parts: [{ type: 'text', text }],
@@ -1250,7 +1265,7 @@ export class ModelAgentRuntime implements AgentRuntime {
providerMessage?.includes('模型接口请求失败')
? '上游图像服务暂时不可用，请稍后重试或联系服务商'
: providerMessage
? redactSensitiveText(providerMessage).slice(0, 1_000)
? providerMessage.slice(0, 1_000)
: '图像生成请求失败'
throw new Error(
`${publicMessage}（HTTP ${response.status}${
@@ -1546,13 +1561,15 @@ export class ModelAgentRuntime implements AgentRuntime {
seenCallIds.add(call.id)
const tool = toolsByName.get(call.name)
const displayName = tool?.displayName ?? call.name.slice(0, 128)
const input = boundedToolDetail(call.arguments, 4_000)
yield {
requestId: request.requestId,
type: 'tool',
callId: call.id,
name: displayName,
state: 'pending',
summary: `直连模型工具：${displayName}`
summary: `直连模型工具：${displayName}`,
input
}
if (!tool) {
yield {
@@ -1561,7 +1578,8 @@ export class ModelAgentRuntime implements AgentRuntime {
callId: call.id,
name: displayName,
state: 'failed',
summary: `直连模型请求了未知工具：${displayName}`
summary: `直连模型请求了未知工具：${displayName}`,
input
}
throw new Error(`模型请求了未知工具「${displayName}」`)
}
@@ -1581,19 +1599,22 @@ export class ModelAgentRuntime implements AgentRuntime {
this.toolProvider.getApproval(
tool,
call.arguments,
safeToolArgumentSummary(call.arguments),
boundedToolDetail(call.arguments, 1_000) ?? '',
toolContext
)
)
}
} catch (error) {
const detail = safeToolErrorDetail(error)
yield {
requestId: request.requestId,
type: 'tool',
callId: call.id,
name: displayName,
state: 'failed',
summary: `直连模型工具审批失败：${displayName}`
summary: `直连模型工具审批失败：${displayName}`,
input,
...(detail ? { error: detail } : {})
}
throw error
}
@@ -1604,7 +1625,8 @@ export class ModelAgentRuntime implements AgentRuntime {
callId: call.id,
name: displayName,
state: 'failed',
summary: `用户拒绝了直连模型工具：${displayName}`
summary: `用户拒绝了直连模型工具：${displayName}`,
input
}
throw new Error(`用户拒绝了工具「${displayName}」`)
}
@@ -1615,7 +1637,8 @@ export class ModelAgentRuntime implements AgentRuntime {
callId: call.id,
name: displayName,
state: 'running',
summary: `正在执行直连模型工具：${displayName}`
summary: `正在执行直连模型工具：${displayName}`,
input
}
let result: ModelToolResult
@@ -1629,6 +1652,7 @@ export class ModelAgentRuntime implements AgentRuntime {
)
} catch (error) {
const recoverable = error instanceof RecoverableModelToolError
const detail = safeToolErrorDetail(error)
yield {
requestId: request.requestId,
type: 'tool',
@@ -1638,7 +1662,9 @@ export class ModelAgentRuntime implements AgentRuntime {
summary:
recoverable
? `直连模型工具需要刷新后重试：${displayName}`
: `直连模型工具执行失败：${displayName}`
: `直连模型工具执行失败：${displayName}`,
input,
...(detail ? { error: detail } : {})
}
if (recoverable) {
result = createRecoverableToolErrorResult(error)
@@ -1657,7 +1683,8 @@ export class ModelAgentRuntime implements AgentRuntime {
callId: call.id,
name: displayName,
state: 'failed',
summary: `直连模型工具结果超过限制：${displayName}`
summary: `直连模型工具结果超过限制：${displayName}`,
input
}
throw new Error('直连模型工具结果总量超过 1MB 安全限制')
}
@@ -1691,7 +1718,9 @@ export class ModelAgentRuntime implements AgentRuntime {
callId: call.id,
name: displayName,
state: 'completed',
summary: `直连模型工具已完成：${displayName}`
summary: `直连模型工具已完成：${displayName}`,
input,
output: getToolResultPreview(result.parts)
}
}
}
src/main/agent/opencode-runtime.test.ts
+17 -5
View File
@@ -141,7 +141,14 @@ function completedToolEvent(
callID: callId,
type: 'tool',
tool,
state: { status: 'completed' }
state: {
status: 'completed',
input: {
command: 'npm test',
token: 'visible-token'
},
output: 'Tests passed\nAuthorization: Bearer secret-token'
}
}
}
}
@@ -1503,7 +1510,11 @@ describe('OpenCodeRuntime embedded permission mediation', () => {
expect.objectContaining({
type: 'tool',
callId: 'call-1',
state: 'completed'
state: 'completed',
input:
'{\n "command": "npm test",\n "token": "visible-token"\n}',
output:
'Tests passed\nAuthorization: Bearer secret-token'
})
)
expect(events).toContainEqual(
@@ -1631,11 +1642,12 @@ describe('OpenCodeRuntime embedded permission mediation', () => {
type: 'tool',
callId: 'call-1',
state: 'failed',
error: 'write failed Authorization: [REDACTED]'
error:
'write failed Authorization: Bearer secret-token'
}
})
await expect(stream.next()).rejects.toThrow(
'write failed Authorization: [REDACTED]'
'write failed Authorization: Bearer secret-token'
)
expect(session.abort).toHaveBeenCalledOnce()
await runtime.dispose()
@@ -1661,7 +1673,7 @@ describe('OpenCodeRuntime embedded permission mediation', () => {
const runtime = embeddedRuntime(client)
await expect(collectRun(runtime)).rejects.toThrow(
'prompt rejected Authorization: [REDACTED]'
'prompt rejected Authorization: Bearer secret-token'
)
await runtime.dispose()
})
src/main/agent/opencode-runtime.ts
+17
View File
@@ -35,6 +35,7 @@ import {
type RuntimeSandboxResolution
} from './runtime-sandbox'
import {
boundedToolDetail,
safeToolErrorDetail
} from './approval-summary'
@@ -978,6 +979,8 @@ export class OpenCodeRuntime implements AgentRuntime {
{
name: string
state: 'pending' | 'running' | 'completed' | 'failed'
input?: string
output?: string
error?: string
}
>()
@@ -1077,9 +1080,19 @@ export class OpenCodeRuntime implements AgentRuntime {
part.state.status === 'error'
? safeToolErrorDetail(part.state.error)
: undefined
const input = isRecord(part.state.input)
? boundedToolDetail(part.state.input, 4_000)
: undefined
const output =
part.state.status === 'completed' &&
typeof part.state.output === 'string'
? part.state.output.slice(0, 16_000)
: undefined
toolStates.set(callId, {
name: toolName,
state,
...(input ? { input } : {}),
...(output ? { output } : {}),
...(error ? { error } : {})
})
yield {
@@ -1089,6 +1102,8 @@ export class OpenCodeRuntime implements AgentRuntime {
name: toolName,
state,
summary: `OpenCode 工具：${toolName}`,
...(input ? { input } : {}),
...(output ? { output } : {}),
...(error ? { error } : {})
}
}
@@ -1309,6 +1324,8 @@ export class OpenCodeRuntime implements AgentRuntime {
name: tool.name,
state: 'failed',
summary: `OpenCode 工具：${tool.name}`,
...(tool.input ? { input: tool.input } : {}),
...(tool.output ? { output: tool.output } : {}),
...(tool.error ? { error: tool.error } : {})
}
}
src/main/browser/browser-model-tools.test.ts
+1
View File
@@ -127,6 +127,7 @@ describe('BrowserModelTools', () => {
})
expect(first.scopeKey).not.toBe(second.scopeKey)
expect(first.allowPermanent).toBe(false)
expect(first.description).toContain('包括密码字段')
expect(JSON.stringify(first)).not.toContain('top-secret')
const result = await tools.callTool(
src/main/browser/browser-model-tools.ts
+2 -2
View File
@@ -84,7 +84,7 @@ const definitions = [
type: 'string',
minLength: 1,
maxLength: 8_192,
description: '完整的公开 HTTP(S) URL'
description: '当前设备可连接的完整 HTTP 或 HTTPS URL'
}
},
required: ['url'],
@@ -267,7 +267,7 @@ export class BrowserModelTools {
scopeKey = `model:browser:click:${currentOrigin}:${input.ref}`
} else if (name === 'browser_type') {
const input = browserTypeInputSchema.parse(argumentsValue)
description = `向 ${currentOrigin} 页面中的元素 ${input.ref} 输入已隐藏的文本。密码、文件和隐藏字段会被拒绝。`
description = `向 ${currentOrigin} 页面中的元素 ${input.ref} 输入已隐藏的文本，包括密码字段；文件、隐藏、禁用和只读字段不支持输入。`
argumentSummary = `元素：${input.ref}；内容：[已隐藏，${input.text.length} 个字符]`
// A session approval must never authorize a later value, even for the
// same element. The nonce intentionally makes this invocation-only.
src/main/browser/browser-service.test.ts
+200 -9
View File
@@ -6,6 +6,7 @@ import {
type BrowserSessionLike
} from './browser-service'
import type { BrowserWebContents } from './electron-browser-session'
import type { BrowserLiveState } from '../../shared/contracts'
type HarnessSlot = {
currentOrigin?: string
@@ -30,19 +31,33 @@ function createHarness(options: {
cleanupTimeoutMs?: number
dispose?: () => Promise<void>
sessionGate?: Promise<void>
captureScreenshot?: (
signal: AbortSignal
) => Promise<{
type: 'image'
mimeType: 'image/jpeg'
data: string
}>
driverScreenshot?: BrowserDriverLike['screenshot']
} = {}) {
const slots: HarnessSlot[] = []
const byContents = new Map<BrowserWebContents, HarnessSlot>()
const createSession = vi.fn(async (): Promise<BrowserSessionLike> => {
await options.sessionGate
const webContents = {} as BrowserWebContents
const slot = {} as HarnessSlot
const webContents = {
getURL: () => `${slot.currentOrigin}/page`
} as BrowserWebContents
const session: BrowserSessionLike = {
webContents,
approveNavigation: vi.fn((target) => {
slot.approvedOrigin = target.origin
}),
getCurrentOrigin: vi.fn(() => slot.currentOrigin),
openInteraction: vi.fn(async () => undefined),
...(options.captureScreenshot
? { captureScreenshot: vi.fn(options.captureScreenshot) }
: {}),
dispose: vi.fn(options.dispose ?? (async () => undefined))
}
const driver: BrowserDriverLike = {
@@ -67,11 +82,14 @@ function createHarness(options: {
slot.currentOrigin = canonicalizeBrowserUrl(target.url).origin
return { url: target.url }
}),
screenshot: vi.fn(async () => ({
type: 'image' as const,
mimeType: 'image/jpeg' as const,
data: '/9j/2Q=='
})),
screenshot: vi.fn(
options.driverScreenshot ??
(async () => ({
type: 'image' as const,
mimeType: 'image/jpeg' as const,
data: '/9j/2Q=='
}))
),
dispose: vi.fn()
}
Object.assign(slot, { session, driver })
@@ -148,8 +166,8 @@ describe('BrowserService', () => {
it('does not publish ready after a session is stopped during frame capture', async () => {
const harness = createHarness()
const signal = new AbortController().signal
const states: string[] = []
harness.service.onState((state) => states.push(state.status))
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
await harness.service.navigate(
'conversation',
'https://example.com/',
@@ -181,7 +199,128 @@ describe('BrowserService', () => {
await harness.service.releaseConversation('conversation')
await expect(click).rejects.toThrow('浏览器会话已释放')
expect(states.at(-1)).toBe('stopped')
expect(states.at(-1)?.status).toBe('stopped')
})
it('falls back to CDP when native capture cannot produce the live frame', async () => {
const nativeCapture = vi.fn(async () => {
throw new Error('native capture unavailable while hidden')
})
const harness = createHarness({
captureScreenshot: nativeCapture
})
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
await harness.service.navigate(
'conversation',
'https://example.com/',
new AbortController().signal
)
expect(nativeCapture).toHaveBeenCalledOnce()
expect(harness.slots[0]?.driver.screenshot).toHaveBeenCalledOnce()
expect(states.at(-1)).toMatchObject({
status: 'ready',
frameDataUrl: 'data:image/jpeg;base64,/9j/2Q=='
})
await harness.service.dispose()
})
it('retries live capture while a newly committed page starts painting', async () => {
let attempts = 0
const harness = createHarness({
captureScreenshot: async () => {
attempts += 1
if (attempts === 1) {
throw new Error('page has not painted yet')
}
return {
type: 'image',
mimeType: 'image/jpeg',
data: '/9j/2Q=='
}
},
driverScreenshot: async () => {
throw new Error('CDP frame not ready')
}
})
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
await harness.service.navigate(
'conversation',
'https://example.com/',
new AbortController().signal
)
expect(attempts).toBe(2)
expect(states.at(-1)).toMatchObject({
status: 'ready',
frameDataUrl: 'data:image/jpeg;base64,/9j/2Q=='
})
await harness.service.dispose()
})
it('reports a live-frame failure instead of waiting indefinitely', async () => {
const harness = createHarness({
captureScreenshot: async () => {
throw new Error('native capture failed')
},
driverScreenshot: async () => {
throw new Error('CDP capture failed')
}
})
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
await harness.service.navigate(
'conversation',
'https://example.com/',
new AbortController().signal
)
expect(states.at(-1)).toMatchObject({
status: 'failed',
error: '页面已就绪，但实时画面捕获失败，请重试浏览器操作'
})
await harness.service.dispose()
})
it('keeps the last frame when a later refresh cannot capture a minimized window', async () => {
let nativeAttempts = 0
const harness = createHarness({
captureScreenshot: async () => {
nativeAttempts += 1
if (nativeAttempts === 1) {
return {
type: 'image',
mimeType: 'image/jpeg',
data: '/9j/2Q=='
}
}
throw new Error('minimized native capture unavailable')
},
driverScreenshot: async () => {
throw new Error('minimized CDP capture unavailable')
}
})
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://example.com/',
signal
)
await harness.service.click('conversation', 'button_ref', signal)
expect(states.at(-1)).toMatchObject({
status: 'ready',
frameDataUrl: 'data:image/jpeg;base64,/9j/2Q=='
})
await harness.service.dispose()
})
it('isolates browser state and drivers by conversation', async () => {
@@ -244,6 +383,58 @@ describe('BrowserService', () => {
await harness.service.dispose()
})
it('pauses agent operations while the user interacts with the same session', async () => {
const harness = createHarness()
const signal = new AbortController().signal
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
await harness.service.navigate(
'conversation',
'https://a.example/',
signal
)
const interactionGate = deferred<
Awaited<ReturnType<BrowserSessionLike['openInteraction']>>
>()
const slot = harness.slots[0]
if (!slot) {
throw new Error('slot missing')
}
vi.mocked(slot.session.openInteraction).mockReturnValueOnce(
interactionGate.promise
)
const interaction = harness.service.interact(
'conversation',
signal
)
await vi.waitFor(() =>
expect(slot.session.openInteraction).toHaveBeenCalledOnce()
)
const snapshot = harness.service.snapshot('conversation', signal)
await new Promise((resolve) => setTimeout(resolve, 10))
expect(slot.driver.snapshot).not.toHaveBeenCalled()
interactionGate.resolve({
type: 'image',
mimeType: 'image/jpeg',
data: 'closing-frame'
})
await interaction
expect(states.slice(-2).map((state) => state.status)).toEqual([
'interactive',
'ready'
])
expect(states.at(-1)?.frameDataUrl).toBe(
'data:image/jpeg;base64,closing-frame'
)
expect(harness.service.getSessionCount()).toBe(1)
expect(slot.session.dispose).not.toHaveBeenCalled()
await snapshot
expect(slot.driver.snapshot).toHaveBeenCalledOnce()
await harness.service.dispose()
})
it('does not let a canceled queued waiter clear the active operation owner', async () => {
const harness = createHarness()
const signal = new AbortController().signal
src/main/browser/browser-service.ts
+121 -35
View File
@@ -7,6 +7,7 @@ import {
import type { BrowserScreenshot } from './browser-screenshot'
import {
ElectronBrowserSession,
type BrowserParentWindowHandle,
type BrowserWebContents
} from './electron-browser-session'
import type { BrowserLiveState } from '../../shared/contracts'
@@ -20,6 +21,7 @@ export type BrowserSessionLike = {
target: Awaited<ReturnType<BrowserUrlPolicy['validate']>>
): void
getCurrentOrigin(): string | undefined
openInteraction(): Promise<BrowserScreenshot | undefined>
captureScreenshot?(signal: AbortSignal): Promise<BrowserScreenshot>
dispose(): Promise<void>
}
@@ -45,6 +47,7 @@ export type BrowserServiceOptions = {
idleTimeoutMs?: number
cleanupTimeoutMs?: number
liveFrameDelayMs?: number
parentWindow?: BrowserParentWindowHandle
createSession?: (
policy: BrowserUrlPolicy,
signal: AbortSignal
@@ -112,9 +115,16 @@ async function boundedCleanup(
async function defaultCreateSession(
policy: BrowserUrlPolicy,
signal: AbortSignal
signal: AbortSignal,
parentWindow?: BrowserParentWindowHandle
): Promise<BrowserSessionLike> {
return ElectronBrowserSession.create({ policy }, signal)
return ElectronBrowserSession.create(
{
policy,
...(parentWindow ? { parentWindow } : {})
},
signal
)
}
function defaultCreateDriver(webContents: BrowserWebContents): BrowserDriverLike {
@@ -152,7 +162,10 @@ export class BrowserService {
this.cleanupTimeoutMs =
options.cleanupTimeoutMs ?? DEFAULT_CLEANUP_TIMEOUT_MS
this.liveFrameDelayMs = options.liveFrameDelayMs ?? 100
this.createSession = options.createSession ?? defaultCreateSession
this.createSession =
options.createSession ??
((policy, signal) =>
defaultCreateSession(policy, signal, options.parentWindow))
this.createDriver = options.createDriver ?? defaultCreateDriver
if (
!Number.isSafeInteger(this.maximumSessions) ||
@@ -198,6 +211,9 @@ export class BrowserService {
conversationId,
status,
...(previous?.url ? { url: previous.url } : {}),
...(status !== 'stopped' && previous?.frameDataUrl
? { frameDataUrl: previous.frameDataUrl }
: {}),
...update,
updatedAt: Date.now()
}
@@ -235,40 +251,73 @@ export class BrowserService {
signal
)
}
const previewController = new AbortController()
const timeout = setTimeout(
() =>
previewController.abort(
new Error('浏览器实时画面捕获超时')
),
2_000
)
try {
const previewSignal = AbortSignal.any([
signal,
previewController.signal
])
frame = slot.session.captureScreenshot
? await slot.session.captureScreenshot(previewSignal)
: await slot.driver.screenshot(previewSignal)
} catch {
signal.throwIfAborted()
// Browser control succeeds even when the optional live frame fails.
} finally {
clearTimeout(timeout)
const captureDeadline = AbortSignal.any([
signal,
AbortSignal.timeout(6_000)
])
for (let attempt = 0; attempt < 3 && !frame; attempt += 1) {
if (attempt > 0) {
try {
await waitFor(
new Promise<void>((resolve) =>
setTimeout(resolve, attempt * 150)
),
captureDeadline
)
} catch {
signal.throwIfAborted()
break
}
}
if (slot.session.captureScreenshot) {
try {
frame = await slot.session.captureScreenshot(
AbortSignal.any([
captureDeadline,
AbortSignal.timeout(1_500)
])
)
} catch {
signal.throwIfAborted()
}
}
if (!frame && !captureDeadline.aborted) {
try {
frame = await slot.driver.screenshot(
AbortSignal.any([
captureDeadline,
AbortSignal.timeout(1_500)
])
)
} catch {
signal.throwIfAborted()
}
}
}
}
signal.throwIfAborted()
if (slot.released || this.slots.get(conversationId) !== slot) {
return
}
if (!frame) {
const previousFrame =
this.liveStates.get(conversationId)?.frameDataUrl
if (previousFrame) {
this.emitState(conversationId, 'ready', {
...(url ? { url } : {}),
frameDataUrl: previousFrame
})
return
}
this.emitState(conversationId, 'failed', {
...(url ? { url } : {}),
error: '页面已就绪，但实时画面捕获失败，请重试浏览器操作'
})
return
}
this.emitState(conversationId, 'ready', {
...(url ? { url } : {}),
...(frame
? {
frameDataUrl: `data:${frame.mimeType};base64,${frame.data}`
}
: {})
frameDataUrl: `data:${frame.mimeType};base64,${frame.data}`
})
}
@@ -429,7 +478,7 @@ export class BrowserService {
slot: BrowserSlot,
signal: AbortSignal,
operation: (effectiveSignal: AbortSignal) => Promise<T>,
status?: 'loading' | 'acting'
status?: 'loading' | 'acting' | 'interactive'
): Promise<T> {
signal.throwIfAborted()
if (slot.released || this.disposed) {
@@ -496,7 +545,7 @@ export class BrowserService {
private async runInSession<T>(
conversationId: string,
signal: AbortSignal,
status: 'loading' | 'acting',
status: 'loading' | 'acting' | 'interactive',
failureStage: string,
operation: (
slot: BrowserSlot,
@@ -715,10 +764,17 @@ export class BrowserService {
'浏览器截图',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
const screenshot =
slot.session.captureScreenshot
? await slot.session.captureScreenshot(effectiveSignal)
: await slot.driver.screenshot(effectiveSignal)
let screenshot: BrowserScreenshot | undefined
if (slot.session.captureScreenshot) {
try {
screenshot = await slot.session.captureScreenshot(
effectiveSignal
)
} catch {
effectiveSignal.throwIfAborted()
}
}
screenshot ??= await slot.driver.screenshot(effectiveSignal)
await this.captureFrame(
conversationId,
slot,
@@ -731,6 +787,36 @@ export class BrowserService {
)
}
async interact(
conversationId: string,
signal: AbortSignal
): Promise<void> {
await this.runInSession(
conversationId,
signal,
'interactive',
'浏览器交互',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
const closingFrame = await waitFor(
slot.session.openInteraction(),
effectiveSignal
)
const currentUrl = canonicalizeBrowserUrl(
slot.session.webContents.getURL()
)
slot.origin = currentUrl.origin
await this.captureFrame(
conversationId,
slot,
effectiveSignal,
currentUrl.href,
closingFrame
)
}
)
}
async releaseConversation(conversationId: string): Promise<void> {
this.releaseRequests.add(conversationId)
let releasedSlot = false
src/main/browser/cdp-browser-driver.test.ts
+6 -5
View File
@@ -536,7 +536,7 @@ describe('CdpBrowserDriver', () => {
driver.dispose()
})
it('rejects password, file, hidden, and stale typing targets', async () => {
it('allows password typing while keeping the password value redacted', async () => {
const harness = createHarness(standardCommand)
const driver = new CdpBrowserDriver(harness.webContents)
const snapshot = await driver.snapshot(new AbortController().signal)
@@ -545,15 +545,16 @@ describe('CdpBrowserDriver', () => {
throw new Error('password missing')
}
await expect(
driver.type(password.ref, 'never-send', new AbortController().signal)
).rejects.toThrow('受保护')
driver.type(password.ref, 'login-secret', new AbortController().signal)
).resolves.toBeUndefined()
expect(
harness.sendCommand.mock.calls.some(
([method, parameters]) =>
method === 'Input.insertText' &&
parameters?.text === 'never-send'
parameters?.text === 'login-secret'
)
).toBe(false)
).toBe(true)
expect(JSON.stringify(snapshot)).not.toContain('secret')
driver.dispose()
})
src/main/browser/cdp-browser-driver.ts
+2 -6
View File
@@ -88,7 +88,6 @@ type RefBinding = {
backendNodeId: number
generation: number
role: string
protected: boolean
}
export type CdpBrowserDriverOptions = {
@@ -591,8 +590,7 @@ export class CdpBrowserDriver {
this.refs.set(ref, {
backendNodeId: node.backendDOMNodeId,
generation: this.generation,
role,
protected: protectedNode
role
})
output.push(item)
}
@@ -648,7 +646,6 @@ export class CdpBrowserDriver {
typeof node.nodeName === 'string' ? node.nodeName.toLowerCase() : ''
const inputType = (attributeMap.get('type') ?? '').toLowerCase()
const blocked =
binding.protected ||
attributeMap.has('hidden') ||
attributeMap.has('disabled') ||
attributeMap.has('inert') ||
@@ -656,10 +653,9 @@ export class CdpBrowserDriver {
attributeMap.get('aria-hidden') === 'true' ||
attributeMap.get('aria-disabled') === 'true' ||
inputType === 'hidden' ||
inputType === 'password' ||
inputType === 'file'
if (blocked) {
throw new Error('浏览器拒绝操作受保护、隐藏或禁用字段')
throw new Error('浏览器拒绝操作隐藏、禁用、只读或文件字段')
}
if (
action === 'type' &&
src/main/browser/electron-browser-session.test.ts
+85
View File
@@ -21,6 +21,7 @@ function createHarness() {
const debuggerEvents = new EventEmitter()
const contentEvents = new EventEmitter()
const partitionEvents = new EventEmitter()
const windowEvents = new EventEmitter()
let currentUrl = ''
let openHandler: ((details: { url: string }) => { action: 'deny' }) | undefined
const sendCommand = vi.fn(async () => ({}))
@@ -71,6 +72,21 @@ function createHarness() {
loadURL: vi.fn(async (url: string) => {
currentUrl = url
}),
show: vi.fn(),
minimize: vi.fn(),
restore: vi.fn(),
isMinimized: vi.fn(() => false),
focus: vi.fn(),
on: (event, listener) =>
windowEvents.on(
event,
listener as (...argumentsValue: unknown[]) => void
),
off: (event, listener) =>
windowEvents.off(
event,
listener as (...argumentsValue: unknown[]) => void
),
destroy: vi.fn(),
isDestroyed: vi.fn(() => false)
}
@@ -125,6 +141,7 @@ function createHarness() {
contentEvents,
debuggerEvents,
partitionEvents,
windowEvents,
partition,
proxy,
policy,
@@ -276,6 +293,74 @@ describe('ElectronBrowserSession', () => {
await session.dispose()
})
it('restores the browser for interaction and minimizes it on close', async () => {
const harness = createHarness()
const parentWindow = {
setEnabled: vi.fn(),
focus: vi.fn(),
isDestroyed: vi.fn(() => false)
}
let createdWindowOptions: Record<string, unknown> | undefined
const createWindow = vi.fn(
async (options: Record<string, unknown>) => {
createdWindowOptions = options
return harness.window
}
)
const session = await ElectronBrowserSession.create({
policy: harness.policy,
parentWindow,
createPartition: async () => harness.partition,
createWindow,
createProxy: () => harness.proxy
})
expect(createWindow).toHaveBeenCalledWith(
expect.objectContaining({
parent: parentWindow,
show: false,
title: 'GoodBuddy 浏览器交互'
})
)
expect(createdWindowOptions?.modal).toBeUndefined()
const interaction = session.openInteraction()
expect(parentWindow.setEnabled).toHaveBeenCalledWith(false)
expect(harness.window.show).toHaveBeenCalledOnce()
expect(harness.window.focus).toHaveBeenCalledOnce()
const closeEvent = { preventDefault: vi.fn() }
harness.windowEvents.emit('close', closeEvent)
await expect(interaction).resolves.toEqual({
type: 'image',
mimeType: 'image/jpeg',
data: '/9j/2Q=='
})
expect(closeEvent.preventDefault).toHaveBeenCalledOnce()
expect(harness.webContents.capturePage).toHaveBeenCalledOnce()
expect(harness.window.minimize).toHaveBeenCalledOnce()
expect(parentWindow.setEnabled).toHaveBeenLastCalledWith(true)
expect(parentWindow.focus).toHaveBeenCalledOnce()
expect(
vi.mocked(harness.webContents.capturePage!).mock
.invocationCallOrder[0]
).toBeLessThan(
vi.mocked(harness.window.minimize).mock.invocationCallOrder[0] ??
Number.POSITIVE_INFINITY
)
const repeatedCloseEvent = { preventDefault: vi.fn() }
harness.windowEvents.emit('close', repeatedCloseEvent)
expect(repeatedCloseEvent.preventDefault).toHaveBeenCalledOnce()
expect(harness.window.minimize).toHaveBeenCalledTimes(2)
expect(harness.window.destroy).not.toHaveBeenCalled()
vi.mocked(harness.window.isMinimized).mockReturnValue(true)
const reopenedInteraction = session.openInteraction()
expect(harness.window.restore).toHaveBeenCalledOnce()
harness.windowEvents.emit('close', { preventDefault: vi.fn() })
await reopenedInteraction
await session.dispose()
expect(harness.window.destroy).toHaveBeenCalledOnce()
})
it('detaches listeners and clears isolated data on idempotent disposal', async () => {
const harness = createHarness()
const session = await ElectronBrowserSession.create({
src/main/browser/electron-browser-session.ts
+132 -2
View File
@@ -49,10 +49,23 @@ export type BrowserWebContents = {
export type BrowserWindowHandle = {
webContents: BrowserWebContents
loadURL(url: string): Promise<unknown>
show(): void
minimize(): void
restore(): void
isMinimized(): boolean
focus(): void
on(event: string, listener: BrowserEventListener): unknown
off(event: string, listener: BrowserEventListener): unknown
destroy(): void
isDestroyed(): boolean
}
export type BrowserParentWindowHandle = {
setEnabled?(enabled: boolean): void
focus?(): void
isDestroyed?(): boolean
}
export type BrowserPartitionSession = {
setPermissionCheckHandler(
handler: (...argumentsValue: never[]) => boolean
@@ -100,6 +113,7 @@ export type ElectronBrowserSessionOptions = {
options: Record<string, unknown>
) => Promise<BrowserWindowHandle>
createProxy?: (policy: BrowserUrlPolicy) => FilteringProxyLike
parentWindow?: BrowserParentWindowHandle
}
type Listener = {
@@ -211,6 +225,11 @@ export class ElectronBrowserSession {
readonly webContents: BrowserWebContents
private approvedOrigin?: string
private readonly listeners: Listener[] = []
private interaction?: {
promise: Promise<BrowserScreenshot | undefined>
resolve(frame?: BrowserScreenshot): void
}
private interactionClosing?: Promise<void>
private disposed = false
private constructor(
@@ -219,7 +238,8 @@ export class ElectronBrowserSession {
private readonly window: BrowserWindowHandle,
private readonly proxy: FilteringProxyLike,
partition: string,
private readonly cleanupTimeoutMs: number
private readonly cleanupTimeoutMs: number,
private readonly parentWindow?: BrowserParentWindowHandle
) {
this.partition = partition
this.webContents = window.webContents
@@ -301,6 +321,13 @@ export class ElectronBrowserSession {
show: false,
width: 1280,
height: 900,
title: 'GoodBuddy 浏览器交互',
autoHideMenuBar: true,
...(options.parentWindow
? {
parent: options.parentWindow
}
: {}),
webPreferences: {
partition,
sandbox: true,
@@ -308,6 +335,7 @@ export class ElectronBrowserSession {
nodeIntegration: false,
nodeIntegrationInSubFrames: false,
nodeIntegrationInWorker: false,
backgroundThrottling: false,
webSecurity: true,
allowRunningInsecureContent: false,
plugins: false,
@@ -337,7 +365,8 @@ export class ElectronBrowserSession {
window,
managedProxy,
partition,
cleanupTimeoutMs
cleanupTimeoutMs,
options.parentWindow
)
setupStage = '初始化浏览器协议'
await boundedSetup(result.initialize(), signal, setupTimeoutMs)
@@ -383,6 +412,21 @@ export class ElectronBrowserSession {
private async initialize(): Promise<void> {
const contents = this.webContents
this.listen(
this.window,
'close',
(event: { preventDefault(): void }) => {
if (this.disposed) {
return
}
event.preventDefault()
if (this.interaction) {
void this.captureAndFinishInteraction()
} else {
this.window.minimize()
}
}
)
contents.setWindowOpenHandler(() => ({ action: 'deny' }))
this.listen(contents, 'will-navigate', (event: { preventDefault(): void }, details: { url?: string } | string) => {
const url = typeof details === 'string' ? details : details.url
@@ -513,12 +557,98 @@ export class ElectronBrowserSession {
this.approvedOrigin = target.origin
}
openInteraction(): Promise<BrowserScreenshot | undefined> {
this.assertOpen()
if (this.interaction) {
this.setParentEnabled(false)
if (this.window.isMinimized()) {
this.window.restore()
}
this.window.show()
this.window.focus()
return this.interaction.promise
}
let resolve!: (frame?: BrowserScreenshot) => void
const promise = new Promise<BrowserScreenshot | undefined>(
(resolvePromise) => {
resolve = resolvePromise
}
)
this.interaction = { promise, resolve }
this.setParentEnabled(false)
try {
if (this.window.isMinimized()) {
this.window.restore()
}
this.window.show()
this.window.focus()
} catch (error) {
this.finishInteraction()
throw error
}
return promise
}
private captureAndFinishInteraction(): Promise<void> {
if (this.interactionClosing) {
return this.interactionClosing
}
const operation = (async (): Promise<void> => {
let frame: BrowserScreenshot | undefined
try {
frame = await this.captureScreenshot(AbortSignal.timeout(2_000))
} catch {
// The session remains usable even if the final visible frame fails.
}
try {
if (!this.disposed && !this.window.isDestroyed()) {
this.window.minimize()
}
} catch {
// Resolving interaction must not depend on native minimize success.
}
this.finishInteraction(frame)
})()
this.interactionClosing = operation
void operation.finally(() => {
if (this.interactionClosing === operation) {
this.interactionClosing = undefined
}
})
return operation
}
private finishInteraction(frame?: BrowserScreenshot): void {
const interaction = this.interaction
this.interaction = undefined
this.setParentEnabled(true)
interaction?.resolve(frame)
}
private setParentEnabled(enabled: boolean): void {
try {
if (
!this.parentWindow ||
this.parentWindow.isDestroyed?.() === true
) {
return
}
this.parentWindow.setEnabled?.(enabled)
if (enabled) {
this.parentWindow.focus?.()
}
} catch {
// Parent-window state must not break browser-session cleanup.
}
}
async dispose(): Promise<void> {
if (this.disposed) {
return
}
this.disposed = true
this.approvedOrigin = undefined
this.finishInteraction()
for (const { target, event, listener } of this.listeners.splice(0)) {
target.off(event, listener)
}
src/main/index.ts
+1 -1
View File
@@ -303,7 +303,7 @@ if (hasSingleInstanceLock) {
const speechTranscriptionService = new SpeechTranscriptionService(
speechModelManager
)
browserService = new BrowserService()
browserService = new BrowserService({ parentWindow: mainWindow })
const bundledRuntimePaths = resolveBundledRuntimePaths({
appPath: app.getAppPath(),
resourcesPath: process.resourcesPath,
src/main/ipc.test.ts
+33 -6
View File
@@ -93,6 +93,7 @@ describe('registerIpcHandlers computer capabilities', () => {
}))
}
const onRuntimeSettingsChanged = vi.fn(async () => {})
const interact = vi.fn(async () => {})
const releaseConversation = vi.fn(async () => {})
let browserStateListener:
| ((state: BrowserLiveState) => void)
@@ -111,6 +112,7 @@ describe('registerIpcHandlers computer capabilities', () => {
onRuntimeSettingsChanged,
undefined,
{
interact,
releaseConversation,
onState: (listener) => {
browserStateListener = listener
@@ -181,6 +183,15 @@ describe('registerIpcHandlers computer capabilities', () => {
expect(releaseConversation).toHaveBeenCalledWith(
'browser-conversation'
)
await expect(
electronMocks.handlers.get(ipcChannels.browserInteract)?.(event, {
conversationId: 'browser-conversation'
})
).resolves.toBeUndefined()
expect(interact).toHaveBeenCalledWith(
'browser-conversation',
expect.any(AbortSignal)
)
expect(() =>
electronMocks.handlers.get(
@@ -2019,7 +2030,25 @@ describe('registerIpcHandlers agent terminal state', () => {
await harness.dispose()
})
it('redacts runtime errors before persistence and renderer delivery', async () => {
it('preserves bounded runtime errors for persistence and renderer delivery', async () => {
const fetchCause = Object.assign(
new Error('connect ECONNREFUSED 127.0.0.1:11434'),
{
code: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 11434
}
)
const expectedError = [
'fetch failed',
'cause:',
'connect ECONNREFUSED 127.0.0.1:11434',
'code: ECONNREFUSED',
'syscall: connect',
'address: 127.0.0.1',
'port: 11434'
].join('\n')
const runtime = {
capability: 'chat',
requiresToolApproval: false,
@@ -2028,9 +2057,7 @@ describe('registerIpcHandlers agent terminal state', () => {
dispose: vi.fn(),
async *run() {
yield* []
throw new Error(
'gateway failed Authorization: Bearer secret-token'
)
throw new TypeError('fetch failed', { cause: fetchCause })
}
}
const harness = createHarness(runtime)
@@ -2047,13 +2074,13 @@ describe('registerIpcHandlers agent terminal state', () => {
expect(harness.assistantDatabase.updateTaskStatus).toHaveBeenCalledWith(
requestId,
'failed',
'gateway failed Authorization: [REDACTED]'
expectedError
)
)
expect(harness.webContents.send).toHaveBeenCalledWith(
ipcChannels.agentEvent,
expect.objectContaining({
message: 'gateway failed Authorization: [REDACTED]'
message: expectedError
})
)
await harness.dispose()
src/main/ipc.ts
+17
View File
@@ -15,6 +15,7 @@ import {
approvalDecisionSchema,
agentQuestionResponseSchema,
agentRequestSchema,
browserInteractRequestSchema,
browserStopRequestSchema,
knowledgeCreateSchema,
knowledgeEntityUpdateSchema,
@@ -491,6 +492,10 @@ export function registerIpcHandlers(
onRuntimeSettingsChanged: () => Promise<void>,
onBeforeClearLocalData?: () => Promise<void>,
browserControl?: {
interact(
conversationId: string,
signal: AbortSignal
): Promise<void>
releaseConversation(conversationId: string): Promise<void>
onState(listener: (state: BrowserLiveState) => void): () => void
},
@@ -1193,6 +1198,18 @@ export function registerIpcHandlers(
])
})
ipcMain.handle(
ipcChannels.browserInteract,
async (event, input: unknown) => {
assertTrustedSender(event, window)
const request = browserInteractRequestSchema.parse(input)
await browserControl?.interact(
request.conversationId,
new AbortController().signal
)
}
)
ipcMain.handle(ipcChannels.agentRun, async (event, input: unknown) => {
assertTrustedSender(event, window)
if (executionPaused || shuttingDown) {