feat: add browser workbar navigation controls

The Browser workbar previously showed the Agent-controlled page but did not let users navigate it directly. It now provides Back, Refresh or Stop Loading, an address field, and Go alongside the existing interaction and close actions.

User and Agent operations share the same per-conversation browser session and serialization path. Loading, committed URL, and history state keep the toolbar accurate, while Stop Loading interrupts page navigation without closing the session or releasing Runtime context.

Direct navigation continues to use the existing built-in-browser capability and URL policy. Closing the browser remains a distinct lifecycle action.

Release note: 浏览器工作栏新增返回、刷新/停止加载、地址栏和前往操作;用户与 Agent 共用同一浏览器会话,停止加载不会关闭会话。
This commit is contained in:
mesalogo
2026-09-01 08:26:57 +08:00
parent 27055693bf
commit 5127f7d045
22 changed files with 2007 additions and 213 deletions
+313 -6
View File
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { BrowserUrlPolicy, canonicalizeBrowserUrl } from './browser-url-policy'
import {
BrowserNavigationStoppedError,
BrowserService,
type BrowserDriverLike,
type BrowserSessionLike
@@ -10,7 +11,9 @@ import type { BrowserLiveState } from '../../shared/contracts'
type HarnessSlot = {
currentOrigin?: string
currentUrl?: string
approvedOrigin?: string
emitLoading(isLoading: boolean): void
session: BrowserSessionLike
driver: BrowserDriverLike
}
@@ -42,19 +45,30 @@ function createHarness(options: {
} = {}) {
const slots: HarnessSlot[] = []
const byContents = new Map<BrowserWebContents, HarnessSlot>()
const dnsResolver = vi.fn(async () => [
{ address: '93.184.216.34', family: 4 as const }
])
const createSession = vi.fn(async (): Promise<BrowserSessionLike> => {
await options.sessionGate
const slot = {} as HarnessSlot
const webContents = {
getURL: () => `${slot.currentOrigin}/page`
getURL: () => slot.currentUrl ?? `${slot.currentOrigin}/page`
} as BrowserWebContents
const loadingListeners = new Set<(isLoading: boolean) => void>()
let isLoading = false
const session: BrowserSessionLike = {
webContents,
approveNavigation: vi.fn((target) => {
slot.approvedOrigin = target.origin
}),
getCurrentOrigin: vi.fn(() => slot.currentOrigin),
isLoading: vi.fn(() => isLoading),
onLoadingChange: vi.fn((listener) => {
loadingListeners.add(listener)
return () => loadingListeners.delete(listener)
}),
openInteraction: vi.fn(async () => undefined),
stopLoading: vi.fn(),
...(options.captureScreenshot
? { captureScreenshot: vi.fn(options.captureScreenshot) }
: {}),
@@ -63,8 +77,12 @@ function createHarness(options: {
const driver: BrowserDriverLike = {
navigate: vi.fn(async (url) => {
slot.currentOrigin = canonicalizeBrowserUrl(url).origin
slot.currentUrl = url
return { url }
}),
reload: vi.fn(async () => ({
url: slot.currentUrl ?? `${slot.currentOrigin}/page`
})),
snapshot: vi.fn(async () => ({
url: `${slot.currentOrigin}/page`,
title: 'Page',
@@ -80,8 +98,13 @@ function createHarness(options: {
})),
backTo: vi.fn(async (target) => {
slot.currentOrigin = canonicalizeBrowserUrl(target.url).origin
slot.currentUrl = target.url
return { url: target.url }
}),
getNavigationMetadata: vi.fn(async () => ({
url: slot.currentUrl ?? `${slot.currentOrigin}/page`,
canGoBack: false
})),
screenshot: vi.fn(
options.driverScreenshot ??
(async () => ({
@@ -92,15 +115,22 @@ function createHarness(options: {
),
dispose: vi.fn()
}
Object.assign(slot, { session, driver })
Object.assign(slot, {
session,
driver,
emitLoading: (loading: boolean) => {
isLoading = loading
for (const listener of loadingListeners) {
listener(loading)
}
}
})
slots.push(slot)
byContents.set(webContents, slot)
return session
})
const service = new BrowserService({
policy: new BrowserUrlPolicy(async () => [
{ address: '93.184.216.34', family: 4 }
]),
policy: new BrowserUrlPolicy(dnsResolver),
maximumSessions: options.maximumSessions,
idleTimeoutMs: options.idleTimeoutMs,
cleanupTimeoutMs: options.cleanupTimeoutMs,
@@ -114,7 +144,7 @@ function createHarness(options: {
return slot.driver
}
})
return { createSession, service, slots }
return { createSession, dnsResolver, service, slots }
}
afterEach(() => {
@@ -149,6 +179,20 @@ describe('BrowserService', () => {
'ready',
'stopped'
])
expect(states[0]).toMatchObject({
sessionActive: false,
isLoading: true,
canGoBack: false
})
expect(states.find((state) => state.status === 'ready')).toMatchObject({
sessionActive: true,
isLoading: false
})
expect(states.at(-1)).toMatchObject({
sessionActive: false,
isLoading: false,
canGoBack: false
})
expect(states.find((state) => state.status === 'ready')?.frameDataUrl).toBe(
'data:image/jpeg;base64,/9j/2Q=='
)
@@ -383,6 +427,43 @@ describe('BrowserService', () => {
await harness.service.dispose()
})
it('serializes user navigation behind an active Agent action', async () => {
const harness = createHarness()
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://a.example/',
signal
)
const clickGate = deferred<void>()
const slot = harness.slots[0]
if (!slot) {
throw new Error('slot missing')
}
vi.mocked(slot.driver.click).mockImplementationOnce(async () =>
clickGate.promise
)
const click = harness.service.click('conversation', 'b_ref', signal)
await vi.waitFor(() => expect(slot.driver.click).toHaveBeenCalled())
const navigation = harness.service.navigate(
'conversation',
'https://b.example/',
signal
)
await new Promise((resolve) => setTimeout(resolve, 10))
expect(slot.driver.navigate).toHaveBeenCalledTimes(1)
clickGate.resolve()
await click
await navigation
expect(slot.driver.navigate).toHaveBeenLastCalledWith(
'https://b.example/',
expect.any(AbortSignal)
)
await harness.service.dispose()
})
it('pauses agent operations while the user interacts with the same session', async () => {
const harness = createHarness()
const signal = new AbortController().signal
@@ -568,6 +649,232 @@ describe('BrowserService', () => {
await harness.service.dispose()
})
it('reloads in the retained session and publishes actual history metadata', async () => {
const harness = createHarness()
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://example.com/first',
signal
)
const slot = harness.slots[0]
if (!slot) {
throw new Error('slot missing')
}
vi.mocked(slot.driver.getNavigationMetadata).mockResolvedValue({
url: 'https://example.com/committed',
canGoBack: true
})
slot.currentUrl = 'https://example.com/committed'
await expect(
harness.service.reload('conversation', signal)
).resolves.toEqual({
url: 'https://example.com/committed',
origin: 'https://example.com'
})
expect(slot.driver.reload).toHaveBeenCalledOnce()
expect(states.at(-1)).toMatchObject({
status: 'ready',
url: 'https://example.com/committed',
canGoBack: true,
sessionActive: true,
isLoading: false
})
await harness.service.dispose()
})
it('stops only active navigation and retains the reusable session', async () => {
const harness = createHarness()
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://example.com/first',
signal
)
const slot = harness.slots[0]
if (!slot) {
throw new Error('slot missing')
}
vi.mocked(slot.driver.navigate).mockImplementationOnce(
async (_url, operationSignal) =>
new Promise<never>((_resolve, reject) => {
operationSignal.addEventListener(
'abort',
() => reject(operationSignal.reason),
{ once: true }
)
})
)
const navigation = harness.service.navigate(
'conversation',
'https://example.com/slow',
signal
)
await vi.waitFor(() =>
expect(slot.driver.navigate).toHaveBeenCalledTimes(2)
)
await expect(
harness.service.stopLoading('conversation')
).resolves.toBe(true)
await expect(
harness.service.stopLoading('conversation')
).resolves.toBe(false)
await expect(navigation).rejects.toBeInstanceOf(
BrowserNavigationStoppedError
)
expect(slot.session.stopLoading).toHaveBeenCalledOnce()
expect(slot.session.dispose).not.toHaveBeenCalled()
expect(harness.service.getSessionCount()).toBe(1)
expect(states.at(-1)).toMatchObject({
status: 'ready',
url: 'https://example.com/first',
sessionActive: true,
isLoading: false
})
await harness.service.dispose()
})
it('stops click-triggered page loading without releasing the session', async () => {
const harness = createHarness()
const states: BrowserLiveState[] = []
harness.service.onState((state) => states.push(state))
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://example.com/',
signal
)
const slot = harness.slots[0]
if (!slot) {
throw new Error('slot missing')
}
vi.mocked(slot.driver.click).mockImplementationOnce(
async (_ref, operationSignal) =>
new Promise<never>((_resolve, reject) => {
slot.emitLoading(true)
operationSignal.addEventListener(
'abort',
() => reject(operationSignal.reason),
{ once: true }
)
})
)
const click = harness.service.click('conversation', 'b_ref', signal)
await vi.waitFor(() =>
expect(states.at(-1)).toMatchObject({
status: 'acting',
isLoading: true
})
)
await expect(
harness.service.stopLoading('conversation')
).resolves.toBe(true)
await expect(click).rejects.toBeInstanceOf(
BrowserNavigationStoppedError
)
expect(slot.session.stopLoading).toHaveBeenCalledOnce()
expect(slot.session.dispose).not.toHaveBeenCalled()
expect(states.at(-1)).toMatchObject({
status: 'ready',
isLoading: false
})
await harness.service.dispose()
})
it('stops a page load that outlives the click operation', async () => {
const harness = createHarness()
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://example.com/',
signal
)
const slot = harness.slots[0]
if (!slot) {
throw new Error('slot missing')
}
await harness.service.click('conversation', 'b_ref', signal)
slot.emitLoading(true)
await expect(
harness.service.stopLoading('conversation')
).resolves.toBe(true)
await expect(
harness.service.stopLoading('conversation')
).resolves.toBe(false)
expect(slot.session.stopLoading).toHaveBeenCalledOnce()
expect(slot.session.dispose).not.toHaveBeenCalled()
slot.emitLoading(false)
slot.emitLoading(true)
await expect(
harness.service.stopLoading('conversation')
).resolves.toBe(true)
expect(slot.session.stopLoading).toHaveBeenCalledTimes(2)
await harness.service.dispose()
})
it('never interrupts noninterruptible operations even when loading events fire', async () => {
const harness = createHarness()
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://example.com/',
signal
)
const slot = harness.slots[0]
if (!slot) {
throw new Error('slot missing')
}
const typeGate = deferred<void>()
vi.mocked(slot.driver.type).mockImplementationOnce(async () => {
slot.emitLoading(true)
await typeGate.promise
})
const typing = harness.service.type(
'conversation',
'input_ref',
'text',
signal
)
await vi.waitFor(() => expect(slot.driver.type).toHaveBeenCalled())
await expect(
harness.service.stopLoading('conversation')
).resolves.toBe(false)
expect(slot.session.stopLoading).not.toHaveBeenCalled()
slot.emitLoading(false)
typeGate.resolve()
await expect(typing).resolves.toBeUndefined()
await harness.service.dispose()
})
it('does not resolve DNS again while refreshing frame metadata', async () => {
const harness = createHarness()
const signal = new AbortController().signal
await harness.service.navigate(
'conversation',
'https://example.com/',
signal
)
expect(harness.dnsResolver).toHaveBeenCalledTimes(2)
await harness.service.click('conversation', 'button_ref', signal)
expect(harness.dnsResolver).toHaveBeenCalledTimes(2)
await harness.service.dispose()
})
it('fails closed and releases a slot when navigation origin does not match', async () => {
const harness = createHarness()
await harness.service.navigate(
+255 -67
View File
@@ -2,6 +2,7 @@ import { BrowserUrlPolicy, canonicalizeBrowserUrl } from './browser-url-policy'
import {
CdpBrowserDriver,
type BrowserHistoryTarget,
type BrowserNavigationMetadata,
type BrowserSnapshot
} from './cdp-browser-driver'
import type { BrowserScreenshot } from './browser-screenshot'
@@ -15,19 +16,30 @@ import type { BrowserLiveState } from '../../shared/contracts'
const DEFAULT_IDLE_TIMEOUT_MS = 10 * 60_000
const DEFAULT_CLEANUP_TIMEOUT_MS = 5_000
export class BrowserNavigationStoppedError extends Error {
constructor() {
super('浏览器导航已停止,可继续使用当前页面')
this.name = 'BrowserNavigationStoppedError'
}
}
export type BrowserSessionLike = {
readonly webContents: BrowserWebContents
approveNavigation(
target: Awaited<ReturnType<BrowserUrlPolicy['validate']>>
): void
getCurrentOrigin(): string | undefined
isLoading(): boolean
onLoadingChange(listener: (isLoading: boolean) => void): () => void
openInteraction(): Promise<BrowserScreenshot | undefined>
stopLoading(): void
captureScreenshot?(signal: AbortSignal): Promise<BrowserScreenshot>
dispose(): Promise<void>
}
export type BrowserDriverLike = {
navigate(url: string, signal: AbortSignal): Promise<{ url: string }>
reload(signal: AbortSignal): Promise<{ url: string }>
snapshot(signal: AbortSignal): Promise<BrowserSnapshot>
click(ref: string, signal: AbortSignal): Promise<void>
type(ref: string, text: string, signal: AbortSignal): Promise<void>
@@ -37,6 +49,9 @@ export type BrowserDriverLike = {
target: BrowserHistoryTarget,
signal: AbortSignal
): Promise<{ url: string }>
getNavigationMetadata(
signal: AbortSignal
): Promise<BrowserNavigationMetadata>
screenshot(signal: AbortSignal): Promise<BrowserScreenshot>
dispose(): void
}
@@ -61,7 +76,14 @@ type BrowserSlot = {
driver: BrowserDriverLike
origin?: string
tail: Promise<void>
active?: AbortController
active?: {
controller: AbortController
stopLoadingAllowed: boolean
status: 'loading' | 'acting' | 'interactive'
}
isLoading: boolean
stopLoadingRequested: boolean
removeLoadingListener: () => void
idleTimer?: ReturnType<typeof setTimeout>
lastUsedAt: number
released: boolean
@@ -203,13 +225,29 @@ export class BrowserService {
conversationId: string,
status: BrowserLiveState['status'],
update: Partial<
Pick<BrowserLiveState, 'url' | 'frameDataUrl' | 'error'>
Pick<
BrowserLiveState,
'url' | 'frameDataUrl' | 'error' | 'canGoBack' | 'isLoading'
>
> = {}
): void {
const previous = this.liveStates.get(conversationId)
const slot = this.slots.get(conversationId)
const state: BrowserLiveState = {
conversationId,
status,
sessionActive:
status !== 'creating' &&
status !== 'stopped' &&
Boolean(slot && !slot.released),
isLoading:
update.isLoading ??
(status === 'creating'
? true
: status === 'stopped'
? false
: slot?.isLoading ?? false),
canGoBack: update.canGoBack ?? previous?.canGoBack ?? false,
...(previous?.url ? { url: previous.url } : {}),
...(status !== 'stopped' && previous?.frameDataUrl
? { frameDataUrl: previous.frameDataUrl }
@@ -241,6 +279,20 @@ export class BrowserService {
url?: string,
screenshot?: BrowserScreenshot
): Promise<void> {
let committedUrl = url
let canGoBack = this.liveStates.get(conversationId)?.canGoBack ?? false
try {
const metadata = await slot.driver.getNavigationMetadata(signal)
const currentTarget = canonicalizeBrowserUrl(metadata.url)
if (slot.session.getCurrentOrigin() !== currentTarget.origin) {
throw new Error('浏览器当前页面来源不一致')
}
slot.origin = currentTarget.origin
committedUrl = currentTarget.href
canGoBack = metadata.canGoBack
} catch {
signal.throwIfAborted()
}
let frame = screenshot
if (!frame) {
if (this.liveFrameDelayMs > 0) {
@@ -304,19 +356,22 @@ export class BrowserService {
this.liveStates.get(conversationId)?.frameDataUrl
if (previousFrame) {
this.emitState(conversationId, 'ready', {
...(url ? { url } : {}),
...(committedUrl ? { url: committedUrl } : {}),
canGoBack,
frameDataUrl: previousFrame
})
return
}
this.emitState(conversationId, 'failed', {
...(url ? { url } : {}),
...(committedUrl ? { url: committedUrl } : {}),
canGoBack,
error: '页面已就绪,但实时画面捕获失败,请重试浏览器操作'
})
return
}
this.emitState(conversationId, 'ready', {
...(url ? { url } : {}),
...(committedUrl ? { url: committedUrl } : {}),
canGoBack,
frameDataUrl: `data:${frame.mimeType};base64,${frame.data}`
})
}
@@ -340,6 +395,9 @@ export class BrowserService {
signal: AbortSignal,
error: unknown
): boolean {
if (error instanceof BrowserNavigationStoppedError) {
return false
}
if (signal.aborted || this.releaseRequests.has(conversationId)) {
return false
}
@@ -439,10 +497,30 @@ export class BrowserService {
session,
driver,
tail: Promise.resolve(),
isLoading: session.isLoading(),
stopLoadingRequested: false,
removeLoadingListener: () => undefined,
lastUsedAt: Date.now(),
released: false
}
this.slots.set(conversationId, slot)
slot.removeLoadingListener = session.onLoadingChange((isLoading) => {
if (!isLoading) {
slot.stopLoadingRequested = false
}
if (
slot.released ||
slot.isLoading === isLoading ||
this.slots.get(conversationId) !== slot
) {
return
}
slot.isLoading = isLoading
const current = this.liveStates.get(conversationId)
if (current) {
this.emitState(conversationId, current.status, { isLoading })
}
})
this.scheduleIdleExpiry(slot)
return slot
}
@@ -478,7 +556,8 @@ export class BrowserService {
slot: BrowserSlot,
signal: AbortSignal,
operation: (effectiveSignal: AbortSignal) => Promise<T>,
status?: 'loading' | 'acting' | 'interactive'
status: 'loading' | 'acting' | 'interactive',
stopLoadingAllowed: boolean
): Promise<T> {
signal.throwIfAborted()
if (slot.released || this.disposed) {
@@ -503,8 +582,13 @@ export class BrowserService {
clearTimeout(slot.idleTimer)
slot.idleTimer = undefined
}
slot.stopLoadingRequested = false
operationController = new AbortController()
slot.active = operationController
slot.active = {
controller: operationController,
stopLoadingAllowed,
status
}
const effectiveSignal = AbortSignal.any([
signal,
this.lifecycle.signal,
@@ -512,7 +596,10 @@ export class BrowserService {
])
return await operation(effectiveSignal)
} finally {
if (operationController && slot.active === operationController) {
if (
operationController &&
slot.active?.controller === operationController
) {
slot.active = undefined
}
releaseGate()
@@ -546,21 +633,32 @@ export class BrowserService {
conversationId: string,
signal: AbortSignal,
status: 'loading' | 'acting' | 'interactive',
stopLoadingAllowed: boolean,
failureStage: string,
operation: (
slot: BrowserSlot,
effectiveSignal: AbortSignal
) => Promise<T>
): Promise<T> {
let slot: BrowserSlot | undefined
try {
const slot = this.requireSlot(conversationId)
const requiredSlot = this.requireSlot(conversationId)
slot = requiredSlot
return await this.serialize(
slot,
requiredSlot,
signal,
(effectiveSignal) => operation(slot, effectiveSignal),
status
(effectiveSignal) => operation(requiredSlot, effectiveSignal),
status,
stopLoadingAllowed
)
} catch (error) {
if (
error instanceof BrowserNavigationStoppedError &&
slot &&
status !== 'loading'
) {
this.recoverStoppedNavigation(slot)
}
if (this.shouldEmitFailure(conversationId, signal, error)) {
this.emitFailure(conversationId, failureStage, error)
}
@@ -577,41 +675,25 @@ export class BrowserService {
!this.slots.has(conversationId) &&
!this.creations.has(conversationId)
) {
this.emitState(conversationId, 'creating', { url })
this.emitState(conversationId, 'creating')
}
try {
const slot = await this.getOrCreateSlot(conversationId, signal)
return await this.serialize(slot, signal, async (effectiveSignal) => {
const target = await this.policy.validate(url, effectiveSignal)
slot.session.approveNavigation(target)
try {
const result = await slot.driver.navigate(
return this.completeNavigation(
conversationId,
slot,
effectiveSignal,
() =>
slot.driver.navigate(
target.url.href,
effectiveSignal
)
const finalTarget = await this.policy.validateRedirect(
result.url,
effectiveSignal
)
if (slot.session.getCurrentOrigin() !== finalTarget.origin) {
throw new Error('浏览器导航结果来源不一致')
}
slot.origin = finalTarget.origin
await this.captureFrame(
conversationId,
slot,
effectiveSignal,
finalTarget.url.href
)
return {
url: finalTarget.url.href,
origin: finalTarget.origin
}
} catch (error) {
await this.releaseSlot(slot).catch(() => undefined)
throw error
}
}, 'loading')
),
'浏览器导航结果来源不一致'
)
}, 'loading', true)
} catch (error) {
if (this.shouldEmitFailure(conversationId, signal, error)) {
this.emitFailure(conversationId, '浏览器导航', error)
@@ -628,6 +710,7 @@ export class BrowserService {
conversationId,
signal,
'acting',
false,
'读取浏览器页面',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
@@ -657,6 +740,7 @@ export class BrowserService {
conversationId,
signal,
'acting',
true,
'浏览器点击',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
@@ -676,6 +760,7 @@ export class BrowserService {
conversationId,
signal,
'acting',
false,
'浏览器输入',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
@@ -695,6 +780,7 @@ export class BrowserService {
conversationId,
signal,
'acting',
false,
'浏览器选择',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
@@ -712,6 +798,7 @@ export class BrowserService {
conversationId,
signal,
'loading',
true,
'浏览器返回',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
@@ -722,33 +809,36 @@ export class BrowserService {
effectiveSignal
)
slot.session.approveNavigation(target)
try {
const result = await slot.driver.backTo(
historyTarget,
effectiveSignal
)
const finalTarget = await this.policy.validateRedirect(
result.url,
effectiveSignal
)
if (slot.session.getCurrentOrigin() !== finalTarget.origin) {
throw new Error('浏览器返回结果来源不一致')
}
slot.origin = finalTarget.origin
await this.captureFrame(
conversationId,
slot,
effectiveSignal,
finalTarget.url.href
)
return {
url: finalTarget.url.href,
origin: finalTarget.origin
}
} catch (error) {
await this.releaseSlot(slot).catch(() => undefined)
throw error
}
return this.completeNavigation(
conversationId,
slot,
effectiveSignal,
() => slot.driver.backTo(historyTarget, effectiveSignal),
'浏览器返回结果来源不一致'
)
}
)
}
async reload(
conversationId: string,
signal: AbortSignal
): Promise<{ url: string; origin: string }> {
return this.runInSession(
conversationId,
signal,
'loading',
true,
'浏览器刷新',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
return this.completeNavigation(
conversationId,
slot,
effectiveSignal,
() => slot.driver.reload(effectiveSignal),
'浏览器刷新结果来源不一致'
)
}
)
}
@@ -761,6 +851,7 @@ export class BrowserService {
conversationId,
signal,
'acting',
false,
'浏览器截图',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
@@ -795,6 +886,7 @@ export class BrowserService {
conversationId,
signal,
'interactive',
false,
'浏览器交互',
async (slot, effectiveSignal) => {
await this.verifyCurrentOriginOrRelease(slot)
@@ -817,6 +909,101 @@ export class BrowserService {
)
}
async stopLoading(conversationId: string): Promise<boolean> {
const slot = this.slots.get(conversationId)
const active = slot?.active
if (!slot || slot.released) {
return false
}
if (slot.stopLoadingRequested) {
return false
}
if (active) {
if (
!active.stopLoadingAllowed ||
active.controller.signal.aborted ||
(active.status !== 'loading' &&
!slot.isLoading &&
!slot.session.isLoading())
) {
return false
}
slot.isLoading = false
slot.stopLoadingRequested = true
slot.session.stopLoading()
active.controller.abort(new BrowserNavigationStoppedError())
return true
}
if (!slot.isLoading && !slot.session.isLoading()) {
return false
}
slot.isLoading = false
slot.stopLoadingRequested = true
slot.session.stopLoading()
this.recoverStoppedNavigation(slot)
return true
}
private recoverStoppedNavigation(slot: BrowserSlot): void {
if (slot.released || this.slots.get(slot.conversationId) !== slot) {
return
}
slot.isLoading = false
const currentUrl = slot.session.webContents.getURL()
try {
const currentTarget = canonicalizeBrowserUrl(currentUrl)
if (slot.session.getCurrentOrigin() !== currentTarget.origin) {
throw new Error('浏览器停止导航后的页面来源不一致')
}
slot.origin = currentTarget.origin
this.emitState(slot.conversationId, 'ready', {
url: currentTarget.href,
isLoading: false
})
} catch {
if (!slot.released && this.slots.get(slot.conversationId) === slot) {
this.emitState(slot.conversationId, 'ready', { isLoading: false })
}
}
}
private async completeNavigation(
conversationId: string,
slot: BrowserSlot,
signal: AbortSignal,
operation: () => Promise<{ url: string }>,
originMismatchMessage: string
): Promise<{ url: string; origin: string }> {
try {
const result = await operation()
const finalTarget = await this.policy.validateRedirect(
result.url,
signal
)
if (slot.session.getCurrentOrigin() !== finalTarget.origin) {
throw new Error(originMismatchMessage)
}
slot.origin = finalTarget.origin
await this.captureFrame(
conversationId,
slot,
signal,
finalTarget.url.href
)
return {
url: finalTarget.url.href,
origin: finalTarget.origin
}
} catch (error) {
if (error instanceof BrowserNavigationStoppedError) {
this.recoverStoppedNavigation(slot)
throw error
}
await this.releaseSlot(slot).catch(() => undefined)
throw error
}
}
async releaseConversation(conversationId: string): Promise<void> {
this.releaseRequests.add(conversationId)
let releasedSlot = false
@@ -855,7 +1042,8 @@ export class BrowserService {
clearTimeout(slot.idleTimer)
slot.idleTimer = undefined
}
slot.active?.abort(new Error('浏览器会话已释放'))
slot.removeLoadingListener()
slot.active?.controller.abort(new Error('浏览器会话已释放'))
try {
slot.driver.dispose()
} finally {
@@ -692,6 +692,63 @@ describe('CdpBrowserDriver', () => {
driver.dispose()
})
it('reloads through CDP with bounded commit and document waits', async () => {
const harness = createHarness(async (method, parameters) =>
method === 'Page.reload'
? {}
: standardCommand(method, parameters)
)
const driver = new CdpBrowserDriver(harness.webContents)
const reload = driver.reload(new AbortController().signal)
await vi.waitFor(() =>
expect(harness.sendCommand).toHaveBeenCalledWith(
'Page.reload',
undefined
)
)
harness.setUrl('https://example.com/reloaded')
harness.contentEvents.emit(
'did-navigate',
{},
'https://example.com/reloaded'
)
await expect(reload).resolves.toEqual({
url: 'https://example.com/reloaded'
})
expect(harness.sendCommand).toHaveBeenCalledWith(
'Runtime.evaluate',
expect.objectContaining({ expression: 'document.readyState' })
)
driver.dispose()
})
it('reports committed URL and actual back availability from history', async () => {
const harness = createHarness(standardCommand)
const driver = new CdpBrowserDriver(harness.webContents)
const historyQuery = vi.spyOn(
driver as unknown as {
getNavigationHistory(signal: AbortSignal): Promise<unknown>
},
'getNavigationHistory'
)
await expect(
driver.getNavigationMetadata(new AbortController().signal)
).resolves.toEqual({
url: 'https://example.com/page',
canGoBack: true
})
await expect(
driver.getBackTarget(new AbortController().signal)
).resolves.toEqual({
entryId: 4,
url: 'https://previous.example/'
})
expect(historyQuery).toHaveBeenCalledTimes(2)
driver.dispose()
})
it('bounds hung commands and removes listeners on disposal', async () => {
const harness = createHarness(async () => new Promise(() => undefined))
const driver = new CdpBrowserDriver(harness.webContents, { timeoutMs: 5 })
+68 -6
View File
@@ -84,6 +84,16 @@ export type BrowserHistoryTarget = {
url: string
}
export type BrowserNavigationMetadata = {
url: string
canGoBack: boolean
}
type BrowserNavigationHistory = {
currentIndex: number
entries: Array<{ id?: number; url?: string }>
}
type RefBinding = {
backendNodeId: number
generation: number
@@ -346,6 +356,25 @@ export class CdpBrowserDriver {
return { url: this.webContents.getURL() || url }
}
async reload(signal: AbortSignal): Promise<{ url: string }> {
const currentUrl = this.webContents.getURL()
if (!currentUrl) {
throw new Error('浏览器当前页面不可重新加载')
}
this.invalidate()
const navigation = this.waitForMainFrameCommit(currentUrl, signal)
try {
await this.command('Page.reload', undefined, signal)
} catch (error) {
navigation.cancel(error)
await navigation.promise.catch(() => undefined)
throw error
}
await navigation.promise
await this.waitForDocument(signal)
return { url: this.webContents.getURL() || currentUrl }
}
private waitForMainFrameCommit(
targetUrl: string,
signal: AbortSignal
@@ -813,12 +842,9 @@ export class CdpBrowserDriver {
}
async getBackTarget(signal: AbortSignal): Promise<BrowserHistoryTarget> {
const history = await this.command<{
currentIndex?: number
entries?: Array<{ id?: number; url?: string }>
}>('Page.getNavigationHistory', undefined, signal)
const index = history.currentIndex ?? -1
const entry = history.entries?.[index - 1]
const history = await this.getNavigationHistory(signal)
const index = history.currentIndex
const entry = history.entries[index - 1]
if (
index < 1 ||
typeof entry?.id !== 'number' ||
@@ -831,6 +857,42 @@ export class CdpBrowserDriver {
return { entryId: entry.id, url: entry.url }
}
private async getNavigationHistory(
signal: AbortSignal
): Promise<BrowserNavigationHistory> {
const history = await this.command<{
currentIndex?: number
entries?: Array<{ id?: number; url?: string }>
}>('Page.getNavigationHistory', undefined, signal)
return {
currentIndex: history.currentIndex ?? -1,
entries: history.entries ?? []
}
}
async getNavigationMetadata(
signal: AbortSignal
): Promise<BrowserNavigationMetadata> {
const history = await this.getNavigationHistory(signal)
const index = history.currentIndex
const entry = history.entries[index]
const url =
typeof entry?.url === 'string' && entry.url.length > 0
? entry.url
: this.webContents.getURL()
if (!url || url.length > 8_192) {
throw new Error('浏览器当前页面 URL 无效')
}
const previousUrl = history.entries[index - 1]?.url
return {
url,
canGoBack:
index > 0 &&
typeof previousUrl === 'string' &&
/^https?:\/\//iu.test(previousUrl)
}
}
async backTo(
target: BrowserHistoryTarget,
signal: AbortSignal
@@ -23,6 +23,7 @@ function createHarness() {
const partitionEvents = new EventEmitter()
const windowEvents = new EventEmitter()
let currentUrl = ''
let loadingMainFrame = false
let openHandler: ((details: { url: string }) => { action: 'deny' }) | undefined
const sendCommand = vi.fn(async () => ({}))
const capturedImage = {
@@ -63,6 +64,7 @@ function createHarness() {
}),
capturePage: vi.fn(async () => capturedImage),
getURL: vi.fn(() => currentUrl),
isLoadingMainFrame: vi.fn(() => loadingMainFrame),
stop: vi.fn(),
destroy: vi.fn(),
isDestroyed: vi.fn(() => false)
@@ -151,6 +153,9 @@ function createHarness() {
setCurrentUrl(value: string) {
currentUrl = value
},
setLoadingMainFrame(value: boolean) {
loadingMainFrame = value
},
getOpenHandler: () => openHandler,
getPermissionCheck: () => permissionCheck,
getPermissionRequest: () => permissionRequest,
@@ -223,6 +228,8 @@ describe('ElectronBrowserSession', () => {
mimeType: 'image/jpeg',
data: '/9j/2Q=='
})
session.stopLoading()
expect(harness.webContents.stop).toHaveBeenCalledOnce()
const downloadEvent = { preventDefault: vi.fn() }
const item = { cancel: vi.fn() }
@@ -293,6 +300,34 @@ describe('ElectronBrowserSession', () => {
await session.dispose()
})
it('reports actual main-frame loading changes and detaches subscribers', async () => {
const harness = createHarness()
harness.setLoadingMainFrame(true)
const session = await ElectronBrowserSession.create({
policy: harness.policy,
createPartition: async () => harness.partition,
createWindow: async () => harness.window,
createProxy: () => harness.proxy
})
expect(session.isLoading()).toBe(true)
const changes: boolean[] = []
const remove = session.onLoadingChange((isLoading) => {
changes.push(isLoading)
})
harness.contentEvents.emit('did-start-loading')
harness.setLoadingMainFrame(false)
harness.contentEvents.emit('did-stop-loading')
expect(session.isLoading()).toBe(false)
expect(changes).toEqual([false])
remove()
harness.setLoadingMainFrame(true)
harness.contentEvents.emit('did-start-loading')
expect(changes).toEqual([false])
await session.dispose()
})
it('restores the browser for interaction and minimizes it on close', async () => {
const harness = createHarness()
const parentWindow = {
@@ -40,6 +40,7 @@ export type BrowserWebContents = {
): void
capturePage?(): Promise<BrowserCapturedImage>
getURL(): string
isLoadingMainFrame?(): boolean
stop(): void
close?(options?: { waitForBeforeUnload?: boolean }): void
destroy(): void
@@ -230,6 +231,8 @@ export class ElectronBrowserSession {
resolve(frame?: BrowserScreenshot): void
}
private interactionClosing?: Promise<void>
private readonly loadingListeners = new Set<(isLoading: boolean) => void>()
private loading = false
private disposed = false
private constructor(
@@ -412,6 +415,13 @@ export class ElectronBrowserSession {
private async initialize(): Promise<void> {
const contents = this.webContents
this.loading = contents.isLoadingMainFrame?.() ?? false
this.listen(contents, 'did-start-loading', () => {
this.setLoading(true)
})
this.listen(contents, 'did-stop-loading', () => {
this.setLoading(contents.isLoadingMainFrame?.() ?? false)
})
this.listen(
this.window,
'close',
@@ -531,6 +541,33 @@ export class ElectronBrowserSession {
}
}
isLoading(): boolean {
return this.loading
}
onLoadingChange(listener: (isLoading: boolean) => void): () => void {
this.assertOpen()
this.loadingListeners.add(listener)
return () => {
this.loadingListeners.delete(listener)
}
}
private setLoading(isLoading: boolean): void {
if (this.loading === isLoading) {
return
}
this.loading = isLoading
for (const listener of this.loadingListeners) {
listener(isLoading)
}
}
stopLoading(): void {
this.assertOpen()
this.webContents.stop()
}
async captureScreenshot(
signal: AbortSignal
): Promise<BrowserScreenshot> {
@@ -648,6 +685,7 @@ export class ElectronBrowserSession {
}
this.disposed = true
this.approvedOrigin = undefined
this.loadingListeners.clear()
this.finishInteraction()
for (const { target, event, listener } of this.listeners.splice(0)) {
target.off(event, listener)
+145
View File
@@ -19,6 +19,7 @@ import type {
} from '../shared/ssh-host-contracts'
import { defaultKnowledgeOntologySettings } from '../shared/knowledge-ontology'
import { AssistantDatabase } from './assistant/assistant-database'
import { BrowserNavigationStoppedError } from './browser/browser-service'
import {
registerIpcHandlers,
sendRemoteEnvironmentUpdateProgress,
@@ -357,6 +358,10 @@ describe('registerIpcHandlers computer capabilities', () => {
setBuiltinMcpServerEnabled: vi.fn(async () => snapshot),
setBuiltinMcpServerAssignments: vi.fn(async () => snapshot),
setComputerCapabilityEnabled: vi.fn(async () => snapshot),
getComputerCapabilityStatus: vi.fn(async () => ({
enabled: true,
supported: true
})),
setWebSearchEnabled: vi.fn(async () => snapshot),
createBrowserProfile: vi.fn(async () => snapshot),
diagnoseComputerCapability: vi.fn(async () => ({
@@ -368,6 +373,19 @@ describe('registerIpcHandlers computer capabilities', () => {
}
const onRuntimeSettingsChanged = vi.fn(async () => {})
const interact = vi.fn(async () => {})
const navigate = vi.fn(async () => ({
url: 'https://example.com/',
origin: 'https://example.com'
}))
const back = vi.fn(async () => ({
url: 'https://example.com/previous',
origin: 'https://example.com'
}))
const reload = vi.fn(async () => ({
url: 'https://example.com/',
origin: 'https://example.com'
}))
const stopLoading = vi.fn(async () => true)
const releaseConversation = vi.fn(async () => {})
const selectFiles = vi.fn(
async (
@@ -430,6 +448,10 @@ describe('registerIpcHandlers computer capabilities', () => {
onRuntimeSettingsChanged,
undefined,
{
navigate,
back,
reload,
stopLoading,
interact,
releaseConversation,
onState: (listener) => {
@@ -573,6 +595,10 @@ describe('registerIpcHandlers computer capabilities', () => {
browserStateListener?.({
conversationId: 'browser-conversation',
status: 'ready',
sessionActive: true,
isLoading: false,
canGoBack: false,
frameDataUrl: 'data:image/jpeg;base64,frame',
updatedAt: 1
})
expect(webContents.send).toHaveBeenCalledWith(
@@ -582,6 +608,38 @@ describe('registerIpcHandlers computer capabilities', () => {
status: 'ready'
})
)
browserStateListener?.({
conversationId: 'browser-conversation',
status: 'acting',
sessionActive: true,
isLoading: false,
canGoBack: false,
frameDataUrl: 'data:image/jpeg;base64,frame',
updatedAt: 2
})
const repeatedFramePayload = webContents.send.mock.calls.at(-1)?.[1]
expect(repeatedFramePayload).not.toHaveProperty('frameDataUrl')
browserStateListener?.({
conversationId: 'browser-conversation',
status: 'stopped',
sessionActive: false,
isLoading: false,
canGoBack: false,
updatedAt: 3
})
browserStateListener?.({
conversationId: 'browser-conversation',
status: 'ready',
sessionActive: true,
isLoading: false,
canGoBack: false,
frameDataUrl: 'data:image/jpeg;base64,frame',
updatedAt: 4
})
expect(webContents.send.mock.calls.at(-1)?.[1]).toHaveProperty(
'frameDataUrl',
'data:image/jpeg;base64,frame'
)
await expect(
electronMocks.handlers.get(ipcChannels.browserStop)?.(event, {
conversationId: 'browser-conversation'
@@ -590,6 +648,42 @@ describe('registerIpcHandlers computer capabilities', () => {
expect(releaseConversation).toHaveBeenCalledWith(
'browser-conversation'
)
await expect(
electronMocks.handlers.get(ipcChannels.browserNavigate)?.(event, {
conversationId: 'browser-conversation',
url: 'https://example.com/'
})
).resolves.toBeUndefined()
expect(navigate).toHaveBeenCalledWith(
'browser-conversation',
'https://example.com/',
expect.any(AbortSignal)
)
await expect(
electronMocks.handlers.get(ipcChannels.browserBack)?.(event, {
conversationId: 'browser-conversation'
})
).resolves.toBeUndefined()
expect(back).toHaveBeenCalledWith(
'browser-conversation',
expect.any(AbortSignal)
)
await expect(
electronMocks.handlers.get(ipcChannels.browserReload)?.(event, {
conversationId: 'browser-conversation'
})
).resolves.toBeUndefined()
expect(reload).toHaveBeenCalledWith(
'browser-conversation',
expect.any(AbortSignal)
)
await expect(
electronMocks.handlers.get(ipcChannels.browserStopLoading)?.(
event,
{ conversationId: 'browser-conversation' }
)
).resolves.toBeUndefined()
expect(stopLoading).toHaveBeenCalledWith('browser-conversation')
await expect(
electronMocks.handlers.get(ipcChannels.browserInteract)?.(event, {
conversationId: 'browser-conversation'
@@ -600,6 +694,57 @@ describe('registerIpcHandlers computer capabilities', () => {
expect.any(AbortSignal)
)
navigate.mockRejectedValueOnce(new BrowserNavigationStoppedError())
await expect(
electronMocks.handlers.get(ipcChannels.browserNavigate)?.(event, {
conversationId: 'browser-conversation',
url: 'https://example.com/slow'
})
).resolves.toBeUndefined()
capabilityService.getComputerCapabilityStatus.mockResolvedValueOnce({
enabled: false,
supported: true
})
await expect(
electronMocks.handlers.get(ipcChannels.browserBack)?.(event, {
conversationId: 'browser-conversation'
})
).rejects.toThrow('尚未启用')
capabilityService.getComputerCapabilityStatus.mockResolvedValueOnce({
enabled: true,
supported: false
})
await expect(
electronMocks.handlers.get(ipcChannels.browserReload)?.(event, {
conversationId: 'browser-conversation'
})
).rejects.toThrow('不支持')
await expect(
electronMocks.handlers.get(ipcChannels.browserNavigate)?.(event, {
conversationId: 'browser-conversation',
url: 'https://example.com/',
debugger: true
})
).rejects.toThrow()
await expect(
electronMocks.handlers.get(ipcChannels.browserStopLoading)?.(
{
sender: {},
senderFrame: webContents.mainFrame
},
{ conversationId: 'browser-conversation' }
)
).rejects.toThrow('拒绝来自未知窗口的 IPC 请求')
await expect(
electronMocks.handlers.get(ipcChannels.browserStopLoading)?.(
event,
{
conversationId: 'browser-conversation',
rawElectron: true
}
)
).rejects.toThrow()
expect(() =>
electronMocks.handlers.get(
ipcChannels.capabilitiesCreateBrowserProfile
+126 -2
View File
@@ -25,7 +25,11 @@ import {
approvalDecisionSchema,
agentQuestionResponseSchema,
agentRequestSchema,
browserBackRequestSchema,
browserInteractRequestSchema,
browserNavigateRequestSchema,
browserReloadRequestSchema,
browserStopLoadingRequestSchema,
browserStopRequestSchema,
conversationQueueUserInputSchema,
defaultRuntimeSettings,
@@ -127,6 +131,7 @@ import {
agentPackageInventorySchema
} from '../shared/agent-package-contracts'
import { assertTrustedSender } from './trusted-ipc-sender'
import { BrowserNavigationStoppedError } from './browser/browser-service'
import type { TerminalSessionManager } from './terminal/terminal-session-manager'
import { releaseNotesAcknowledgeSchema } from '../shared/release-notes-contracts'
import {
@@ -1050,6 +1055,20 @@ export function registerIpcHandlers(
onRuntimeSettingsChanged: () => Promise<void>,
onBeforeClearLocalData?: () => Promise<void>,
browserControl?: {
navigate(
conversationId: string,
url: string,
signal: AbortSignal
): Promise<{ url: string; origin: string }>
back(
conversationId: string,
signal: AbortSignal
): Promise<{ url: string; origin: string }>
reload(
conversationId: string,
signal: AbortSignal
): Promise<{ url: string; origin: string }>
stopLoading(conversationId: string): boolean | Promise<boolean>
interact(
conversationId: string,
signal: AbortSignal
@@ -1314,9 +1333,25 @@ export function registerIpcHandlers(
}
window.on('maximize', notifyMaximizedChanged)
window.on('unmaximize', notifyMaximizedChanged)
const lastSentBrowserFrames = new Map<string, string>()
const removeBrowserStateListener = browserControl?.onState((state) => {
if (!window.isDestroyed()) {
window.webContents.send(ipcChannels.browserState, state)
const frame = state.frameDataUrl
let payload: BrowserLiveState = state
if (
frame &&
lastSentBrowserFrames.get(state.conversationId) === frame
) {
payload = { ...state }
delete payload.frameDataUrl
}
if (frame) {
lastSentBrowserFrames.set(state.conversationId, frame)
}
if (state.status === 'stopped') {
lastSentBrowserFrames.delete(state.conversationId)
}
window.webContents.send(ipcChannels.browserState, payload)
}
})
const abortActiveRequests = (
@@ -3364,6 +3399,93 @@ export function registerIpcHandlers(
])
})
const requireBrowserControl = async (): Promise<
NonNullable<typeof browserControl>
> => {
if (!browserControl) {
throw new Error('浏览器控制当前不可用')
}
const capability =
await capabilityService.getComputerCapabilityStatus(
'host-browser-control'
)
if (!capability.supported) {
throw new Error('当前平台不支持浏览器控制')
}
if (!capability.enabled) {
throw new Error('浏览器控制能力尚未启用')
}
return browserControl
}
const runUiBrowserNavigation = async (
operation: (
control: NonNullable<typeof browserControl>,
signal: AbortSignal
) => Promise<unknown>
): Promise<void> => {
try {
const control = await requireBrowserControl()
await operation(control, new AbortController().signal)
} catch (error) {
if (!(error instanceof BrowserNavigationStoppedError)) {
throw error
}
}
}
registerHandler(
ipcChannels.browserNavigate,
async (event, input: unknown) => {
assertTrustedSender(event, window)
const request = browserNavigateRequestSchema.parse(input)
await runUiBrowserNavigation((control, signal) =>
control.navigate(
request.conversationId,
request.url,
signal
)
)
}
)
registerHandler(
ipcChannels.browserBack,
async (event, input: unknown) => {
assertTrustedSender(event, window)
const request = browserBackRequestSchema.parse(input)
await runUiBrowserNavigation((control, signal) =>
control.back(
request.conversationId,
signal
)
)
}
)
registerHandler(
ipcChannels.browserReload,
async (event, input: unknown) => {
assertTrustedSender(event, window)
const request = browserReloadRequestSchema.parse(input)
await runUiBrowserNavigation((control, signal) =>
control.reload(
request.conversationId,
signal
)
)
}
)
registerHandler(
ipcChannels.browserStopLoading,
async (event, input: unknown) => {
assertTrustedSender(event, window)
const request = browserStopLoadingRequestSchema.parse(input)
await browserControl?.stopLoading(request.conversationId)
}
)
registerHandler(
ipcChannels.terminalCreate,
async (event, input: unknown) => {
@@ -3455,7 +3577,8 @@ export function registerIpcHandlers(
async (event, input: unknown) => {
assertTrustedSender(event, window)
const request = browserInteractRequestSchema.parse(input)
await browserControl?.interact(
const control = await requireBrowserControl()
await control.interact(
request.conversationId,
new AbortController().signal
)
@@ -7837,6 +7960,7 @@ export function registerIpcHandlers(
return async () => {
shuttingDown = true
removeBrowserStateListener?.()
lastSentBrowserFrames.clear()
removeRemoteAgentConnectionStatusListener?.()
clearInterval(scheduleInterval)
for (const timeout of queueDispatchTimers.values()) {