diff --git a/UI-DESIGN.md b/UI-DESIGN.md index baa5f133..ed25dbb8 100644 --- a/UI-DESIGN.md +++ b/UI-DESIGN.md @@ -340,6 +340,16 @@ - 下一次请求使用的附件和知识库由会话输入区展示与管理，不在工作栏重复提供“上下文”栏目； 记忆功能实现前不展示占位内容，未来的记忆与历史执行上下文归入关联 Task 的任务中心详情。 - 徽标可以提示未解决意见、等待审批、失败或连接状态，但不能成为唯一状态信号，也不能无条件抢占当前栏目。 +- 浏览器 Tab 的内容顶部固定使用一条紧凑工具栏，按“返回、刷新/停止加载、地址输入、 + 前往、交互、关闭浏览器”排列。“刷新”在当前页面加载时原位切换为“停止加载”； + “关闭浏览器”必须使用明确文字和与页面导航控件清楚区分的危险操作视觉，不能继续标为 + 含义模糊的“停止浏览器”。地址输入显示浏览器权威提交 URL，但用户编辑期间保留本地草稿， + 不被画面或状态刷新覆盖；提交、失焦后的同步规则不得让尚未提交的文字改变 Agent 看到的页面。 +- 浏览器工具栏操作的是当前 Conversation 中 Agent 使用的同一隔离浏览器会话，用户导航会 + 有意改变 Agent 下一步看到的页面。创建中、Agent 操作中或已进入完整交互窗口时禁用其余 + 工具栏控件；仅当前导航仍在加载时保留“停止加载”，且该操作只中止页面加载，不关闭浏览器、 + 不释放 Runtime 上下文。没有会话的活动 Conversation 可以直接在地址栏输入 URL 并“前往” + 创建会话，但必须复用 Agent 浏览器相同的 URL 策略与能力路径。 - 所有窗口宽度下均停靠在主工作区右侧并参与水平布局回流；展开时压缩主工作区，不能覆盖、 遮罩或隔离其内容。分栏计算排除最左侧主导航，主工作区与工作栏采用相同的 `300px` 最小宽度；拖动保存两者在可用分栏宽度中的比例，任一侧到达最小值后都不得继续侵占。 diff --git a/docs/prd/assistant-experience/assistant-workbar-and-execution-spaces-prd.md b/docs/prd/assistant-experience/assistant-workbar-and-execution-spaces-prd.md index 9b25a20b..bc3d5695 100644 --- a/docs/prd/assistant-experience/assistant-workbar-and-execution-spaces-prd.md +++ b/docs/prd/assistant-experience/assistant-workbar-and-execution-spaces-prd.md @@ -376,10 +376,43 @@ Renderer 不接收任意系统 PID 控制能力。控制动作引用 Main 签发 - 创建新的 GoodBuddy 隔离浏览器会话。 - 选择当前会话或其他受控浏览器会话。 - 查看状态、当前 URL、有界画面和错误。 -- 进入明确的交互模式或停止会话。 +- 通过顶部浏览器工具栏返回、刷新或停止加载、编辑地址并前往。 +- 进入明确的交互模式或关闭浏览器。 -没有浏览器会话时显示“新建浏览器会话”，而不是隐藏能力。模型或后台浏览器活动可以更新 -徽标，但不得无条件打开面板或切换用户当前面板。 +顶部工具栏按“返回、刷新/停止加载、地址输入、前往、交互、关闭浏览器”组织。“刷新”在 +页面加载期间原位切换为“停止加载”；“关闭浏览器”是清楚独立于导航的会话生命周期操作， +必须与“停止加载”在文案和视觉上明确区分。现有“停止浏览器”语义统一改名为“关闭浏览器”。 + +浏览器面板和 Agent 操作同一条按 Conversation 归属的浏览器会话，不创建用户预览专用的 +第二个页面。用户执行前往、返回或刷新会有意改变 Agent 下一步读取和操作的页面。所有 +前往、返回和刷新请求都进入 `BrowserService`，并与 Agent 发起的浏览器操作串行执行；不在 +Renderer 建立第二套导航队列、抢占规则或恢复状态机。 + +浏览器工具栏读取的权威状态至少包含： + +```ts +type BrowserToolbarState = { + committedUrl: string + loading: boolean + canGoBack: boolean + sessionActivity: 'inactive' | 'creating' | 'ready' | 'agent_action' | 'full_interaction' +} +``` + +`committedUrl` 是浏览器已经提交的权威 URL。地址输入在未编辑时与它同步；获得编辑意图后， +输入文字保持为 Renderer 本地草稿，画面、加载状态和 URL 更新不得覆盖草稿，直到用户“前往” +或放弃编辑。地址草稿不属于浏览器会话持久状态，也不需要恢复或版本化机制。 + +创建会话、Agent 操作和完整交互窗口期间，返回、刷新、地址提交、前往、交互和关闭浏览器 +控件禁用，避免与同一会话并发操作。例外是活动导航处于加载中时，“停止加载”保持可用， +可中断当前页面加载； +它只取消页面加载，不销毁浏览器会话、不执行“关闭浏览器”，也不释放当前 Runtime 上下文。 +“关闭浏览器”沿用现有停止会话的生命周期语义。 + +没有浏览器会话时仍显示工具栏。活动 Conversation 可以直接在地址栏输入非空 URL 并“前往”， +由 `BrowserService` 通过 Agent 浏览器相同的能力检查和 URL 策略创建会话并导航；不得为这一 +空状态增加旁路 WebView 或宽松 URL 入口。没有活动 Conversation 时说明需要先选择或创建会话。 +模型或后台浏览器活动可以更新徽标，但不得无条件打开面板或切换用户当前面板。 浏览器面板只管理 GoodBuddy 受控浏览器，不表示可以控制用户已安装的浏览器。 @@ -925,6 +958,11 @@ Electron、ChildProcess、PTY、SSH Client、Socket 或文件句柄。 - [ ] 仅使用键盘可以选择能力、面板实例、目标、执行空间和控制动作。 - [ ] 状态不只依赖颜色，徽标具有文字或可访问名称。 - [ ] 终端、HTML、监督证据和高频日志具有可访问替代或降噪行为。 +- [ ] 浏览器 Tab 顶部工具栏始终清楚区分“停止加载”和“关闭浏览器”；地址编辑草稿不会被 + 权威状态刷新覆盖，用户与 Agent 的导航通过同一 `BrowserService` 会话串行执行。 +- [ ] 创建中、Agent 操作中和完整交互期间禁用浏览器其余工具栏控件；活动加载仍可单独停止， + 且停止加载不会关闭会话或释放 Runtime 上下文。 +- [ ] 活动 Conversation 没有浏览器会话时，可从同一地址栏按 Agent 相同 URL 策略启动会话。 ## 18. 相关文档的职责 diff --git a/docs/status/computer-control-implementation-status.md b/docs/status/computer-control-implementation-status.md index c090c540..4ae3352d 100644 --- a/docs/status/computer-control-implementation-status.md +++ b/docs/status/computer-control-implementation-status.md @@ -93,11 +93,27 @@ Linux x64 和 Linux arm64。 新增“浏览器”页签；活动对话启动浏览器时会自动打开该页签，并显示创建中、 加载中、操作中、用户交互中、就绪、失败和已停止状态。用户可点击“交互”打开 同一会话的子窗口辅助 Agent；交互时主窗口暂时禁用，关闭时先刷新最终画面再将 -浏览器窗口最小化，页面和会话继续保留。用户也可立即停止当前对话的浏览器会话。 - +浏览器窗口最小化，页面和会话继续保留。用户也可明确关闭当前对话的浏览器会话。 当前实现按导航、快照、点击、输入、选择、返回和截图操作刷新画面，而不是创建 第二个预览浏览器，因此显示内容与模型受控页面一致。 +### 已实现：浏览器工具栏 + +浏览器 Tab 顶部已增加“返回、刷新/停止加载、地址输入、前往、交互、关闭浏览器”工具栏。 +它继续操作 Agent 在当前 Conversation 中使用的同一浏览器会话；用户导航会有意改变 Agent +下一步看到的页面。前往、返回和刷新通过 `BrowserService` 与 Agent 浏览器操作串行执行， +不会创建第二套 Renderer 导航路径。 + +工具栏状态以浏览器提交后的权威 URL、是否加载中、能否返回和会话活动状态为准；地址输入 +在用户编辑期间保留本地草稿，不被状态刷新覆盖。创建中、Agent 操作中和完整交互窗口期间禁用 +其余工具栏控件，但活动导航仍可用“停止加载”中断。停止加载只取消页面加载，不关闭浏览器，也 +不释放 Runtime 上下文；原“停止浏览器”会话生命周期操作已统一标为“关闭浏览器”，并在 +视觉和文案上与停止加载清楚区分。 + +活动 Conversation 尚无浏览器会话时，可直接从地址栏输入 URL 并“前往”，通过现有浏览器 +能力检查和同一 URL 策略创建会话。实现复用现有每 Conversation 串行队列和有界状态，不引入 +恢复、版本化或第二套会话框架。 + ### P0：百度输入时元素引用失效，已修复 截图中的“浏览器元素引用已失效，请重新获取快照”来自过宽的导航监听：广告或 diff --git a/src/main/browser/browser-service.test.ts b/src/main/browser/browser-service.test.ts index 03f19c90..a018af3b 100644 --- a/src/main/browser/browser-service.test.ts +++ b/src/main/browser/browser-service.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { BrowserUrlPolicy, canonicalizeBrowserUrl } from './browser-url-policy' import { + BrowserNavigationStoppedError, BrowserService, type BrowserDriverLike, type BrowserSessionLike @@ -10,7 +11,9 @@ import type { BrowserLiveState } from '../../shared/contracts' type HarnessSlot = { currentOrigin?: string + currentUrl?: string approvedOrigin?: string + emitLoading(isLoading: boolean): void session: BrowserSessionLike driver: BrowserDriverLike } @@ -42,19 +45,30 @@ function createHarness(options: { } = {}) { const slots: HarnessSlot[] = [] const byContents = new Map() + const dnsResolver = vi.fn(async () => [ + { address: '93.184.216.34', family: 4 as const } + ]) const createSession = vi.fn(async (): Promise => { await options.sessionGate const slot = {} as HarnessSlot const webContents = { - getURL: () => `${slot.currentOrigin}/page` + getURL: () => slot.currentUrl ?? `${slot.currentOrigin}/page` } as BrowserWebContents + const loadingListeners = new Set<(isLoading: boolean) => void>() + let isLoading = false const session: BrowserSessionLike = { webContents, approveNavigation: vi.fn((target) => { slot.approvedOrigin = target.origin }), getCurrentOrigin: vi.fn(() => slot.currentOrigin), + isLoading: vi.fn(() => isLoading), + onLoadingChange: vi.fn((listener) => { + loadingListeners.add(listener) + return () => loadingListeners.delete(listener) + }), openInteraction: vi.fn(async () => undefined), + stopLoading: vi.fn(), ...(options.captureScreenshot ? { captureScreenshot: vi.fn(options.captureScreenshot) } : {}), @@ -63,8 +77,12 @@ function createHarness(options: { const driver: BrowserDriverLike = { navigate: vi.fn(async (url) => { slot.currentOrigin = canonicalizeBrowserUrl(url).origin + slot.currentUrl = url return { url } }), + reload: vi.fn(async () => ({ + url: slot.currentUrl ?? `${slot.currentOrigin}/page` + })), snapshot: vi.fn(async () => ({ url: `${slot.currentOrigin}/page`, title: 'Page', @@ -80,8 +98,13 @@ function createHarness(options: { })), backTo: vi.fn(async (target) => { slot.currentOrigin = canonicalizeBrowserUrl(target.url).origin + slot.currentUrl = target.url return { url: target.url } }), + getNavigationMetadata: vi.fn(async () => ({ + url: slot.currentUrl ?? `${slot.currentOrigin}/page`, + canGoBack: false + })), screenshot: vi.fn( options.driverScreenshot ?? (async () => ({ @@ -92,15 +115,22 @@ function createHarness(options: { ), dispose: vi.fn() } - Object.assign(slot, { session, driver }) + Object.assign(slot, { + session, + driver, + emitLoading: (loading: boolean) => { + isLoading = loading + for (const listener of loadingListeners) { + listener(loading) + } + } + }) slots.push(slot) byContents.set(webContents, slot) return session }) const service = new BrowserService({ - policy: new BrowserUrlPolicy(async () => [ - { address: '93.184.216.34', family: 4 } - ]), + policy: new BrowserUrlPolicy(dnsResolver), maximumSessions: options.maximumSessions, idleTimeoutMs: options.idleTimeoutMs, cleanupTimeoutMs: options.cleanupTimeoutMs, @@ -114,7 +144,7 @@ function createHarness(options: { return slot.driver } }) - return { createSession, service, slots } + return { createSession, dnsResolver, service, slots } } afterEach(() => { @@ -149,6 +179,20 @@ describe('BrowserService', () => { 'ready', 'stopped' ]) + expect(states[0]).toMatchObject({ + sessionActive: false, + isLoading: true, + canGoBack: false + }) + expect(states.find((state) => state.status === 'ready')).toMatchObject({ + sessionActive: true, + isLoading: false + }) + expect(states.at(-1)).toMatchObject({ + sessionActive: false, + isLoading: false, + canGoBack: false + }) expect(states.find((state) => state.status === 'ready')?.frameDataUrl).toBe( 'data:image/jpeg;base64,/9j/2Q==' ) @@ -383,6 +427,43 @@ describe('BrowserService', () => { await harness.service.dispose() }) + it('serializes user navigation behind an active Agent action', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://a.example/', + signal + ) + const clickGate = deferred() + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + vi.mocked(slot.driver.click).mockImplementationOnce(async () => + clickGate.promise + ) + + const click = harness.service.click('conversation', 'b_ref', signal) + await vi.waitFor(() => expect(slot.driver.click).toHaveBeenCalled()) + const navigation = harness.service.navigate( + 'conversation', + 'https://b.example/', + signal + ) + await new Promise((resolve) => setTimeout(resolve, 10)) + expect(slot.driver.navigate).toHaveBeenCalledTimes(1) + + clickGate.resolve() + await click + await navigation + expect(slot.driver.navigate).toHaveBeenLastCalledWith( + 'https://b.example/', + expect.any(AbortSignal) + ) + await harness.service.dispose() + }) + it('pauses agent operations while the user interacts with the same session', async () => { const harness = createHarness() const signal = new AbortController().signal @@ -568,6 +649,232 @@ describe('BrowserService', () => { await harness.service.dispose() }) + it('reloads in the retained session and publishes actual history metadata', async () => { + const harness = createHarness() + const states: BrowserLiveState[] = [] + harness.service.onState((state) => states.push(state)) + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://example.com/first', + signal + ) + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + vi.mocked(slot.driver.getNavigationMetadata).mockResolvedValue({ + url: 'https://example.com/committed', + canGoBack: true + }) + slot.currentUrl = 'https://example.com/committed' + + await expect( + harness.service.reload('conversation', signal) + ).resolves.toEqual({ + url: 'https://example.com/committed', + origin: 'https://example.com' + }) + expect(slot.driver.reload).toHaveBeenCalledOnce() + expect(states.at(-1)).toMatchObject({ + status: 'ready', + url: 'https://example.com/committed', + canGoBack: true, + sessionActive: true, + isLoading: false + }) + await harness.service.dispose() + }) + + it('stops only active navigation and retains the reusable session', async () => { + const harness = createHarness() + const states: BrowserLiveState[] = [] + harness.service.onState((state) => states.push(state)) + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://example.com/first', + signal + ) + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + vi.mocked(slot.driver.navigate).mockImplementationOnce( + async (_url, operationSignal) => + new Promise((_resolve, reject) => { + operationSignal.addEventListener( + 'abort', + () => reject(operationSignal.reason), + { once: true } + ) + }) + ) + + const navigation = harness.service.navigate( + 'conversation', + 'https://example.com/slow', + signal + ) + await vi.waitFor(() => + expect(slot.driver.navigate).toHaveBeenCalledTimes(2) + ) + await expect( + harness.service.stopLoading('conversation') + ).resolves.toBe(true) + await expect( + harness.service.stopLoading('conversation') + ).resolves.toBe(false) + + await expect(navigation).rejects.toBeInstanceOf( + BrowserNavigationStoppedError + ) + expect(slot.session.stopLoading).toHaveBeenCalledOnce() + expect(slot.session.dispose).not.toHaveBeenCalled() + expect(harness.service.getSessionCount()).toBe(1) + expect(states.at(-1)).toMatchObject({ + status: 'ready', + url: 'https://example.com/first', + sessionActive: true, + isLoading: false + }) + await harness.service.dispose() + }) + + it('stops click-triggered page loading without releasing the session', async () => { + const harness = createHarness() + const states: BrowserLiveState[] = [] + harness.service.onState((state) => states.push(state)) + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://example.com/', + signal + ) + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + vi.mocked(slot.driver.click).mockImplementationOnce( + async (_ref, operationSignal) => + new Promise((_resolve, reject) => { + slot.emitLoading(true) + operationSignal.addEventListener( + 'abort', + () => reject(operationSignal.reason), + { once: true } + ) + }) + ) + const click = harness.service.click('conversation', 'b_ref', signal) + await vi.waitFor(() => + expect(states.at(-1)).toMatchObject({ + status: 'acting', + isLoading: true + }) + ) + + await expect( + harness.service.stopLoading('conversation') + ).resolves.toBe(true) + await expect(click).rejects.toBeInstanceOf( + BrowserNavigationStoppedError + ) + expect(slot.session.stopLoading).toHaveBeenCalledOnce() + expect(slot.session.dispose).not.toHaveBeenCalled() + expect(states.at(-1)).toMatchObject({ + status: 'ready', + isLoading: false + }) + await harness.service.dispose() + }) + + it('stops a page load that outlives the click operation', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://example.com/', + signal + ) + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + + await harness.service.click('conversation', 'b_ref', signal) + slot.emitLoading(true) + + await expect( + harness.service.stopLoading('conversation') + ).resolves.toBe(true) + await expect( + harness.service.stopLoading('conversation') + ).resolves.toBe(false) + expect(slot.session.stopLoading).toHaveBeenCalledOnce() + expect(slot.session.dispose).not.toHaveBeenCalled() + + slot.emitLoading(false) + slot.emitLoading(true) + await expect( + harness.service.stopLoading('conversation') + ).resolves.toBe(true) + expect(slot.session.stopLoading).toHaveBeenCalledTimes(2) + await harness.service.dispose() + }) + + it('never interrupts noninterruptible operations even when loading events fire', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://example.com/', + signal + ) + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + const typeGate = deferred() + vi.mocked(slot.driver.type).mockImplementationOnce(async () => { + slot.emitLoading(true) + await typeGate.promise + }) + + const typing = harness.service.type( + 'conversation', + 'input_ref', + 'text', + signal + ) + await vi.waitFor(() => expect(slot.driver.type).toHaveBeenCalled()) + + await expect( + harness.service.stopLoading('conversation') + ).resolves.toBe(false) + expect(slot.session.stopLoading).not.toHaveBeenCalled() + slot.emitLoading(false) + typeGate.resolve() + await expect(typing).resolves.toBeUndefined() + await harness.service.dispose() + }) + + it('does not resolve DNS again while refreshing frame metadata', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://example.com/', + signal + ) + expect(harness.dnsResolver).toHaveBeenCalledTimes(2) + + await harness.service.click('conversation', 'button_ref', signal) + + expect(harness.dnsResolver).toHaveBeenCalledTimes(2) + await harness.service.dispose() + }) + it('fails closed and releases a slot when navigation origin does not match', async () => { const harness = createHarness() await harness.service.navigate( diff --git a/src/main/browser/browser-service.ts b/src/main/browser/browser-service.ts index 4295e991..908edd45 100644 --- a/src/main/browser/browser-service.ts +++ b/src/main/browser/browser-service.ts @@ -2,6 +2,7 @@ import { BrowserUrlPolicy, canonicalizeBrowserUrl } from './browser-url-policy' import { CdpBrowserDriver, type BrowserHistoryTarget, + type BrowserNavigationMetadata, type BrowserSnapshot } from './cdp-browser-driver' import type { BrowserScreenshot } from './browser-screenshot' @@ -15,19 +16,30 @@ import type { BrowserLiveState } from '../../shared/contracts' const DEFAULT_IDLE_TIMEOUT_MS = 10 * 60_000 const DEFAULT_CLEANUP_TIMEOUT_MS = 5_000 +export class BrowserNavigationStoppedError extends Error { + constructor() { + super('浏览器导航已停止，可继续使用当前页面') + this.name = 'BrowserNavigationStoppedError' + } +} + export type BrowserSessionLike = { readonly webContents: BrowserWebContents approveNavigation( target: Awaited> ): void getCurrentOrigin(): string | undefined + isLoading(): boolean + onLoadingChange(listener: (isLoading: boolean) => void): () => void openInteraction(): Promise + stopLoading(): void captureScreenshot?(signal: AbortSignal): Promise dispose(): Promise } export type BrowserDriverLike = { navigate(url: string, signal: AbortSignal): Promise<{ url: string }> + reload(signal: AbortSignal): Promise<{ url: string }> snapshot(signal: AbortSignal): Promise click(ref: string, signal: AbortSignal): Promise type(ref: string, text: string, signal: AbortSignal): Promise @@ -37,6 +49,9 @@ export type BrowserDriverLike = { target: BrowserHistoryTarget, signal: AbortSignal ): Promise<{ url: string }> + getNavigationMetadata( + signal: AbortSignal + ): Promise screenshot(signal: AbortSignal): Promise dispose(): void } @@ -61,7 +76,14 @@ type BrowserSlot = { driver: BrowserDriverLike origin?: string tail: Promise - active?: AbortController + active?: { + controller: AbortController + stopLoadingAllowed: boolean + status: 'loading' | 'acting' | 'interactive' + } + isLoading: boolean + stopLoadingRequested: boolean + removeLoadingListener: () => void idleTimer?: ReturnType lastUsedAt: number released: boolean @@ -203,13 +225,29 @@ export class BrowserService { conversationId: string, status: BrowserLiveState['status'], update: Partial< - Pick + Pick< + BrowserLiveState, + 'url' | 'frameDataUrl' | 'error' | 'canGoBack' | 'isLoading' + > > = {} ): void { const previous = this.liveStates.get(conversationId) + const slot = this.slots.get(conversationId) const state: BrowserLiveState = { conversationId, status, + sessionActive: + status !== 'creating' && + status !== 'stopped' && + Boolean(slot && !slot.released), + isLoading: + update.isLoading ?? + (status === 'creating' + ? true + : status === 'stopped' + ? false + : slot?.isLoading ?? false), + canGoBack: update.canGoBack ?? previous?.canGoBack ?? false, ...(previous?.url ? { url: previous.url } : {}), ...(status !== 'stopped' && previous?.frameDataUrl ? { frameDataUrl: previous.frameDataUrl } @@ -241,6 +279,20 @@ export class BrowserService { url?: string, screenshot?: BrowserScreenshot ): Promise { + let committedUrl = url + let canGoBack = this.liveStates.get(conversationId)?.canGoBack ?? false + try { + const metadata = await slot.driver.getNavigationMetadata(signal) + const currentTarget = canonicalizeBrowserUrl(metadata.url) + if (slot.session.getCurrentOrigin() !== currentTarget.origin) { + throw new Error('浏览器当前页面来源不一致') + } + slot.origin = currentTarget.origin + committedUrl = currentTarget.href + canGoBack = metadata.canGoBack + } catch { + signal.throwIfAborted() + } let frame = screenshot if (!frame) { if (this.liveFrameDelayMs > 0) { @@ -304,19 +356,22 @@ export class BrowserService { this.liveStates.get(conversationId)?.frameDataUrl if (previousFrame) { this.emitState(conversationId, 'ready', { - ...(url ? { url } : {}), + ...(committedUrl ? { url: committedUrl } : {}), + canGoBack, frameDataUrl: previousFrame }) return } this.emitState(conversationId, 'failed', { - ...(url ? { url } : {}), + ...(committedUrl ? { url: committedUrl } : {}), + canGoBack, error: '页面已就绪，但实时画面捕获失败，请重试浏览器操作' }) return } this.emitState(conversationId, 'ready', { - ...(url ? { url } : {}), + ...(committedUrl ? { url: committedUrl } : {}), + canGoBack, frameDataUrl: `data:${frame.mimeType};base64,${frame.data}` }) } @@ -340,6 +395,9 @@ export class BrowserService { signal: AbortSignal, error: unknown ): boolean { + if (error instanceof BrowserNavigationStoppedError) { + return false + } if (signal.aborted || this.releaseRequests.has(conversationId)) { return false } @@ -439,10 +497,30 @@ export class BrowserService { session, driver, tail: Promise.resolve(), + isLoading: session.isLoading(), + stopLoadingRequested: false, + removeLoadingListener: () => undefined, lastUsedAt: Date.now(), released: false } this.slots.set(conversationId, slot) + slot.removeLoadingListener = session.onLoadingChange((isLoading) => { + if (!isLoading) { + slot.stopLoadingRequested = false + } + if ( + slot.released || + slot.isLoading === isLoading || + this.slots.get(conversationId) !== slot + ) { + return + } + slot.isLoading = isLoading + const current = this.liveStates.get(conversationId) + if (current) { + this.emitState(conversationId, current.status, { isLoading }) + } + }) this.scheduleIdleExpiry(slot) return slot } @@ -478,7 +556,8 @@ export class BrowserService { slot: BrowserSlot, signal: AbortSignal, operation: (effectiveSignal: AbortSignal) => Promise, - status?: 'loading' | 'acting' | 'interactive' + status: 'loading' | 'acting' | 'interactive', + stopLoadingAllowed: boolean ): Promise { signal.throwIfAborted() if (slot.released || this.disposed) { @@ -503,8 +582,13 @@ export class BrowserService { clearTimeout(slot.idleTimer) slot.idleTimer = undefined } + slot.stopLoadingRequested = false operationController = new AbortController() - slot.active = operationController + slot.active = { + controller: operationController, + stopLoadingAllowed, + status + } const effectiveSignal = AbortSignal.any([ signal, this.lifecycle.signal, @@ -512,7 +596,10 @@ export class BrowserService { ]) return await operation(effectiveSignal) } finally { - if (operationController && slot.active === operationController) { + if ( + operationController && + slot.active?.controller === operationController + ) { slot.active = undefined } releaseGate() @@ -546,21 +633,32 @@ export class BrowserService { conversationId: string, signal: AbortSignal, status: 'loading' | 'acting' | 'interactive', + stopLoadingAllowed: boolean, failureStage: string, operation: ( slot: BrowserSlot, effectiveSignal: AbortSignal ) => Promise ): Promise { + let slot: BrowserSlot | undefined try { - const slot = this.requireSlot(conversationId) + const requiredSlot = this.requireSlot(conversationId) + slot = requiredSlot return await this.serialize( - slot, + requiredSlot, signal, - (effectiveSignal) => operation(slot, effectiveSignal), - status + (effectiveSignal) => operation(requiredSlot, effectiveSignal), + status, + stopLoadingAllowed ) } catch (error) { + if ( + error instanceof BrowserNavigationStoppedError && + slot && + status !== 'loading' + ) { + this.recoverStoppedNavigation(slot) + } if (this.shouldEmitFailure(conversationId, signal, error)) { this.emitFailure(conversationId, failureStage, error) } @@ -577,41 +675,25 @@ export class BrowserService { !this.slots.has(conversationId) && !this.creations.has(conversationId) ) { - this.emitState(conversationId, 'creating', { url }) + this.emitState(conversationId, 'creating') } try { const slot = await this.getOrCreateSlot(conversationId, signal) return await this.serialize(slot, signal, async (effectiveSignal) => { const target = await this.policy.validate(url, effectiveSignal) slot.session.approveNavigation(target) - try { - const result = await slot.driver.navigate( + return this.completeNavigation( + conversationId, + slot, + effectiveSignal, + () => + slot.driver.navigate( target.url.href, effectiveSignal - ) - const finalTarget = await this.policy.validateRedirect( - result.url, - effectiveSignal - ) - if (slot.session.getCurrentOrigin() !== finalTarget.origin) { - throw new Error('浏览器导航结果来源不一致') - } - slot.origin = finalTarget.origin - await this.captureFrame( - conversationId, - slot, - effectiveSignal, - finalTarget.url.href - ) - return { - url: finalTarget.url.href, - origin: finalTarget.origin - } - } catch (error) { - await this.releaseSlot(slot).catch(() => undefined) - throw error - } - }, 'loading') + ), + '浏览器导航结果来源不一致' + ) + }, 'loading', true) } catch (error) { if (this.shouldEmitFailure(conversationId, signal, error)) { this.emitFailure(conversationId, '浏览器导航', error) @@ -628,6 +710,7 @@ export class BrowserService { conversationId, signal, 'acting', + false, '读取浏览器页面', async (slot, effectiveSignal) => { await this.verifyCurrentOriginOrRelease(slot) @@ -657,6 +740,7 @@ export class BrowserService { conversationId, signal, 'acting', + true, '浏览器点击', async (slot, effectiveSignal) => { await this.verifyCurrentOriginOrRelease(slot) @@ -676,6 +760,7 @@ export class BrowserService { conversationId, signal, 'acting', + false, '浏览器输入', async (slot, effectiveSignal) => { await this.verifyCurrentOriginOrRelease(slot) @@ -695,6 +780,7 @@ export class BrowserService { conversationId, signal, 'acting', + false, '浏览器选择', async (slot, effectiveSignal) => { await this.verifyCurrentOriginOrRelease(slot) @@ -712,6 +798,7 @@ export class BrowserService { conversationId, signal, 'loading', + true, '浏览器返回', async (slot, effectiveSignal) => { await this.verifyCurrentOriginOrRelease(slot) @@ -722,33 +809,36 @@ export class BrowserService { effectiveSignal ) slot.session.approveNavigation(target) - try { - const result = await slot.driver.backTo( - historyTarget, - effectiveSignal - ) - const finalTarget = await this.policy.validateRedirect( - result.url, - effectiveSignal - ) - if (slot.session.getCurrentOrigin() !== finalTarget.origin) { - throw new Error('浏览器返回结果来源不一致') - } - slot.origin = finalTarget.origin - await this.captureFrame( - conversationId, - slot, - effectiveSignal, - finalTarget.url.href - ) - return { - url: finalTarget.url.href, - origin: finalTarget.origin - } - } catch (error) { - await this.releaseSlot(slot).catch(() => undefined) - throw error - } + return this.completeNavigation( + conversationId, + slot, + effectiveSignal, + () => slot.driver.backTo(historyTarget, effectiveSignal), + '浏览器返回结果来源不一致' + ) + } + ) + } + + async reload( + conversationId: string, + signal: AbortSignal + ): Promise<{ url: string; origin: string }> { + return this.runInSession( + conversationId, + signal, + 'loading', + true, + '浏览器刷新', + async (slot, effectiveSignal) => { + await this.verifyCurrentOriginOrRelease(slot) + return this.completeNavigation( + conversationId, + slot, + effectiveSignal, + () => slot.driver.reload(effectiveSignal), + '浏览器刷新结果来源不一致' + ) } ) } @@ -761,6 +851,7 @@ export class BrowserService { conversationId, signal, 'acting', + false, '浏览器截图', async (slot, effectiveSignal) => { await this.verifyCurrentOriginOrRelease(slot) @@ -795,6 +886,7 @@ export class BrowserService { conversationId, signal, 'interactive', + false, '浏览器交互', async (slot, effectiveSignal) => { await this.verifyCurrentOriginOrRelease(slot) @@ -817,6 +909,101 @@ export class BrowserService { ) } + async stopLoading(conversationId: string): Promise { + const slot = this.slots.get(conversationId) + const active = slot?.active + if (!slot || slot.released) { + return false + } + if (slot.stopLoadingRequested) { + return false + } + if (active) { + if ( + !active.stopLoadingAllowed || + active.controller.signal.aborted || + (active.status !== 'loading' && + !slot.isLoading && + !slot.session.isLoading()) + ) { + return false + } + slot.isLoading = false + slot.stopLoadingRequested = true + slot.session.stopLoading() + active.controller.abort(new BrowserNavigationStoppedError()) + return true + } + if (!slot.isLoading && !slot.session.isLoading()) { + return false + } + slot.isLoading = false + slot.stopLoadingRequested = true + slot.session.stopLoading() + this.recoverStoppedNavigation(slot) + return true + } + + private recoverStoppedNavigation(slot: BrowserSlot): void { + if (slot.released || this.slots.get(slot.conversationId) !== slot) { + return + } + slot.isLoading = false + const currentUrl = slot.session.webContents.getURL() + try { + const currentTarget = canonicalizeBrowserUrl(currentUrl) + if (slot.session.getCurrentOrigin() !== currentTarget.origin) { + throw new Error('浏览器停止导航后的页面来源不一致') + } + slot.origin = currentTarget.origin + this.emitState(slot.conversationId, 'ready', { + url: currentTarget.href, + isLoading: false + }) + } catch { + if (!slot.released && this.slots.get(slot.conversationId) === slot) { + this.emitState(slot.conversationId, 'ready', { isLoading: false }) + } + } + } + + private async completeNavigation( + conversationId: string, + slot: BrowserSlot, + signal: AbortSignal, + operation: () => Promise<{ url: string }>, + originMismatchMessage: string + ): Promise<{ url: string; origin: string }> { + try { + const result = await operation() + const finalTarget = await this.policy.validateRedirect( + result.url, + signal + ) + if (slot.session.getCurrentOrigin() !== finalTarget.origin) { + throw new Error(originMismatchMessage) + } + slot.origin = finalTarget.origin + await this.captureFrame( + conversationId, + slot, + signal, + finalTarget.url.href + ) + return { + url: finalTarget.url.href, + origin: finalTarget.origin + } + } catch (error) { + if (error instanceof BrowserNavigationStoppedError) { + this.recoverStoppedNavigation(slot) + throw error + } + await this.releaseSlot(slot).catch(() => undefined) + throw error + } + } + async releaseConversation(conversationId: string): Promise { this.releaseRequests.add(conversationId) let releasedSlot = false @@ -855,7 +1042,8 @@ export class BrowserService { clearTimeout(slot.idleTimer) slot.idleTimer = undefined } - slot.active?.abort(new Error('浏览器会话已释放')) + slot.removeLoadingListener() + slot.active?.controller.abort(new Error('浏览器会话已释放')) try { slot.driver.dispose() } finally { diff --git a/src/main/browser/cdp-browser-driver.test.ts b/src/main/browser/cdp-browser-driver.test.ts index 967cc441..3c79f51b 100644 --- a/src/main/browser/cdp-browser-driver.test.ts +++ b/src/main/browser/cdp-browser-driver.test.ts @@ -692,6 +692,63 @@ describe('CdpBrowserDriver', () => { driver.dispose() }) + it('reloads through CDP with bounded commit and document waits', async () => { + const harness = createHarness(async (method, parameters) => + method === 'Page.reload' + ? {} + : standardCommand(method, parameters) + ) + const driver = new CdpBrowserDriver(harness.webContents) + const reload = driver.reload(new AbortController().signal) + await vi.waitFor(() => + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Page.reload', + undefined + ) + ) + harness.setUrl('https://example.com/reloaded') + harness.contentEvents.emit( + 'did-navigate', + {}, + 'https://example.com/reloaded' + ) + + await expect(reload).resolves.toEqual({ + url: 'https://example.com/reloaded' + }) + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Runtime.evaluate', + expect.objectContaining({ expression: 'document.readyState' }) + ) + driver.dispose() + }) + + it('reports committed URL and actual back availability from history', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents) + const historyQuery = vi.spyOn( + driver as unknown as { + getNavigationHistory(signal: AbortSignal): Promise + }, + 'getNavigationHistory' + ) + + await expect( + driver.getNavigationMetadata(new AbortController().signal) + ).resolves.toEqual({ + url: 'https://example.com/page', + canGoBack: true + }) + await expect( + driver.getBackTarget(new AbortController().signal) + ).resolves.toEqual({ + entryId: 4, + url: 'https://previous.example/' + }) + expect(historyQuery).toHaveBeenCalledTimes(2) + driver.dispose() + }) + it('bounds hung commands and removes listeners on disposal', async () => { const harness = createHarness(async () => new Promise(() => undefined)) const driver = new CdpBrowserDriver(harness.webContents, { timeoutMs: 5 }) diff --git a/src/main/browser/cdp-browser-driver.ts b/src/main/browser/cdp-browser-driver.ts index 8f24aa78..b76cedee 100644 --- a/src/main/browser/cdp-browser-driver.ts +++ b/src/main/browser/cdp-browser-driver.ts @@ -84,6 +84,16 @@ export type BrowserHistoryTarget = { url: string } +export type BrowserNavigationMetadata = { + url: string + canGoBack: boolean +} + +type BrowserNavigationHistory = { + currentIndex: number + entries: Array<{ id?: number; url?: string }> +} + type RefBinding = { backendNodeId: number generation: number @@ -346,6 +356,25 @@ export class CdpBrowserDriver { return { url: this.webContents.getURL() || url } } + async reload(signal: AbortSignal): Promise<{ url: string }> { + const currentUrl = this.webContents.getURL() + if (!currentUrl) { + throw new Error('浏览器当前页面不可重新加载') + } + this.invalidate() + const navigation = this.waitForMainFrameCommit(currentUrl, signal) + try { + await this.command('Page.reload', undefined, signal) + } catch (error) { + navigation.cancel(error) + await navigation.promise.catch(() => undefined) + throw error + } + await navigation.promise + await this.waitForDocument(signal) + return { url: this.webContents.getURL() || currentUrl } + } + private waitForMainFrameCommit( targetUrl: string, signal: AbortSignal @@ -813,12 +842,9 @@ export class CdpBrowserDriver { } async getBackTarget(signal: AbortSignal): Promise { - const history = await this.command<{ - currentIndex?: number - entries?: Array<{ id?: number; url?: string }> - }>('Page.getNavigationHistory', undefined, signal) - const index = history.currentIndex ?? -1 - const entry = history.entries?.[index - 1] + const history = await this.getNavigationHistory(signal) + const index = history.currentIndex + const entry = history.entries[index - 1] if ( index < 1 || typeof entry?.id !== 'number' || @@ -831,6 +857,42 @@ export class CdpBrowserDriver { return { entryId: entry.id, url: entry.url } } + private async getNavigationHistory( + signal: AbortSignal + ): Promise { + const history = await this.command<{ + currentIndex?: number + entries?: Array<{ id?: number; url?: string }> + }>('Page.getNavigationHistory', undefined, signal) + return { + currentIndex: history.currentIndex ?? -1, + entries: history.entries ?? [] + } + } + + async getNavigationMetadata( + signal: AbortSignal + ): Promise { + const history = await this.getNavigationHistory(signal) + const index = history.currentIndex + const entry = history.entries[index] + const url = + typeof entry?.url === 'string' && entry.url.length > 0 + ? entry.url + : this.webContents.getURL() + if (!url || url.length > 8_192) { + throw new Error('浏览器当前页面 URL 无效') + } + const previousUrl = history.entries[index - 1]?.url + return { + url, + canGoBack: + index > 0 && + typeof previousUrl === 'string' && + /^https?:\/\//iu.test(previousUrl) + } + } + async backTo( target: BrowserHistoryTarget, signal: AbortSignal diff --git a/src/main/browser/electron-browser-session.test.ts b/src/main/browser/electron-browser-session.test.ts index a23d2877..3c34dd0f 100644 --- a/src/main/browser/electron-browser-session.test.ts +++ b/src/main/browser/electron-browser-session.test.ts @@ -23,6 +23,7 @@ function createHarness() { const partitionEvents = new EventEmitter() const windowEvents = new EventEmitter() let currentUrl = '' + let loadingMainFrame = false let openHandler: ((details: { url: string }) => { action: 'deny' }) | undefined const sendCommand = vi.fn(async () => ({})) const capturedImage = { @@ -63,6 +64,7 @@ function createHarness() { }), capturePage: vi.fn(async () => capturedImage), getURL: vi.fn(() => currentUrl), + isLoadingMainFrame: vi.fn(() => loadingMainFrame), stop: vi.fn(), destroy: vi.fn(), isDestroyed: vi.fn(() => false) @@ -151,6 +153,9 @@ function createHarness() { setCurrentUrl(value: string) { currentUrl = value }, + setLoadingMainFrame(value: boolean) { + loadingMainFrame = value + }, getOpenHandler: () => openHandler, getPermissionCheck: () => permissionCheck, getPermissionRequest: () => permissionRequest, @@ -223,6 +228,8 @@ describe('ElectronBrowserSession', () => { mimeType: 'image/jpeg', data: '/9j/2Q==' }) + session.stopLoading() + expect(harness.webContents.stop).toHaveBeenCalledOnce() const downloadEvent = { preventDefault: vi.fn() } const item = { cancel: vi.fn() } @@ -293,6 +300,34 @@ describe('ElectronBrowserSession', () => { await session.dispose() }) + it('reports actual main-frame loading changes and detaches subscribers', async () => { + const harness = createHarness() + harness.setLoadingMainFrame(true) + const session = await ElectronBrowserSession.create({ + policy: harness.policy, + createPartition: async () => harness.partition, + createWindow: async () => harness.window, + createProxy: () => harness.proxy + }) + expect(session.isLoading()).toBe(true) + const changes: boolean[] = [] + const remove = session.onLoadingChange((isLoading) => { + changes.push(isLoading) + }) + + harness.contentEvents.emit('did-start-loading') + harness.setLoadingMainFrame(false) + harness.contentEvents.emit('did-stop-loading') + expect(session.isLoading()).toBe(false) + expect(changes).toEqual([false]) + + remove() + harness.setLoadingMainFrame(true) + harness.contentEvents.emit('did-start-loading') + expect(changes).toEqual([false]) + await session.dispose() + }) + it('restores the browser for interaction and minimizes it on close', async () => { const harness = createHarness() const parentWindow = { diff --git a/src/main/browser/electron-browser-session.ts b/src/main/browser/electron-browser-session.ts index c48f79c1..f27a8f07 100644 --- a/src/main/browser/electron-browser-session.ts +++ b/src/main/browser/electron-browser-session.ts @@ -40,6 +40,7 @@ export type BrowserWebContents = { ): void capturePage?(): Promise getURL(): string + isLoadingMainFrame?(): boolean stop(): void close?(options?: { waitForBeforeUnload?: boolean }): void destroy(): void @@ -230,6 +231,8 @@ export class ElectronBrowserSession { resolve(frame?: BrowserScreenshot): void } private interactionClosing?: Promise + private readonly loadingListeners = new Set<(isLoading: boolean) => void>() + private loading = false private disposed = false private constructor( @@ -412,6 +415,13 @@ export class ElectronBrowserSession { private async initialize(): Promise { const contents = this.webContents + this.loading = contents.isLoadingMainFrame?.() ?? false + this.listen(contents, 'did-start-loading', () => { + this.setLoading(true) + }) + this.listen(contents, 'did-stop-loading', () => { + this.setLoading(contents.isLoadingMainFrame?.() ?? false) + }) this.listen( this.window, 'close', @@ -531,6 +541,33 @@ export class ElectronBrowserSession { } } + isLoading(): boolean { + return this.loading + } + + onLoadingChange(listener: (isLoading: boolean) => void): () => void { + this.assertOpen() + this.loadingListeners.add(listener) + return () => { + this.loadingListeners.delete(listener) + } + } + + private setLoading(isLoading: boolean): void { + if (this.loading === isLoading) { + return + } + this.loading = isLoading + for (const listener of this.loadingListeners) { + listener(isLoading) + } + } + + stopLoading(): void { + this.assertOpen() + this.webContents.stop() + } + async captureScreenshot( signal: AbortSignal ): Promise { @@ -648,6 +685,7 @@ export class ElectronBrowserSession { } this.disposed = true this.approvedOrigin = undefined + this.loadingListeners.clear() this.finishInteraction() for (const { target, event, listener } of this.listeners.splice(0)) { target.off(event, listener) diff --git a/src/main/ipc.test.ts b/src/main/ipc.test.ts index c7fb140d..ad422a90 100644 --- a/src/main/ipc.test.ts +++ b/src/main/ipc.test.ts @@ -19,6 +19,7 @@ import type { } from '../shared/ssh-host-contracts' import { defaultKnowledgeOntologySettings } from '../shared/knowledge-ontology' import { AssistantDatabase } from './assistant/assistant-database' +import { BrowserNavigationStoppedError } from './browser/browser-service' import { registerIpcHandlers, sendRemoteEnvironmentUpdateProgress, @@ -357,6 +358,10 @@ describe('registerIpcHandlers computer capabilities', () => { setBuiltinMcpServerEnabled: vi.fn(async () => snapshot), setBuiltinMcpServerAssignments: vi.fn(async () => snapshot), setComputerCapabilityEnabled: vi.fn(async () => snapshot), + getComputerCapabilityStatus: vi.fn(async () => ({ + enabled: true, + supported: true + })), setWebSearchEnabled: vi.fn(async () => snapshot), createBrowserProfile: vi.fn(async () => snapshot), diagnoseComputerCapability: vi.fn(async () => ({ @@ -368,6 +373,19 @@ describe('registerIpcHandlers computer capabilities', () => { } const onRuntimeSettingsChanged = vi.fn(async () => {}) const interact = vi.fn(async () => {}) + const navigate = vi.fn(async () => ({ + url: 'https://example.com/', + origin: 'https://example.com' + })) + const back = vi.fn(async () => ({ + url: 'https://example.com/previous', + origin: 'https://example.com' + })) + const reload = vi.fn(async () => ({ + url: 'https://example.com/', + origin: 'https://example.com' + })) + const stopLoading = vi.fn(async () => true) const releaseConversation = vi.fn(async () => {}) const selectFiles = vi.fn( async ( @@ -430,6 +448,10 @@ describe('registerIpcHandlers computer capabilities', () => { onRuntimeSettingsChanged, undefined, { + navigate, + back, + reload, + stopLoading, interact, releaseConversation, onState: (listener) => { @@ -573,6 +595,10 @@ describe('registerIpcHandlers computer capabilities', () => { browserStateListener?.({ conversationId: 'browser-conversation', status: 'ready', + sessionActive: true, + isLoading: false, + canGoBack: false, + frameDataUrl: 'data:image/jpeg;base64,frame', updatedAt: 1 }) expect(webContents.send).toHaveBeenCalledWith( @@ -582,6 +608,38 @@ describe('registerIpcHandlers computer capabilities', () => { status: 'ready' }) ) + browserStateListener?.({ + conversationId: 'browser-conversation', + status: 'acting', + sessionActive: true, + isLoading: false, + canGoBack: false, + frameDataUrl: 'data:image/jpeg;base64,frame', + updatedAt: 2 + }) + const repeatedFramePayload = webContents.send.mock.calls.at(-1)?.[1] + expect(repeatedFramePayload).not.toHaveProperty('frameDataUrl') + browserStateListener?.({ + conversationId: 'browser-conversation', + status: 'stopped', + sessionActive: false, + isLoading: false, + canGoBack: false, + updatedAt: 3 + }) + browserStateListener?.({ + conversationId: 'browser-conversation', + status: 'ready', + sessionActive: true, + isLoading: false, + canGoBack: false, + frameDataUrl: 'data:image/jpeg;base64,frame', + updatedAt: 4 + }) + expect(webContents.send.mock.calls.at(-1)?.[1]).toHaveProperty( + 'frameDataUrl', + 'data:image/jpeg;base64,frame' + ) await expect( electronMocks.handlers.get(ipcChannels.browserStop)?.(event, { conversationId: 'browser-conversation' @@ -590,6 +648,42 @@ describe('registerIpcHandlers computer capabilities', () => { expect(releaseConversation).toHaveBeenCalledWith( 'browser-conversation' ) + await expect( + electronMocks.handlers.get(ipcChannels.browserNavigate)?.(event, { + conversationId: 'browser-conversation', + url: 'https://example.com/' + }) + ).resolves.toBeUndefined() + expect(navigate).toHaveBeenCalledWith( + 'browser-conversation', + 'https://example.com/', + expect.any(AbortSignal) + ) + await expect( + electronMocks.handlers.get(ipcChannels.browserBack)?.(event, { + conversationId: 'browser-conversation' + }) + ).resolves.toBeUndefined() + expect(back).toHaveBeenCalledWith( + 'browser-conversation', + expect.any(AbortSignal) + ) + await expect( + electronMocks.handlers.get(ipcChannels.browserReload)?.(event, { + conversationId: 'browser-conversation' + }) + ).resolves.toBeUndefined() + expect(reload).toHaveBeenCalledWith( + 'browser-conversation', + expect.any(AbortSignal) + ) + await expect( + electronMocks.handlers.get(ipcChannels.browserStopLoading)?.( + event, + { conversationId: 'browser-conversation' } + ) + ).resolves.toBeUndefined() + expect(stopLoading).toHaveBeenCalledWith('browser-conversation') await expect( electronMocks.handlers.get(ipcChannels.browserInteract)?.(event, { conversationId: 'browser-conversation' @@ -600,6 +694,57 @@ describe('registerIpcHandlers computer capabilities', () => { expect.any(AbortSignal) ) + navigate.mockRejectedValueOnce(new BrowserNavigationStoppedError()) + await expect( + electronMocks.handlers.get(ipcChannels.browserNavigate)?.(event, { + conversationId: 'browser-conversation', + url: 'https://example.com/slow' + }) + ).resolves.toBeUndefined() + capabilityService.getComputerCapabilityStatus.mockResolvedValueOnce({ + enabled: false, + supported: true + }) + await expect( + electronMocks.handlers.get(ipcChannels.browserBack)?.(event, { + conversationId: 'browser-conversation' + }) + ).rejects.toThrow('尚未启用') + capabilityService.getComputerCapabilityStatus.mockResolvedValueOnce({ + enabled: true, + supported: false + }) + await expect( + electronMocks.handlers.get(ipcChannels.browserReload)?.(event, { + conversationId: 'browser-conversation' + }) + ).rejects.toThrow('不支持') + await expect( + electronMocks.handlers.get(ipcChannels.browserNavigate)?.(event, { + conversationId: 'browser-conversation', + url: 'https://example.com/', + debugger: true + }) + ).rejects.toThrow() + await expect( + electronMocks.handlers.get(ipcChannels.browserStopLoading)?.( + { + sender: {}, + senderFrame: webContents.mainFrame + }, + { conversationId: 'browser-conversation' } + ) + ).rejects.toThrow('拒绝来自未知窗口的 IPC 请求') + await expect( + electronMocks.handlers.get(ipcChannels.browserStopLoading)?.( + event, + { + conversationId: 'browser-conversation', + rawElectron: true + } + ) + ).rejects.toThrow() + expect(() => electronMocks.handlers.get( ipcChannels.capabilitiesCreateBrowserProfile diff --git a/src/main/ipc.ts b/src/main/ipc.ts index 39de491d..cee18ac6 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -25,7 +25,11 @@ import { approvalDecisionSchema, agentQuestionResponseSchema, agentRequestSchema, + browserBackRequestSchema, browserInteractRequestSchema, + browserNavigateRequestSchema, + browserReloadRequestSchema, + browserStopLoadingRequestSchema, browserStopRequestSchema, conversationQueueUserInputSchema, defaultRuntimeSettings, @@ -127,6 +131,7 @@ import { agentPackageInventorySchema } from '../shared/agent-package-contracts' import { assertTrustedSender } from './trusted-ipc-sender' +import { BrowserNavigationStoppedError } from './browser/browser-service' import type { TerminalSessionManager } from './terminal/terminal-session-manager' import { releaseNotesAcknowledgeSchema } from '../shared/release-notes-contracts' import { @@ -1050,6 +1055,20 @@ export function registerIpcHandlers( onRuntimeSettingsChanged: () => Promise, onBeforeClearLocalData?: () => Promise, browserControl?: { + navigate( + conversationId: string, + url: string, + signal: AbortSignal + ): Promise<{ url: string; origin: string }> + back( + conversationId: string, + signal: AbortSignal + ): Promise<{ url: string; origin: string }> + reload( + conversationId: string, + signal: AbortSignal + ): Promise<{ url: string; origin: string }> + stopLoading(conversationId: string): boolean | Promise interact( conversationId: string, signal: AbortSignal @@ -1314,9 +1333,25 @@ export function registerIpcHandlers( } window.on('maximize', notifyMaximizedChanged) window.on('unmaximize', notifyMaximizedChanged) + const lastSentBrowserFrames = new Map() const removeBrowserStateListener = browserControl?.onState((state) => { if (!window.isDestroyed()) { - window.webContents.send(ipcChannels.browserState, state) + const frame = state.frameDataUrl + let payload: BrowserLiveState = state + if ( + frame && + lastSentBrowserFrames.get(state.conversationId) === frame + ) { + payload = { ...state } + delete payload.frameDataUrl + } + if (frame) { + lastSentBrowserFrames.set(state.conversationId, frame) + } + if (state.status === 'stopped') { + lastSentBrowserFrames.delete(state.conversationId) + } + window.webContents.send(ipcChannels.browserState, payload) } }) const abortActiveRequests = ( @@ -3364,6 +3399,93 @@ export function registerIpcHandlers( ]) }) + const requireBrowserControl = async (): Promise< + NonNullable + > => { + if (!browserControl) { + throw new Error('浏览器控制当前不可用') + } + const capability = + await capabilityService.getComputerCapabilityStatus( + 'host-browser-control' + ) + if (!capability.supported) { + throw new Error('当前平台不支持浏览器控制') + } + if (!capability.enabled) { + throw new Error('浏览器控制能力尚未启用') + } + return browserControl + } + + const runUiBrowserNavigation = async ( + operation: ( + control: NonNullable, + signal: AbortSignal + ) => Promise + ): Promise => { + try { + const control = await requireBrowserControl() + await operation(control, new AbortController().signal) + } catch (error) { + if (!(error instanceof BrowserNavigationStoppedError)) { + throw error + } + } + } + + registerHandler( + ipcChannels.browserNavigate, + async (event, input: unknown) => { + assertTrustedSender(event, window) + const request = browserNavigateRequestSchema.parse(input) + await runUiBrowserNavigation((control, signal) => + control.navigate( + request.conversationId, + request.url, + signal + ) + ) + } + ) + + registerHandler( + ipcChannels.browserBack, + async (event, input: unknown) => { + assertTrustedSender(event, window) + const request = browserBackRequestSchema.parse(input) + await runUiBrowserNavigation((control, signal) => + control.back( + request.conversationId, + signal + ) + ) + } + ) + + registerHandler( + ipcChannels.browserReload, + async (event, input: unknown) => { + assertTrustedSender(event, window) + const request = browserReloadRequestSchema.parse(input) + await runUiBrowserNavigation((control, signal) => + control.reload( + request.conversationId, + signal + ) + ) + } + ) + + registerHandler( + ipcChannels.browserStopLoading, + async (event, input: unknown) => { + assertTrustedSender(event, window) + const request = browserStopLoadingRequestSchema.parse(input) + await browserControl?.stopLoading(request.conversationId) + } + ) + registerHandler( ipcChannels.terminalCreate, async (event, input: unknown) => { @@ -3455,7 +3577,8 @@ export function registerIpcHandlers( async (event, input: unknown) => { assertTrustedSender(event, window) const request = browserInteractRequestSchema.parse(input) - await browserControl?.interact( + const control = await requireBrowserControl() + await control.interact( request.conversationId, new AbortController().signal ) @@ -7837,6 +7960,7 @@ export function registerIpcHandlers( return async () => { shuttingDown = true removeBrowserStateListener?.() + lastSentBrowserFrames.clear() removeRemoteAgentConnectionStatusListener?.() clearInterval(scheduleInterval) for (const timeout of queueDispatchTimers.values()) { diff --git a/src/preload/index.ts b/src/preload/index.ts index 97b35290..03082608 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -290,6 +290,27 @@ const desktopApi: DesktopApi = { } }, browser: { + navigate: async (conversationId: string, url: string) => { + await ipcRenderer.invoke(ipcChannels.browserNavigate, { + conversationId, + url + }) + }, + back: async (conversationId: string) => { + await ipcRenderer.invoke(ipcChannels.browserBack, { + conversationId + }) + }, + reload: async (conversationId: string) => { + await ipcRenderer.invoke(ipcChannels.browserReload, { + conversationId + }) + }, + stopLoading: async (conversationId: string) => { + await ipcRenderer.invoke(ipcChannels.browserStopLoading, { + conversationId + }) + }, interact: async (conversationId: string) => { await ipcRenderer.invoke( ipcChannels.browserInteract, diff --git a/src/preload/preload-sandbox.test.ts b/src/preload/preload-sandbox.test.ts index a8bfa95e..e2a33192 100644 --- a/src/preload/preload-sandbox.test.ts +++ b/src/preload/preload-sandbox.test.ts @@ -21,6 +21,31 @@ describe('sandboxed preload', () => { expect(source).not.toMatch(/\bfrom\s+['"]zod['"]/u) }) + it('exposes only narrow browser-control methods', () => { + const source = readFileSync( + join(process.cwd(), 'src', 'preload', 'index.ts'), + 'utf8' + ) + const browser = + source.match( + /browser: \{(?[\s\S]*?)\r?\n {2}\},\r?\n {2}terminal:/u + )?.groups?.body ?? '' + for (const method of [ + 'navigate:', + 'back:', + 'reload:', + 'stopLoading:', + 'interact:', + 'stop:', + 'onState:' + ]) { + expect(browser).toContain(method) + } + expect(browser).not.toMatch( + /\b(?:webContents|debugger|session|partition|cookie)\b/iu + ) + }) + it('exposes only explicit computer capability and managed profile IPC methods', () => { const source = readFileSync( join(process.cwd(), 'src', 'preload', 'index.ts'), diff --git a/src/renderer/src/App.test.tsx b/src/renderer/src/App.test.tsx index 5aecf662..0a271f45 100644 --- a/src/renderer/src/App.test.tsx +++ b/src/renderer/src/App.test.tsx @@ -245,6 +245,10 @@ const api: DesktopApi = { }) }, browser: { + navigate: vi.fn(async () => {}), + back: vi.fn(async () => {}), + reload: vi.fn(async () => {}), + stopLoading: vi.fn(async () => {}), interact: vi.fn(async () => {}), stop: vi.fn(async () => {}), onState: vi.fn((listener) => { @@ -963,6 +967,21 @@ function deferred(): { return { promise, resolve, reject } } +function createBrowserState( + conversationId: string, + overrides: Partial = {} +): BrowserLiveState { + return { + conversationId, + status: 'ready', + sessionActive: true, + isLoading: false, + canGoBack: false, + updatedAt: Date.now(), + ...overrides + } +} + function installRemoteProjectsSetting(enabled: boolean): { updateSettings: NonNullable['updateSettings'] } { @@ -9571,7 +9590,140 @@ describe('App', () => { ).toBeInTheDocument() }) - it('opens the live browser tab for the active conversation and can stop it', async () => { + it('shows the browser toolbar and starts an empty session by address', async () => { + render() + + fireEvent.click(screen.getByLabelText('切换助手工作栏')) + fireEvent.click(screen.getByRole('tab', { name: '浏览器' })) + + const toolbar = screen.getByRole('form', { + name: '浏览器工具栏' + }) + const address = within(toolbar).getByRole('textbox', { + name: '浏览器地址' + }) + const go = within(toolbar).getByRole('button', { name: '前往' }) + expect(within(toolbar).getByRole('button', { name: '返回' })) + .toBeDisabled() + expect(within(toolbar).getByRole('button', { name: '刷新' })) + .toBeDisabled() + expect(within(toolbar).getByRole('button', { name: '交互' })) + .toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '关闭浏览器' }) + ).toBeDisabled() + expect(go).toBeDisabled() + expect(screen.getByText(/输入地址并前往可打开页面/u)) + .toBeInTheDocument() + + fireEvent.change(address, { + target: { value: 'https://example.com/start' } + }) + expect(go).toBeEnabled() + fireEvent.click(go) + await waitFor(() => + expect(api.browser.navigate).toHaveBeenLastCalledWith( + expect.any(String), + 'https://example.com/start' + ) + ) + expect(address).toHaveValue('https://example.com/start') + + fireEvent.change(address, { + target: { value: 'https://example.com/enter' } + }) + fireEvent.keyDown(address, { key: 'Enter' }) + await waitFor(() => + expect(api.browser.navigate).toHaveBeenLastCalledWith( + expect.any(String), + 'https://example.com/enter' + ) + ) + expect(api.browser.navigate).toHaveBeenCalledTimes(2) + }) + + it('suppresses duplicate browser actions while keeping stop loading available', async () => { + const navigation = deferred() + vi.mocked(api.browser.navigate).mockReturnValueOnce( + navigation.promise + ) + render() + + fireEvent.click(screen.getByLabelText('切换助手工作栏')) + fireEvent.click(screen.getByRole('tab', { name: '浏览器' })) + + const toolbar = screen.getByRole('form', { + name: '浏览器工具栏' + }) + const address = within(toolbar).getByRole('textbox', { + name: '浏览器地址' + }) + const go = within(toolbar).getByRole('button', { name: '前往' }) + fireEvent.change(address, { + target: { value: 'https://example.com/pending' } + }) + fireEvent.click(go) + fireEvent.click(go) + fireEvent.keyDown(address, { key: 'Enter' }) + + expect(api.browser.navigate).toHaveBeenCalledOnce() + const conversationId = + vi.mocked(api.browser.navigate).mock.calls[0]?.[0] ?? '' + act(() => { + browserListener?.(createBrowserState(conversationId, { + status: 'loading', + isLoading: true, + url: 'https://example.com/pending', + updatedAt: Date.now() + })) + }) + + expect( + within(toolbar).getByRole('button', { name: '返回' }) + ).toBeDisabled() + expect(address).toBeDisabled() + expect(go).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '交互' }) + ).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '关闭浏览器' }) + ).toBeDisabled() + const stopLoading = within(toolbar).getByRole('button', { + name: '停止加载' + }) + expect(stopLoading).toBeEnabled() + fireEvent.click(stopLoading) + expect(api.browser.stopLoading).toHaveBeenCalledOnce() + + await act(async () => { + navigation.resolve() + await navigation.promise + }) + const back = within(toolbar).getByRole('button', { name: '返回' }) + const backNavigation = deferred() + vi.mocked(api.browser.back).mockReturnValueOnce( + backNavigation.promise + ) + act(() => { + browserListener?.(createBrowserState(conversationId, { + canGoBack: true, + url: 'https://example.com/pending', + updatedAt: Date.now() + 1 + })) + }) + expect(back).toBeEnabled() + fireEvent.click(back) + fireEvent.click(back) + expect(api.browser.back).toHaveBeenCalledOnce() + + await act(async () => { + backNavigation.resolve() + await backNavigation.promise + }) + }) + + it('controls the active live browser without overwriting its address draft', async () => { render() fireEvent.change(screen.getByLabelText('向 GoodBuddy 提问'), { @@ -9583,13 +9735,12 @@ describe('App', () => { expect(conversationId).toBeTruthy() act(() => { - browserListener?.({ - conversationId: conversationId ?? '', - status: 'ready', + browserListener?.(createBrowserState(conversationId ?? '', { + canGoBack: false, url: 'https://example.com/', frameDataUrl: 'data:image/jpeg;base64,/9j/2Q==', updatedAt: Date.now() - }) + })) }) expect(screen.getByLabelText('助手工作栏')).toHaveClass( @@ -9604,14 +9755,191 @@ describe('App', () => { 'src', 'data:image/jpeg;base64,/9j/2Q==' ) + const toolbar = screen.getByRole('form', { + name: '浏览器工具栏' + }) + const address = within(toolbar).getByRole('textbox', { + name: '浏览器地址' + }) + const back = within(toolbar).getByRole('button', { name: '返回' }) + expect(address).toHaveValue('https://example.com/') + expect(back).toBeDisabled() + + fireEvent.focus(address) + fireEvent.change(address, { + target: { value: 'https://typed.example/draft' } + }) + act(() => { + browserListener?.(createBrowserState(conversationId ?? '', { + canGoBack: true, + url: 'https://example.com/redirected', + updatedAt: Date.now() + 1 + })) + }) + expect(address).toHaveValue('https://typed.example/draft') + expect( + screen.getByAltText('Agent 实时浏览器画面') + ).toHaveAttribute( + 'src', + 'data:image/jpeg;base64,/9j/2Q==' + ) + + fireEvent.keyDown(address, { key: 'Escape' }) + expect(address).toHaveValue('https://example.com/redirected') + expect(address).not.toHaveFocus() + expect(back).toBeEnabled() + fireEvent.click(back) + await waitFor(() => + expect(api.browser.back).toHaveBeenCalledWith(conversationId) + ) + fireEvent.click( - screen.getByRole('button', { name: '交互' }) + within(toolbar).getByRole('button', { name: '刷新' }) + ) + await waitFor(() => + expect(api.browser.reload).toHaveBeenCalledWith(conversationId) + ) + fireEvent.change(address, { + target: { value: 'https://example.com/manual' } + }) + fireEvent.click( + within(toolbar).getByRole('button', { name: '前往' }) + ) + await waitFor(() => + expect(api.browser.navigate).toHaveBeenCalledWith( + conversationId, + 'https://example.com/manual' + ) + ) + expect(address).toHaveValue('https://example.com/redirected') + act(() => { + browserListener?.(createBrowserState(conversationId ?? '', { + canGoBack: true, + url: 'https://example.com/canonical/', + updatedAt: Date.now() + 2 + })) + }) + expect(address).toHaveValue('https://example.com/canonical/') + fireEvent.focus(address) + fireEvent.change(address, { + target: { value: 'https://example.com/canonical' } + }) + fireEvent.click( + within(toolbar).getByRole('button', { name: '前往' }) + ) + await waitFor(() => + expect(api.browser.navigate).toHaveBeenCalledWith( + conversationId, + 'https://example.com/canonical' + ) + ) + expect(address).toHaveValue('https://example.com/canonical/') + fireEvent.click( + within(toolbar).getByRole('button', { name: '交互' }) ) await waitFor(() => expect(api.browser.interact).toHaveBeenCalledWith(conversationId) ) + + act(() => { + browserListener?.(createBrowserState(conversationId ?? '', { + status: 'loading', + isLoading: false, + canGoBack: true, + url: 'https://example.com/canonical/', + updatedAt: Date.now() + 3 + })) + }) + expect(back).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '刷新' }) + ).toBeDisabled() + expect(address).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '交互' }) + ).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '关闭浏览器' }) + ).toBeDisabled() + + act(() => { + browserListener?.(createBrowserState(conversationId ?? '', { + status: 'acting', + canGoBack: true, + url: 'https://example.com/canonical/', + updatedAt: Date.now() + 4 + })) + }) + expect(back).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '刷新' }) + ).toBeDisabled() + expect(address).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '前往' }) + ).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '交互' }) + ).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '关闭浏览器' }) + ).toBeDisabled() + + act(() => { + browserListener?.(createBrowserState(conversationId ?? '', { + status: 'acting', + isLoading: true, + canGoBack: true, + url: 'https://example.com/canonical/', + updatedAt: Date.now() + 5 + })) + }) + expect( + within(toolbar).queryByRole('button', { name: '刷新' }) + ).not.toBeInTheDocument() + const stopLoading = within(toolbar).getByRole('button', { + name: '停止加载' + }) + expect(stopLoading).toBeEnabled() + expect( + within(toolbar).getByRole('button', { name: '关闭浏览器' }) + ).toBeDisabled() + fireEvent.click(stopLoading) + await waitFor(() => + expect(api.browser.stopLoading).toHaveBeenCalledWith( + conversationId + ) + ) + + act(() => { + browserListener?.(createBrowserState(conversationId ?? '', { + status: 'interactive', + canGoBack: true, + url: 'https://example.com/canonical/', + updatedAt: Date.now() + 6 + })) + }) + expect(back).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '刷新' }) + ).toBeDisabled() + expect(address).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '交互' }) + ).toBeDisabled() + expect( + within(toolbar).getByRole('button', { name: '关闭浏览器' }) + ).toBeDisabled() + + act(() => { + browserListener?.(createBrowserState(conversationId ?? '', { + canGoBack: true, + url: 'https://example.com/canonical/', + updatedAt: Date.now() + 7 + })) + }) fireEvent.click( - screen.getByRole('button', { name: '停止浏览器' }) + within(toolbar).getByRole('button', { name: '关闭浏览器' }) ) await waitFor(() => expect(api.browser.stop).toHaveBeenCalledWith(conversationId) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 0e34a02d..6e7ac519 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -8203,6 +8203,25 @@ function App(): React.JSX.Element { ? t('runtime.unavailable') : t('runtime.state.ready')) const composerContextErrorId = 'composer-context-error' + const runBrowserCommand = async ( + command: ( + browserApi: NonNullable, + conversationId: string + ) => Promise + ): Promise => { + if (!activeId) { + return + } + const browserApi = window.goodbuddy.browser + if (!browserApi) { + notify({ + tone: 'error', + message: t('notices.browserControlUnavailable') + }) + return + } + await command(browserApi, activeId) + } return (
@@ -10822,6 +10841,7 @@ function App(): React.JSX.Element { /> )} { - if (!activeId) { - return - } - const browserApi = window.goodbuddy.browser - if (!browserApi) { - notify({ - tone: 'error', - message: t('notices.browserControlUnavailable') - }) - return - } - await browserApi.interact(activeId) - }} - onStopBrowser={async () => { - if (!activeId) { - return - } - const browserApi = window.goodbuddy.browser - if (!browserApi) { - notify({ - tone: 'error', - message: t('notices.browserControlUnavailable') - }) - return - } - try { - await browserApi.stop(activeId) - } catch { - notify({ - tone: 'error', - message: t('notices.browserStopFailed') - }) - } - }} + onBackBrowser={() => + runBrowserCommand((browserApi, conversationId) => + browserApi.back(conversationId) + ) + } + onInteractBrowser={() => + runBrowserCommand((browserApi, conversationId) => + browserApi.interact(conversationId) + ) + } + onNavigateBrowser={(url) => + runBrowserCommand((browserApi, conversationId) => + browserApi.navigate(conversationId, url) + ) + } + onReloadBrowser={() => + runBrowserCommand((browserApi, conversationId) => + browserApi.reload(conversationId) + ) + } + onStopLoadingBrowser={() => + runBrowserCommand((browserApi, conversationId) => + browserApi.stopLoading(conversationId) + ) + } + onStopBrowser={() => + runBrowserCommand((browserApi, conversationId) => + browserApi.stop(conversationId) + ) + } onImportArtifacts={async () => { const imported = await window.goodbuddy.artifacts.importFiles( activeProjectId || undefined diff --git a/src/renderer/src/RightAssistantSidebar.tsx b/src/renderer/src/RightAssistantSidebar.tsx index f329e9d3..4c4e87ff 100644 --- a/src/renderer/src/RightAssistantSidebar.tsx +++ b/src/renderer/src/RightAssistantSidebar.tsx @@ -1,4 +1,6 @@ import { + ArrowLeft, + ArrowRight, ChevronLeft, ChevronRight, CircleAlert, @@ -10,7 +12,9 @@ import { Plus, RefreshCw, ShieldAlert, - Upload + Square, + Upload, + X } from 'lucide-react' import { lazy, @@ -99,10 +103,15 @@ type RightAssistantSidebarProps = { selectedTaskId?: string workspaceChanges?: WorkspaceChanges workspaceProjectId?: string + activeConversationId?: string browserState?: BrowserLiveState currentProject?: AssistantProject restoreFocusRef?: { current: HTMLElement | null } + onBackBrowser?: () => Promise onInteractBrowser: () => Promise + onNavigateBrowser?: (url: string) => Promise + onReloadBrowser?: () => Promise + onStopLoadingBrowser?: () => Promise onStopBrowser: () => Promise onCreateCustomTask: () => void onImportArtifacts: () => Promise @@ -228,6 +237,226 @@ function measureSplitLayoutWidth( return measuredWidth > 0 ? measuredWidth : fallbackWidth } +function BrowserToolbar({ + activeConversationId, + browserState, + onBack, + onInteract, + onNavigate, + onReload, + onStop, + onStopLoading +}: { + activeConversationId?: string + browserState?: BrowserLiveState + onBack: () => Promise + onInteract: () => Promise + onNavigate: (url: string) => Promise + onReload: () => Promise + onStop: () => Promise + onStopLoading: () => Promise +}): React.JSX.Element { + const { t } = useTranslation('workspace') + const inputRef = useRef(null) + const actionPendingRef = useRef(false) + const [addressDraft, setAddressDraft] = useState( + browserState?.url ?? '' + ) + const [editing, setEditing] = useState(false) + const [dirty, setDirty] = useState(false) + const [actionPending, setActionPending] = useState(false) + const authoritativeUrl = browserState?.url ?? '' + const hasBrowserState = browserState !== undefined + const displayedAddress = + hasBrowserState && !editing && !dirty + ? authoritativeUrl + : addressDraft + const sessionActive = browserState?.sessionActive === true + const isLoading = browserState?.isLoading === true + const status = browserState?.status + const isCreating = status === 'creating' + const isActing = status === 'acting' + const isInteractive = status === 'interactive' + const operationBlocked = + actionPending || + isCreating || + status === 'loading' || + isActing || + isInteractive + const addressDisabled = operationBlocked || isLoading + const backDisabled = + operationBlocked || + !sessionActive || + browserState?.canGoBack !== true || + addressDisabled + const refreshDisabled = + operationBlocked || + !sessionActive || + isLoading + const interactDisabled = + operationBlocked || + !sessionActive || + isLoading || + (status !== 'ready' && status !== 'failed') + const closeDisabled = + operationBlocked || + !sessionActive || + isLoading + const goDisabled = + actionPending || + !activeConversationId || + addressDisabled || + displayedAddress.trim().length === 0 + + const runToolbarAction = async ( + action: () => Promise + ): Promise => { + if (actionPendingRef.current) { + return false + } + actionPendingRef.current = true + setActionPending(true) + try { + return await action() + } finally { + actionPendingRef.current = false + setActionPending(false) + } + } + + const submitAddress = async (): Promise => { + const url = displayedAddress.trim() + if (goDisabled || !url) { + return + } + if (await runToolbarAction(() => onNavigate(url))) { + setDirty(false) + setEditing(false) + inputRef.current?.blur() + } + } + + const refreshLabel = isLoading + ? t('sidebar.browser.toolbar.stopLoading') + : t('sidebar.browser.toolbar.refresh') + + return ( +
{ + event.preventDefault() + void submitAddress() + }} + > + + + + + + +
+ ) +} + export function RightAssistantSidebar({ open, tab, @@ -240,10 +469,15 @@ export function RightAssistantSidebar({ selectedTaskId, workspaceChanges, workspaceProjectId, + activeConversationId, browserState, currentProject, restoreFocusRef, + onBackBrowser = async () => {}, onInteractBrowser, + onNavigateBrowser = async () => {}, + onReloadBrowser = async () => {}, + onStopLoadingBrowser = async () => {}, onStopBrowser, onCreateCustomTask, onImportArtifacts, @@ -817,19 +1051,22 @@ export function RightAssistantSidebar({ }) } - const runAction = ( + const runAction = async ( action: () => Promise, fallback: string, onSuccess?: () => void - ): void => { + ): Promise => { setActionError('') - void action() - .then(onSuccess) - .catch((reason: unknown) => { - setActionError( - reason instanceof Error ? reason.message : fallback - ) - }) + try { + await action() + onSuccess?.() + return true + } catch (reason) { + setActionError( + reason instanceof Error ? reason.message : fallback + ) + return false + } } return ( @@ -1243,7 +1480,7 @@ export function RightAssistantSidebar({ disabled={!workspaceProjectId} onClick={() => { setWorkspaceRefreshVersion((current) => current + 1) - runAction( + void runAction( onRefreshChanges, t('sidebar.errors.refreshWorkspace') ) @@ -1347,7 +1584,7 @@ export function RightAssistantSidebar({ - -
- )} {!browserState ? (

@@ -1471,14 +1715,6 @@ export function RightAssistantSidebar({ t('sidebar.browser.statuses.failed') : t('sidebar.browser.statuses.stopped')} - {browserState.url && ( -

- {browserState.url} -
- )} {browserState.frameDataUrl ? ( {t('sidebar.browser.frameAlt')} svg { gap: var(--space-2); } -.assistant-sidebar__browser-actions { +.assistant-sidebar__browser-toolbar { display: flex; + min-width: 0; align-items: center; - gap: var(--space-2); + flex-wrap: wrap; + padding-bottom: var(--space-3); + border-bottom: 1px solid var(--border-subtle); + gap: var(--space-1); } -.assistant-sidebar__browser > header .secondary-button { - display: inline-flex; - min-height: 28px; - align-items: center; - padding: 4px 8px; - flex: none; - font-size: 10px; +.assistant-sidebar__browser-tool-button, +.assistant-sidebar__browser-go, +.assistant-sidebar__browser-interact, +.assistant-sidebar__browser-close { gap: var(--space-1); } +.assistant-sidebar__browser-tool-button { + width: var(--control-height); + padding: 0; + flex: 0 0 var(--control-height); +} + +.assistant-sidebar__browser-address { + min-width: 112px; + flex: 1 1 128px; +} + +.assistant-sidebar__browser-address input { + width: 100%; + min-width: 0; + min-height: var(--control-height); + padding: 0 var(--space-2); + border: 1px solid var(--border-control); + border-radius: var(--radius-control); + outline: none; + background: var(--surface-raised); + color: var(--text-primary); + font-family: var(--font-family-ui); + font-size: var(--font-caption); +} + +.assistant-sidebar__browser-address input:focus-visible { + border-color: var(--accent); + box-shadow: var(--focus-ring); +} + +.assistant-sidebar__browser-go, +.assistant-sidebar__browser-interact, +.assistant-sidebar__browser-close { + padding: 0 var(--space-2); + flex: 0 0 auto; +} + +.assistant-sidebar__browser-tool-button:hover:not(:disabled), +.assistant-sidebar__browser-go:hover:not(:disabled), +.assistant-sidebar__browser-interact:hover:not(:disabled) { + border-color: var(--accent-selected); + background: var(--accent-subtle); + color: var(--accent); +} + +.assistant-sidebar__browser-address input:disabled { + cursor: not-allowed; + opacity: 0.55; +} + +.assistant-sidebar__browser-toolbar + .assistant-sidebar__browser-close { + padding: 0 var(--space-2); + border-color: var(--danger-border); + margin-left: var(--space-1); + color: var(--danger); +} + .assistant-sidebar__browser-status { padding: var(--space-2) var(--space-3); border: 1px solid var(--border-default); @@ -4832,15 +4891,6 @@ button > svg { color: var(--danger); } -.assistant-sidebar__browser-url { - overflow: hidden; - color: var(--text-muted); - font-family: "Cascadia Code", "SFMono-Regular", Consolas, monospace; - font-size: 9px; - text-overflow: ellipsis; - white-space: nowrap; -} - .assistant-sidebar__browser-frame, .assistant-sidebar__browser-placeholder { width: 100%; @@ -15626,10 +15676,8 @@ details.settings-section > :not(summary) + :not(summary) { .assistant-sidebar__approval p, .assistant-sidebar__approval-actions button, .assistant-sidebar__badge, -.assistant-sidebar__browser > header .secondary-button, .assistant-sidebar__browser-placeholder, .assistant-sidebar__browser-status, -.assistant-sidebar__browser-url, .assistant-sidebar__diff, .assistant-sidebar__empty, .assistant-sidebar__import, diff --git a/src/shared/contracts.test.ts b/src/shared/contracts.test.ts index 58d63b2f..11a474b0 100644 --- a/src/shared/contracts.test.ts +++ b/src/shared/contracts.test.ts @@ -1,11 +1,64 @@ import { describe, expect, it } from 'vitest' import { agentRequestSchema, + browserLiveStateSchema, + browserNavigateRequestSchema, + browserStopLoadingRequestSchema, builtinEmbeddingConnectionId, legacyEmbeddingConnectionId, runtimeSettingsInputSchema } from './contracts' +describe('browser control contracts', () => { + it('requires authoritative toolbar metadata and accepts committed URLs up to 8192 characters', () => { + const base = { + conversationId: 'conversation', + status: 'ready' as const, + sessionActive: true, + isLoading: false, + canGoBack: true, + updatedAt: 1 + } + expect( + browserLiveStateSchema.safeParse({ + ...base, + url: `https://example.com/${'a'.repeat(8_172)}` + }).success + ).toBe(true) + expect( + browserLiveStateSchema.safeParse({ + conversationId: 'conversation', + status: 'ready', + updatedAt: 1 + }).success + ).toBe(false) + }) + + it('keeps browser navigation and stop-loading requests narrow', () => { + expect( + browserNavigateRequestSchema.parse({ + conversationId: 'conversation', + url: 'https://example.com/' + }) + ).toEqual({ + conversationId: 'conversation', + url: 'https://example.com/' + }) + expect( + browserNavigateRequestSchema.safeParse({ + conversationId: 'conversation', + url: 'https://example.com/', + rawElectron: true + }).success + ).toBe(false) + expect( + browserStopLoadingRequestSchema.safeParse({ + conversationId: 'conversation' + }).success + ).toBe(true) + }) +}) + const baseInput = { provider: 'model' as const, modelBaseUrl: 'https://bigtoken.ai', diff --git a/src/shared/contracts.ts b/src/shared/contracts.ts index b677107d..eedae7f0 100644 --- a/src/shared/contracts.ts +++ b/src/shared/contracts.ts @@ -1346,7 +1346,10 @@ export const browserLiveStateSchema = z 'failed', 'stopped' ]), - url: z.string().max(2_048).optional(), + sessionActive: z.boolean(), + isLoading: z.boolean(), + canGoBack: z.boolean(), + url: z.string().max(8_192).optional(), frameDataUrl: z .string() .max(400_000) @@ -1369,6 +1372,16 @@ export const browserStopRequestSchema = z .strict() export const browserInteractRequestSchema = browserStopRequestSchema +export const browserBackRequestSchema = browserStopRequestSchema +export const browserReloadRequestSchema = browserStopRequestSchema +export const browserStopLoadingRequestSchema = browserStopRequestSchema + +export const browserNavigateRequestSchema = z + .object({ + conversationId: conversationIdSchema, + url: z.string().min(1).max(8_192) + }) + .strict() export const knowledgeIdSchema = z.string().uuid() export const knowledgeCreateSchema = z @@ -1554,6 +1567,10 @@ export type DesktopApi = { onEvent: (listener: (event: AgentEvent) => void) => () => void } browser: { + navigate: (conversationId: string, url: string) => Promise + back: (conversationId: string) => Promise + reload: (conversationId: string) => Promise + stopLoading: (conversationId: string) => Promise interact: (conversationId: string) => Promise stop: (conversationId: string) => Promise onState: (listener: (state: BrowserLiveState) => void) => () => void diff --git a/src/shared/ipc-channels.ts b/src/shared/ipc-channels.ts index 454dce8d..4e3c7974 100644 --- a/src/shared/ipc-channels.ts +++ b/src/shared/ipc-channels.ts @@ -20,6 +20,10 @@ export const ipcChannels = { agentQuestionRespond: 'agent:question:respond', agentCompactConversation: 'agent:context:compact', agentEvent: 'agent:event', + browserNavigate: 'browser:navigate', + browserBack: 'browser:back', + browserReload: 'browser:reload', + browserStopLoading: 'browser:stop-loading', browserInteract: 'browser:interact', browserStop: 'browser:stop', browserState: 'browser:state',