diff --git a/AGENTS.md b/AGENTS.md index b3e29b6..7b02e47 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,34 @@ Keep Electron security boundaries intact: - Keep UI accessible with labels, keyboard behavior, semantic roles, and visible focus states. +## Release Packaging + +- `.github/workflows/packages.yml` is the canonical cross-platform packaging + workflow. It validates and builds `out` once, then packages on six native + runners: Windows, macOS, and Linux, each for x64 and arm64. +- Run the unified packager with + `npm run release:package -- --platform --arch `. It only + packages for the native host and writes to + `dist/release/-`. +- Default deliverables are NSIS and portable EXE for Windows, DMG and ZIP for + macOS, and AppImage and DEB for Linux. Every target includes + `release-manifest.json` with SHA-256 hashes. +- `build/build-release.cjs` verifies the unpacked application, `app.asar`, + bundled Continue and OpenCode runtimes, executable architecture, and package + signatures before atomically replacing a release directory. +- Keep electron-builder invocations on `--publish never`. Current CI uploads + 30-day GitHub Actions artifacts and does not create GitHub Release assets. + Signing and macOS notarization are not configured. +- Keep `ELECTRON_CACHE` and `ELECTRON_BUILDER_CACHE` under + `${{ runner.temp }}` in step-level workflow contexts. A cache beneath the + repository inherits the root `"type": "module"` and breaks electron-builder's + CommonJS macOS icon tool. +- Tag builds must use `v${package.version}`. The workflow also supports manual + dispatch and main-branch changes to release tooling. +- Verified baseline on 2026-08-04: commit `2f54938`, GitHub Actions run + `30893805567` succeeded for validation and all six package targets, producing + six release artifacts plus the shared production bundle. + ## Validation Run all validators after source changes: diff --git a/BUILD.md b/BUILD.md index 5fbefb7..1f90b89 100644 --- a/BUILD.md +++ b/BUILD.md @@ -137,7 +137,7 @@ npm run dist:linux:arm64 2. 中文输入法、窗口缩放和高分屏显示。 3. 系统密钥环和模型连接。 4. 本地知识库导入、检索和知识图谱。 -5. Ask、Plan、Execute 的权限边界。 -6. OpenCode 与 Continue 的审批、取消和超时。 +5. Ask、Execute 的权限边界与旧版 Plan 数据兼容。 +6. OpenCode 与 Continue 的权限边界、取消和超时。 7. 智能心跳的创建、暂停、恢复和历史记录。 8. 应用退出后无残留 Runtime 子进程。 diff --git a/README.md b/README.md index c275b26..74dc87a 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,9 @@ GoodBuddy 将模型连接、Agent Runtime、本地知识库、知识图谱、任 GoodBuddy 通过统一的 Agent Runtime 控制层接入直连模型、OpenCode 和 Continue。工具不会被直接暴露给界面，所有执行都受到工作模式、权限审批和运行边界约束。 -- `Ask`：只读问答，不调用写入工具。 -- `Plan`：先分析和制定计划，不修改工作区。 -- `Execute`：在用户授权范围内执行受控操作。 -- 支持拒绝、仅此次、当前会话和永久授权。 +- `Ask`：只读问答，不调用工具。 +- `Execute`：选择该模式即授权当前交互运行使用已启用的受控工具。 +- 可在设置中禁止直连模型执行所有工具；工具调用仍记录到活动。 - 统一处理取消、超时、输出边界、进程退出和异常恢复。 ### 数据主权与本地优先 @@ -72,16 +71,16 @@ GoodBuddy 通过统一的 Agent Runtime 控制层接入直连模型、OpenCode - 展示心跳健康、记忆沉淀、洞察发现和行动转化。 - 提供成长趋势、最新报告和可审计的运行轨迹。 - 建议记忆可确认或忽略。 -- 后续任务可带入 Plan 对话、标记完成或忽略。 +- 后续任务可带入 Ask 对话、标记完成或忽略。 - 支持手动运行、暂停、恢复和安全删除计划。 ### 多 Runtime 与模型连接 | 能力 | 适用场景 | 控制方式 | | --- | --- | --- | -| 直连模型 | 问答、规划、知识总结、图像生成 | 协议校验，不开放本地工具 | -| OpenCode | 完整编码与工作区任务 | 整次执行审批、受控目录与工具策略 | -| Continue | 逐工具 Agent 任务 | 独立宿主、逐工具审批与隔离权限 | +| 直连模型 | 问答、知识总结、受控工具执行、图像生成 | Ask 只读；Execute 自动授权已启用的工作区、浏览器与 MCP 工具，可设置为全部禁止 | +| OpenCode | 完整编码与工作区任务 | Execute 不弹 GoodBuddy 审批，保留 Runtime 自身权限、取消和活动记录 | +| Continue | Agent 编码与工作区任务 | Execute 不弹 GoodBuddy 审批，使用独立宿主、取消和活动记录 | ## 隐私说明 diff --git a/continue-runtime-web-demo/index.html b/continue-runtime-web-demo/index.html new file mode 100644 index 0000000..c24e1eb --- /dev/null +++ b/continue-runtime-web-demo/index.html @@ -0,0 +1,308 @@ + + + + + + GoodBuddy 控制中心 · AI 运维仪表盘 + + + + + + +
+
+
+ + GoodBuddy 控制中心 +
+ + + + + +
+ +
+
+
+ +
+
+
+

今日运维概览

+

实时掌握 AI 代理集群健康状况与任务执行进度

+ +
+
+

总任务数

+

128

+

较昨日 +12

+
+
+

运行中代理

+

6

+

全部在线

+
+
+

已完成任务

+

96

+

完成率 75%

+
+
+

平均响应时间

+

340ms

+

较昨日 -18ms

+
+
+
+
+ +
+
+

代理状态

+
    +
  • +
    + + 运行中 +
    +

    Agent Alpha

    +

    负责：数据同步 & 报表生成

    +

    最近活跃：2 分钟前

    +
    • +
  • +
    + + 空闲 +
    +

    Agent Beta

    +

    负责：客户工单处理

    +

    最近活跃：15 分钟前

    +
    • +
  • +
    + + 维护中 +
    +

    Agent Gamma

    +

    负责：库存预警检测

    +

    最近活跃：1 小时前

    +
    • +
  • +
    + + 运行中 +
    +

    Agent Delta

    +

    负责：日志汇总分析

    +

    最近活跃：刚刚

    +
    • +
+
+
+ +
+
+
+

任务看板

+ +
+ +
+ + + +
+ + + +
    +
  • +
    +

    数据同步任务

    + 运行中 +
    +

    负责人：Agent Alpha

    +

    截止时间：今天 18:00

    +
    • +
  • +
    +

    客户工单自动回复

    + 已完成 +
    +

    负责人：Agent Beta

    +

    完成时间：今天 09:24

    +
    • +
  • +
    +

    每日日志汇总

    + 运行中 +
    +

    负责人：Agent Delta

    +

    截止时间：今天 23:00

    +
    • +
  • +
    +

    库存预警检测

    + 已完成 +
    +

    负责人：Agent Gamma

    +

    完成时间：昨天 21:10

    +
    • +
  • +
    +

    周报生成

    + 运行中 +
    +

    负责人：Agent Alpha

    +

    截止时间：本周五 12:00

    +
    • +
+
+
+ +
+
+

活动日志

+ +
+ + +
+ + + +
    +
  • + 10:42 + Agent Alpha 完成了「数据同步任务」的第 3 阶段 +
    • +
  • + 10:31 + Agent Beta 自动回复了 12 条客户工单 +
    • +
  • + 09:58 + Agent Gamma 触发了库存预警检测维护流程 +
    • +
  • + 09:20 + Agent Delta 生成了每日日志汇总报告 +
    • +
  • + 08:47 + 系统检测到 Agent Alpha 响应时间恢复正常 +
    • +
  • + 08:02 + Agent Delta 启动，开始监听新任务队列 +
    • +
+
+
+
+ +
+
+

© 2026 GoodBuddy Control Center. 保留所有权利。

+

CONTINUE_WEB_DEMO_OK

+
+
+ + + + + + + diff --git a/continue-runtime-web-demo/script.js b/continue-runtime-web-demo/script.js new file mode 100644 index 0000000..03472e7 --- /dev/null +++ b/continue-runtime-web-demo/script.js @@ -0,0 +1,382 @@ +/*! + * GoodBuddy Control Center — script.js + * Vanilla JS only. No external dependencies. + * CONTINUE_WEB_DEMO_OK + */ +/* global document, window */ +(function () { + "use strict"; + + /* --------------------------------------------------------- + * Utilities + * ------------------------------------------------------- */ + + /** + * Escapes HTML-sensitive characters in a string so that it is + * safe to insert as text content inside markup. + * @param {string} value + * @returns {string} + */ + function escapeHtml(value) { + return String(value) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + } + + function qs(selector, scope) { + return (scope || document).querySelector(selector); + } + + function qsa(selector, scope) { + return Array.prototype.slice.call((scope || document).querySelectorAll(selector)); + } + + /* --------------------------------------------------------- + * Theme toggle (persisted via localStorage) + * ------------------------------------------------------- */ + + var THEME_STORAGE_KEY = "goodbuddy-theme"; + + function initTheme() { + var themeToggleBtn = qs("#themeToggleBtn"); + var iconEl = qs(".theme-toggle-icon", themeToggleBtn); + var labelEl = qs(".theme-toggle-label", themeToggleBtn); + + function applyTheme(theme) { + var label; + if (theme === "dark") { + document.documentElement.setAttribute("data-theme", "dark"); + themeToggleBtn.setAttribute("aria-pressed", "true"); + if (iconEl) iconEl.textContent = "☀️"; + label = "浅色模式"; + } else { + document.documentElement.removeAttribute("data-theme"); + themeToggleBtn.setAttribute("aria-pressed", "false"); + if (iconEl) iconEl.textContent = "🌙"; + label = "深色模式"; + } + if (labelEl) labelEl.textContent = label; + // .theme-toggle-label is visually hidden on narrow viewports and the + // icon is aria-hidden, so without an explicit aria-label the button + // has no accessible name on mobile. Keep it in sync with the visible + // desktop label on every theme change. + themeToggleBtn.setAttribute("aria-label", label); + } + + var stored = null; + try { + stored = window.localStorage.getItem(THEME_STORAGE_KEY); + } catch { + /* localStorage unavailable, use the default theme */ + } + + var prefersDark = + window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches; + var initialTheme = stored || (prefersDark ? "dark" : "light"); + applyTheme(initialTheme); + + themeToggleBtn.addEventListener("click", function () { + var isDark = document.documentElement.getAttribute("data-theme") === "dark"; + var nextTheme = isDark ? "light" : "dark"; + applyTheme(nextTheme); + try { + window.localStorage.setItem(THEME_STORAGE_KEY, nextTheme); + } catch { + /* localStorage unavailable — ignore silently */ + } + }); + } + + /* --------------------------------------------------------- + * Mobile navigation toggle + * ------------------------------------------------------- */ + + function initNavToggle() { + var navToggleBtn = qs("#navToggleBtn"); + var primaryNav = qs("#primaryNav"); + if (!navToggleBtn || !primaryNav) return; + + navToggleBtn.addEventListener("click", function () { + var isOpen = primaryNav.classList.toggle("is-open"); + navToggleBtn.setAttribute("aria-expanded", String(isOpen)); + }); + + qsa("a", primaryNav).forEach(function (link) { + link.addEventListener("click", function () { + primaryNav.classList.remove("is-open"); + navToggleBtn.setAttribute("aria-expanded", "false"); + }); + }); + } + + /* --------------------------------------------------------- + * Task board: filtering + adding tasks + * ------------------------------------------------------- */ + + var STATUS_LABELS = { + running: "运行中", + completed: "已完成" + }; + + var STATUS_BADGE_CLASS = { + running: "status-running", + completed: "status-completed" + }; + + function initTaskBoard() { + var taskList = qs("#taskList"); + var filterButtons = qsa(".filter-btn"); + var emptyState = qs("#taskEmptyState"); + + function currentFilter() { + var activeBtn = qs(".filter-btn.is-active"); + return activeBtn ? activeBtn.getAttribute("data-filter") : "all"; + } + + function applyFilter() { + var filter = currentFilter(); + var cards = qsa(".task-card", taskList); + var visibleCount = 0; + + cards.forEach(function (card) { + var status = card.getAttribute("data-status"); + var matches = filter === "all" || status === filter; + card.hidden = !matches; + if (matches) visibleCount += 1; + }); + + if (emptyState) { + emptyState.hidden = visibleCount !== 0; + } + } + + filterButtons.forEach(function (btn) { + btn.addEventListener("click", function () { + filterButtons.forEach(function (b) { + b.classList.remove("is-active"); + b.setAttribute("aria-pressed", "false"); + }); + btn.classList.add("is-active"); + btn.setAttribute("aria-pressed", "true"); + applyFilter(); + }); + }); + + /** + * Builds a task card list item using safe DOM APIs so that + * user-provided text can never be interpreted as markup. + */ + function createTaskCard(title, assignee, status) { + var li = document.createElement("li"); + li.className = "task-card"; + li.setAttribute("data-status", status); + + var top = document.createElement("div"); + top.className = "task-card-top"; + + var heading = document.createElement("h3"); + heading.className = "task-title"; + // textContent assigns the raw string as plain text — the browser + // never parses it as markup, so no HTML-escaping is needed (or + // wanted) here. Escaping first would make textContent render the + // escaped entities literally (e.g. "<img...") instead of the + // user's exact text. + heading.textContent = title; + + var badge = document.createElement("span"); + badge.className = "status-badge " + STATUS_BADGE_CLASS[status]; + badge.textContent = STATUS_LABELS[status]; + + top.appendChild(heading); + top.appendChild(badge); + + var meta = document.createElement("p"); + meta.className = "task-meta"; + meta.textContent = "负责人：" + assignee; + + var metaTime = document.createElement("p"); + metaTime.className = "task-meta"; + var now = new Date(); + var timeLabel = + status === "completed" + ? "完成时间：刚刚" + : "截止时间：" + + String(now.getHours()).padStart(2, "0") + + ":" + + String(now.getMinutes()).padStart(2, "0"); + metaTime.textContent = timeLabel; + + li.appendChild(top); + li.appendChild(meta); + li.appendChild(metaTime); + + return li; + } + + applyFilter(); + + return { + addTask: function (title, assignee, status) { + // Pass the trimmed, un-escaped user text straight through. + // createTaskCard assigns it via textContent (never innerHTML), + // so it is rendered as inert plain text and can never execute as + // markup/script regardless of its contents. + var card = createTaskCard(title.trim(), assignee, status); + taskList.insertBefore(card, taskList.firstChild); + applyFilter(); + return card; + } + }; + } + + /* --------------------------------------------------------- + * Activity log search + * ------------------------------------------------------- */ + + function initActivitySearch() { + var searchInput = qs("#activitySearch"); + var activityList = qs("#activityList"); + var emptyState = qs("#activityEmptyState"); + if (!searchInput || !activityList) return; + + function runSearch() { + var query = searchInput.value.trim().toLowerCase(); + var items = qsa(".activity-item", activityList); + var visibleCount = 0; + + items.forEach(function (item) { + var text = item.textContent.toLowerCase(); + var matches = query === "" || text.indexOf(query) !== -1; + item.hidden = !matches; + if (matches) visibleCount += 1; + }); + + if (emptyState) { + emptyState.hidden = visibleCount !== 0; + } + } + + searchInput.addEventListener("input", runSearch); + } + + /* --------------------------------------------------------- + * Add-task modal (accessible dialog) + * ------------------------------------------------------- */ + + function initTaskModal(taskBoard) { + var overlay = qs("#taskModalOverlay"); + var modal = qs("#taskModal"); + var openBtn = qs("#addTaskBtn"); + var closeBtn = qs("#modalCloseBtn"); + var cancelBtn = qs("#modalCancelBtn"); + var form = qs("#taskForm"); + var titleInput = qs("#taskTitleInput"); + var assigneeSelect = qs("#taskAssigneeSelect"); + var statusSelect = qs("#taskStatusSelect"); + var errorMsg = qs("#taskFormError"); + + if (!overlay || !modal || !openBtn || !form) return; + + var lastFocusedElement = null; + + function getFocusableElements() { + return qsa( + 'a[href], button:not([disabled]), textarea, input, select, [tabindex]:not([tabindex="-1"])', + modal + ).filter(function (el) { + return el.offsetParent !== null; + }); + } + + function openModal() { + lastFocusedElement = document.activeElement; + overlay.hidden = false; + modal.hidden = false; + errorMsg.hidden = true; + form.reset(); + document.body.style.overflow = "hidden"; + titleInput.focus(); + document.addEventListener("keydown", handleKeydown); + } + + function closeModal() { + overlay.hidden = true; + modal.hidden = true; + document.body.style.overflow = ""; + document.removeEventListener("keydown", handleKeydown); + if (lastFocusedElement && typeof lastFocusedElement.focus === "function") { + lastFocusedElement.focus(); + } + } + + function handleKeydown(event) { + if (event.key === "Escape") { + event.preventDefault(); + closeModal(); + return; + } + + if (event.key === "Tab") { + var focusable = getFocusableElements(); + if (focusable.length === 0) return; + + var first = focusable[0]; + var last = focusable[focusable.length - 1]; + + if (event.shiftKey && document.activeElement === first) { + event.preventDefault(); + last.focus(); + } else if (!event.shiftKey && document.activeElement === last) { + event.preventDefault(); + first.focus(); + } + } + } + + openBtn.addEventListener("click", openModal); + closeBtn.addEventListener("click", closeModal); + cancelBtn.addEventListener("click", closeModal); + overlay.addEventListener("click", closeModal); + + form.addEventListener("submit", function (event) { + event.preventDefault(); + + var title = titleInput.value.trim(); + if (title === "") { + errorMsg.hidden = false; + titleInput.focus(); + return; + } + + errorMsg.hidden = true; + taskBoard.addTask(title, assigneeSelect.value, statusSelect.value); + closeModal(); + }); + } + + /* --------------------------------------------------------- + * Bootstrap + * ------------------------------------------------------- */ + + function init() { + initTheme(); + initNavToggle(); + var taskBoard = initTaskBoard(); + initActivitySearch(); + initTaskModal(taskBoard); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", init); + } else { + init(); + } + + // Expose escapeHtml for testability / verification purposes only. + window.__goodbuddyEscapeHtml = escapeHtml; +})(); + +// CONTINUE_WEB_DEMO_OK diff --git a/continue-runtime-web-demo/styles.css b/continue-runtime-web-demo/styles.css new file mode 100644 index 0000000..b9c8e35 --- /dev/null +++ b/continue-runtime-web-demo/styles.css @@ -0,0 +1,782 @@ +/* ========================================================= + GoodBuddy Control Center — styles.css + ========================================================= */ + +:root { + color-scheme: light; + + --color-bg: #f4f6fb; + --color-bg-elevated: #ffffff; + --color-bg-header: rgba(255, 255, 255, 0.92); + --color-border: #dde3ee; + --color-text: #1c2333; + --color-text-muted: #5a6479; + --color-primary: #3a5bfd; + --color-primary-hover: #2a45d6; + --color-primary-contrast: #ffffff; + --color-success: #12a454; + --color-success-bg: #e4f8ec; + --color-warning: #b8860b; + --color-warning-bg: #fbf1d9; + --color-info: #2a6df4; + --color-info-bg: #e6edff; + --color-danger: #d43b3b; + --color-danger-bg: #fdeaea; + --shadow-sm: 0 1px 2px rgba(20, 26, 46, 0.06); + --shadow-md: 0 8px 24px rgba(20, 26, 46, 0.08); + --shadow-lg: 0 20px 48px rgba(20, 26, 46, 0.16); + --radius-sm: 8px; + --radius-md: 14px; + --radius-lg: 20px; + --transition-fast: 150ms ease; + --transition-base: 220ms ease; + --focus-ring: 0 0 0 3px rgba(58, 91, 253, 0.35); + --max-width: 1180px; +} + +[data-theme="dark"] { + color-scheme: dark; + + --color-bg: #0f1220; + --color-bg-elevated: #1a1f33; + --color-bg-header: rgba(15, 18, 32, 0.9); + --color-border: #2b3150; + --color-text: #eef1fb; + --color-text-muted: #a3aac4; + --color-primary: #7b93ff; + --color-primary-hover: #93a7ff; + --color-primary-contrast: #0f1220; + --color-success: #35d17d; + --color-success-bg: rgba(53, 209, 125, 0.14); + --color-warning: #e8c15b; + --color-warning-bg: rgba(232, 193, 91, 0.14); + --color-info: #7ea2ff; + --color-info-bg: rgba(126, 162, 255, 0.14); + --color-danger: #f27272; + --color-danger-bg: rgba(242, 114, 114, 0.14); + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.4); + --shadow-md: 0 8px 24px rgba(0, 0, 0, 0.45); + --shadow-lg: 0 20px 48px rgba(0, 0, 0, 0.55); + --focus-ring: 0 0 0 3px rgba(123, 147, 255, 0.45); +} + +/* ---------- Reset & base ---------- */ +*, +*::before, +*::after { + box-sizing: border-box; +} + +html { + scroll-behavior: smooth; +} + +body { + margin: 0; + min-height: 100vh; + background: var(--color-bg); + color: var(--color-text); + font-family: "Segoe UI", "PingFang SC", "Microsoft YaHei", "Noto Sans SC", system-ui, + -apple-system, sans-serif; + line-height: 1.6; + transition: background-color var(--transition-base), color var(--transition-base); +} + +h1, +h2, +h3, +p { + margin: 0 0 0.5em; +} + +ul { + list-style: none; + margin: 0; + padding: 0; +} + +button { + font-family: inherit; +} + +input, +select { + font-family: inherit; +} + +.section-inner { + max-width: var(--max-width); + margin: 0 auto; + padding: 0 1.5rem; +} + +/* ---------- Accessibility helpers ---------- */ +.visually-hidden { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +.skip-link { + position: absolute; + top: -100px; + left: 1rem; + background: var(--color-primary); + color: var(--color-primary-contrast); + padding: 0.6rem 1rem; + border-radius: var(--radius-sm); + z-index: 1000; + transition: top var(--transition-fast); +} + +.skip-link:focus { + top: 1rem; +} + +:focus-visible { + outline: none; + box-shadow: var(--focus-ring); + border-radius: var(--radius-sm); +} + +a { + color: var(--color-primary); +} + +/* ---------- Header ---------- */ +.site-header { + position: sticky; + top: 0; + z-index: 100; + background: var(--color-bg-header); + backdrop-filter: blur(10px); + border-bottom: 1px solid var(--color-border); +} + +.header-inner { + max-width: var(--max-width); + margin: 0 auto; + padding: 0.85rem 1.5rem; + display: flex; + align-items: center; + gap: 1rem; +} + +.brand { + display: flex; + align-items: center; + gap: 0.6rem; + font-weight: 700; + font-size: 1.05rem; + white-space: nowrap; +} + +.brand-mark { + display: inline-flex; + align-items: center; + justify-content: center; + width: 36px; + height: 36px; + border-radius: 10px; + background: linear-gradient(135deg, var(--color-primary), #7d5bfd); + color: #fff; + font-size: 0.85rem; + font-weight: 800; +} + +.primary-nav { + flex: 1; +} + +.primary-nav ul { + display: flex; + gap: 1.5rem; +} + +.primary-nav a { + text-decoration: none; + color: var(--color-text-muted); + font-weight: 600; + padding: 0.4rem 0.2rem; + border-radius: var(--radius-sm); + transition: color var(--transition-fast); +} + +.primary-nav a:hover { + color: var(--color-primary); +} + +.header-actions { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.theme-toggle { + display: inline-flex; + align-items: center; + gap: 0.5rem; + padding: 0.5rem 0.9rem; + border-radius: 999px; + border: 1px solid var(--color-border); + background: var(--color-bg-elevated); + color: var(--color-text); + cursor: pointer; + font-weight: 600; + font-size: 0.9rem; + transition: transform var(--transition-fast), background-color var(--transition-fast); +} + +.theme-toggle:hover { + transform: translateY(-1px); +} + +.nav-toggle { + display: none; + flex-direction: column; + gap: 4px; + padding: 0.5rem; + background: transparent; + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + cursor: pointer; +} + +.nav-toggle-bar { + display: block; + width: 20px; + height: 2px; + background: var(--color-text); + border-radius: 2px; +} + +/* ---------- Hero ---------- */ +.hero { + padding: 3rem 0 2.5rem; +} + +.hero-subtitle { + color: var(--color-text-muted); + font-size: 1.05rem; + max-width: 46ch; +} + +.metrics { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 1rem; + margin-top: 1.75rem; +} + +.metric-card { + background: var(--color-bg-elevated); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + padding: 1.25rem; + box-shadow: var(--shadow-sm); + transition: box-shadow var(--transition-fast), transform var(--transition-fast); +} + +.metric-card:hover { + box-shadow: var(--shadow-md); + transform: translateY(-2px); +} + +.metric-label { + color: var(--color-text-muted); + font-size: 0.85rem; + font-weight: 600; +} + +.metric-value { + font-size: 2rem; + font-weight: 800; +} + +.metric-delta { + font-size: 0.82rem; + font-weight: 700; +} + +.metric-delta--up { + color: var(--color-success); +} + +.metric-delta--down { + color: var(--color-danger); +} + +/* ---------- Sections generic ---------- */ +.agents, +.tasks, +.activity { + padding: 2.5rem 0; +} + +.agents h2, +.tasks h2, +.activity h2 { + font-size: 1.5rem; +} + +/* ---------- Agent grid ---------- */ +.agent-grid { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 1rem; + margin-top: 1.25rem; +} + +.agent-card { + background: var(--color-bg-elevated); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + padding: 1.25rem; + box-shadow: var(--shadow-sm); + transition: transform var(--transition-fast), box-shadow var(--transition-fast); +} + +.agent-card:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-md); +} + +.agent-card-top { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 0.75rem; +} + +.agent-avatar { + display: inline-flex; + align-items: center; + justify-content: center; + width: 42px; + height: 42px; + border-radius: 50%; + background: var(--color-info-bg); + color: var(--color-info); + font-weight: 800; + font-size: 0.85rem; +} + +.agent-name { + font-size: 1.05rem; +} + +.agent-meta { + color: var(--color-text-muted); + font-size: 0.88rem; + margin-bottom: 0.2rem; +} + +/* ---------- Status badges ---------- */ +.status-badge { + display: inline-flex; + align-items: center; + padding: 0.28rem 0.7rem; + border-radius: 999px; + font-size: 0.78rem; + font-weight: 700; +} + +.status-running { + background: var(--color-info-bg); + color: var(--color-info); +} + +.status-completed { + background: var(--color-success-bg); + color: var(--color-success); +} + +.status-idle { + background: var(--color-warning-bg); + color: var(--color-warning); +} + +.status-maintenance { + background: var(--color-danger-bg); + color: var(--color-danger); +} + +/* ---------- Buttons ---------- */ +.btn { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 0.6rem 1.1rem; + border-radius: var(--radius-sm); + border: 1px solid transparent; + font-weight: 700; + font-size: 0.92rem; + cursor: pointer; + transition: background-color var(--transition-fast), transform var(--transition-fast), + box-shadow var(--transition-fast); +} + +.btn:hover { + transform: translateY(-1px); +} + +.btn-primary { + background: var(--color-primary); + color: var(--color-primary-contrast); + box-shadow: var(--shadow-sm); +} + +.btn-primary:hover { + background: var(--color-primary-hover); +} + +.btn-ghost { + background: transparent; + border-color: var(--color-border); + color: var(--color-text); +} + +.btn-ghost:hover { + background: var(--color-bg); +} + +/* ---------- Tasks ---------- */ +.tasks-header { + display: flex; + align-items: center; + justify-content: space-between; + flex-wrap: wrap; + gap: 1rem; +} + +.filter-group { + display: flex; + gap: 0.6rem; + margin: 1.25rem 0 1rem; + flex-wrap: wrap; +} + +.filter-btn { + padding: 0.5rem 1rem; + border-radius: 999px; + border: 1px solid var(--color-border); + background: var(--color-bg-elevated); + color: var(--color-text-muted); + font-weight: 700; + font-size: 0.88rem; + cursor: pointer; + transition: all var(--transition-fast); +} + +.filter-btn:hover { + color: var(--color-primary); + border-color: var(--color-primary); +} + +.filter-btn.is-active { + background: var(--color-primary); + border-color: var(--color-primary); + color: var(--color-primary-contrast); +} + +.task-list { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 1rem; +} + +.task-card { + background: var(--color-bg-elevated); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + padding: 1.1rem 1.25rem; + box-shadow: var(--shadow-sm); + transition: transform var(--transition-fast), box-shadow var(--transition-fast); +} + +.task-card:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-md); +} + +.task-card-top { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.75rem; + margin-bottom: 0.4rem; +} + +.task-title { + font-size: 1rem; + word-break: break-word; +} + +.task-meta { + color: var(--color-text-muted); + font-size: 0.85rem; + margin-bottom: 0.15rem; +} + +.empty-state { + color: var(--color-text-muted); + font-style: italic; + padding: 1rem 0; +} + +/* ---------- Activity ---------- */ +.search-field { + margin: 1.25rem 0; +} + +.search-input { + width: 100%; + max-width: 420px; + padding: 0.65rem 1rem; + border-radius: var(--radius-sm); + border: 1px solid var(--color-border); + background: var(--color-bg-elevated); + color: var(--color-text); + font-size: 0.95rem; +} + +.activity-list { + display: flex; + flex-direction: column; + gap: 0.6rem; +} + +.activity-item { + display: flex; + gap: 1rem; + align-items: baseline; + padding: 0.75rem 1rem; + background: var(--color-bg-elevated); + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + box-shadow: var(--shadow-sm); +} + +.activity-time { + flex-shrink: 0; + font-variant-numeric: tabular-nums; + color: var(--color-text-muted); + font-weight: 700; + font-size: 0.85rem; + min-width: 3.2rem; +} + +.activity-text { + color: var(--color-text); +} + +/* ---------- Footer ---------- */ +.site-footer { + border-top: 1px solid var(--color-border); + padding: 1.5rem 0 2.5rem; + color: var(--color-text-muted); + font-size: 0.85rem; +} + +.footer-marker { + opacity: 0.55; + font-size: 0.75rem; + letter-spacing: 0.02em; +} + +/* ---------- Modal ---------- */ +.modal-overlay { + position: fixed; + inset: 0; + background: rgba(10, 12, 24, 0.5); + z-index: 200; +} + +.modal { + position: fixed; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + width: min(480px, calc(100vw - 2rem)); + max-height: calc(100vh - 3rem); + overflow-y: auto; + background: var(--color-bg-elevated); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-lg); + z-index: 201; + border: 1px solid var(--color-border); +} + +.modal[hidden], +.modal-overlay[hidden] { + display: none; +} + +.modal-form { + display: flex; + flex-direction: column; +} + +.modal-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 1.25rem 1.5rem; + border-bottom: 1px solid var(--color-border); +} + +.modal-header h2 { + font-size: 1.15rem; + margin: 0; +} + +.modal-close { + background: transparent; + border: none; + font-size: 1.4rem; + line-height: 1; + color: var(--color-text-muted); + cursor: pointer; + padding: 0.25rem 0.5rem; + border-radius: var(--radius-sm); +} + +.modal-close:hover { + color: var(--color-danger); +} + +.modal-body { + padding: 1.25rem 1.5rem; + display: flex; + flex-direction: column; + gap: 1rem; +} + +.form-field { + display: flex; + flex-direction: column; + gap: 0.4rem; +} + +.form-field label { + font-weight: 700; + font-size: 0.9rem; +} + +.form-field input, +.form-field select { + padding: 0.6rem 0.75rem; + border-radius: var(--radius-sm); + border: 1px solid var(--color-border); + background: var(--color-bg); + color: var(--color-text); + font-size: 0.95rem; +} + +.form-error { + color: var(--color-danger); + font-size: 0.85rem; + font-weight: 700; + margin: 0; +} + +.modal-footer { + display: flex; + justify-content: flex-end; + gap: 0.75rem; + padding: 1.1rem 1.5rem; + border-top: 1px solid var(--color-border); +} + +/* ---------- Responsive ---------- */ +@media (max-width: 900px) { + .metrics { + grid-template-columns: repeat(2, 1fr); + } + + .agent-grid { + grid-template-columns: repeat(2, 1fr); + } + + .task-list { + grid-template-columns: 1fr; + } +} + +@media (max-width: 720px) { + .nav-toggle { + display: inline-flex; + } + + .primary-nav { + position: absolute; + top: 100%; + left: 0; + right: 0; + background: var(--color-bg-elevated); + border-bottom: 1px solid var(--color-border); + box-shadow: var(--shadow-md); + display: none; + } + + .primary-nav.is-open { + display: block; + } + + .primary-nav ul { + flex-direction: column; + padding: 1rem 1.5rem; + gap: 0.25rem; + } + + .primary-nav a { + display: block; + padding: 0.6rem 0.2rem; + } + + .header-inner { + flex-wrap: wrap; + position: relative; + } + + .theme-toggle-label { + display: none; + } +} + +@media (max-width: 560px) { + .metrics { + grid-template-columns: 1fr; + } + + .agent-grid { + grid-template-columns: 1fr; + } + + .tasks-header { + flex-direction: column; + align-items: flex-start; + } +} + +/* ---------- Reduced motion ---------- */ +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + animation-duration: 0.001ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.001ms !important; + scroll-behavior: auto !important; + } + + html { + scroll-behavior: auto; + } +} diff --git a/docs/电脑控制开发进度.md b/docs/电脑控制开发进度.md new file mode 100644 index 0000000..359c2e9 --- /dev/null +++ b/docs/电脑控制开发进度.md @@ -0,0 +1,131 @@ +# GoodBuddy 电脑控制开发进度 + +最后更新：2026-08-05 + +## 范围 + +本阶段不包含 LoongArch 和厂商互认证。目标平台仍为 Windows、macOS、 +Linux x64 和 Linux arm64。 + +## 已完成 + +### 直连模型与浏览器 + +- 已实现宿主进程拥有的隔离浏览器服务。 +- 使用 Electron 自带 Chromium 和 CDP，不下载额外浏览器。 +- 已实现导航、可访问性快照、点击、输入、选择、返回和有界截图工具。 +- 浏览器工具只在直连模型的 Execute 模式中提供；用户选择 Execute 即授权 + 本次交互运行，不再逐个弹出 GoodBuddy 工具审批。 +- 已实现公网 URL、DNS、重定向、私有地址和元数据地址限制。 +- 已实现回环过滤代理、下载和文件选择器阻止、权限拒绝、会话取消、 + 空闲回收和应用退出清理。 +- 直连模型工具循环已提高到适合浏览器任务的有界上限，并包含重复调用和 + 无进展保护。 + +### MCP + +- MCP 图片结果已支持 PNG、JPEG 和 WebP 的严格 Base64、文件签名和大小 + 校验。 +- 已支持 Anthropic Messages、OpenAI Responses 和兼容 Chat Completions + 的多模态工具结果。 +- 已支持 MCP 进度续期、硬超时、Tasks、取消和关闭。 +- 自定义 stdio MCP 不继承桌面会话变量。 +- 已增加受审计 curated MCP 启动描述符，但在没有签名和打包的 helper + 前不启用外部桌面 helper。 + +### Computer Control Broker + +- 已实现动作契约、风险分类、窗口租约、感知版本、过期引用、单次变更、 + 审批、超时和取消核心。 +- 注入开始后的不确定失败记录为 `outcome_unknown`，防止重试造成重复提交。 +- 已实现 SQLite 持久审计，输入文本只保存长度和 SHA-256 摘要。 + +### Linux 技术预览 + +- 已实现 AT-SPI 语义核心、Portal 会话状态机、PipeWire 元数据校验、 + 坐标映射和输入路由核心。 +- DDE、Treeland、UKUI、GNOME 和 KDE 名称只用于诊断，不用于假定能力。 +- 当前没有打包真实 D-Bus、PipeWire/libei 或 XTest 原生适配器，因此 Linux + 桌面控制保持不可启用，不能表述为生产可用。 + +### 设置与安全边界 + +- 能力存储已迁移到版本 2。 +- 已增加电脑控制能力卡片、诊断、托管浏览器配置元数据、IPC 和 preload + 契约。 +- 电脑控制能力默认关闭；启用、配置变更和本地数据清除会替换或清理相关 + Runtime 和会话。 + +## 本轮已修复缺陷 + +### P0：隔离浏览器创建失败，已修复 + +便携版中的两次失败都发生在浏览器导航工具进入运行状态约 15 秒后。 +隔离 Electron 探针已定位到： + +1. 过滤代理启动成功。 +2. 隔离 Session 创建成功。 +3. 隐藏 BrowserWindow 创建成功。 +4. `webContents.debugger.attach('1.3')` 成功。 +5. 第一个 `Page.enable` CDP 命令一直不返回，最终触发 15 秒会话创建超时。 + +现在会先加载并等待 `about:blank` 初始文档，再附加并启用 CDP 域，同时保留 +超时后的迟到资源清理。真实 Electron 探针已确认 `Page.enable`、访问 +`https://example.com/` 和 PNG 画面捕获均成功。 + +### P0：右侧没有浏览器实时画面，已修复 + +浏览器窗口仍使用 `show: false`，但 BrowserService 现在从模型实际操作的同一 +会话捕获页面帧，并通过受限 IPC 发送状态、当前 URL 和 PNG 画面。右侧工作栏 +新增“浏览器”页签；活动对话启动浏览器时会自动打开该页签，并显示创建中、 +加载中、操作中、就绪、失败和已停止状态。用户可在页签内立即停止当前对话的 +浏览器会话。 + +当前实现按导航、快照、点击、输入、选择、返回和截图操作刷新画面，而不是创建 +第二个预览浏览器，因此显示内容与模型受控页面一致。 + +### P0：百度输入时元素引用失效，已修复 + +截图中的“浏览器元素引用已失效，请重新获取快照”来自过宽的导航监听：广告或 +其他子框架开始导航、重定向时，也会清空主文档刚生成的元素引用。驱动现在只在 +主框架导航和重定向时清空引用，子框架活动不再中断主页面输入和点击。 + +如果主页面确实已经变化，过期引用会作为“可重试”工具结果返回给直连模型，并 +明确要求重新调用 `browser_snapshot`；模型可以用新引用继续操作，不再让整次 +任务直接失败。百度搜索结果页的原始可访问性树约为 782KB，因此内部有界读取 +上限调整为 1MB，返回给模型的节点数和最终快照大小仍分别受独立上限约束。 + +真实 Electron 探针已完成“打开百度、获取快照、输入阿里云、点击百度一下、 +读取结果页”，结果页标题为“阿里云_百度搜索”，返回约 330 个节点并正确标记 +为已截断。 + +### P1：工具错误信息过度包装，已改善 + +浏览器会话创建错误现在包含“启动代理、创建隔离会话、配置网络代理、创建窗口、 +加载初始页面、初始化浏览器协议”等有界阶段信息；实时页签也显示当前操作阶段 +和截断后的错误原因。 + +浏览器底层错误目前被包装为“工具『浏览器导航』执行失败”，数据库和界面无法 +区分代理、分区、窗口、CDP 初始化、DNS 或页面导航失败。需要增加有界、脱敏的 +阶段错误码，并保留用户可执行的修复建议。 + +### P1：命名浏览器配置尚未用于执行 + +设置中的命名浏览器配置当前只保存未来托管隔离所需的元数据。实际执行仍使用 +每个会话随机创建的临时分区，不复用登录状态。 + +## 下一步顺序 + +1. 在便携版真实模型流程中继续验证返回、取消和右侧画面刷新。 +2. 再设计命名隔离 Profile 的持久登录态、引用和清理策略。 +3. 分别实现并真机验证 Windows UIA、macOS AX 和 Linux 原生适配器。 + +## 最近验证 + +- `npm test`：591 项通过，6 项跳过。 +- `npm run typecheck`：通过。 +- `npm run lint`：通过。 +- `npm run build`：通过。 + +以上自动化结果覆盖协议、安全边界和模拟驱动，不替代真实 Electron +BrowserWindow/WebContentsView、目标网站和目标 Linux 桌面环境的集成验证。 diff --git a/docs/长期助手功能规划.md b/docs/长期助手功能规划.md index f585c66..f988dcd 100644 --- a/docs/长期助手功能规划.md +++ b/docs/长期助手功能规划.md @@ -240,7 +240,7 @@ GoodBuddy 应能够： ### 阶段 1：长期工作区骨架 - Projects 与会话归属。 -- Ask、Plan、Execute 工作模式。 +- Ask、Execute 工作模式与旧版 Plan 数据兼容。 - 全局右侧栏。 - 任务、上下文、成果、文件更改和预览页签。 diff --git a/src/main/agent/create-runtime.test.ts b/src/main/agent/create-runtime.test.ts index 06b5f73..333844f 100644 --- a/src/main/agent/create-runtime.test.ts +++ b/src/main/agent/create-runtime.test.ts @@ -1,6 +1,25 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import type { ResolvedRuntimeSettings } from '../runtime-settings-store' +import type { BrowserToolService } from '../browser/browser-model-tools' import { createAgentRuntime } from './create-runtime' +import { AgentRuntimeController } from './runtime-controller' + +function createBrowserService(): BrowserToolService & { + dispose: ReturnType +} { + return { + getOrigin: vi.fn(() => undefined), + navigate: vi.fn(), + snapshot: vi.fn(), + click: vi.fn(), + type: vi.fn(), + select: vi.fn(), + back: vi.fn(), + screenshot: vi.fn(), + releaseConversation: vi.fn(async () => undefined), + dispose: vi.fn(async () => undefined) + } +} function settings( overrides: Partial = {} @@ -41,6 +60,50 @@ describe('createAgentRuntime model compatibility', () => { await runtime.dispose() }) + it('shares injected browser service without runtime-owned disposal', async () => { + const browserService = createBrowserService() + const first = createAgentRuntime(process.cwd(), settings(), { + browserService + }) + const second = createAgentRuntime(process.cwd(), settings(), { + browserService + }) + const controller = new AgentRuntimeController(first) + + await controller.releaseConversation('conversation-one') + await controller.replace(second) + await controller.releaseConversation('conversation-two') + await controller.dispose() + + expect(browserService.releaseConversation).toHaveBeenNthCalledWith( + 1, + 'conversation-one' + ) + expect(browserService.releaseConversation).toHaveBeenNthCalledWith( + 2, + 'conversation-two' + ) + expect(browserService.dispose).not.toHaveBeenCalled() + }) + + it('does not expose the browser service to OpenCode runtimes', async () => { + const browserService = createBrowserService() + const runtime = createAgentRuntime( + process.cwd(), + settings({ + provider: 'opencode', + opencodeBaseUrl: 'http://127.0.0.1:4096' + }), + { browserService } + ) + + await runtime.releaseConversation?.('opencode-conversation') + await runtime.dispose() + + expect(browserService.releaseConversation).not.toHaveBeenCalled() + expect(browserService.dispose).not.toHaveBeenCalled() + }) + it('keeps OpenCode independent profiles Anthropic API-key only', () => { expect(() => createAgentRuntime( diff --git a/src/main/agent/create-runtime.ts b/src/main/agent/create-runtime.ts index 1a0d4f1..2c0c246 100644 --- a/src/main/agent/create-runtime.ts +++ b/src/main/agent/create-runtime.ts @@ -9,6 +9,7 @@ import type { ResolvedMcpServer } from '../capabilities/capability-service' import type { BundledRuntimePaths } from './bundled-runtimes' import type { ContinueHostLauncher } from './continue-host-adapter' import { resolveRuntimeSandbox } from './runtime-sandbox' +import type { BrowserToolService } from '../browser/browser-model-tools' export type AgentCapabilityContext = { skillInstructions?: string @@ -16,6 +17,7 @@ export type AgentCapabilityContext = { continueHostCacheRoot?: string bundledRuntimePaths?: BundledRuntimePaths continueHostLauncher?: ContinueHostLauncher + browserService?: BrowserToolService } export function createAgentRuntime( @@ -127,7 +129,8 @@ export function createAgentRuntime( authentication: modelAuthentication, skillInstructions: capabilities.skillInstructions, defaultWorkspace: workspace, - mcpServers: capabilities.mcpServers + mcpServers: capabilities.mcpServers, + browserService: capabilities.browserService }) } diff --git a/src/main/agent/model-runtime.test.ts b/src/main/agent/model-runtime.test.ts index 6048853..04b1dd5 100644 --- a/src/main/agent/model-runtime.test.ts +++ b/src/main/agent/model-runtime.test.ts @@ -1,10 +1,39 @@ import { describe, expect, it, vi } from 'vitest' -import type { - ModelToolDefinition, - ModelToolProviderLike +import { + RecoverableModelToolError, + type ModelToolDefinition, + type ModelToolProviderLike, + type ModelToolResult } from './model-tool-provider' import { ModelAgentRuntime } from './model-runtime' +const toolPng = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a +]).toString('base64') + +function createTextToolResult(text: string): ModelToolResult { + return { + parts: [{ type: 'text', text }], + contextBytes: Buffer.byteLength(text) + } +} + +function createMultimodalToolResult(): ModelToolResult { + return { + parts: [ + { type: 'text', text: 'tool result' }, + { + type: 'image', + mimeType: 'image/png', + data: toolPng + } + ], + contextBytes: + Buffer.byteLength('tool result') + Buffer.byteLength(toolPng) + } +} + function createEventStream(text: string): string { return [ 'event: message_start', @@ -90,7 +119,8 @@ function createToolProvider( toolName: '读取工作区文本', argumentSummary: summary })), - callTool: vi.fn(async () => 'tool result'), + callTool: vi.fn(async () => createTextToolResult('tool result')), + releaseConversation: vi.fn(async () => {}), dispose: vi.fn(async () => {}), ...overrides } @@ -357,6 +387,42 @@ describe('ModelAgentRuntime', () => { expect(toolProvider.listTools).not.toHaveBeenCalled() }) + it.each(['ask', 'plan'] as const)( + 'keeps browser and workspace tools out of %s mode', + async (workMode) => { + const fetcher = vi.fn(async () => + new Response('data: {"choices":[{"delta":{"content":"只读回答"}}]}\n\ndata: [DONE]\n\n', { + status: 200, + headers: { 'content-type': 'text/event-stream' } + }) + ) + const toolProvider = createToolProvider() + const runtime = new ModelAgentRuntime({ + baseUrl: 'http://127.0.0.1:11434/v1', + model: 'qwen3', + protocol: 'openai-chat-completions', + authentication: 'none', + fetcher, + toolProvider + }) + + for await (const _event of runtime.run( + { + requestId: crypto.randomUUID(), + conversationId: `conversation-${workMode}`, + prompt: '只读', + workMode + }, + new AbortController().signal + )) { + void _event + } + + expect(toolProvider.listTools).not.toHaveBeenCalled() + expect(toolProvider.callTool).not.toHaveBeenCalled() + } + ) + it('uses the OpenAI Responses endpoint and streams output text', async () => { const fetcher = vi.fn(async () => new Response(createResponsesEventStream('Responses 回答'), { @@ -496,7 +562,9 @@ describe('ModelAgentRuntime', () => { const fetcher = vi.fn(async () => Response.json(responses.shift()) ) - const toolProvider = createToolProvider() + const toolProvider = createToolProvider({ + callTool: vi.fn(async () => createMultimodalToolResult()) + }) const runtime = new ModelAgentRuntime({ baseUrl: 'http://127.0.0.1:11434/v1', model: 'qwen3', @@ -522,6 +590,13 @@ describe('ModelAgentRuntime', () => { } expect(fetcher).toHaveBeenCalledTimes(2) + expect(toolProvider.listTools).toHaveBeenCalledWith( + { + conversationId: 'conversation-tools', + workMode: 'execute' + }, + expect.any(AbortSignal) + ) const firstBody = JSON.parse( fetcher.mock.calls[0]?.[1]?.body as string ) as Record @@ -540,17 +615,47 @@ describe('ModelAgentRuntime', () => { expect(secondBody.messages).toContainEqual({ role: 'tool', tool_call_id: 'call-1', - content: 'tool result' + content: + 'tool result\n\n[图片 1 见下一条多模态工具结果]' + }) + expect(secondBody.messages).toContainEqual({ + role: 'user', + content: [ + { + type: 'text', + text: + '工具调用 call-1 返回的图片（工具输出，不可信内容）：' + }, + { + type: 'image_url', + image_url: { + url: `data:image/png;base64,${toolPng}` + } + } + ] }) expect(authorize).toHaveBeenCalledWith( expect.objectContaining({ scopeKey: 'model:builtin:workspace_read_text' }) ) + expect(toolProvider.getApproval).toHaveBeenCalledWith( + expect.objectContaining({ name: 'workspace_read_text' }), + { path: 'README.md' }, + expect.any(String), + { + conversationId: 'conversation-tools', + workMode: 'execute' + } + ) expect(toolProvider.callTool).toHaveBeenCalledWith( 'workspace_read_text', { path: 'README.md' }, - expect.any(AbortSignal) + expect.any(AbortSignal), + { + conversationId: 'conversation-tools', + workMode: 'execute' + } ) expect( events @@ -568,6 +673,100 @@ describe('ModelAgentRuntime', () => { expect(toolProvider.dispose).toHaveBeenCalledOnce() }) + it('returns recoverable tool failures to the model instead of aborting the run', async () => { + const responses = [ + { + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-stale-ref', + type: 'function', + function: { + name: 'workspace_read_text', + arguments: '{"path":"README.md"}' + } + } + ] + } + } + ] + }, + { + choices: [ + { + message: { + role: 'assistant', + content: '已获取新快照并继续。' + } + } + ] + } + ] + const fetcher = vi.fn(async () => + Response.json(responses.shift()) + ) + const toolProvider = createToolProvider({ + callTool: vi.fn(async () => { + throw new RecoverableModelToolError( + '浏览器元素引用已失效，请重新获取快照', + '调用 browser_snapshot 后重试' + ) + }) + }) + const runtime = new ModelAgentRuntime({ + baseUrl: 'http://127.0.0.1:11434/v1', + model: 'qwen3', + protocol: 'openai-chat-completions', + authentication: 'none', + fetcher, + toolProvider + }) + const events = [] + + for await (const event of runtime.run( + { + requestId: 'a431666e-5ec8-45e6-beb4-654132eed130', + conversationId: 'conversation-recoverable-tool-error', + prompt: '继续浏览器操作', + workMode: 'execute' + }, + new AbortController().signal, + async () => 'once' + )) { + events.push(event) + } + + expect(fetcher).toHaveBeenCalledTimes(2) + const secondBody = JSON.parse( + fetcher.mock.calls[1]?.[1]?.body as string + ) as { messages: Array> } + const toolMessage = secondBody.messages.find( + (message) => message.role === 'tool' + ) + expect(JSON.parse(toolMessage?.content as string)).toEqual({ + ok: false, + recoverable: true, + error: '浏览器元素引用已失效，请重新获取快照', + nextAction: '调用 browser_snapshot 后重试' + }) + expect( + events + .filter((event) => event.type === 'tool') + .map((event) => event.state) + ).toEqual(['pending', 'running', 'recoverable']) + expect(events).toContainEqual( + expect.objectContaining({ + type: 'text', + delta: '已获取新快照并继续。' + }) + ) + expect(events.at(-1)).toMatchObject({ type: 'done' }) + }) + it('continues OpenAI Responses with function_call_output', async () => { const responses = [ { @@ -611,7 +810,9 @@ describe('ModelAgentRuntime', () => { protocol: 'openai-responses', authentication: 'api-key', fetcher, - toolProvider: createToolProvider() + toolProvider: createToolProvider({ + callTool: vi.fn(async () => createMultimodalToolResult()) + }) }) const events = [] @@ -651,7 +852,16 @@ describe('ModelAgentRuntime', () => { { type: 'function_call_output', call_id: 'call-responses-1', - output: 'tool result' + output: [ + { + type: 'input_text', + text: 'tool result' + }, + { + type: 'input_image', + image_url: `data:image/png;base64,${toolPng}` + } + ] } ] }) @@ -758,7 +968,9 @@ describe('ModelAgentRuntime', () => { protocol: 'anthropic-messages', authentication: 'api-key', fetcher, - toolProvider: createToolProvider() + toolProvider: createToolProvider({ + callTool: vi.fn(async () => createMultimodalToolResult()) + }) }) for await (const _event of runtime.run( @@ -795,12 +1007,277 @@ describe('ModelAgentRuntime', () => { { type: 'tool_result', tool_use_id: 'toolu-1', - content: 'tool result' + content: [ + { + type: 'text', + text: 'tool result' + }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: toolPng + } + } + ] } ] }) }) + it('does not issue a follow-up model request after tool cancellation', async () => { + const response = { + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-aborted', + type: 'function', + function: { + name: 'workspace_read_text', + arguments: '{}' + } + } + ] + } + } + ] + } + const fetcher = vi.fn(async () => Response.json(response)) + const controller = new AbortController() + const runtime = new ModelAgentRuntime({ + baseUrl: 'http://127.0.0.1:11434/v1', + model: 'qwen3', + protocol: 'openai-chat-completions', + authentication: 'none', + fetcher, + toolProvider: createToolProvider({ + callTool: vi.fn(async () => { + controller.abort() + return createTextToolResult('late result') + }) + }) + }) + const consume = async (): Promise => { + for await (const _event of runtime.run( + { + requestId: crypto.randomUUID(), + conversationId: crypto.randomUUID(), + prompt: 'run', + workMode: 'execute' + }, + controller.signal, + async () => 'once' + )) { + void _event + } + } + + await expect(consume()).rejects.toThrow() + expect(fetcher).toHaveBeenCalledOnce() + }) + + it('terminates repeated identical tool rounds without exhausting hard limits', async () => { + let callId = 0 + const fetcher = vi.fn(async () => { + callId += 1 + return Response.json({ + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: `call-repeat-${callId}`, + type: 'function', + function: { + name: 'workspace_read_text', + arguments: '{"path":"README.md"}' + } + } + ] + } + } + ] + }) + }) + const toolProvider = createToolProvider() + const runtime = new ModelAgentRuntime({ + baseUrl: 'http://127.0.0.1:11434/v1', + model: 'qwen3', + protocol: 'openai-chat-completions', + authentication: 'none', + fetcher, + toolProvider + }) + const consume = async (): Promise => { + for await (const _event of runtime.run( + { + requestId: crypto.randomUUID(), + conversationId: 'conversation-repeat', + prompt: 'repeat', + workMode: 'execute' + }, + new AbortController().signal, + async () => 'once' + )) { + void _event + } + } + + await expect(consume()).rejects.toThrow('没有取得进展') + expect(fetcher).toHaveBeenCalledTimes(3) + expect(toolProvider.callTool).toHaveBeenCalledTimes(2) + }) + + it('releases provider state for only the requested conversation', async () => { + const toolProvider = createToolProvider() + const runtime = new ModelAgentRuntime({ + baseUrl: 'http://127.0.0.1:11434/v1', + model: 'qwen3', + protocol: 'openai-chat-completions', + authentication: 'none', + toolProvider + }) + + await runtime.releaseConversation('conversation-release') + + expect(toolProvider.releaseConversation).toHaveBeenCalledOnce() + expect(toolProvider.releaseConversation).toHaveBeenCalledWith( + 'conversation-release' + ) + }) + + it('releases known conversations before provider disposal and permits replacement reuse', async () => { + const lifecycle: string[] = [] + const released = new Set() + const createProvider = (): ModelToolProviderLike => + createToolProvider({ + releaseConversation: vi.fn(async (conversationId) => { + lifecycle.push(`release:${conversationId}`) + released.add(conversationId) + }), + dispose: vi.fn(async () => { + lifecycle.push('dispose') + }) + }) + const createRuntime = (toolProvider: ModelToolProviderLike) => + new ModelAgentRuntime({ + baseUrl: 'http://127.0.0.1:11434/v1', + model: 'qwen3', + protocol: 'openai-chat-completions', + authentication: 'none', + fetcher: vi.fn(async () => + new Response( + 'data: {"choices":[{"delta":{"content":"ok"}}]}\n\ndata: [DONE]\n\n', + { + status: 200, + headers: { 'content-type': 'text/event-stream' } + } + ) + ), + toolProvider + }) + const request = { + requestId: crypto.randomUUID(), + conversationId: 'conversation-replacement', + prompt: 'hello', + workMode: 'ask' as const + } + const firstProvider = createProvider() + const firstRuntime = createRuntime(firstProvider) + for await (const _event of firstRuntime.run( + request, + new AbortController().signal + )) { + void _event + } + + await firstRuntime.dispose() + expect(lifecycle).toEqual([ + 'release:conversation-replacement', + 'dispose' + ]) + expect(released).toContain('conversation-replacement') + + const replacement = createRuntime(createProvider()) + const replacementEvents = [] + for await (const event of replacement.run( + { ...request, requestId: crypto.randomUUID() }, + new AbortController().signal + )) { + replacementEvents.push(event) + } + expect(replacementEvents.at(-1)).toMatchObject({ type: 'done' }) + await replacement.dispose() + }) + + it('counts image base64 data against the aggregate tool context limit', async () => { + const response = { + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-large-image', + type: 'function', + function: { + name: 'workspace_read_text', + arguments: '{}' + } + } + ] + } + } + ] + } + const imageData = Buffer.alloc(1024 * 1024 + 1).toString('base64') + const fetcher = vi.fn(async () => Response.json(response)) + const runtime = new ModelAgentRuntime({ + baseUrl: 'http://127.0.0.1:11434/v1', + model: 'qwen3', + protocol: 'openai-chat-completions', + authentication: 'none', + fetcher, + toolProvider: createToolProvider({ + callTool: vi.fn(async () => ({ + parts: [ + { + type: 'image' as const, + mimeType: 'image/png' as const, + data: imageData + } + ], + contextBytes: Buffer.byteLength(imageData) + })) + }) + }) + const consume = async (): Promise => { + for await (const _event of runtime.run( + { + requestId: crypto.randomUUID(), + conversationId: crypto.randomUUID(), + prompt: 'run', + workMode: 'execute' + }, + new AbortController().signal, + async () => 'once' + )) { + void _event + } + } + + await expect(consume()).rejects.toThrow('结果总量超过 1MB') + expect(fetcher).toHaveBeenCalledOnce() + }) + it('generates a bounded image through the BigToken-compatible endpoint', async () => { const png = Buffer.from([ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, diff --git a/src/main/agent/model-runtime.ts b/src/main/agent/model-runtime.ts index 930adc9..e912cfd 100644 --- a/src/main/agent/model-runtime.ts +++ b/src/main/agent/model-runtime.ts @@ -5,11 +5,16 @@ import type { ModelProtocol } from '../../shared/contracts' import type { ResolvedMcpServer } from '../capabilities/capability-service' +import type { BrowserToolService } from '../browser/browser-model-tools' import { createAnthropicMessagesUrl } from './anthropic-endpoint' import { ModelToolProvider, + RecoverableModelToolError, + type ModelToolCallContext, type ModelToolDefinition, - type ModelToolProviderLike + type ModelToolProviderLike, + type ModelToolResult, + type ModelToolResultPart } from './model-tool-provider' import { createOpenAIChatCompletionsUrl, @@ -86,8 +91,10 @@ const maxImageResponseBytes = 5_300_000 const maxChatResponseBytes = 2 * 1024 * 1024 const maxToolArgumentBytes = 128 * 1024 const maxToolContextBytes = 1024 * 1024 -const maxToolCallsPerRun = 12 -const maxToolRounds = 8 +const maxToolCallsPerRun = 40 +const maxToolRounds = 24 +const maxRepeatedIdenticalCalls = 3 +const maxIdenticalRoundsWithoutProgress = 2 export type ModelRuntimeOptions = { apiKey?: string @@ -98,6 +105,7 @@ export type ModelRuntimeOptions = { skillInstructions?: string defaultWorkspace?: string mcpServers?: ResolvedMcpServer[] + browserService?: BrowserToolService toolProvider?: ModelToolProviderLike fetcher?: typeof fetch } @@ -459,6 +467,152 @@ function parseToolArguments(value: unknown): Record { return parsed as Record } +function canonicalizeToolArguments(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(canonicalizeToolArguments) + } + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => [key, canonicalizeToolArguments(item)]) + ) + } + return value +} + +function getToolCallFingerprint(call: ModelToolCall): string { + return `${call.name}:${JSON.stringify( + canonicalizeToolArguments(call.arguments) + )}` +} + +function validateToolResult(result: ModelToolResult): number { + if ( + !Array.isArray(result.parts) || + result.parts.length === 0 || + !Number.isSafeInteger(result.contextBytes) || + result.contextBytes < 0 + ) { + throw new Error('直连模型工具返回了无效结果') + } + let contextBytes = 0 + for (const part of result.parts) { + if (part.type === 'text') { + if (typeof part.text !== 'string') { + throw new Error('直连模型工具返回了无效文本结果') + } + contextBytes += Buffer.byteLength(part.text) + } else if ( + part.type === 'image' && + (part.mimeType === 'image/png' || + part.mimeType === 'image/jpeg' || + part.mimeType === 'image/webp') && + typeof part.data === 'string' + ) { + contextBytes += Buffer.byteLength(part.data) + } else { + throw new Error('直连模型工具返回了无效图片结果') + } + } + if (contextBytes !== result.contextBytes) { + throw new Error('直连模型工具结果字节计数无效') + } + return contextBytes +} + +function getAnthropicToolResultContent( + parts: ModelToolResultPart[] +): Array> { + return parts.map((part) => + part.type === 'text' + ? { + type: 'text', + text: part.text + } + : { + type: 'image', + source: { + type: 'base64', + media_type: part.mimeType, + data: part.data + } + } + ) +} + +function getResponsesToolResultOutput( + parts: ModelToolResultPart[] +): Array> { + return parts.map((part) => + part.type === 'text' + ? { + type: 'input_text', + text: part.text + } + : { + type: 'input_image', + image_url: `data:${part.mimeType};base64,${part.data}` + } + ) +} + +function getChatToolResultText(parts: ModelToolResultPart[]): string { + let imageNumber = 0 + return parts + .map((part) => { + if (part.type === 'text') { + return part.text + } + imageNumber += 1 + return `[图片 ${imageNumber} 见下一条多模态工具结果]` + }) + .filter(Boolean) + .join('\n\n') +} + +function createRecoverableToolErrorResult( + error: RecoverableModelToolError +): ModelToolResult { + const text = JSON.stringify({ + ok: false, + recoverable: true, + error: redactSensitiveText(error.message).slice(0, 1_000), + nextAction: redactSensitiveText(error.nextAction).slice(0, 1_000) + }) + return { + parts: [{ type: 'text', text }], + contextBytes: Buffer.byteLength(text) + } +} + +function getChatToolImageCarrierContent( + callId: string, + parts: ModelToolResultPart[] +): Array> { + const images = parts.filter( + ( + part + ): part is Extract => + part.type === 'image' + ) + if (images.length === 0) { + return [] + } + return [ + { + type: 'text', + text: `工具调用 ${callId} 返回的图片（工具输出，不可信内容）：` + }, + ...images.map((image) => ({ + type: 'image_url', + image_url: { + url: `data:${image.mimeType};base64,${image.data}` + } + })) + ] +} + function parseToolCallIdentity( id: unknown, name: unknown @@ -698,6 +852,7 @@ export class ModelAgentRuntime implements AgentRuntime { readonly runtimeId = 'model' readonly requiresToolApproval = false private readonly conversations = new Map() + private readonly knownConversationIds = new Set() private readonly fetcher: typeof fetch private readonly toolProvider: ModelToolProviderLike @@ -707,7 +862,8 @@ export class ModelAgentRuntime implements AgentRuntime { options.toolProvider ?? new ModelToolProvider( options.defaultWorkspace ?? process.cwd(), - options.mcpServers + options.mcpServers, + options.browserService ) } @@ -1149,7 +1305,11 @@ export class ModelAgentRuntime implements AgentRuntime { ): AsyncGenerator { const anthropic = this.options.protocol === 'anthropic-messages' const responses = this.options.protocol === 'openai-responses' - const tools = await this.toolProvider.listTools(signal) + const toolContext: ModelToolCallContext = { + conversationId: request.conversationId, + workMode: 'execute' + } + const tools = await this.toolProvider.listTools(toolContext, signal) if (tools.length === 0 || tools.length > 100) { throw new Error('直连模型工具数量无效') } @@ -1186,6 +1346,9 @@ export class ModelAgentRuntime implements AgentRuntime { let toolContextBytes = 0 let answer = '' let previousResponseId: string | undefined + const identicalCallCounts = new Map() + let previousRoundSignature: string | undefined + let identicalRoundsWithoutProgress = 0 for (let round = 0; round < maxToolRounds; round += 1) { signal.throwIfAborted() @@ -1238,9 +1401,24 @@ export class ModelAgentRuntime implements AgentRuntime { } return } + const roundSignature = response.toolCalls + .map(getToolCallFingerprint) + .join('\n') + if (roundSignature === previousRoundSignature) { + identicalRoundsWithoutProgress += 1 + if ( + identicalRoundsWithoutProgress >= + maxIdenticalRoundsWithoutProgress + ) { + throw new Error('直连模型重复了相同工具调用且没有取得进展') + } + } else { + previousRoundSignature = roundSignature + identicalRoundsWithoutProgress = 0 + } totalToolCalls += response.toolCalls.length if (totalToolCalls > maxToolCallsPerRun) { - throw new Error('直连模型单次运行的工具调用超过 12 个') + throw new Error('直连模型单次运行的工具调用超过 40 个') } if (responses) { if (!response.responseId) { @@ -1254,8 +1432,16 @@ export class ModelAgentRuntime implements AgentRuntime { } const anthropicResults: Array> = [] const responsesResults: Array> = [] + const chatImageCarrierContent: Array> = [] for (const call of response.toolCalls) { signal.throwIfAborted() + const callFingerprint = getToolCallFingerprint(call) + const identicalCallCount = + (identicalCallCounts.get(callFingerprint) ?? 0) + 1 + identicalCallCounts.set(callFingerprint, identicalCallCount) + if (identicalCallCount > maxRepeatedIdenticalCalls) { + throw new Error('直连模型重复请求了完全相同的工具调用') + } if (seenCallIds.has(call.id)) { throw new Error('模型重复使用了工具调用 ID') } @@ -1291,7 +1477,8 @@ export class ModelAgentRuntime implements AgentRuntime { this.toolProvider.getApproval( tool, call.arguments, - safeToolArgumentSummary(call.arguments) + safeToolArgumentSummary(call.arguments), + toolContext ) ) } catch (error) { @@ -1316,6 +1503,7 @@ export class ModelAgentRuntime implements AgentRuntime { } throw new Error(`用户拒绝了工具「${displayName}」`) } + signal.throwIfAborted() yield { requestId: request.requestId, type: 'tool', @@ -1325,27 +1513,38 @@ export class ModelAgentRuntime implements AgentRuntime { summary: `正在执行直连模型工具：${displayName}` } - let result: string + let result: ModelToolResult + let toolFailed = false try { result = await this.toolProvider.callTool( tool.name, call.arguments, - signal + signal, + toolContext ) } catch (error) { + const recoverable = error instanceof RecoverableModelToolError yield { requestId: request.requestId, type: 'tool', callId: call.id, name: displayName, - state: 'failed', - summary: `直连模型工具执行失败：${displayName}` + state: recoverable ? 'recoverable' : 'failed', + summary: + recoverable + ? `直连模型工具需要刷新后重试：${displayName}` + : `直连模型工具执行失败：${displayName}` + } + if (recoverable) { + result = createRecoverableToolErrorResult(error) + toolFailed = true + } else { + throw new Error(`工具「${displayName}」执行失败`, { + cause: error + }) } - throw new Error(`工具「${displayName}」执行失败`, { - cause: error - }) } - toolContextBytes += Buffer.byteLength(result) + toolContextBytes += validateToolResult(result) if (toolContextBytes > maxToolContextBytes) { yield { requestId: request.requestId, @@ -1361,28 +1560,34 @@ export class ModelAgentRuntime implements AgentRuntime { responsesResults.push({ type: 'function_call_output', call_id: call.id, - output: result + output: getResponsesToolResultOutput(result.parts) }) } else if (anthropic) { anthropicResults.push({ type: 'tool_result', tool_use_id: call.id, - content: result + content: getAnthropicToolResultContent(result.parts), + ...(toolFailed ? { is_error: true } : {}) }) } else { messages.push({ role: 'tool', tool_call_id: call.id, - content: result + content: getChatToolResultText(result.parts) }) + chatImageCarrierContent.push( + ...getChatToolImageCarrierContent(call.id, result.parts) + ) } - yield { - requestId: request.requestId, - type: 'tool', - callId: call.id, - name: displayName, - state: 'completed', - summary: `直连模型工具已完成：${displayName}` + if (!toolFailed) { + yield { + requestId: request.requestId, + type: 'tool', + callId: call.id, + name: displayName, + state: 'completed', + summary: `直连模型工具已完成：${displayName}` + } } } if (anthropic) { @@ -1392,9 +1597,15 @@ export class ModelAgentRuntime implements AgentRuntime { }) } else if (responses) { messages.splice(0, messages.length, ...responsesResults) + } else if (chatImageCarrierContent.length > 0) { + messages.push({ + role: 'user', + content: chatImageCarrierContent + }) } + signal.throwIfAborted() } - throw new Error('直连模型工具调用轮次超过 8 轮') + throw new Error('直连模型工具调用轮次超过 24 轮') } async *run( @@ -1402,6 +1613,7 @@ export class ModelAgentRuntime implements AgentRuntime { signal: AbortSignal, authorize?: RuntimeAuthorizer ): AsyncGenerator { + this.knownConversationIds.add(request.conversationId) if (!this.isConfigured()) { throw new Error('请先在设置中配置模型接口 API Key') } @@ -1575,12 +1787,26 @@ export class ModelAgentRuntime implements AgentRuntime { } async dispose(): Promise { + const conversationIds = new Set([ + ...this.knownConversationIds, + ...this.conversations.keys() + ]) + await Promise.allSettled( + [...conversationIds].map((conversationId) => + this.toolProvider.releaseConversation(conversationId) + ) + ) + this.knownConversationIds.clear() this.conversations.clear() await this.toolProvider.dispose() } - releaseConversation(conversationId: string): Promise { + async releaseConversation(conversationId: string): Promise { this.conversations.delete(conversationId) - return Promise.resolve() + try { + await this.toolProvider.releaseConversation(conversationId) + } finally { + this.knownConversationIds.delete(conversationId) + } } } diff --git a/src/main/agent/model-tool-provider.test.ts b/src/main/agent/model-tool-provider.test.ts index 58060e2..00e6d51 100644 --- a/src/main/agent/model-tool-provider.test.ts +++ b/src/main/agent/model-tool-provider.test.ts @@ -9,16 +9,24 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ResolvedMcpServer } from '../capabilities/capability-service' +import type { BrowserToolService } from '../browser/browser-model-tools' +import { BrowserStaleReferenceError } from '../browser/cdp-browser-driver' const mocks = vi.hoisted(() => { + const tasks = { + callToolStream: vi.fn(), + cancelTask: vi.fn() + } const client = { connect: vi.fn(), listTools: vi.fn(), callTool: vi.fn(), + experimental: { tasks }, close: vi.fn() } return { client, + tasks, Client: vi.fn(function Client() { return client }), @@ -33,9 +41,63 @@ vi.mock('../capabilities/mcp-client-transport', () => ({ createMcpTransport: mocks.createMcpTransport })) -import { ModelToolProvider } from './model-tool-provider' +import { + ModelToolProvider, + type ModelToolCallContext +} from './model-tool-provider' const temporaryDirectories: string[] = [] +const png = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a +]).toString('base64') +const toolContext = { + conversationId: 'provider-test-conversation', + workMode: 'execute' +} satisfies ModelToolCallContext + +function createBrowserService(): BrowserToolService { + return { + getOrigin: vi.fn(() => 'https://example.com'), + navigate: vi.fn(async (_conversationId, url) => ({ + url, + origin: 'https://example.com' + })), + snapshot: vi.fn(async () => ({ + url: 'https://example.com/', + title: 'Example', + nodes: [], + truncated: false + })), + click: vi.fn(async () => undefined), + type: vi.fn(async () => undefined), + select: vi.fn(async () => undefined), + back: vi.fn(async () => ({ + url: 'https://previous.example/', + origin: 'https://previous.example' + })), + screenshot: vi.fn(async () => ({ + type: 'image' as const, + mimeType: 'image/png' as const, + data: png + })), + releaseConversation: vi.fn(async () => undefined) + } +} + +function createMcpServer(): ResolvedMcpServer { + return { + id: 'd2ef774b-146c-4467-a909-6feb112a9c2c', + name: 'Search MCP', + description: '', + enabled: true, + assignments: ['model'], + secretConfigured: false, + transport: 'stdio', + command: 'node', + args: ['server.js'] + } +} async function createWorkspace(): Promise { const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-tools-')) @@ -51,6 +113,13 @@ describe('ModelToolProvider', () => { mocks.client.callTool.mockResolvedValue({ content: [{ type: 'text', text: 'MCP result' }] }) + mocks.tasks.callToolStream.mockImplementation(async function* () { + yield { + type: 'result', + result: { content: [{ type: 'text', text: 'MCP task result' }] } + } + }) + mocks.tasks.cancelTask.mockResolvedValue({}) mocks.client.close.mockResolvedValue(undefined) }) @@ -71,7 +140,7 @@ describe('ModelToolProvider', () => { const provider = new ModelToolProvider(workspace) const signal = new AbortController().signal - await expect(provider.listTools(signal)).resolves.toEqual( + await expect(provider.listTools(toolContext, signal)).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ name: 'workspace_read_text' }), expect.objectContaining({ name: 'workspace_list_directory' }), @@ -82,23 +151,37 @@ describe('ModelToolProvider', () => { provider.callTool( 'workspace_read_text', { path: 'docs/note.txt' }, - signal + signal, + toolContext ) - ).resolves.toBe('hello') - await expect( - provider.callTool( - 'workspace_list_directory', - { path: 'docs' }, - signal - ) - ).resolves.toContain('"note.txt"') - await expect( - provider.callTool( - 'workspace_write_text', - { path: 'docs/output.txt', content: 'saved' }, - signal - ) - ).resolves.toContain('"bytesWritten":5') + ).resolves.toEqual({ + parts: [{ type: 'text', text: 'hello' }], + contextBytes: 5 + }) + const listing = await provider.callTool( + 'workspace_list_directory', + { path: 'docs' }, + signal, + toolContext + ) + expect(listing.parts).toEqual([ + expect.objectContaining({ + type: 'text', + text: expect.stringContaining('"note.txt"') + }) + ]) + const written = await provider.callTool( + 'workspace_write_text', + { path: 'docs/output.txt', content: 'saved' }, + signal, + toolContext + ) + expect(written.parts).toEqual([ + expect.objectContaining({ + type: 'text', + text: expect.stringContaining('"bytesWritten":5') + }) + ]) await expect( readFile(join(workspace, 'docs', 'output.txt'), 'utf8') ).resolves.toBe('saved') @@ -112,11 +195,116 @@ describe('ModelToolProvider', () => { provider.callTool( 'workspace_read_text', { path: '../outside.txt' }, - new AbortController().signal + new AbortController().signal, + toolContext ) ).rejects.toThrow('不能超出工作区') }) + it('delegates browser tools with per-call conversation context', async () => { + const workspace = await createWorkspace() + const browserService = createBrowserService() + const provider = new ModelToolProvider(workspace, [], browserService) + const firstContext = { + conversationId: 'browser-conversation-one', + workMode: 'execute' + } satisfies ModelToolCallContext + const secondContext = { + conversationId: 'browser-conversation-two', + workMode: 'execute' + } satisfies ModelToolCallContext + const signal = new AbortController().signal + + for (const workMode of ['ask', 'plan'] as const) { + const readOnlyContext = { + conversationId: `browser-${workMode}`, + workMode + } satisfies ModelToolCallContext + await expect( + provider.listTools(readOnlyContext, signal) + ).resolves.not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'browser_screenshot' }) + ]) + ) + await expect( + provider.callTool( + 'browser_screenshot', + {}, + signal, + readOnlyContext + ) + ).rejects.toThrow('未知工具') + } + expect(browserService.screenshot).not.toHaveBeenCalled() + + const tools = await provider.listTools(firstContext, signal) + expect( + tools.filter((tool) => tool.name.startsWith('browser_')) + ).toHaveLength(7) + const navigate = tools.find((tool) => tool.name === 'browser_navigate') + expect( + provider.getApproval( + navigate!, + { url: 'https://example.com/path?secret=value' }, + 'runtime summary', + firstContext + ) + ).toMatchObject({ + scopeKey: 'model:browser:navigate:https://example.com', + argumentSummary: 'https://example.com/path?[查询参数已隐藏]', + allowPermanent: false + }) + + await expect( + provider.callTool('browser_screenshot', {}, signal, firstContext) + ).resolves.toEqual({ + parts: [{ type: 'image', mimeType: 'image/png', data: png }], + contextBytes: Buffer.byteLength(png) + }) + await provider.callTool('browser_screenshot', {}, signal, secondContext) + expect(browserService.screenshot).toHaveBeenNthCalledWith( + 1, + firstContext.conversationId, + signal + ) + expect(browserService.screenshot).toHaveBeenNthCalledWith( + 2, + secondContext.conversationId, + signal + ) + + await provider.releaseConversation(firstContext.conversationId) + expect(browserService.releaseConversation).toHaveBeenCalledWith( + firstContext.conversationId + ) + expect(browserService.releaseConversation).not.toHaveBeenCalledWith( + secondContext.conversationId + ) + }) + + it('marks stale browser references as recoverable model tool errors', async () => { + const workspace = await createWorkspace() + const browserService = createBrowserService() + vi.mocked(browserService.click).mockRejectedValue( + new BrowserStaleReferenceError() + ) + const provider = new ModelToolProvider(workspace, [], browserService) + + await expect( + provider.callTool( + 'browser_click', + { ref: 'b_currentReference' }, + new AbortController().signal, + toolContext + ) + ).rejects.toMatchObject({ + name: 'RecoverableModelToolError', + message: '浏览器元素引用已失效，请重新获取快照', + nextAction: expect.stringContaining('browser_snapshot') + }) + }) + it('loads and invokes configured MCP tools through provider-safe names', async () => { const workspace = await createWorkspace() mocks.client.listTools.mockResolvedValue({ @@ -132,21 +320,10 @@ describe('ModelToolProvider', () => { } ] }) - const server = { - id: 'd2ef774b-146c-4467-a909-6feb112a9c2c', - name: 'Search MCP', - description: '', - enabled: true, - assignments: ['model'], - secretConfigured: false, - transport: 'stdio', - command: 'node', - args: ['server.js'] - } satisfies ResolvedMcpServer - const provider = new ModelToolProvider(workspace, [server]) + const provider = new ModelToolProvider(workspace, [createMcpServer()]) const signal = new AbortController().signal - const tools = await provider.listTools(signal) + const tools = await provider.listTools(toolContext, signal) const mcpTool = tools.find((tool) => tool.source === 'mcp') expect(mcpTool).toMatchObject({ displayName: 'Search MCP / search-web', @@ -157,9 +334,13 @@ describe('ModelToolProvider', () => { provider.callTool( mcpTool?.name ?? '', { query: 'GoodBuddy' }, - signal + signal, + toolContext ) - ).resolves.toBe('MCP result') + ).resolves.toEqual({ + parts: [{ type: 'text', text: 'MCP result' }], + contextBytes: 10 + }) expect(mocks.client.callTool).toHaveBeenCalledWith( { name: 'search-web', @@ -168,11 +349,308 @@ describe('ModelToolProvider', () => { undefined, expect.objectContaining({ timeout: 30_000, - signal + signal, + resetTimeoutOnProgress: true, + maxTotalTimeout: 300_000, + onprogress: expect.any(Function) }) ) await provider.dispose() expect(mocks.client.close).toHaveBeenCalledOnce() }) + + it('preserves ordered bounded MCP text, image, and unsupported audio parts', async () => { + const workspace = await createWorkspace() + mocks.client.listTools.mockResolvedValue({ + tools: [ + { + name: 'capture', + inputSchema: { type: 'object' } + } + ] + }) + mocks.client.callTool.mockResolvedValue({ + content: [ + { type: 'text', text: 'before' }, + { type: 'image', mimeType: 'image/png', data: png }, + { type: 'audio', mimeType: 'audio/wav', data: 'ignored' }, + { type: 'text', text: 'after' } + ] + }) + const provider = new ModelToolProvider(workspace, [createMcpServer()]) + const tools = await provider.listTools(toolContext, new AbortController().signal) + const tool = tools.find((candidate) => candidate.source === 'mcp') + + await expect( + provider.callTool( + tool?.name ?? '', + {}, + new AbortController().signal, + toolContext + ) + ).resolves.toEqual({ + parts: [ + { type: 'text', text: 'before' }, + { + type: 'image', + mimeType: 'image/png', + data: png + }, + { type: 'text', text: '[audio result unsupported]' }, + { type: 'text', text: 'after' } + ], + contextBytes: + Buffer.byteLength('before') + + Buffer.byteLength(png) + + Buffer.byteLength('[audio result unsupported]') + + Buffer.byteLength('after') + }) + }) + + it.each([ + { + mimeType: 'image/jpeg', + data: Buffer.from([0xff, 0xd8, 0xff]).toString('base64') + }, + { + mimeType: 'image/webp', + data: Buffer.from([ + 0x52, 0x49, 0x46, 0x46, + 0x00, 0x00, 0x00, 0x00, + 0x57, 0x45, 0x42, 0x50 + ]).toString('base64') + } + ])('accepts a valid $mimeType signature', async ({ mimeType, data }) => { + const workspace = await createWorkspace() + mocks.client.listTools.mockResolvedValue({ + tools: [{ name: 'capture', inputSchema: { type: 'object' } }] + }) + mocks.client.callTool.mockResolvedValue({ + content: [{ type: 'image', mimeType, data }] + }) + const provider = new ModelToolProvider(workspace, [createMcpServer()]) + const tools = await provider.listTools(toolContext, new AbortController().signal) + const tool = tools.find((candidate) => candidate.source === 'mcp') + + await expect( + provider.callTool( + tool?.name ?? '', + {}, + new AbortController().signal, + toolContext + ) + ).resolves.toEqual({ + parts: [{ type: 'image', mimeType, data }], + contextBytes: Buffer.byteLength(data) + }) + }) + + it.each([ + { + name: 'malformed base64', + image: { + type: 'image', + mimeType: 'image/png', + data: `${png.slice(0, -1)}!` + }, + message: '无效的 base64' + }, + { + name: 'MIME signature mismatch', + image: { + type: 'image', + mimeType: 'image/jpeg', + data: png + }, + message: 'MIME 类型与文件签名不匹配' + }, + { + name: 'unsupported MIME type', + image: { + type: 'image', + mimeType: 'image/gif', + data: png + }, + message: '不支持的图片格式' + } + ])('rejects $name in MCP image blocks', async ({ image, message }) => { + const workspace = await createWorkspace() + mocks.client.listTools.mockResolvedValue({ + tools: [{ name: 'capture', inputSchema: { type: 'object' } }] + }) + mocks.client.callTool.mockResolvedValue({ content: [image] }) + const provider = new ModelToolProvider(workspace, [createMcpServer()]) + const tools = await provider.listTools(toolContext, new AbortController().signal) + const tool = tools.find((candidate) => candidate.source === 'mcp') + + await expect( + provider.callTool( + tool?.name ?? '', + {}, + new AbortController().signal, + toolContext + ) + ).rejects.toThrow(message) + }) + + it('counts encoded and decoded image data against the MCP result budget', async () => { + const workspace = await createWorkspace() + mocks.client.listTools.mockResolvedValue({ + tools: [{ name: 'capture', inputSchema: { type: 'object' } }] + }) + const encodedContextOversizedPng = Buffer.concat([ + Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a + ]), + Buffer.alloc(200 * 1024) + ]).toString('base64') + mocks.client.callTool.mockResolvedValue({ + content: [ + { + type: 'image', + mimeType: 'image/png', + data: encodedContextOversizedPng + } + ] + }) + const provider = new ModelToolProvider(workspace, [createMcpServer()]) + const tools = await provider.listTools(toolContext, new AbortController().signal) + const tool = tools.find((candidate) => candidate.source === 'mcp') + + await expect( + provider.callTool( + tool?.name ?? '', + {}, + new AbortController().signal, + toolContext + ) + ).rejects.toThrow('工具结果超过 256KB') + + const decodedOversizedPng = Buffer.concat([ + Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a + ]), + Buffer.alloc(256 * 1024) + ]).toString('base64') + mocks.client.callTool.mockResolvedValue({ + content: [ + { + type: 'image', + mimeType: 'image/png', + data: decodedOversizedPng + } + ] + }) + await expect( + provider.callTool( + tool?.name ?? '', + {}, + new AbortController().signal, + toolContext + ) + ).rejects.toThrow('过大的 base64 图片') + }) + + it('bounds MCP content block and image counts', async () => { + const workspace = await createWorkspace() + mocks.client.listTools.mockResolvedValue({ + tools: [{ name: 'capture', inputSchema: { type: 'object' } }] + }) + const provider = new ModelToolProvider(workspace, [createMcpServer()]) + const tools = await provider.listTools(toolContext, new AbortController().signal) + const tool = tools.find((candidate) => candidate.source === 'mcp') + + mocks.client.callTool.mockResolvedValue({ + content: Array.from({ length: 101 }, () => ({ + type: 'text', + text: 'x' + })) + }) + await expect( + provider.callTool( + tool?.name ?? '', + {}, + new AbortController().signal, + toolContext + ) + ).rejects.toThrow('内容块数量超过安全限制') + + mocks.client.callTool.mockResolvedValue({ + content: Array.from({ length: 9 }, () => ({ + type: 'image', + mimeType: 'image/png', + data: png + })) + }) + await expect( + provider.callTool( + tool?.name ?? '', + {}, + new AbortController().signal, + toolContext + ) + ).rejects.toThrow('图片数量超过安全限制') + }) + + it('streams required task tools and best-effort cancels their MCP task', async () => { + const workspace = await createWorkspace() + mocks.client.listTools.mockResolvedValue({ + tools: [ + { + name: 'long-job', + inputSchema: { type: 'object' }, + execution: { taskSupport: 'required' } + } + ] + }) + const controller = new AbortController() + mocks.tasks.callToolStream.mockImplementation(async function* ( + _params, + _schema, + options + ) { + yield { + type: 'taskCreated', + task: { taskId: 'task-1', status: 'working' } + } + controller.abort() + throw options.signal.reason + }) + const provider = new ModelToolProvider(workspace, [createMcpServer()]) + const tools = await provider.listTools(toolContext, controller.signal) + const tool = tools.find((candidate) => candidate.source === 'mcp') + expect(tool?.taskSupport).toBe('required') + + await expect( + provider.callTool( + tool?.name ?? '', + {}, + controller.signal, + toolContext + ) + ).rejects.toThrow() + expect(mocks.tasks.callToolStream).toHaveBeenCalledWith( + { + name: 'long-job', + arguments: {} + }, + undefined, + expect.objectContaining({ + timeout: 30_000, + signal: controller.signal, + resetTimeoutOnProgress: true, + maxTotalTimeout: 300_000 + }) + ) + expect(mocks.tasks.cancelTask).toHaveBeenCalledWith( + 'task-1', + { + timeout: 5_000, + maxTotalTimeout: 5_000 + } + ) + }) }) diff --git a/src/main/agent/model-tool-provider.ts b/src/main/agent/model-tool-provider.ts index 8d28011..d4be715 100644 --- a/src/main/agent/model-tool-provider.ts +++ b/src/main/agent/model-tool-provider.ts @@ -14,6 +14,7 @@ import { resolve } from 'node:path' import { z } from 'zod' +import { builtinModelTools } from '../../shared/builtin-model-tools' import type { ResolvedMcpServer } from '../capabilities/capability-service' import { createMcpTransport } from '../capabilities/mcp-client-transport' import { @@ -23,6 +24,11 @@ import { readBoundedUtf8File } from '../workspace-file-access' import type { RuntimeApprovalRequest } from './runtime' +import { + BrowserModelTools, + type BrowserToolService +} from '../browser/browser-model-tools' +import { BrowserStaleReferenceError } from '../browser/cdp-browser-driver' const MAX_MODEL_TOOLS = 100 const MAX_MCP_SERVERS = 16 @@ -31,6 +37,15 @@ const MAX_TOOL_RESULT_BYTES = 256 * 1024 const MAX_READ_BYTES = 256 * 1024 const MAX_WRITE_BYTES = 512 * 1024 const MCP_TIMEOUT_MS = 30_000 +const MCP_CALL_MAX_TOTAL_TIMEOUT_MS = 5 * 60_000 +const MCP_TASK_CANCEL_TIMEOUT_MS = 5_000 +const MAX_MCP_CONTENT_BLOCKS = 100 +const MAX_MCP_IMAGES = 8 +const [ + workspaceReadTextTool, + workspaceListDirectoryTool, + workspaceWriteTextTool +] = builtinModelTools const workspacePathSchema = z .string() @@ -66,20 +81,62 @@ export type ModelToolDefinition = { inputSchema: Record source: 'builtin' | 'mcp' serverName?: string + taskSupport?: 'forbidden' | 'optional' | 'required' +} + +export type ModelToolResultPart = + | { + type: 'text' + text: string + } + | { + type: 'image' + mimeType: 'image/png' | 'image/jpeg' | 'image/webp' + data: string + } + +export type ModelToolResult = { + parts: ModelToolResultPart[] + contextBytes: number +} + +export type ModelToolCallContext = { + conversationId: string + workMode: 'ask' | 'plan' | 'execute' +} + +export class RecoverableModelToolError extends Error { + readonly nextAction: string + + constructor( + message: string, + nextAction: string, + options?: ErrorOptions + ) { + super(message, options) + this.name = 'RecoverableModelToolError' + this.nextAction = nextAction + } } export interface ModelToolProviderLike { - listTools(signal: AbortSignal): Promise + listTools( + context: ModelToolCallContext, + signal: AbortSignal + ): Promise getApproval( tool: ModelToolDefinition, argumentsValue: Record, - argumentSummary: string + argumentSummary: string, + context: ModelToolCallContext ): RuntimeApprovalRequest callTool( name: string, argumentsValue: Record, - signal: AbortSignal - ): Promise + signal: AbortSignal, + context: ModelToolCallContext + ): Promise + releaseConversation(conversationId: string): Promise dispose(): Promise } @@ -151,64 +208,177 @@ function createMcpToolName(serverId: string, originalName: string): string { return `mcp_${serverHash}_${toolHash}_${readable}`.slice(0, 64) } -function getMcpResultText(result: unknown): string { +function createTextToolResult(text: string): ModelToolResult { + const contextBytes = Buffer.byteLength(text) + if (contextBytes > MAX_TOOL_RESULT_BYTES) { + throw new Error('工具结果超过 256KB 安全限制') + } + return { + parts: [{ type: 'text', text }], + contextBytes + } +} + +function parseMcpImage( + content: Record +): Extract { + const mimeType = content.mimeType + if ( + mimeType !== 'image/png' && + mimeType !== 'image/jpeg' && + mimeType !== 'image/webp' + ) { + throw new Error('MCP 工具返回了不支持的图片格式') + } + if ( + typeof content.data !== 'string' || + content.data.length === 0 || + content.data.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test( + content.data + ) + ) { + throw new Error('MCP 工具返回了无效的 base64 图片') + } + const decoded = Buffer.from(content.data, 'base64') + if ( + decoded.length === 0 || + decoded.length > MAX_TOOL_RESULT_BYTES || + decoded.toString('base64') !== content.data + ) { + throw new Error('MCP 工具返回了无效或过大的 base64 图片') + } + const signatureMatches = + mimeType === 'image/png' + ? decoded.length >= 8 && + decoded.subarray(0, 8).equals( + Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a + ]) + ) + : mimeType === 'image/jpeg' + ? decoded.length >= 3 && + decoded[0] === 0xff && + decoded[1] === 0xd8 && + decoded[2] === 0xff + : decoded.length >= 12 && + decoded.subarray(0, 4).toString('ascii') === 'RIFF' && + decoded.subarray(8, 12).toString('ascii') === 'WEBP' + if (!signatureMatches) { + throw new Error('MCP 工具图片的 MIME 类型与文件签名不匹配') + } + return { + type: 'image', + mimeType, + data: content.data + } +} + +function normalizeMcpResult(result: unknown): ModelToolResult { if (!result || typeof result !== 'object') { - return boundedJson(result, 'MCP 工具结果无法序列化') + return createTextToolResult( + boundedJson(result, 'MCP 工具结果无法序列化') + ) } const record = result as Record if (record.isError === true) { throw new Error('MCP Server 报告工具执行失败') } if ('toolResult' in record) { - return boundedJson(record.toolResult, 'MCP 工具结果无法序列化') + if ( + record.toolResult && + typeof record.toolResult === 'object' && + ( + Array.isArray( + (record.toolResult as Record).content + ) || + 'structuredContent' in + (record.toolResult as Record) || + 'isError' in (record.toolResult as Record) + ) + ) { + return normalizeMcpResult(record.toolResult) + } + return createTextToolResult( + boundedJson(record.toolResult, 'MCP 工具结果无法序列化') + ) } - const sections: string[] = [] + const parts: ModelToolResultPart[] = [] if ( record.structuredContent && typeof record.structuredContent === 'object' ) { - sections.push( - boundedJson( + parts.push({ + type: 'text', + text: boundedJson( record.structuredContent, 'MCP 结构化工具结果无法序列化' ) - ) + }) } if (Array.isArray(record.content)) { - for (const item of record.content.slice(0, 100)) { + if (record.content.length > MAX_MCP_CONTENT_BLOCKS) { + throw new Error('MCP 工具结果内容块数量超过安全限制') + } + let imageCount = 0 + for (const item of record.content) { if (!item || typeof item !== 'object') { continue } const content = item as Record if (content.type === 'text' && typeof content.text === 'string') { - sections.push(content.text) + parts.push({ type: 'text', text: content.text }) } else if ( content.type === 'resource' && content.resource && typeof content.resource === 'object' && typeof (content.resource as Record).text === 'string' ) { - sections.push( - (content.resource as Record).text as string - ) + parts.push({ + type: 'text', + text: (content.resource as Record).text as string + }) } else if (content.type === 'resource_link') { - sections.push( - boundedJson(content, 'MCP 资源链接无法序列化') - ) - } else if (content.type === 'image' || content.type === 'audio') { - sections.push(`[${String(content.type)} result omitted]`) + parts.push({ + type: 'text', + text: boundedJson(content, 'MCP 资源链接无法序列化') + }) + } else if (content.type === 'image') { + imageCount += 1 + if (imageCount > MAX_MCP_IMAGES) { + throw new Error('MCP 工具结果图片数量超过安全限制') + } + parts.push(parseMcpImage(content)) + } else if (content.type === 'audio') { + parts.push({ + type: 'text', + text: '[audio result unsupported]' + }) } } } - const text = sections.join('\n\n').trim() - if (!text) { - return '{}' + if (parts.length === 0) { + return createTextToolResult('{}') } - if (Buffer.byteLength(text) > MAX_TOOL_RESULT_BYTES) { + let contextBytes = 0 + let decodedImageBytes = 0 + for (const part of parts) { + contextBytes += Buffer.byteLength( + part.type === 'text' ? part.text : part.data + ) + if (part.type === 'image') { + decodedImageBytes += Buffer.from(part.data, 'base64').length + } + } + if ( + contextBytes > MAX_TOOL_RESULT_BYTES || + decodedImageBytes > MAX_TOOL_RESULT_BYTES + ) { throw new Error('工具结果超过 256KB 安全限制') } - return text + return { parts, contextBytes } } export class ModelToolProvider implements ModelToolProviderLike { @@ -218,9 +388,21 @@ export class ModelToolProvider implements ModelToolProviderLike { constructor( private readonly workspace: string, - private readonly mcpServers: ResolvedMcpServer[] = [] + private readonly mcpServers: ResolvedMcpServer[] = [], + private readonly browserService?: BrowserToolService ) {} + private getBrowserTools( + context: ModelToolCallContext + ): BrowserModelTools | undefined { + return this.browserService && context.workMode === 'execute' + ? new BrowserModelTools({ + service: this.browserService, + conversationId: context.conversationId + }) + : undefined + } + private async getWorkspace(): Promise { this.canonicalWorkspace ??= getCanonicalWorkspace( this.workspace, @@ -289,10 +471,9 @@ export class ModelToolProvider implements ModelToolProviderLike { private getBuiltinTools(): ModelToolDefinition[] { return [ { - name: 'workspace_read_text', - displayName: '读取工作区文本', - description: - '读取当前工作区内一个不超过 256KB 的 UTF-8 文本文件。', + name: workspaceReadTextTool.name, + displayName: workspaceReadTextTool.displayName, + description: workspaceReadTextTool.description, inputSchema: { type: 'object', properties: { @@ -307,10 +488,9 @@ export class ModelToolProvider implements ModelToolProviderLike { source: 'builtin' }, { - name: 'workspace_list_directory', - displayName: '列出工作区目录', - description: - '列出当前工作区内目录的直属内容，最多返回 200 项。', + name: workspaceListDirectoryTool.name, + displayName: workspaceListDirectoryTool.displayName, + description: workspaceListDirectoryTool.description, inputSchema: { type: 'object', properties: { @@ -324,10 +504,9 @@ export class ModelToolProvider implements ModelToolProviderLike { source: 'builtin' }, { - name: 'workspace_write_text', - displayName: '写入工作区文本', - description: - '在当前工作区内新建或覆盖一个不超过 512KB 的 UTF-8 文本文件；父目录必须已存在。', + name: workspaceWriteTextTool.name, + displayName: workspaceWriteTextTool.displayName, + description: workspaceWriteTextTool.description, inputSchema: { type: 'object', properties: { @@ -366,7 +545,9 @@ export class ModelToolProvider implements ModelToolProviderLike { timeout: MCP_TIMEOUT_MS, signal }) - if (result.tools.length > MAX_MODEL_TOOLS - 3) { + const builtinToolCount = + this.getBuiltinTools().length + (this.browserService ? 7 : 0) + if (result.tools.length > MAX_MODEL_TOOLS - builtinToolCount) { throw new Error( `MCP Server「${server.name}」提供的工具数量超过安全限制` ) @@ -386,7 +567,8 @@ export class ModelToolProvider implements ModelToolProviderLike { .slice(0, 1_000), inputSchema: normalizeToolSchema(tool.inputSchema), source: 'mcp', - serverName: server.name + serverName: server.name, + taskSupport: tool.execution?.taskSupport } })) if ( @@ -423,9 +605,11 @@ export class ModelToolProvider implements ModelToolProviderLike { ) .then((connections) => { const bindings = new Map() + const builtinToolCount = + this.getBuiltinTools().length + (this.browserService ? 7 : 0) for (const connection of connections) { for (const binding of connection.tools) { - if (bindings.size + 3 >= MAX_MODEL_TOOLS) { + if (bindings.size + builtinToolCount >= MAX_MODEL_TOOLS) { throw new Error('直连模型工具总数超过 100 个安全限制') } if (bindings.has(binding.definition.name)) { @@ -448,11 +632,16 @@ export class ModelToolProvider implements ModelToolProviderLike { return this.mcpBindings } - async listTools(signal: AbortSignal): Promise { + async listTools( + context: ModelToolCallContext, + signal: AbortSignal + ): Promise { signal.throwIfAborted() const bindings = await this.getMcpBindings(signal) + const browserTools = this.getBrowserTools(context) return [ ...this.getBuiltinTools(), + ...(browserTools?.listTools() ?? []), ...[...bindings.values()].map((binding) => binding.definition) ] } @@ -460,8 +649,17 @@ export class ModelToolProvider implements ModelToolProviderLike { getApproval( tool: ModelToolDefinition, argumentsValue: Record, - argumentSummary: string + argumentSummary: string, + context: ModelToolCallContext ): RuntimeApprovalRequest { + const browserTools = this.getBrowserTools(context) + if (browserTools?.ownsTool(tool.name)) { + return browserTools.getApproval( + tool, + argumentsValue, + argumentSummary + ) + } const path = typeof argumentsValue.path === 'string' ? argumentsValue.path.slice(0, 500) @@ -490,20 +688,38 @@ export class ModelToolProvider implements ModelToolProviderLike { async callTool( name: string, argumentsValue: Record, - signal: AbortSignal - ): Promise { + signal: AbortSignal, + context: ModelToolCallContext + ): Promise { signal.throwIfAborted() + const browserTools = this.getBrowserTools(context) + if (browserTools?.ownsTool(name)) { + try { + return await browserTools.callTool(name, argumentsValue, signal) + } catch (error) { + if (error instanceof BrowserStaleReferenceError) { + throw new RecoverableModelToolError( + error.message, + '调用 browser_snapshot 获取新快照，然后用新引用重试刚才的操作', + { cause: error } + ) + } + throw error + } + } if (name === 'workspace_read_text') { const input = readInputSchema.parse(argumentsValue) const filePath = await this.resolveExistingPath(input.path, 'file') - return ( - await readBoundedUtf8File( - filePath, - MAX_READ_BYTES, - '工作区文本文件超过 256KB 安全限制', - '工作区读取目标不是有效 UTF-8 文本' - ) - ).content + return createTextToolResult( + ( + await readBoundedUtf8File( + filePath, + MAX_READ_BYTES, + '工作区文本文件超过 256KB 安全限制', + '工作区读取目标不是有效 UTF-8 文本' + ) + ).content + ) } if (name === 'workspace_list_directory') { const input = listInputSchema.parse(argumentsValue) @@ -515,21 +731,23 @@ export class ModelToolProvider implements ModelToolProviderLike { directoryPath, 200 ) - return boundedJson( - { - entries: listing.entries - .sort((left, right) => left.name.localeCompare(right.name)) - .map((entry) => ({ - name: entry.name, - type: entry.isDirectory() - ? 'directory' - : entry.isFile() - ? 'file' - : 'other' - })), - truncated: listing.truncated - }, - '工作区目录结果无法序列化' + return createTextToolResult( + boundedJson( + { + entries: listing.entries + .sort((left, right) => left.name.localeCompare(right.name)) + .map((entry) => ({ + name: entry.name, + type: entry.isDirectory() + ? 'directory' + : entry.isFile() + ? 'file' + : 'other' + })), + truncated: listing.truncated + }, + '工作区目录结果无法序列化' + ) ) } if (name === 'workspace_write_text') { @@ -551,12 +769,14 @@ export class ModelToolProvider implements ModelToolProviderLike { await rm(temporaryPath, { force: true }).catch(() => undefined) throw new Error('无法安全写入工作区文件', { cause: error }) } - return boundedJson( - { - path: input.path, - bytesWritten: Buffer.byteLength(input.content) - }, - '工作区写入结果无法序列化' + return createTextToolResult( + boundedJson( + { + path: input.path, + bytesWritten: Buffer.byteLength(input.content) + }, + '工作区写入结果无法序列化' + ) ) } @@ -564,18 +784,52 @@ export class ModelToolProvider implements ModelToolProviderLike { if (!binding) { throw new Error('模型请求了未知工具') } - const result = await binding.client.callTool( - { - name: binding.originalName, - arguments: argumentsValue - }, - undefined, - { - timeout: MCP_TIMEOUT_MS, - signal + const params = { + name: binding.originalName, + arguments: argumentsValue + } + const options = { + timeout: MCP_TIMEOUT_MS, + signal, + onprogress: () => undefined, + resetTimeoutOnProgress: true, + maxTotalTimeout: MCP_CALL_MAX_TOTAL_TIMEOUT_MS + } + if (binding.definition.taskSupport !== 'required') { + return normalizeMcpResult( + await binding.client.callTool(params, undefined, options) + ) + } + + let taskId: string | undefined + try { + for await (const message of binding.client.experimental.tasks.callToolStream( + params, + undefined, + options + )) { + if ( + (message.type === 'taskCreated' || + message.type === 'taskStatus') && + typeof message.task.taskId === 'string' + ) { + taskId = message.task.taskId + } else if (message.type === 'result') { + return normalizeMcpResult(message.result) + } else if (message.type === 'error') { + throw message.error + } } - ) - return getMcpResultText(result) + throw new Error('MCP 任务工具未返回最终结果') + } catch (error) { + if (taskId) { + await binding.client.experimental.tasks.cancelTask(taskId, { + timeout: MCP_TASK_CANCEL_TIMEOUT_MS, + maxTotalTimeout: MCP_TASK_CANCEL_TIMEOUT_MS + }).catch(() => undefined) + } + throw error + } } async dispose(): Promise { @@ -584,4 +838,14 @@ export class ModelToolProvider implements ModelToolProviderLike { this.mcpBindings = undefined await Promise.allSettled(clients.map((client) => client.close())) } + + async releaseConversation(conversationId: string): Promise { + if (!this.browserService) { + return + } + await new BrowserModelTools({ + service: this.browserService, + conversationId + }).release() + } } diff --git a/src/main/assistant/assistant-database.test.ts b/src/main/assistant/assistant-database.test.ts index 710ccfc..53a0c21 100644 --- a/src/main/assistant/assistant-database.test.ts +++ b/src/main/assistant/assistant-database.test.ts @@ -24,7 +24,7 @@ async function createDatabase(): Promise { } describe('AssistantDatabase', () => { - it('migrates existing databases to schema version 5', async () => { + it('migrates existing databases to schema version 6', async () => { const directory = await mkdtemp( join(tmpdir(), 'goodbuddy-assistant-migration-') ) @@ -52,7 +52,7 @@ describe('AssistantDatabase', () => { user_version: number } ).user_version - ).toBe(5) + ).toBe(6) expect( current .prepare( @@ -95,6 +95,74 @@ describe('AssistantDatabase', () => { current.close() }) + it('idempotently migrates version 5 databases to computer control audit schema', async () => { + const directory = await mkdtemp( + join(tmpdir(), 'goodbuddy-control-audit-migration-') + ) + temporaryDirectories.push(directory) + const databasePath = join(directory, 'assistant.sqlite') + const initial = new AssistantDatabase(databasePath) + initial.initialize('C:\\Workspace') + initial.close() + + const versionFive = new DatabaseSync(databasePath) + versionFive.exec(` + DROP TABLE computer_control_actions; + PRAGMA user_version = 5; + `) + versionFive.close() + + for (let attempt = 0; attempt < 2; attempt += 1) { + const migrated = new AssistantDatabase(databasePath) + migrated.initialize('C:\\Workspace') + migrated.close() + } + + const current = new DatabaseSync(databasePath) + expect( + ( + current.prepare('PRAGMA user_version').get() as { + user_version: number + } + ).user_version + ).toBe(6) + expect( + current + .prepare( + `SELECT name FROM sqlite_master + WHERE type = 'table' + AND name = 'computer_control_actions'` + ) + .get() + ).toEqual({ name: 'computer_control_actions' }) + expect( + current + .prepare( + `SELECT name FROM sqlite_master + WHERE type = 'index' + AND name = 'computer_control_actions_recent_idx'` + ) + .get() + ).toEqual({ name: 'computer_control_actions_recent_idx' }) + expect( + current + .prepare( + 'PRAGMA foreign_key_list(computer_control_actions)' + ) + .all() + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + table: 'tasks', + from: 'task_id', + to: 'id', + on_delete: 'CASCADE' + }) + ]) + ) + current.close() + }) + it('creates a default project and persists project updates', async () => { const database = await createDatabase() const [defaultProject] = database.listProjects() @@ -138,6 +206,38 @@ describe('AssistantDatabase', () => { database.close() }) + it('creates, updates, and soft-deletes expert roles', async () => { + const database = await createDatabase() + const expert = database.createExpert({ + name: '代码审查专家', + description: '检查代码正确性', + systemInstructions: 'Review code for actionable bugs.' + }) + + const updated = database.updateExpert(expert.id, { + name: '高级代码审查专家', + description: '检查正确性和安全性', + systemInstructions: 'Review correctness and security risks.' + }) + expect(updated).toMatchObject({ + id: expert.id, + name: '高级代码审查专家', + description: '检查正确性和安全性', + systemInstructions: 'Review correctness and security risks.', + enabled: true + }) + + database.removeExpert(expert.id) + + expect( + database.listExperts().some((item) => item.id === expert.id) + ).toBe(false) + expect(() => database.getExpert(expert.id)).toThrow( + '专家不存在或已停用' + ) + database.close() + }) + it('persists task lifecycle and events', async () => { const database = await createDatabase() const project = database.listProjects()[0]! diff --git a/src/main/assistant/assistant-database.ts b/src/main/assistant/assistant-database.ts index 4d34b55..2288cfc 100644 --- a/src/main/assistant/assistant-database.ts +++ b/src/main/assistant/assistant-database.ts @@ -12,6 +12,7 @@ import type { AssistantTask, ConversationSnapshot, ExpertCreateInput, + ExpertUpdateInput, HeartbeatCreateInput, HeartbeatSummaryOutput, HeartbeatUpdateInput, @@ -22,6 +23,13 @@ import type { TokenUsageRecord, TokenUsageSummary } from '../../shared/assistant-contracts' +import { + computerControlErrorCodeSchema, + computerControlRiskSchema, + type ComputerControlErrorCode, + type ComputerControlRisk +} from '../../shared/computer-control-contracts' +import type { ComputerControlAuditEvent } from '../computer-control/audit' import { computeNextHeartbeatRun } from './heartbeat-recurrence' type ProjectRow = { @@ -188,6 +196,23 @@ type TokenUsageRecordRow = { cache_write_tokens: number } +type ComputerControlActionRow = { + command_id: string + task_id: string + conversation_id: string + lease_id: string + action: ComputerControlAuditEvent['action'] + risk: ComputerControlRisk + outcome: ComputerControlAuditEvent['outcome'] + error_code: ComputerControlErrorCode | null + occurred_at: number + text_length: number | null + text_digest: string | null +} + +export type PersistedComputerControlAuditEvent = + ComputerControlAuditEvent + export type ClaimedHeartbeatRun = { config: AssistantHeartbeatConfig run: AssistantHeartbeatRun @@ -407,6 +432,76 @@ function validateTokenCount(value: number, label: string): number { return value } +const computerControlActions = [ + 'observe', + 'activate', + 'replace_text', + 'select_option', + 'scroll' +] as const +const computerControlOutcomes = [ + 'completed', + 'denied', + 'failed', + 'outcome_unknown' +] as const +function validateComputerControlId( + value: string, + label: string, + maximumLength: number, + opaque = false +): string { + if (typeof value !== 'string') { + throw new TypeError(`${label} must be a string`) + } + if ( + value.length < (opaque ? 16 : 1) || + value.length > maximumLength || + (opaque + ? !/^[A-Za-z0-9_-]+$/.test(value) + : [...value].some((character) => { + const code = character.charCodeAt(0) + return code <= 31 || code === 127 + })) + ) { + throw new RangeError(`Invalid ${label}`) + } + return value +} + +function validateComputerControlEnum( + value: string, + label: string, + allowed: readonly T[] +): T { + if (!allowed.includes(value as T)) { + throw new RangeError(`Invalid ${label}`) + } + return value as T +} + +function toComputerControlAuditEvent( + row: ComputerControlActionRow +): PersistedComputerControlAuditEvent { + return { + timestamp: row.occurred_at, + taskId: row.task_id, + conversationId: row.conversation_id, + leaseId: row.lease_id, + commandId: row.command_id, + action: row.action, + risk: row.risk, + outcome: row.outcome, + ...(row.error_code ? { errorCode: row.error_code } : {}), + ...(row.text_length === null + ? {} + : { textLength: row.text_length }), + ...(row.text_digest === null + ? {} + : { textDigest: row.text_digest }) + } +} + const interruptedTaskError = '应用退出时任务仍在运行' const interruptedMessageStatus = '上次运行意外中断，可以重新发送问题' @@ -575,6 +670,7 @@ export class AssistantDatabase { 'schedules', 'memory_items', 'artifacts', + 'computer_control_actions', 'task_events', 'runs', 'model_usage_calls', @@ -1112,6 +1208,165 @@ export class AssistantDatabase { ) } + persistComputerControlAudit( + event: ComputerControlAuditEvent + ): void { + if (!event || typeof event !== 'object') { + throw new TypeError('Computer control audit event is required') + } + const taskId = validateComputerControlId( + event.taskId, + 'taskId', + 128 + ) + const conversationId = validateComputerControlId( + event.conversationId, + 'conversationId', + 128 + ) + const leaseId = validateComputerControlId( + event.leaseId, + 'leaseId', + 160, + true + ) + const commandId = validateComputerControlId( + event.commandId, + 'commandId', + 160, + true + ) + const action = validateComputerControlEnum( + event.action, + 'action', + computerControlActions + ) + const risk = computerControlRiskSchema.parse(event.risk) + const outcome = validateComputerControlEnum( + event.outcome, + 'outcome', + computerControlOutcomes + ) + if ( + !Number.isSafeInteger(event.timestamp) || + event.timestamp < 0 + ) { + throw new RangeError('Invalid timestamp') + } + const errorCode = + event.errorCode === undefined + ? undefined + : computerControlErrorCodeSchema.parse(event.errorCode) + if ( + (outcome === 'completed' && errorCode !== undefined) || + (outcome !== 'completed' && errorCode === undefined) + ) { + throw new RangeError('Invalid outcome and errorCode combination') + } + const hasTextMetadata = + event.textLength !== undefined || + event.textDigest !== undefined + if ( + (action === 'replace_text') !== hasTextMetadata || + (hasTextMetadata && + (!Number.isInteger(event.textLength) || + event.textLength! < 0 || + event.textLength! > 4_096 || + typeof event.textDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(event.textDigest))) + ) { + throw new RangeError('Invalid redacted text metadata') + } + + const database = this.requireDatabase() + const createdAt = new Date().toISOString() + database.exec('BEGIN IMMEDIATE') + try { + const inserted = database + .prepare( + `INSERT OR IGNORE INTO computer_control_actions + (command_id, task_id, conversation_id, lease_id, action, risk, + outcome, error_code, occurred_at, text_length, text_digest, + created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + commandId, + taskId, + conversationId, + leaseId, + action, + risk, + outcome, + errorCode ?? null, + event.timestamp, + event.textLength ?? null, + event.textDigest ?? null, + createdAt + ) + if (inserted.changes === 1) { + database + .prepare( + `INSERT INTO task_events + (task_id, run_id, kind, payload_json, created_at) + VALUES (?, NULL, 'computer_control', ?, ?)` + ) + .run( + taskId, + JSON.stringify({ + commandId, + action, + risk, + outcome, + ...(errorCode ? { errorCode } : {}), + ...(event.textLength === undefined + ? {} + : { textLength: event.textLength }), + ...(event.textDigest === undefined + ? {} + : { textDigest: event.textDigest }) + }), + createdAt + ) + database + .prepare( + `DELETE FROM computer_control_actions + WHERE command_id IN ( + SELECT command_id FROM computer_control_actions + ORDER BY occurred_at DESC, created_at DESC + LIMIT -1 OFFSET 10000 + )` + ) + .run() + } + database.exec('COMMIT') + } catch (error) { + database.exec('ROLLBACK') + throw error + } + } + + listRecentComputerControlAudit( + limit = 100 + ): PersistedComputerControlAuditEvent[] { + if (!Number.isInteger(limit) || limit < 1 || limit > 500) { + throw new RangeError( + 'Computer control audit limit must be between 1 and 500' + ) + } + const rows = this.requireDatabase() + .prepare( + `SELECT command_id, task_id, conversation_id, lease_id, action, + risk, outcome, error_code, occurred_at, text_length, + text_digest + FROM computer_control_actions + ORDER BY occurred_at DESC, created_at DESC + LIMIT ?` + ) + .all(limit) as ComputerControlActionRow[] + return rows.map(toComputerControlAuditEvent) + } + listArtifacts(projectId?: string, limit = 100): AssistantArtifact[] { const safeLimit = Math.max(1, Math.min(500, Math.trunc(limit))) const columns = `id, project_id, task_id, kind, title, mime_type, @@ -2304,6 +2559,43 @@ export class AssistantDatabase { return this.getExpert(id) } + updateExpert( + expertId: string, + input: ExpertUpdateInput + ): AssistantExpert { + const result = this.requireDatabase() + .prepare( + `UPDATE experts + SET name = ?, description = ?, system_instructions = ?, + updated_at = ? + WHERE id = ? AND enabled = 1` + ) + .run( + input.name, + input.description, + input.systemInstructions, + new Date().toISOString(), + expertId + ) + if (result.changes === 0) { + throw new Error('专家不存在或已停用') + } + return this.getExpert(expertId) + } + + removeExpert(expertId: string): void { + const result = this.requireDatabase() + .prepare( + `UPDATE experts + SET enabled = 0, updated_at = ? + WHERE id = ? AND enabled = 1` + ) + .run(new Date().toISOString(), expertId) + if (result.changes === 0) { + throw new Error('专家不存在或已停用') + } + } + getExpert(expertId: string): AssistantExpert { const row = this.requireDatabase() .prepare('SELECT * FROM experts WHERE id = ? AND enabled = 1') @@ -2358,7 +2650,7 @@ export class AssistantDatabase { const version = database .prepare('PRAGMA user_version') .get() as { user_version: number } - if (version.user_version >= 5) { + if (version.user_version >= 6) { return } if (version.user_version < 1) { @@ -2620,9 +2912,10 @@ export class AssistantDatabase { COMMIT; `) } - database.exec(` - BEGIN IMMEDIATE; - CREATE TABLE IF NOT EXISTS model_usage_calls ( + if (version.user_version < 4) { + database.exec(` + BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS model_usage_calls ( request_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, call_id TEXT NOT NULL, @@ -2642,17 +2935,81 @@ export class AssistantDatabase { CREATE INDEX IF NOT EXISTS model_usage_calls_dimensions_idx ON model_usage_calls(runtime, provider, model); PRAGMA user_version = 4; - COMMIT; - `) - database.exec(` - BEGIN IMMEDIATE; - CREATE INDEX IF NOT EXISTS tasks_status_idx + COMMIT; + `) + } + if (version.user_version < 5) { + database.exec(` + BEGIN IMMEDIATE; + CREATE INDEX IF NOT EXISTS tasks_status_idx ON tasks(status); CREATE INDEX IF NOT EXISTS messages_state_idx ON messages(state); PRAGMA user_version = 5; - COMMIT; - `) + COMMIT; + `) + } + if (version.user_version < 6) { + database.exec(` + BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS computer_control_actions ( + command_id TEXT PRIMARY KEY + CHECK(length(command_id) BETWEEN 16 AND 160), + task_id TEXT NOT NULL + REFERENCES tasks(id) ON DELETE CASCADE + CHECK(length(task_id) BETWEEN 1 AND 128), + conversation_id TEXT NOT NULL + CHECK(length(conversation_id) BETWEEN 1 AND 128), + lease_id TEXT NOT NULL + CHECK(length(lease_id) BETWEEN 16 AND 160), + action TEXT NOT NULL + CHECK(action IN ('observe', 'activate', 'replace_text', + 'select_option', 'scroll')), + risk TEXT NOT NULL + CHECK(risk IN ('observe', 'navigate', 'input', 'commit', + 'forbidden')), + outcome TEXT NOT NULL + CHECK(outcome IN ('completed', 'denied', 'failed', + 'outcome_unknown')), + error_code TEXT + CHECK(error_code IS NULL OR error_code IN ( + 'invalid_request', 'driver_unavailable', 'driver_timeout', + 'lease_not_found', 'lease_expired', 'lease_mismatch', + 'observation_not_found', 'observation_stale', + 'observation_consumed', 'element_not_found', + 'window_not_foreground', 'element_identity_changed', + 'focus_failed', 'forbidden', 'approval_denied', + 'approval_timeout', 'cancelled', 'command_id_conflict', + 'outcome_unknown', 'internal_error')), + occurred_at INTEGER NOT NULL + CHECK(occurred_at BETWEEN 0 AND 9007199254740991), + text_length INTEGER + CHECK(text_length IS NULL OR + text_length BETWEEN 0 AND 4096), + text_digest TEXT + CHECK(text_digest IS NULL OR ( + length(text_digest) = 64 AND + text_digest NOT GLOB '*[^0-9a-f]*')), + created_at TEXT NOT NULL, + CHECK( + (outcome = 'completed' AND error_code IS NULL) OR + (outcome <> 'completed' AND error_code IS NOT NULL) + ), + CHECK( + (action = 'replace_text' AND text_length IS NOT NULL AND + text_digest IS NOT NULL) OR + (action <> 'replace_text' AND text_length IS NULL AND + text_digest IS NULL) + ) + ); + CREATE INDEX IF NOT EXISTS computer_control_actions_recent_idx + ON computer_control_actions(occurred_at DESC, created_at DESC); + CREATE INDEX IF NOT EXISTS computer_control_actions_task_idx + ON computer_control_actions(task_id, occurred_at DESC); + PRAGMA user_version = 6; + COMMIT; + `) + } } private requireDatabase(): DatabaseSync { diff --git a/src/main/assistant/heartbeat-database.test.ts b/src/main/assistant/heartbeat-database.test.ts index f15d1dd..a48b895 100644 --- a/src/main/assistant/heartbeat-database.test.ts +++ b/src/main/assistant/heartbeat-database.test.ts @@ -91,7 +91,7 @@ describe('AssistantDatabase heartbeat persistence', () => { user_version: number } ).user_version - ).toBe(5) + ).toBe(6) expect( ( check diff --git a/src/main/browser/browser-model-tools.test.ts b/src/main/browser/browser-model-tools.test.ts new file mode 100644 index 0000000..d900ec1 --- /dev/null +++ b/src/main/browser/browser-model-tools.test.ts @@ -0,0 +1,210 @@ +import { describe, expect, it, vi } from 'vitest' +import { + BrowserModelTools, + browserBackInputSchema, + browserClickInputSchema, + browserNavigateInputSchema, + browserScreenshotInputSchema, + browserSelectInputSchema, + browserSnapshotInputSchema, + browserTypeInputSchema, + type BrowserToolService +} from './browser-model-tools' + +function createService(): BrowserToolService { + return { + getOrigin: vi.fn(() => 'https://example.com'), + navigate: vi.fn(async (_conversationId, url) => ({ + url, + origin: 'https://example.com' + })), + snapshot: vi.fn(async () => ({ + url: 'https://example.com/', + title: 'Example', + nodes: [], + truncated: false + })), + click: vi.fn(async () => undefined), + type: vi.fn(async () => undefined), + select: vi.fn(async () => undefined), + back: vi.fn(async () => ({ + url: 'https://previous.example/', + origin: 'https://previous.example' + })), + screenshot: vi.fn(async () => ({ + type: 'image' as const, + mimeType: 'image/png' as const, + data: 'iVBORw0KGgo=' + })), + releaseConversation: vi.fn(async () => undefined) + } +} + +const signal = new AbortController().signal +const ref = 'b_abcdefghijklmnop' + +describe('BrowserModelTools', () => { + it('publishes seven strict, bounded builtin tool definitions', () => { + const tools = new BrowserModelTools({ + service: createService(), + conversationId: 'conversation' + }) + const definitions = tools.listTools() + expect(definitions.map((definition) => definition.name)).toEqual([ + 'browser_navigate', + 'browser_snapshot', + 'browser_click', + 'browser_type', + 'browser_select', + 'browser_back', + 'browser_screenshot' + ]) + expect( + definitions.every( + (definition) => + definition.source === 'builtin' && + definition.inputSchema.additionalProperties === false + ) + ).toBe(true) + expect(tools.ownsTool('browser_upload')).toBe(false) + expect(tools.ownsTool('browser_download')).toBe(false) + }) + + it('uses strict Zod parsing for every operation', () => { + const cases: Array<[typeof browserSnapshotInputSchema, unknown]> = [ + [browserNavigateInputSchema, { url: 'https://example.com', extra: true }], + [browserSnapshotInputSchema, { extra: true }], + [browserClickInputSchema, { ref: 'not-a-ref' }], + [browserTypeInputSchema, { ref, text: '', extra: true }], + [browserSelectInputSchema, { ref, value: '', extra: true }], + [browserBackInputSchema, { extra: true }], + [browserScreenshotInputSchema, { extra: true }] + ] + for (const [schema, value] of cases) { + expect(() => schema.parse(value)).toThrow() + } + expect(() => + browserNavigateInputSchema.parse({ url: 'file:///etc/passwd' }) + ).not.toThrow() + // Zod limits shape and size; the URL policy is deliberately applied by + // approval/call handling so non-HTTP schemes still fail before execution. + }) + + it('creates dynamic origin-scoped navigation approvals without exposing query values', () => { + const tools = new BrowserModelTools({ + service: createService(), + conversationId: 'conversation' + }) + const approval = tools.getApproval('browser_navigate', { + url: 'https://example.com/path?token=top-secret' + }) + expect(approval).toMatchObject({ + scopeKey: 'model:browser:navigate:https://example.com', + allowPermanent: false + }) + expect(JSON.stringify(approval)).not.toContain('top-secret') + expect(approval.argumentSummary).toContain('[查询参数已隐藏]') + expect(() => + tools.getApproval('browser_navigate', { + url: 'file:///etc/passwd' + }) + ).toThrow('HTTP(S)') + }) + + it('redacts typed and selected values and prevents session-grant reuse', async () => { + const service = createService() + const tools = new BrowserModelTools({ + service, + conversationId: 'conversation' + }) + const first = tools.getApproval('browser_type', { + ref, + text: 'top-secret' + }) + const second = tools.getApproval('browser_type', { + ref, + text: 'top-secret' + }) + expect(first.scopeKey).not.toBe(second.scopeKey) + expect(first.allowPermanent).toBe(false) + expect(JSON.stringify(first)).not.toContain('top-secret') + + const result = await tools.callTool( + 'browser_type', + { ref, text: 'top-secret' }, + signal + ) + expect(service.type).toHaveBeenCalledWith( + 'conversation', + ref, + 'top-secret', + signal + ) + expect(JSON.stringify(result)).not.toContain('top-secret') + expect(result.parts[0]).toMatchObject({ + type: 'text', + text: expect.stringContaining('[已隐藏]') + }) + + const selectApproval = tools.getApproval('browser_select', { + ref, + value: 'private-value' + }) + expect(JSON.stringify(selectApproval)).not.toContain('private-value') + }) + + it('dispatches safe operations and returns correctly counted results', async () => { + const service = createService() + const tools = new BrowserModelTools({ + service, + conversationId: 'conversation' + }) + const navigate = await tools.callTool( + 'browser_navigate', + { url: 'https://example.com/' }, + signal + ) + const snapshot = await tools.callTool('browser_snapshot', {}, signal) + const click = await tools.callTool('browser_click', { ref }, signal) + const back = await tools.callTool('browser_back', {}, signal) + for (const result of [navigate, snapshot, click, back]) { + const part = result.parts[0] + if (!part || part.type !== 'text') { + throw new Error('expected text result') + } + expect(result.contextBytes).toBe(Buffer.byteLength(part.text)) + } + + const screenshot = await tools.callTool('browser_screenshot', {}, signal) + expect(screenshot).toEqual({ + parts: [ + { + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=' + } + ], + contextBytes: Buffer.byteLength('iVBORw0KGgo=') + }) + await tools.release() + expect(service.releaseConversation).toHaveBeenCalledWith('conversation') + }) + + it('rejects unknown tools, extra fields, malformed refs, and cancellation', async () => { + const tools = new BrowserModelTools({ + service: createService(), + conversationId: 'conversation' + }) + expect(() => + tools.getApproval('browser_click', { ref, extra: true }) + ).toThrow() + await expect( + tools.callTool('browser_upload', { path: 'secret.txt' }, signal) + ).rejects.toThrow('未知浏览器工具') + const controller = new AbortController() + controller.abort() + await expect( + tools.callTool('browser_snapshot', {}, controller.signal) + ).rejects.toHaveProperty('name', 'AbortError') + }) +}) diff --git a/src/main/browser/browser-model-tools.ts b/src/main/browser/browser-model-tools.ts new file mode 100644 index 0000000..6b6077c --- /dev/null +++ b/src/main/browser/browser-model-tools.ts @@ -0,0 +1,379 @@ +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { builtinModelTools } from '../../shared/builtin-model-tools' +import type { + ModelToolDefinition, + ModelToolResult +} from '../agent/model-tool-provider' +import type { RuntimeApprovalRequest } from '../agent/runtime' +import { canonicalizeBrowserUrl } from './browser-url-policy' +import type { BrowserService } from './browser-service' + +const MAX_REF_LENGTH = 64 +const MAX_INPUT_LENGTH = 16_384 +const MAX_SELECT_LENGTH = 1_024 + +const refSchema = z + .string() + .min(1) + .max(MAX_REF_LENGTH) + .regex(/^b_[A-Za-z0-9_-]{1,61}$/u, '元素引用格式无效') + +export const browserNavigateInputSchema = z + .object({ + url: z.string().min(1).max(8_192) + }) + .strict() + +export const browserSnapshotInputSchema = z.object({}).strict() + +export const browserClickInputSchema = z + .object({ + ref: refSchema + }) + .strict() + +export const browserTypeInputSchema = z + .object({ + ref: refSchema, + text: z.string().min(1).max(MAX_INPUT_LENGTH) + }) + .strict() + +export const browserSelectInputSchema = z + .object({ + ref: refSchema, + value: z.string().min(1).max(MAX_SELECT_LENGTH) + }) + .strict() + +export const browserBackInputSchema = z.object({}).strict() +export const browserScreenshotInputSchema = z.object({}).strict() + +type BrowserToolName = + | 'browser_navigate' + | 'browser_snapshot' + | 'browser_click' + | 'browser_type' + | 'browser_select' + | 'browser_back' + | 'browser_screenshot' + +function getBrowserToolMetadata(name: BrowserToolName) { + const summary = builtinModelTools.find((tool) => tool.name === name) + if (!summary) { + throw new Error(`缺少内置浏览器工具定义：${name}`) + } + return { + name, + displayName: summary.displayName, + description: summary.description, + source: 'builtin' as const + } +} + +const definitions = [ + { + ...getBrowserToolMetadata('browser_navigate'), + inputSchema: { + type: 'object', + properties: { + url: { + type: 'string', + minLength: 1, + maxLength: 8_192, + description: '完整的公开 HTTP(S) URL' + } + }, + required: ['url'], + additionalProperties: false + }, + }, + { + ...getBrowserToolMetadata('browser_snapshot'), + inputSchema: { + type: 'object', + properties: {}, + additionalProperties: false + } + }, + { + ...getBrowserToolMetadata('browser_click'), + inputSchema: { + type: 'object', + properties: { + ref: { + type: 'string', + pattern: '^b_[A-Za-z0-9_-]{1,61}$', + maxLength: MAX_REF_LENGTH + } + }, + required: ['ref'], + additionalProperties: false + } + }, + { + ...getBrowserToolMetadata('browser_type'), + inputSchema: { + type: 'object', + properties: { + ref: { + type: 'string', + pattern: '^b_[A-Za-z0-9_-]{1,61}$', + maxLength: MAX_REF_LENGTH + }, + text: { + type: 'string', + minLength: 1, + maxLength: MAX_INPUT_LENGTH, + description: '要输入的文本（审批界面不会显示内容）' + } + }, + required: ['ref', 'text'], + additionalProperties: false + } + }, + { + ...getBrowserToolMetadata('browser_select'), + inputSchema: { + type: 'object', + properties: { + ref: { + type: 'string', + pattern: '^b_[A-Za-z0-9_-]{1,61}$', + maxLength: MAX_REF_LENGTH + }, + value: { + type: 'string', + minLength: 1, + maxLength: MAX_SELECT_LENGTH + } + }, + required: ['ref', 'value'], + additionalProperties: false + } + }, + { + ...getBrowserToolMetadata('browser_back'), + inputSchema: { + type: 'object', + properties: {}, + additionalProperties: false + } + }, + { + ...getBrowserToolMetadata('browser_screenshot'), + inputSchema: { + type: 'object', + properties: {}, + additionalProperties: false + } + } +] as const satisfies readonly ModelToolDefinition[] + +export type BrowserToolService = Pick< + BrowserService, + | 'getOrigin' + | 'navigate' + | 'snapshot' + | 'click' + | 'type' + | 'select' + | 'back' + | 'screenshot' + | 'releaseConversation' +> + +export type BrowserModelToolsOptions = { + service: BrowserToolService + conversationId: string +} + +function createTextResult(value: unknown): ModelToolResult { + const text = JSON.stringify(value) + return { + parts: [{ type: 'text', text }], + contextBytes: Buffer.byteLength(text) + } +} + +function safeOrigin(value: string | undefined): string { + return value ?? '尚未导航' +} + +function navigationLabel(url: URL): string { + const pathname = + url.pathname.length > 500 ? `${url.pathname.slice(0, 500)}…` : url.pathname + return `${url.origin}${pathname}${url.search ? '?[查询参数已隐藏]' : ''}` +} + +export class BrowserModelTools { + private readonly service: BrowserToolService + private readonly conversationId: string + + constructor(options: BrowserModelToolsOptions) { + this.service = options.service + this.conversationId = options.conversationId + if (!this.conversationId || this.conversationId.length > 500) { + throw new Error('浏览器对话标识无效') + } + } + + listTools(): ModelToolDefinition[] { + return definitions.map((definition) => ({ + ...definition, + inputSchema: { ...definition.inputSchema } + })) + } + + ownsTool(name: string): name is BrowserToolName { + return definitions.some((definition) => definition.name === name) + } + + getApproval( + tool: ModelToolDefinition | BrowserToolName, + argumentsValue: Record, + argumentSummaryFromRuntime?: string + ): RuntimeApprovalRequest { + void argumentSummaryFromRuntime + const name = typeof tool === 'string' ? tool : tool.name + if (!this.ownsTool(name)) { + throw new Error(`未知浏览器工具：${name}`) + } + const currentOrigin = safeOrigin( + this.service.getOrigin(this.conversationId) + ) + let description: string + let argumentSummary: string + let scopeKey: string + if (name === 'browser_navigate') { + const input = browserNavigateInputSchema.parse(argumentsValue) + const target = canonicalizeBrowserUrl(input.url) + const label = navigationLabel(target) + description = `将在隔离浏览器中访问 ${label}。仅允许公开 HTTP(S) 地址。` + argumentSummary = label + scopeKey = `model:browser:navigate:${target.origin}` + } else if (name === 'browser_snapshot') { + browserSnapshotInputSchema.parse(argumentsValue) + description = `读取 ${currentOrigin} 的页面结构；可编辑字段值会被隐藏。` + argumentSummary = `来源：${currentOrigin}` + scopeKey = `model:browser:snapshot:${currentOrigin}` + } else if (name === 'browser_click') { + const input = browserClickInputSchema.parse(argumentsValue) + description = `点击 ${currentOrigin} 页面中的元素 ${input.ref}。` + argumentSummary = `元素：${input.ref}；来源：${currentOrigin}` + scopeKey = `model:browser:click:${currentOrigin}:${input.ref}` + } else if (name === 'browser_type') { + const input = browserTypeInputSchema.parse(argumentsValue) + description = `向 ${currentOrigin} 页面中的元素 ${input.ref} 输入已隐藏的文本。密码、文件和隐藏字段会被拒绝。` + argumentSummary = `元素：${input.ref}；内容：[已隐藏，${input.text.length} 个字符]` + // A session approval must never authorize a later value, even for the + // same element. The nonce intentionally makes this invocation-only. + scopeKey = `model:browser:type:${randomUUID()}` + } else if (name === 'browser_select') { + const input = browserSelectInputSchema.parse(argumentsValue) + description = `在 ${currentOrigin} 页面中的选择控件 ${input.ref} 选择已隐藏的值。` + argumentSummary = `元素：${input.ref}；选项值：[已隐藏，${input.value.length} 个字符]` + scopeKey = `model:browser:select:${randomUUID()}` + } else if (name === 'browser_back') { + browserBackInputSchema.parse(argumentsValue) + description = `从 ${currentOrigin} 返回浏览器历史记录中的上一页。目标仍需通过 URL 安全策略。` + argumentSummary = `当前来源：${currentOrigin}` + scopeKey = `model:browser:back:${randomUUID()}` + } else { + browserScreenshotInputSchema.parse(argumentsValue) + description = `截取 ${currentOrigin} 当前可见页面区域。` + argumentSummary = `来源：${currentOrigin}` + scopeKey = `model:browser:screenshot:${currentOrigin}` + } + const definition = definitions.find((item) => item.name === name) + if (!definition) { + throw new Error(`未知浏览器工具：${name}`) + } + return { + scopeKey, + title: `允许${definition.displayName}？`, + description, + toolName: definition.displayName, + argumentSummary, + allowPermanent: false + } + } + + async callTool( + name: string, + argumentsValue: Record, + signal: AbortSignal + ): Promise { + signal.throwIfAborted() + if (!this.ownsTool(name)) { + throw new Error(`未知浏览器工具：${name}`) + } + if (name === 'browser_navigate') { + const input = browserNavigateInputSchema.parse(argumentsValue) + return createTextResult( + await this.service.navigate(this.conversationId, input.url, signal) + ) + } + if (name === 'browser_snapshot') { + browserSnapshotInputSchema.parse(argumentsValue) + return createTextResult( + await this.service.snapshot(this.conversationId, signal) + ) + } + if (name === 'browser_click') { + const input = browserClickInputSchema.parse(argumentsValue) + await this.service.click(this.conversationId, input.ref, signal) + return createTextResult({ clicked: input.ref }) + } + if (name === 'browser_type') { + const input = browserTypeInputSchema.parse(argumentsValue) + await this.service.type( + this.conversationId, + input.ref, + input.text, + signal + ) + return createTextResult({ + typed: input.ref, + text: '[已隐藏]', + characters: input.text.length + }) + } + if (name === 'browser_select') { + const input = browserSelectInputSchema.parse(argumentsValue) + await this.service.select( + this.conversationId, + input.ref, + input.value, + signal + ) + return createTextResult({ selected: input.ref, value: '[已隐藏]' }) + } + if (name === 'browser_back') { + browserBackInputSchema.parse(argumentsValue) + return createTextResult( + await this.service.back(this.conversationId, signal) + ) + } + browserScreenshotInputSchema.parse(argumentsValue) + const screenshot = await this.service.screenshot( + this.conversationId, + signal + ) + return { + parts: [ + { + type: 'image', + mimeType: screenshot.mimeType, + data: screenshot.data + } + ], + contextBytes: Buffer.byteLength(screenshot.data) + } + } + + async release(): Promise { + await this.service.releaseConversation(this.conversationId) + } +} diff --git a/src/main/browser/browser-service.test.ts b/src/main/browser/browser-service.test.ts new file mode 100644 index 0000000..b4d8f5f --- /dev/null +++ b/src/main/browser/browser-service.test.ts @@ -0,0 +1,473 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { BrowserUrlPolicy, canonicalizeBrowserUrl } from './browser-url-policy' +import { + BrowserService, + type BrowserDriverLike, + type BrowserSessionLike +} from './browser-service' +import type { BrowserWebContents } from './electron-browser-session' + +type HarnessSlot = { + currentOrigin?: string + approvedOrigin?: string + session: BrowserSessionLike + driver: BrowserDriverLike +} + +function deferred() { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +function createHarness(options: { + maximumSessions?: number + idleTimeoutMs?: number + cleanupTimeoutMs?: number + dispose?: () => Promise + sessionGate?: Promise +} = {}) { + const slots: HarnessSlot[] = [] + const byContents = new Map() + const createSession = vi.fn(async (): Promise => { + await options.sessionGate + const webContents = {} as BrowserWebContents + const slot = {} as HarnessSlot + const session: BrowserSessionLike = { + webContents, + approveNavigation: vi.fn((target) => { + slot.approvedOrigin = target.origin + }), + getCurrentOrigin: vi.fn(() => slot.currentOrigin), + dispose: vi.fn(options.dispose ?? (async () => undefined)) + } + const driver: BrowserDriverLike = { + navigate: vi.fn(async (url) => { + slot.currentOrigin = canonicalizeBrowserUrl(url).origin + return { url } + }), + snapshot: vi.fn(async () => ({ + url: `${slot.currentOrigin}/page`, + title: 'Page', + nodes: [], + truncated: false + })), + click: vi.fn(async () => undefined), + type: vi.fn(async () => undefined), + select: vi.fn(async () => undefined), + getBackTarget: vi.fn(async () => ({ + entryId: 4, + url: 'https://previous.example/back' + })), + backTo: vi.fn(async (target) => { + slot.currentOrigin = canonicalizeBrowserUrl(target.url).origin + return { url: target.url } + }), + screenshot: vi.fn(async () => ({ + type: 'image' as const, + mimeType: 'image/png' as const, + data: 'iVBORw0KGgo=' + })), + dispose: vi.fn() + } + Object.assign(slot, { session, driver }) + slots.push(slot) + byContents.set(webContents, slot) + return session + }) + const service = new BrowserService({ + policy: new BrowserUrlPolicy(async () => [ + { address: '93.184.216.34', family: 4 } + ]), + maximumSessions: options.maximumSessions, + idleTimeoutMs: options.idleTimeoutMs, + cleanupTimeoutMs: options.cleanupTimeoutMs, + liveFrameDelayMs: 0, + createSession, + createDriver: (contents) => { + const slot = byContents.get(contents) + if (!slot) { + throw new Error('unknown contents') + } + return slot.driver + } + }) + return { createSession, service, slots } +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('BrowserService', () => { + it('publishes browser status and live frames through session cleanup', async () => { + const harness = createHarness() + const states: Array<{ + status: string + frameDataUrl?: string + }> = [] + const removeListener = harness.service.onState((state) => { + states.push(state) + }) + const signal = new AbortController().signal + + await harness.service.navigate( + 'conversation', + 'https://example.com/', + signal + ) + await harness.service.click('conversation', 'button_ref', signal) + await harness.service.releaseConversation('conversation') + + expect(states.map((state) => state.status)).toEqual([ + 'creating', + 'loading', + 'ready', + 'acting', + 'ready', + 'stopped' + ]) + expect(states.find((state) => state.status === 'ready')?.frameDataUrl).toBe( + 'data:image/png;base64,iVBORw0KGgo=' + ) + expect(states.at(-1)?.frameDataUrl).toBeUndefined() + const replayed: string[] = [] + const removeReplayListener = harness.service.onState((state) => { + replayed.push(state.status) + }) + expect(replayed).toEqual([]) + removeReplayListener() + removeListener() + await harness.service.dispose() + }) + + it('does not publish ready after a session is stopped during frame capture', async () => { + const harness = createHarness() + const signal = new AbortController().signal + const states: string[] = [] + harness.service.onState((state) => states.push(state.status)) + await harness.service.navigate( + 'conversation', + 'https://example.com/', + signal + ) + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + vi.mocked(slot.driver.screenshot).mockImplementationOnce( + async (operationSignal) => + new Promise((_resolve, reject) => { + operationSignal.addEventListener( + 'abort', + () => reject(operationSignal.reason), + { once: true } + ) + }) + ) + + const click = harness.service.click( + 'conversation', + 'button_ref', + signal + ) + await vi.waitFor(() => + expect(slot.driver.screenshot).toHaveBeenCalledTimes(2) + ) + await harness.service.releaseConversation('conversation') + + await expect(click).rejects.toThrow('浏览器会话已释放') + expect(states.at(-1)).toBe('stopped') + }) + + it('isolates browser state and drivers by conversation', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate('conversation-a', 'https://a.example/', signal) + await harness.service.navigate('conversation-b', 'https://b.example/', signal) + await harness.service.snapshot('conversation-a', signal) + + expect(harness.service.getSessionCount()).toBe(2) + expect(harness.service.getOrigin('conversation-a')).toBe( + 'https://a.example' + ) + expect(harness.service.getOrigin('conversation-b')).toBe( + 'https://b.example' + ) + expect(harness.slots[0]?.driver.snapshot).toHaveBeenCalledOnce() + expect(harness.slots[1]?.driver.snapshot).not.toHaveBeenCalled() + await harness.service.dispose() + }) + + it('enforces a hard maximum of three sessions', async () => { + const harness = createHarness({ maximumSessions: 3 }) + const signal = new AbortController().signal + for (const id of ['one', 'two', 'three']) { + await harness.service.navigate(id, `https://${id}.example/`, signal) + } + await expect( + harness.service.navigate('four', 'https://four.example/', signal) + ).rejects.toThrow('3 个上限') + expect(harness.createSession).toHaveBeenCalledTimes(3) + await harness.service.dispose() + }) + + it('serializes operations in one conversation and lets queued callers cancel', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate('conversation', 'https://a.example/', signal) + const clickGate = deferred() + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + vi.mocked(slot.driver.click).mockImplementationOnce(async () => + clickGate.promise + ) + + const click = harness.service.click('conversation', 'b_ref', signal) + await vi.waitFor(() => expect(slot.driver.click).toHaveBeenCalled()) + const queuedController = new AbortController() + const queued = harness.service.snapshot( + 'conversation', + queuedController.signal + ) + queuedController.abort(new Error('cancel queued')) + await expect(queued).rejects.toThrow('cancel queued') + expect(slot.driver.snapshot).not.toHaveBeenCalled() + clickGate.resolve() + await click + await harness.service.dispose() + }) + + it('does not let a canceled queued waiter clear the active operation owner', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate('conversation', 'https://a.example/', signal) + const clickGate = deferred() + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + let activeSignal: AbortSignal | undefined + vi.mocked(slot.driver.click).mockImplementationOnce( + async (_ref, operationSignal) => { + activeSignal = operationSignal + await clickGate.promise + } + ) + + const click = harness.service.click('conversation', 'b_ref', signal) + await vi.waitFor(() => expect(activeSignal).toBeDefined()) + const queuedController = new AbortController() + const queued = harness.service.snapshot( + 'conversation', + queuedController.signal + ) + queuedController.abort(new Error('cancel queued')) + await expect(queued).rejects.toThrow('cancel queued') + const clickResult = expect(click).rejects.toThrow( + '浏览器会话已释放' + ) + await harness.service.releaseConversation('conversation') + expect(activeSignal?.aborted).toBe(true) + clickGate.resolve() + await clickResult + await harness.service.dispose() + }) + + it('abandons a sole canceled creation without retaining or consuming a slot', async () => { + const creationGate = deferred() + const harness = createHarness({ + maximumSessions: 1, + sessionGate: creationGate.promise + }) + const canceledController = new AbortController() + const canceled = harness.service.navigate( + 'canceled', + 'https://canceled.example/', + canceledController.signal + ) + await vi.waitFor(() => expect(harness.createSession).toHaveBeenCalledOnce()) + canceledController.abort(new Error('cancel creation')) + await expect(canceled).rejects.toThrow('cancel creation') + expect(harness.service.getSessionCount()).toBe(0) + + const replacement = harness.service.navigate( + 'replacement', + 'https://replacement.example/', + new AbortController().signal + ) + await vi.waitFor(() => + expect(harness.createSession).toHaveBeenCalledTimes(2) + ) + creationGate.resolve() + await expect(replacement).resolves.toMatchObject({ + origin: 'https://replacement.example' + }) + expect(harness.service.getSessionCount()).toBe(1) + expect(harness.slots[0]?.session.dispose).toHaveBeenCalledOnce() + await harness.service.dispose() + }) + + it('preserves a shared creation while another waiter cancels', async () => { + const creationGate = deferred() + const harness = createHarness({ sessionGate: creationGate.promise }) + const canceledController = new AbortController() + const canceled = harness.service.navigate( + 'conversation', + 'https://example.com/first', + canceledController.signal + ) + const shared = harness.service.navigate( + 'conversation', + 'https://example.com/second', + new AbortController().signal + ) + await vi.waitFor(() => expect(harness.createSession).toHaveBeenCalledOnce()) + canceledController.abort(new Error('cancel one waiter')) + await expect(canceled).rejects.toThrow('cancel one waiter') + + creationGate.resolve() + await expect(shared).resolves.toMatchObject({ + origin: 'https://example.com' + }) + expect(harness.service.getSessionCount()).toBe(1) + expect(harness.slots[0]?.session.dispose).not.toHaveBeenCalled() + await harness.service.dispose() + }) + + it('expires idle sessions and clears their isolated resources', async () => { + vi.useFakeTimers() + const harness = createHarness({ idleTimeoutMs: 100 }) + await harness.service.navigate( + 'conversation', + 'https://a.example/', + new AbortController().signal + ) + await vi.advanceTimersByTimeAsync(101) + await vi.waitFor(() => expect(harness.service.getSessionCount()).toBe(0)) + expect(harness.slots[0]?.driver.dispose).toHaveBeenCalledOnce() + expect(harness.slots[0]?.session.dispose).toHaveBeenCalledOnce() + await harness.service.dispose() + }) + + it('tracks an approved origin across validated back navigation', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://current.example/', + signal + ) + await expect(harness.service.back('conversation', signal)).resolves.toEqual({ + url: 'https://previous.example/back', + origin: 'https://previous.example' + }) + expect(harness.service.getOrigin('conversation')).toBe( + 'https://previous.example' + ) + expect(harness.slots[0]?.approvedOrigin).toBe( + 'https://previous.example' + ) + await harness.service.dispose() + }) + + it('fails closed and releases a slot when navigation origin does not match', async () => { + const harness = createHarness() + await harness.service.navigate( + 'conversation', + 'https://example.com/', + new AbortController().signal + ) + const slot = harness.slots[0] + if (!slot) { + throw new Error('slot missing') + } + slot.currentOrigin = 'https://attacker.example' + await expect( + harness.service.snapshot( + 'conversation', + new AbortController().signal + ) + ).rejects.toThrow('来源已改变') + expect(harness.service.getSessionCount()).toBe(0) + expect(slot.session.dispose).toHaveBeenCalled() + await harness.service.dispose() + }) + + it('bounds cleanup and makes release and dispose idempotent', async () => { + const harness = createHarness({ + cleanupTimeoutMs: 5, + dispose: async () => new Promise(() => undefined) + }) + await harness.service.navigate( + 'conversation', + 'https://example.com/', + new AbortController().signal + ) + await expect( + harness.service.releaseConversation('conversation') + ).rejects.toThrow('清理超时') + expect(harness.service.getSessionCount()).toBe(0) + await harness.service.releaseConversation('conversation') + await harness.service.dispose() + await harness.service.dispose() + }) + + it('settles a creation and release race without blocking later reuse', async () => { + const creationGate = deferred() + const harness = createHarness({ sessionGate: creationGate.promise }) + const firstNavigation = harness.service.navigate( + 'conversation', + 'https://example.com/', + new AbortController().signal + ) + await vi.waitFor(() => expect(harness.createSession).toHaveBeenCalledOnce()) + + const release = harness.service.releaseConversation('conversation') + creationGate.resolve() + await release + await expect(firstNavigation).rejects.toThrow() + expect(harness.service.getSessionCount()).toBe(0) + expect(harness.slots[0]?.session.dispose).toHaveBeenCalledOnce() + + await expect( + harness.service.navigate( + 'conversation', + 'https://example.com/new', + new AbortController().signal + ) + ).resolves.toMatchObject({ origin: 'https://example.com' }) + expect(harness.createSession).toHaveBeenCalledTimes(2) + await harness.service.dispose() + }) + + it('clears current sessions and remains reusable', async () => { + const harness = createHarness() + const signal = new AbortController().signal + await harness.service.navigate( + 'conversation', + 'https://example.com/', + signal + ) + + await harness.service.clearSessions() + expect(harness.service.getSessionCount()).toBe(0) + expect(harness.slots[0]?.session.dispose).toHaveBeenCalledOnce() + + await expect( + harness.service.navigate( + 'conversation', + 'https://example.com/again', + signal + ) + ).resolves.toMatchObject({ origin: 'https://example.com' }) + expect(harness.createSession).toHaveBeenCalledTimes(2) + await harness.service.dispose() + }) +}) diff --git a/src/main/browser/browser-service.ts b/src/main/browser/browser-service.ts new file mode 100644 index 0000000..607f621 --- /dev/null +++ b/src/main/browser/browser-service.ts @@ -0,0 +1,863 @@ +import { BrowserUrlPolicy, canonicalizeBrowserUrl } from './browser-url-policy' +import { + CdpBrowserDriver, + type BrowserHistoryTarget, + type BrowserScreenshot, + type BrowserSnapshot +} from './cdp-browser-driver' +import { + ElectronBrowserSession, + type BrowserWebContents +} from './electron-browser-session' +import type { BrowserLiveState } from '../../shared/contracts' + +const DEFAULT_IDLE_TIMEOUT_MS = 10 * 60_000 +const DEFAULT_CLEANUP_TIMEOUT_MS = 5_000 + +export type BrowserSessionLike = { + readonly webContents: BrowserWebContents + approveNavigation( + target: Awaited> + ): void + getCurrentOrigin(): string | undefined + captureScreenshot?(signal: AbortSignal): Promise + dispose(): Promise +} + +export type BrowserDriverLike = { + navigate(url: string, signal: AbortSignal): Promise<{ url: string }> + snapshot(signal: AbortSignal): Promise + click(ref: string, signal: AbortSignal): Promise + type(ref: string, text: string, signal: AbortSignal): Promise + select(ref: string, value: string, signal: AbortSignal): Promise + getBackTarget(signal: AbortSignal): Promise + backTo( + target: BrowserHistoryTarget, + signal: AbortSignal + ): Promise<{ url: string }> + screenshot(signal: AbortSignal): Promise + dispose(): void +} + +export type BrowserServiceOptions = { + policy?: BrowserUrlPolicy + maximumSessions?: number + idleTimeoutMs?: number + cleanupTimeoutMs?: number + liveFrameDelayMs?: number + createSession?: ( + policy: BrowserUrlPolicy, + signal: AbortSignal + ) => Promise + createDriver?: (webContents: BrowserWebContents) => BrowserDriverLike +} + +type BrowserSlot = { + conversationId: string + session: BrowserSessionLike + driver: BrowserDriverLike + origin?: string + tail: Promise + active?: AbortController + idleTimer?: ReturnType + lastUsedAt: number + released: boolean +} + +type SlotCreation = { + controller: AbortController + promise: Promise + waiters: Set +} + +function waitFor(promise: Promise, signal: AbortSignal): Promise { + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const abort = (): void => reject(signal.reason) + signal.addEventListener('abort', abort, { once: true }) + void promise.then( + (value) => { + signal.removeEventListener('abort', abort) + resolve(value) + }, + (error: unknown) => { + signal.removeEventListener('abort', abort) + reject(error) + } + ) + }) +} + +async function boundedCleanup( + cleanup: Promise, + timeoutMs: number +): Promise { + let timer: ReturnType | undefined + try { + await Promise.race([ + cleanup, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error('浏览器会话清理超时')), + timeoutMs + ) + }) + ]) + } finally { + if (timer) { + clearTimeout(timer) + } + } +} + +async function defaultCreateSession( + policy: BrowserUrlPolicy, + signal: AbortSignal +): Promise { + return ElectronBrowserSession.create({ policy }, signal) +} + +function defaultCreateDriver(webContents: BrowserWebContents): BrowserDriverLike { + return new CdpBrowserDriver(webContents) +} + +export class BrowserService { + private readonly policy: BrowserUrlPolicy + private readonly maximumSessions: number + private readonly idleTimeoutMs: number + private readonly cleanupTimeoutMs: number + private readonly liveFrameDelayMs: number + private readonly createSession: NonNullable< + BrowserServiceOptions['createSession'] + > + private readonly createDriver: NonNullable< + BrowserServiceOptions['createDriver'] + > + private readonly slots = new Map() + private readonly creations = new Map() + private readonly releaseRequests = new Set() + private readonly stateListeners = new Set< + (state: BrowserLiveState) => void + >() + private readonly liveStates = new Map() + private lifecycle = new AbortController() + private clearOperation?: Promise + private clearing = false + private disposed = false + + constructor(options: BrowserServiceOptions = {}) { + this.policy = options.policy ?? new BrowserUrlPolicy() + this.maximumSessions = options.maximumSessions ?? 3 + this.idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS + this.cleanupTimeoutMs = + options.cleanupTimeoutMs ?? DEFAULT_CLEANUP_TIMEOUT_MS + this.liveFrameDelayMs = options.liveFrameDelayMs ?? 100 + this.createSession = options.createSession ?? defaultCreateSession + this.createDriver = options.createDriver ?? defaultCreateDriver + if ( + !Number.isSafeInteger(this.maximumSessions) || + this.maximumSessions < 1 || + !Number.isSafeInteger(this.idleTimeoutMs) || + this.idleTimeoutMs < 1 || + !Number.isSafeInteger(this.cleanupTimeoutMs) || + this.cleanupTimeoutMs < 1 || + !Number.isSafeInteger(this.liveFrameDelayMs) || + this.liveFrameDelayMs < 0 + ) { + throw new Error('浏览器服务限制配置无效') + } + } + + getOrigin(conversationId: string): string | undefined { + return this.slots.get(conversationId)?.origin + } + + getSessionCount(): number { + return this.slots.size + } + + onState(listener: (state: BrowserLiveState) => void): () => void { + this.stateListeners.add(listener) + for (const state of this.liveStates.values()) { + listener(state) + } + return () => { + this.stateListeners.delete(listener) + } + } + + private emitState( + conversationId: string, + status: BrowserLiveState['status'], + update: Partial< + Pick + > = {} + ): void { + const previous = this.liveStates.get(conversationId) + const state: BrowserLiveState = { + conversationId, + status, + ...(previous?.url ? { url: previous.url } : {}), + ...update, + updatedAt: Date.now() + } + if (status !== 'failed') { + delete state.error + } + if (status === 'stopped') { + this.liveStates.delete(conversationId) + } else { + this.liveStates.set(conversationId, state) + } + for (const listener of this.stateListeners) { + try { + listener(state) + } catch { + // A UI observer must not interrupt browser control. + } + } + } + + private async captureFrame( + conversationId: string, + slot: BrowserSlot, + signal: AbortSignal, + url?: string, + screenshot?: BrowserScreenshot + ): Promise { + let frame = screenshot + if (!frame) { + if (this.liveFrameDelayMs > 0) { + await waitFor( + new Promise((resolve) => + setTimeout(resolve, this.liveFrameDelayMs) + ), + signal + ) + } + const previewController = new AbortController() + const timeout = setTimeout( + () => + previewController.abort( + new Error('浏览器实时画面捕获超时') + ), + 2_000 + ) + try { + const previewSignal = AbortSignal.any([ + signal, + previewController.signal + ]) + frame = slot.session.captureScreenshot + ? await slot.session.captureScreenshot(previewSignal) + : await slot.driver.screenshot(previewSignal) + } catch { + signal.throwIfAborted() + // Browser control succeeds even when the optional live frame fails. + } finally { + clearTimeout(timeout) + } + } + signal.throwIfAborted() + if (slot.released || this.slots.get(conversationId) !== slot) { + return + } + this.emitState(conversationId, 'ready', { + ...(url ? { url } : {}), + ...(frame + ? { + frameDataUrl: `data:${frame.mimeType};base64,${frame.data}` + } + : {}) + }) + } + + private emitFailure( + conversationId: string, + stage: string, + error: unknown + ): void { + const detail = + error instanceof Error && error.message + ? error.message.slice(0, 180) + : '未知错误' + this.emitState(conversationId, 'failed', { + error: `${stage}失败：${detail}`.slice(0, 240) + }) + } + + private shouldEmitFailure( + conversationId: string, + signal: AbortSignal, + error: unknown + ): boolean { + if (signal.aborted || this.releaseRequests.has(conversationId)) { + return false + } + const message = error instanceof Error ? error.message : '' + return ![ + '浏览器会话已释放', + '浏览器会话已清除', + '浏览器会话已关闭', + '浏览器服务已关闭' + ].some((reason) => message.includes(reason)) + } + + private async getOrCreateSlot( + conversationId: string, + signal: AbortSignal + ): Promise { + if (this.disposed || this.clearing || this.lifecycle.signal.aborted) { + throw new Error('浏览器服务已关闭') + } + const existing = this.slots.get(conversationId) + if (existing && !existing.released) { + return existing + } + let creation = this.creations.get(conversationId) + const pendingCreations = [...this.creations.keys()].filter( + (id) => !this.slots.has(id) + ).length + if ( + !creation && + this.slots.size + pendingCreations >= this.maximumSessions + ) { + throw new Error(`浏览器会话已达到 ${this.maximumSessions} 个上限`) + } + if (!creation) { + const controller = new AbortController() + const waiters = new Set() + const promise = this.createSlot( + conversationId, + AbortSignal.any([this.lifecycle.signal, controller.signal]), + () => waiters.size > 0 + ) + const currentCreation = { controller, promise, waiters } + creation = currentCreation + this.creations.set(conversationId, creation) + const removeCreation = (): void => { + if (this.creations.get(conversationId) === creation) { + this.creations.delete(conversationId) + } + } + void promise.then(removeCreation, removeCreation) + } + const waiter = Symbol(conversationId) + creation.waiters.add(waiter) + try { + return await waitFor(creation.promise, signal) + } finally { + creation.waiters.delete(waiter) + if ( + creation.waiters.size === 0 && + this.creations.get(conversationId) === creation + ) { + this.creations.delete(conversationId) + creation.controller.abort(new Error('浏览器会话创建已取消')) + } + } + } + + private async createSlot( + conversationId: string, + signal: AbortSignal, + hasWaiters: () => boolean + ): Promise { + const session = await this.createSession( + this.policy, + signal + ) + if ( + this.disposed || + signal.aborted || + !hasWaiters() || + this.releaseRequests.has(conversationId) + ) { + await boundedCleanup(session.dispose(), this.cleanupTimeoutMs) + throw new Error('浏览器服务已关闭') + } + let driver: BrowserDriverLike + try { + driver = this.createDriver(session.webContents) + } catch (error) { + await boundedCleanup(session.dispose(), this.cleanupTimeoutMs).catch( + () => undefined + ) + throw error + } + const slot: BrowserSlot = { + conversationId, + session, + driver, + tail: Promise.resolve(), + lastUsedAt: Date.now(), + released: false + } + this.slots.set(conversationId, slot) + this.scheduleIdleExpiry(slot) + return slot + } + + private requireSlot(conversationId: string): BrowserSlot { + if (this.disposed) { + throw new Error('浏览器服务已关闭') + } + const slot = this.slots.get(conversationId) + if (!slot || slot.released || !slot.origin) { + throw new Error('当前对话尚未建立浏览器会话，请先导航') + } + return slot + } + + private scheduleIdleExpiry(slot: BrowserSlot): void { + if (slot.idleTimer) { + clearTimeout(slot.idleTimer) + } + if (slot.released || this.disposed) { + return + } + slot.idleTimer = setTimeout(() => { + if (Date.now() - slot.lastUsedAt < this.idleTimeoutMs) { + this.scheduleIdleExpiry(slot) + return + } + void this.releaseSlot(slot).catch(() => undefined) + }, this.idleTimeoutMs) + } + + private async serialize( + slot: BrowserSlot, + signal: AbortSignal, + operation: (effectiveSignal: AbortSignal) => Promise, + status?: 'loading' | 'acting' + ): Promise { + signal.throwIfAborted() + if (slot.released || this.disposed) { + throw new Error('浏览器会话已关闭') + } + let releaseGate!: () => void + const gate = new Promise((resolve) => { + releaseGate = resolve + }) + const predecessor = slot.tail.catch(() => undefined) + slot.tail = predecessor.then(() => gate) + let operationController: AbortController | undefined + try { + await waitFor(predecessor, signal) + if (slot.released || this.disposed) { + throw new Error('浏览器会话已关闭') + } + if (status) { + this.emitState(slot.conversationId, status) + } + if (slot.idleTimer) { + clearTimeout(slot.idleTimer) + slot.idleTimer = undefined + } + operationController = new AbortController() + slot.active = operationController + const effectiveSignal = AbortSignal.any([ + signal, + this.lifecycle.signal, + operationController.signal + ]) + return await operation(effectiveSignal) + } finally { + if (operationController && slot.active === operationController) { + slot.active = undefined + } + releaseGate() + if (operationController) { + slot.lastUsedAt = Date.now() + this.scheduleIdleExpiry(slot) + } + } + } + + private verifyCurrentOrigin(slot: BrowserSlot): string { + const current = slot.session.getCurrentOrigin() + if (!slot.origin || current !== slot.origin) { + throw new Error('浏览器页面来源已改变，会话已被拒绝') + } + return current + } + + private async verifyCurrentOriginOrRelease( + slot: BrowserSlot + ): Promise { + try { + return this.verifyCurrentOrigin(slot) + } catch (error) { + await this.releaseSlot(slot).catch(() => undefined) + throw error + } + } + + private async runInSession( + conversationId: string, + signal: AbortSignal, + status: 'loading' | 'acting', + failureStage: string, + operation: ( + slot: BrowserSlot, + effectiveSignal: AbortSignal + ) => Promise + ): Promise { + try { + const slot = this.requireSlot(conversationId) + return await this.serialize( + slot, + signal, + (effectiveSignal) => operation(slot, effectiveSignal), + status + ) + } catch (error) { + if (this.shouldEmitFailure(conversationId, signal, error)) { + this.emitFailure(conversationId, failureStage, error) + } + throw error + } + } + + async navigate( + conversationId: string, + url: string, + signal: AbortSignal + ): Promise<{ url: string; origin: string }> { + if ( + !this.slots.has(conversationId) && + !this.creations.has(conversationId) + ) { + this.emitState(conversationId, 'creating', { url }) + } + try { + const slot = await this.getOrCreateSlot(conversationId, signal) + return await this.serialize(slot, signal, async (effectiveSignal) => { + const target = await this.policy.validate(url, effectiveSignal) + slot.session.approveNavigation(target) + try { + const result = await slot.driver.navigate( + target.url.href, + effectiveSignal + ) + const finalTarget = await this.policy.validateRedirect( + result.url, + target.origin, + effectiveSignal + ) + if (slot.session.getCurrentOrigin() !== finalTarget.origin) { + throw new Error('浏览器导航结果来源不一致') + } + slot.origin = finalTarget.origin + await this.captureFrame( + conversationId, + slot, + effectiveSignal, + finalTarget.url.href + ) + return { + url: finalTarget.url.href, + origin: finalTarget.origin + } + } catch (error) { + await this.releaseSlot(slot).catch(() => undefined) + throw error + } + }, 'loading') + } catch (error) { + if (this.shouldEmitFailure(conversationId, signal, error)) { + this.emitFailure(conversationId, '浏览器导航', error) + } + throw error + } + } + + async snapshot( + conversationId: string, + signal: AbortSignal + ): Promise { + return this.runInSession( + conversationId, + signal, + 'acting', + '读取浏览器页面', + async (slot, effectiveSignal) => { + await this.verifyCurrentOriginOrRelease(slot) + const snapshot = await slot.driver.snapshot(effectiveSignal) + const target = canonicalizeBrowserUrl(snapshot.url) + if (target.origin !== slot.origin) { + await this.releaseSlot(slot).catch(() => undefined) + throw new Error('浏览器快照来源与当前会话不一致') + } + await this.captureFrame( + conversationId, + slot, + effectiveSignal, + target.href + ) + return { ...snapshot, url: target.href } + } + ) + } + + async click( + conversationId: string, + ref: string, + signal: AbortSignal + ): Promise { + await this.runInSession( + conversationId, + signal, + 'acting', + '浏览器点击', + async (slot, effectiveSignal) => { + await this.verifyCurrentOriginOrRelease(slot) + await slot.driver.click(ref, effectiveSignal) + await this.captureFrame(conversationId, slot, effectiveSignal) + } + ) + } + + async type( + conversationId: string, + ref: string, + text: string, + signal: AbortSignal + ): Promise { + await this.runInSession( + conversationId, + signal, + 'acting', + '浏览器输入', + async (slot, effectiveSignal) => { + await this.verifyCurrentOriginOrRelease(slot) + await slot.driver.type(ref, text, effectiveSignal) + await this.captureFrame(conversationId, slot, effectiveSignal) + } + ) + } + + async select( + conversationId: string, + ref: string, + value: string, + signal: AbortSignal + ): Promise { + await this.runInSession( + conversationId, + signal, + 'acting', + '浏览器选择', + async (slot, effectiveSignal) => { + await this.verifyCurrentOriginOrRelease(slot) + await slot.driver.select(ref, value, effectiveSignal) + await this.captureFrame(conversationId, slot, effectiveSignal) + } + ) + } + + async back( + conversationId: string, + signal: AbortSignal + ): Promise<{ url: string; origin: string }> { + return this.runInSession( + conversationId, + signal, + 'loading', + '浏览器返回', + async (slot, effectiveSignal) => { + await this.verifyCurrentOriginOrRelease(slot) + const historyTarget = + await slot.driver.getBackTarget(effectiveSignal) + const target = await this.policy.validate( + historyTarget.url, + effectiveSignal + ) + slot.session.approveNavigation(target) + try { + const result = await slot.driver.backTo( + historyTarget, + effectiveSignal + ) + const finalTarget = await this.policy.validateRedirect( + result.url, + target.origin, + effectiveSignal + ) + if (slot.session.getCurrentOrigin() !== finalTarget.origin) { + throw new Error('浏览器返回结果来源不一致') + } + slot.origin = finalTarget.origin + await this.captureFrame( + conversationId, + slot, + effectiveSignal, + finalTarget.url.href + ) + return { + url: finalTarget.url.href, + origin: finalTarget.origin + } + } catch (error) { + await this.releaseSlot(slot).catch(() => undefined) + throw error + } + } + ) + } + + async screenshot( + conversationId: string, + signal: AbortSignal + ): Promise { + return this.runInSession( + conversationId, + signal, + 'acting', + '浏览器截图', + async (slot, effectiveSignal) => { + await this.verifyCurrentOriginOrRelease(slot) + const screenshot = + await slot.driver.screenshot(effectiveSignal) + await this.captureFrame( + conversationId, + slot, + effectiveSignal, + undefined, + screenshot + ) + return screenshot + } + ) + } + + async releaseConversation(conversationId: string): Promise { + this.releaseRequests.add(conversationId) + let releasedSlot = false + try { + const creation = this.creations.get(conversationId) + if (creation) { + creation.controller.abort(new Error('浏览器会话已释放')) + const slot = await creation.promise.catch(() => undefined) + if (slot) { + await this.releaseSlot(slot) + releasedSlot = true + } + } + const slot = this.slots.get(conversationId) + if (slot) { + await this.releaseSlot(slot) + releasedSlot = true + } + if (!releasedSlot) { + this.emitState(conversationId, 'stopped') + } + } finally { + if (!this.disposed) { + this.releaseRequests.delete(conversationId) + } + } + } + + private async releaseSlot(slot: BrowserSlot): Promise { + if (slot.released) { + return + } + slot.released = true + this.slots.delete(slot.conversationId) + if (slot.idleTimer) { + clearTimeout(slot.idleTimer) + slot.idleTimer = undefined + } + slot.active?.abort(new Error('浏览器会话已释放')) + try { + slot.driver.dispose() + } finally { + try { + await boundedCleanup(slot.session.dispose(), this.cleanupTimeoutMs) + } finally { + this.emitState(slot.conversationId, 'stopped') + } + } + } + + clearSessions(): Promise { + if (this.disposed) { + return Promise.resolve() + } + if (this.clearOperation) { + return this.clearOperation + } + const operation = this.performClearSessions() + this.clearOperation = operation + void operation.then( + () => { + if (this.clearOperation === operation) { + this.clearOperation = undefined + } + }, + () => { + if (this.clearOperation === operation) { + this.clearOperation = undefined + } + } + ) + return operation + } + + private async performClearSessions(): Promise { + this.clearing = true + const lifecycle = this.lifecycle + lifecycle.abort(new Error('浏览器会话已清除')) + const requestedReleases = new Set(this.creations.keys()) + for (const conversationId of requestedReleases) { + this.releaseRequests.add(conversationId) + } + const slots = new Set(this.slots.values()) + try { + await Promise.allSettled( + [...this.creations.values()].map((creation) => creation.promise) + ) + for (const slot of this.slots.values()) { + slots.add(slot) + } + await Promise.allSettled( + [...slots].map((slot) => this.releaseSlot(slot)) + ) + this.slots.clear() + } finally { + if (!this.disposed) { + for (const conversationId of requestedReleases) { + this.releaseRequests.delete(conversationId) + } + this.lifecycle = new AbortController() + this.clearing = false + } + } + } + + async dispose(): Promise { + if (this.disposed) { + return + } + this.disposed = true + this.clearing = true + this.lifecycle.abort(new Error('浏览器服务已关闭')) + const slots = new Set(this.slots.values()) + for (const [conversationId, creation] of this.creations) { + this.releaseRequests.add(conversationId) + creation.controller.abort(new Error('浏览器服务已关闭')) + const created = await creation.promise.catch(() => undefined) + if (created) { + slots.add(created) + } + } + for (const slot of this.slots.values()) { + slots.add(slot) + } + await Promise.allSettled([...slots].map((slot) => this.releaseSlot(slot))) + this.slots.clear() + this.stateListeners.clear() + this.liveStates.clear() + } +} diff --git a/src/main/browser/browser-url-policy.test.ts b/src/main/browser/browser-url-policy.test.ts new file mode 100644 index 0000000..53a11d8 --- /dev/null +++ b/src/main/browser/browser-url-policy.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from 'vitest' +import { + BrowserUrlPolicy, + canonicalizeBrowserUrl, + isPublicBrowserAddress +} from './browser-url-policy' + +const signal = new AbortController().signal + +describe('BrowserUrlPolicy', () => { + it.each([ + 'file:///etc/passwd', + 'data:text/html,hello', + 'javascript:alert(1)', + 'ssh://example.com', + 'https://user:secret@example.com/', + 'http://localhost/', + 'http://printer/', + 'http://service.local/', + 'http://metadata.google.internal/', + 'http://169.254.169.254/latest/meta-data/', + 'http://[::1]/' + ])('rejects unsafe URL %s', (url) => { + expect(() => canonicalizeBrowserUrl(url)).toThrow() + }) + + it.each([ + '0.0.0.0', + '10.0.0.1', + '100.64.0.1', + '127.0.0.1', + '169.254.169.254', + '172.20.1.1', + '192.168.1.1', + '192.0.2.1', + '224.0.0.1', + '::', + '::1', + '::ffff:127.0.0.1', + 'fc00::1', + 'fe80::1', + 'ff02::1', + '2001:db8::1' + ])('classifies %s as non-public', (address) => { + expect(isPublicBrowserAddress(address)).toBe(false) + }) + + it('accepts canonical public HTTP(S) URLs and strips fragments', async () => { + const resolver = vi.fn(async () => [ + { address: '93.184.216.34', family: 4 as const }, + { address: '2606:2800:220:1:248:1893:25c8:1946', family: 6 as const } + ]) + const policy = new BrowserUrlPolicy(resolver) + + await expect( + policy.validate('https://example.com:8443/docs?q=1#section', signal) + ).resolves.toMatchObject({ + origin: 'https://example.com:8443', + url: expect.objectContaining({ + href: 'https://example.com:8443/docs?q=1' + }) + }) + expect(resolver).toHaveBeenCalledWith( + 'example.com', + expect.any(AbortSignal) + ) + }) + + it('rejects empty, private, malformed, and mixed DNS answers', async () => { + for (const answers of [ + [], + [{ address: '10.0.0.2', family: 4 as const }], + [ + { address: '93.184.216.34', family: 4 as const }, + { address: '127.0.0.1', family: 4 as const } + ], + [{ address: 'not-an-address', family: 4 as const }] + ]) { + const policy = new BrowserUrlPolicy(async () => answers) + await expect(policy.validate('https://example.com', signal)).rejects.toThrow( + '混合地址' + ) + } + }) + + it('validates redirects and keeps them on the approved origin', async () => { + const policy = new BrowserUrlPolicy(async () => [ + { address: '93.184.216.34', family: 4 } + ]) + await expect( + policy.validateRedirect( + 'https://example.com/next', + 'https://example.com', + signal + ) + ).resolves.toMatchObject({ origin: 'https://example.com' }) + await expect( + policy.validateRedirect( + 'https://other.example/next', + 'https://example.com', + signal + ) + ).rejects.toThrow('超出已批准来源') + }) + + it('honors cancellation before and after DNS resolution', async () => { + const before = new AbortController() + before.abort() + await expect( + new BrowserUrlPolicy(vi.fn()).validate('https://example.com', before.signal) + ).rejects.toHaveProperty('name', 'AbortError') + + const after = new AbortController() + const policy = new BrowserUrlPolicy(async () => { + after.abort() + return [{ address: '93.184.216.34', family: 4 }] + }) + await expect( + policy.validate('https://example.com', after.signal) + ).rejects.toHaveProperty('name', 'AbortError') + }) +}) diff --git a/src/main/browser/browser-url-policy.ts b/src/main/browser/browser-url-policy.ts new file mode 100644 index 0000000..6130d84 --- /dev/null +++ b/src/main/browser/browser-url-policy.ts @@ -0,0 +1,350 @@ +import { lookup as dnsLookup } from 'node:dns/promises' +import { isIP } from 'node:net' + +export type BrowserResolvedAddress = { + address: string + family: 4 | 6 +} + +export type BrowserDnsResolver = ( + hostname: string, + signal: AbortSignal +) => Promise + +export type ValidatedBrowserUrl = { + url: URL + origin: string + addresses: readonly BrowserResolvedAddress[] +} + +const LOCAL_HOST_SUFFIXES = [ + '.home', + '.internal', + '.invalid', + '.lan', + '.local', + '.localdomain', + '.localhost', + '.test' +] + +const BLOCKED_HOSTS = new Set([ + 'instance-data', + 'instance-data.ec2.internal', + 'metadata', + 'metadata.aws.internal', + 'metadata.google.internal' +]) + +function ipv4Number(address: string): number | undefined { + if (isIP(address) !== 4) { + return undefined + } + const octets = address.split('.').map(Number) + if (octets.length !== 4) { + return undefined + } + return ( + (((octets[0] ?? 0) << 24) | + ((octets[1] ?? 0) << 16) | + ((octets[2] ?? 0) << 8) | + (octets[3] ?? 0)) >>> + 0 + ) +} + +function inIpv4Range(value: number, base: number, prefix: number): boolean { + const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0 + return (value & mask) === (base & mask) +} + +function isPublicIpv4(address: string): boolean { + const value = ipv4Number(address) + if (value === undefined) { + return false + } + const blocked: Array<[number, number]> = [ + [0x00000000, 8], + [0x0a000000, 8], + [0x64400000, 10], + [0x7f000000, 8], + [0xa9fe0000, 16], + [0xac100000, 12], + [0xc0000000, 24], + [0xc0000200, 24], + [0xc0586300, 24], + [0xc0a80000, 16], + [0xc6120000, 15], + [0xc6336400, 24], + [0xcb007100, 24], + [0xe0000000, 4], + [0xf0000000, 4] + ] + return !blocked.some(([base, prefix]) => + inIpv4Range(value, base, prefix) + ) +} + +function expandIpv6(address: string): readonly number[] | undefined { + const withoutZone = address.toLowerCase().split('%', 1)[0] ?? '' + if (isIP(withoutZone) !== 6) { + return undefined + } + let normalized = withoutZone + const ipv4Match = normalized.match(/(\d+\.\d+\.\d+\.\d+)$/u) + if (ipv4Match) { + const ipv4 = ipv4Number(ipv4Match[1] ?? '') + if (ipv4 === undefined) { + return undefined + } + normalized = normalized.replace( + ipv4Match[1] ?? '', + `${((ipv4 >>> 16) & 0xffff).toString(16)}:${(ipv4 & 0xffff).toString(16)}` + ) + } + const halves = normalized.split('::') + if (halves.length > 2) { + return undefined + } + const left = (halves[0] ?? '').split(':').filter(Boolean) + const right = (halves[1] ?? '').split(':').filter(Boolean) + const missing = 8 - left.length - right.length + if ( + (halves.length === 1 && missing !== 0) || + (halves.length === 2 && missing < 1) + ) { + return undefined + } + const groups = [ + ...left, + ...Array.from({ length: Math.max(0, missing) }, () => '0'), + ...right + ].map((group) => Number.parseInt(group, 16)) + return groups.length === 8 && + groups.every((group) => Number.isInteger(group) && group <= 0xffff) + ? groups + : undefined +} + +function ipv6Prefix( + groups: readonly number[], + expected: readonly number[], + prefixBits: number +): boolean { + let remaining = prefixBits + for (let index = 0; remaining > 0; index += 1) { + const bits = Math.min(16, remaining) + const mask = (0xffff << (16 - bits)) & 0xffff + if (((groups[index] ?? 0) & mask) !== ((expected[index] ?? 0) & mask)) { + return false + } + remaining -= bits + } + return true +} + +function isPublicIpv6(address: string): boolean { + const groups = expandIpv6(address) + if (!groups) { + return false + } + if (groups.slice(0, 5).every((group) => group === 0)) { + const sixth = groups[5] ?? 0 + if (sixth === 0xffff) { + const mapped = `${(groups[6] ?? 0) >>> 8}.${(groups[6] ?? 0) & 0xff}.${(groups[7] ?? 0) >>> 8}.${(groups[7] ?? 0) & 0xff}` + return isPublicIpv4(mapped) + } + if (sixth === 0) { + return false + } + } + const blocked: Array<[readonly number[], number]> = [ + [[0, 0, 0, 0, 0, 0, 0, 0], 128], + [[0, 0, 0, 0, 0, 0, 0, 1], 128], + [[0x64, 0xff9b, 0, 0, 0, 0, 0, 0], 96], + [[0x64, 0xff9b, 1, 0, 0, 0, 0, 0], 48], + [[0x100, 0, 0, 0, 0, 0, 0, 0], 64], + [[0x2001, 0, 0, 0, 0, 0, 0, 0], 32], + [[0x2001, 2, 0, 0, 0, 0, 0, 0], 48], + [[0x2001, 0x10, 0, 0, 0, 0, 0, 0], 28], + [[0x2001, 0x20, 0, 0, 0, 0, 0, 0], 28], + [[0x2001, 0xdb8, 0, 0, 0, 0, 0, 0], 32], + [[0x2002, 0, 0, 0, 0, 0, 0, 0], 16], + [[0x3fff, 0, 0, 0, 0, 0, 0, 0], 20], + [[0x5f00, 0, 0, 0, 0, 0, 0, 0], 16], + [[0xfc00, 0, 0, 0, 0, 0, 0, 0], 7], + [[0xfe80, 0, 0, 0, 0, 0, 0, 0], 10], + [[0xfec0, 0, 0, 0, 0, 0, 0, 0], 10], + [[0xff00, 0, 0, 0, 0, 0, 0, 0], 8] + ] + return !blocked.some(([prefix, bits]) => + ipv6Prefix(groups, prefix, bits) + ) +} + +export function isPublicBrowserAddress(address: string): boolean { + const family = isIP(address.split('%', 1)[0] ?? '') + return family === 4 + ? isPublicIpv4(address) + : family === 6 + ? isPublicIpv6(address) + : false +} + +export function canonicalizeBrowserUrl(input: string): URL { + if (input !== input.trim() || input.length === 0 || input.length > 8_192) { + throw new Error('浏览器 URL 无效') + } + let url: URL + try { + url = new URL(input) + } catch { + throw new Error('浏览器 URL 无效') + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('浏览器仅支持 HTTP(S) URL') + } + if (url.username || url.password || !url.hostname || url.origin === 'null') { + throw new Error('浏览器 URL 不允许包含凭据或无效来源') + } + const rawHostname = url.hostname.toLowerCase() + const hostname = ( + rawHostname.startsWith('[') && rawHostname.endsWith(']') + ? rawHostname.slice(1, -1) + : rawHostname + ).replace(/\.$/u, '') + if ( + hostname !== ( + rawHostname.startsWith('[') && rawHostname.endsWith(']') + ? rawHostname.slice(1, -1) + : rawHostname + ) || + (!hostname.includes('.') && isIP(hostname) === 0) || + BLOCKED_HOSTS.has(hostname) || + LOCAL_HOST_SUFFIXES.some( + (suffix) => hostname === suffix.slice(1) || hostname.endsWith(suffix) + ) + ) { + throw new Error('浏览器 URL 不允许访问本机或内部名称') + } + if (isIP(hostname) !== 0 && !isPublicBrowserAddress(hostname)) { + throw new Error('浏览器 URL 不允许访问私有或保留地址') + } + url.hash = '' + return url +} + +async function defaultResolver( + hostname: string, + signal: AbortSignal +): Promise { + signal.throwIfAborted() + const result = await dnsLookup(hostname, { + all: true, + verbatim: true + }) + signal.throwIfAborted() + return result + .filter( + (entry): entry is { address: string; family: 4 | 6 } => + entry.family === 4 || entry.family === 6 + ) + .map((entry) => ({ address: entry.address, family: entry.family })) +} + +export class BrowserUrlPolicy { + constructor( + private readonly resolveDns: BrowserDnsResolver = defaultResolver, + private readonly resolutionTimeoutMs = 10_000 + ) { + if ( + !Number.isSafeInteger(resolutionTimeoutMs) || + resolutionTimeoutMs < 1 + ) { + throw new Error('浏览器 DNS 解析期限无效') + } + } + + private async resolve( + hostname: string, + signal: AbortSignal + ): Promise { + const timeout = AbortSignal.timeout(this.resolutionTimeoutMs) + const effectiveSignal = AbortSignal.any([signal, timeout]) + const resolution = this.resolveDns(hostname, effectiveSignal) + return new Promise((resolve, reject) => { + const abort = (): void => { + reject( + signal.aborted + ? signal.reason + : new Error(`浏览器 DNS 解析超时（${this.resolutionTimeoutMs}ms）`) + ) + } + effectiveSignal.addEventListener('abort', abort, { once: true }) + void resolution.then( + (addresses) => { + effectiveSignal.removeEventListener('abort', abort) + if (effectiveSignal.aborted) { + abort() + } else { + resolve(addresses) + } + }, + (error: unknown) => { + effectiveSignal.removeEventListener('abort', abort) + reject(error) + } + ) + }) + } + + async validate( + input: string | URL, + signal: AbortSignal + ): Promise { + signal.throwIfAborted() + const url = canonicalizeBrowserUrl( + typeof input === 'string' ? input : input.toString() + ) + const literalHostname = + url.hostname.startsWith('[') && url.hostname.endsWith(']') + ? url.hostname.slice(1, -1) + : url.hostname + const literalFamily = isIP(literalHostname) + const addresses = + literalFamily === 4 || literalFamily === 6 + ? [{ + address: literalHostname, + family: literalFamily + } as const] + : await this.resolve(url.hostname, signal) + signal.throwIfAborted() + if ( + addresses.length === 0 || + addresses.some( + (entry) => + entry.family !== isIP(entry.address) || + !isPublicBrowserAddress(entry.address) + ) + ) { + throw new Error('浏览器目标解析到私有、保留或混合地址') + } + return { + url, + origin: url.origin, + addresses: [...addresses] + } + } + + async validateRedirect( + input: string, + approvedOrigin: string, + signal: AbortSignal + ): Promise { + const target = await this.validate(input, signal) + if (target.origin !== approvedOrigin) { + throw new Error('浏览器重定向超出已批准来源') + } + return target + } +} diff --git a/src/main/browser/cdp-browser-driver.test.ts b/src/main/browser/cdp-browser-driver.test.ts new file mode 100644 index 0000000..135c813 --- /dev/null +++ b/src/main/browser/cdp-browser-driver.test.ts @@ -0,0 +1,470 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' +import { CdpBrowserDriver } from './cdp-browser-driver' +import type { + BrowserDebugger, + BrowserWebContents +} from './electron-browser-session' + +function createHarness( + command: ( + method: string, + parameters?: Record + ) => Promise +) { + const contentEvents = new EventEmitter() + const debuggerEvents = new EventEmitter() + let currentUrl = 'https://example.com/page' + const sendCommand = vi.fn(command) + const browserDebugger: BrowserDebugger = { + attach: vi.fn(), + detach: vi.fn(), + isAttached: vi.fn(() => true), + sendCommand, + on: (event, listener) => + debuggerEvents.on( + event, + listener as (...argumentsValue: unknown[]) => void + ), + off: (event, listener) => + debuggerEvents.off( + event, + listener as (...argumentsValue: unknown[]) => void + ) + } + const webContents: BrowserWebContents = { + debugger: browserDebugger, + on: (event, listener) => + contentEvents.on( + event, + listener as (...argumentsValue: unknown[]) => void + ), + off: (event, listener) => + contentEvents.off( + event, + listener as (...argumentsValue: unknown[]) => void + ), + setWindowOpenHandler: vi.fn(), + getURL: vi.fn(() => currentUrl), + stop: vi.fn(), + destroy: vi.fn(), + isDestroyed: vi.fn(() => false) + } + return { + browserDebugger, + contentEvents, + debuggerEvents, + sendCommand, + webContents, + setUrl(url: string) { + currentUrl = url + } + } +} + +function standardCommand( + method: string, + parameters?: Record +): Promise { + if (method === 'Accessibility.getFullAXTree') { + return Promise.resolve({ + nodes: [ + { + nodeId: 'root', + backendDOMNodeId: 10, + role: { value: 'RootWebArea' }, + name: { value: 'Example' } + }, + { + nodeId: 'button', + parentId: 'root', + backendDOMNodeId: 11, + role: { value: 'button' }, + name: { value: 'Submit' } + }, + { + nodeId: 'input', + parentId: 'root', + backendDOMNodeId: 12, + role: { value: 'textbox' }, + name: { value: 'Email' }, + value: { value: 'typed-secret@example.com' }, + properties: [{ name: 'editable', value: { value: true } }] + }, + { + nodeId: 'password', + parentId: 'root', + backendDOMNodeId: 13, + role: { value: 'password' }, + name: { value: 'Password' }, + value: { value: 'secret' } + } + ] + }) + } + if (method === 'Runtime.evaluate') { + return Promise.resolve( + parameters?.expression === 'document.readyState' + ? { result: { value: 'complete' } } + : { + result: { + value: { + title: 'Example', + url: 'https://example.com/page' + } + } + } + ) + } + if (method === 'DOM.describeNode') { + const backendNodeId = parameters?.backendNodeId + return Promise.resolve({ + node: { + backendNodeId, + nodeName: + backendNodeId === 12 + ? 'INPUT' + : backendNodeId === 13 + ? 'INPUT' + : 'BUTTON', + attributes: + backendNodeId === 12 + ? ['type', 'text'] + : backendNodeId === 13 + ? ['type', 'password'] + : [] + } + }) + } + if (method === 'DOM.getBoxModel') { + return Promise.resolve({ + model: { content: [10, 20, 110, 20, 110, 60, 10, 60] } + }) + } + if (method === 'Page.getLayoutMetrics') { + return Promise.resolve({ + cssVisualViewport: { clientWidth: 800, clientHeight: 600 } + }) + } + if (method === 'Page.getNavigationHistory') { + return Promise.resolve({ + currentIndex: 1, + entries: [ + { id: 4, url: 'https://previous.example/' }, + { id: 5, url: 'https://example.com/page' } + ] + }) + } + if (method === 'Page.captureScreenshot') { + return Promise.resolve({ + data: Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a + ]).toString('base64') + }) + } + return Promise.resolve({}) +} + +function selectCommand( + selection: { selected: boolean; value: string } +): ( + method: string, + parameters?: Record +) => Promise { + return async (method, parameters) => { + if (method === 'Accessibility.getFullAXTree') { + return { + nodes: [ + { + nodeId: 'root', + backendDOMNodeId: 20, + role: { value: 'RootWebArea' }, + name: { value: 'Example' } + }, + { + nodeId: 'select', + parentId: 'root', + backendDOMNodeId: 21, + role: { value: 'combobox' }, + name: { value: 'Region' } + } + ] + } + } + if (method === 'DOM.describeNode') { + return { + node: { + backendNodeId: parameters?.backendNodeId, + nodeName: 'SELECT', + attributes: [] + } + } + } + if (method === 'DOM.resolveNode') { + return { object: { objectId: 'select-object' } } + } + if (method === 'Runtime.callFunctionOn') { + return { result: { value: selection } } + } + return standardCommand(method, parameters) + } +} + +describe('CdpBrowserDriver', () => { + it('creates opaque refs and redacts editable and protected values', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents) + const snapshot = await driver.snapshot(new AbortController().signal) + + expect(snapshot).toMatchObject({ + url: 'https://example.com/page', + title: 'Example', + truncated: false + }) + expect(snapshot.nodes).toHaveLength(4) + expect(snapshot.nodes.every((node) => /^b_[A-Za-z0-9_-]+$/u.test(node.ref))) + .toBe(true) + expect(snapshot.nodes.find((node) => node.name === 'Email')?.value) + .toBeUndefined() + expect(snapshot.nodes.find((node) => node.name === 'Password')?.value) + .toBeUndefined() + expect(JSON.stringify(snapshot)).not.toContain('typed-secret') + driver.dispose() + }) + + it('rejects accessibility trees above the configured byte limit', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents, { + maximumAxBytes: 100 + }) + + await expect( + driver.snapshot(new AbortController().signal) + ).rejects.toThrow('可访问性树超过安全限制') + driver.dispose() + }) + + it('keeps refs for subframe navigation and invalidates them for main-frame navigation', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents) + const snapshot = await driver.snapshot(new AbortController().signal) + const button = snapshot.nodes.find((node) => node.name === 'Submit') + if (!button) { + throw new Error('button missing') + } + await driver.click(button.ref, new AbortController().signal) + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Input.dispatchMouseEvent', + expect.objectContaining({ type: 'mousePressed', x: 60, y: 40 }) + ) + + harness.contentEvents.emit( + 'did-start-navigation', + {}, + 'https://ads.example/frame', + false, + false, + 1, + 2 + ) + await expect( + driver.click(button.ref, new AbortController().signal) + ).resolves.toBeUndefined() + + harness.contentEvents.emit( + 'did-start-navigation', + {}, + 'https://example.com/next', + false, + true, + 1, + 1 + ) + await expect( + driver.click(button.ref, new AbortController().signal) + ).rejects.toThrow('引用已失效') + driver.dispose() + }) + + it('keeps refs for subframe redirects and invalidates them for main-frame redirects', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents) + const snapshot = await driver.snapshot(new AbortController().signal) + const button = snapshot.nodes.find((node) => node.name === 'Submit') + if (!button) { + throw new Error('button missing') + } + + harness.contentEvents.emit( + 'will-redirect', + {}, + 'https://ads.example/redirect', + false, + false, + 1, + 2 + ) + await expect( + driver.click(button.ref, new AbortController().signal) + ).resolves.toBeUndefined() + + harness.contentEvents.emit( + 'will-redirect', + {}, + 'https://example.com/redirect', + false, + true, + 1, + 1 + ) + await expect( + driver.click(button.ref, new AbortController().signal) + ).rejects.toThrow('引用已失效') + driver.dispose() + }) + + it('rejects password, file, hidden, and stale typing targets', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents) + const snapshot = await driver.snapshot(new AbortController().signal) + const password = snapshot.nodes.find((node) => node.name === 'Password') + if (!password) { + throw new Error('password missing') + } + await expect( + driver.type(password.ref, 'never-send', new AbortController().signal) + ).rejects.toThrow('受保护') + expect( + harness.sendCommand.mock.calls.some( + ([method, parameters]) => + method === 'Input.insertText' && + parameters?.text === 'never-send' + ) + ).toBe(false) + driver.dispose() + }) + + it('selects an exact native option value with fixed internal DOM code', async () => { + const selectedValue = `us-west'); globalThis.compromised = true; ('` + const harness = createHarness( + selectCommand({ selected: true, value: selectedValue }) + ) + const driver = new CdpBrowserDriver(harness.webContents) + const snapshot = await driver.snapshot(new AbortController().signal) + const select = snapshot.nodes.find((node) => node.name === 'Region') + if (!select) { + throw new Error('select missing') + } + + await driver.select( + select.ref, + selectedValue, + new AbortController().signal + ) + + const call = harness.sendCommand.mock.calls.find( + ([method]) => method === 'Runtime.callFunctionOn' + ) + expect(call?.[1]).toMatchObject({ + objectId: 'select-object', + arguments: [{ value: selectedValue }], + returnByValue: true + }) + expect(call?.[1]?.functionDeclaration).toEqual(expect.any(String)) + expect(String(call?.[1]?.functionDeclaration)).not.toContain(selectedValue) + expect(String(call?.[1]?.functionDeclaration)).toContain( + "new Event('input'" + ) + expect(String(call?.[1]?.functionDeclaration)).toContain( + "new Event('change'" + ) + expect( + harness.sendCommand.mock.calls.some( + ([method]) => + method === 'Input.insertText' || + method === 'Input.dispatchKeyEvent' + ) + ).toBe(false) + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Runtime.releaseObject', + { objectId: 'select-object' } + ) + driver.dispose() + }) + + it('rejects a native select result that is not an exact value match', async () => { + const harness = createHarness( + selectCommand({ selected: false, value: 'partial-match' }) + ) + const driver = new CdpBrowserDriver(harness.webContents) + const snapshot = await driver.snapshot(new AbortController().signal) + const select = snapshot.nodes.find((node) => node.name === 'Region') + if (!select) { + throw new Error('select missing') + } + + await expect( + driver.select( + select.ref, + 'partial-match-longer', + new AbortController().signal + ) + ).rejects.toThrow('完全匹配') + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Runtime.releaseObject', + { objectId: 'select-object' } + ) + driver.dispose() + }) + + it('bounds screenshots and returns only validated PNG data', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents) + await expect( + driver.screenshot(new AbortController().signal) + ).resolves.toMatchObject({ + type: 'image', + mimeType: 'image/png' + }) + harness.sendCommand.mockImplementation(async (method) => + method === 'Page.captureScreenshot' ? { data: 'bm90LXBuZw==' } : {} + ) + await expect( + driver.screenshot(new AbortController().signal) + ).rejects.toThrow('截图无效') + driver.dispose() + }) + + it('validates a history target again before returning', async () => { + const harness = createHarness(standardCommand) + const driver = new CdpBrowserDriver(harness.webContents) + const target = await driver.getBackTarget(new AbortController().signal) + expect(target).toEqual({ + entryId: 4, + url: 'https://previous.example/' + }) + harness.setUrl('https://previous.example/') + await expect( + driver.backTo(target, new AbortController().signal) + ).resolves.toEqual({ url: 'https://previous.example/' }) + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Page.navigateToHistoryEntry', + { entryId: 4 } + ) + driver.dispose() + }) + + it('bounds hung commands and removes listeners on disposal', async () => { + const harness = createHarness(async () => new Promise(() => undefined)) + const driver = new CdpBrowserDriver(harness.webContents, { timeoutMs: 5 }) + await expect( + driver.screenshot(new AbortController().signal) + ).rejects.toThrow('超时') + driver.dispose() + expect(harness.contentEvents.listenerCount('did-start-navigation')).toBe(0) + expect(harness.debuggerEvents.listenerCount('detach')).toBe(0) + await expect( + driver.screenshot(new AbortController().signal) + ).rejects.toThrow('不可用') + }) +}) diff --git a/src/main/browser/cdp-browser-driver.ts b/src/main/browser/cdp-browser-driver.ts new file mode 100644 index 0000000..6da4455 --- /dev/null +++ b/src/main/browser/cdp-browser-driver.ts @@ -0,0 +1,797 @@ +import { createHash, randomBytes } from 'node:crypto' +import type { + BrowserDebugger, + BrowserEventListener, + BrowserWebContents +} from './electron-browser-session' + +const DEFAULT_TIMEOUT_MS = 15_000 +const MAX_AX_NODES = 500 +const MAX_AX_DEPTH = 20 +const MAX_AX_BYTES = 1024 * 1024 +const MAX_SNAPSHOT_BYTES = 128 * 1024 +const MAX_SCREENSHOT_BYTES = 512 * 1024 +const MAX_INPUT_LENGTH = 16_384 +const MAX_SELECT_LENGTH = 1_024 +const SELECT_OPTION_FUNCTION = `function (expectedValue) { + const options = Array.from(this.options); + const option = options.find((candidate) => candidate.value === expectedValue); + if (!option) { + return { selected: false, value: this.value }; + } + this.value = expectedValue; + const selected = this.value === expectedValue; + if (selected) { + this.dispatchEvent(new Event('input', { bubbles: true })); + this.dispatchEvent(new Event('change', { bubbles: true })); + } + return { selected, value: this.value }; +}` + +type CdpAxValue = { + type?: string + value?: unknown +} + +type CdpAxProperty = { + name?: string + value?: CdpAxValue +} + +type CdpAxNode = { + nodeId?: string + backendDOMNodeId?: number + parentId?: string + ignored?: boolean + role?: CdpAxValue + name?: CdpAxValue + value?: CdpAxValue + properties?: CdpAxProperty[] +} + +export type BrowserSnapshotNode = { + ref: string + role: string + name: string + value?: string + disabled?: boolean + focused?: boolean + editable?: boolean +} + +export type BrowserSnapshot = { + url: string + title: string + nodes: BrowserSnapshotNode[] + truncated: boolean +} + +export type BrowserScreenshot = { + type: 'image' + mimeType: 'image/png' + data: string +} + +export class BrowserStaleReferenceError extends Error { + constructor(message = '浏览器元素引用已失效，请重新获取快照') { + super(message) + this.name = 'BrowserStaleReferenceError' + } +} + +export type BrowserHistoryTarget = { + entryId: number + url: string +} + +type RefBinding = { + backendNodeId: number + generation: number + role: string + protected: boolean +} + +export type CdpBrowserDriverOptions = { + timeoutMs?: number + maximumAxNodes?: number + maximumAxDepth?: number + maximumAxBytes?: number + maximumSnapshotBytes?: number + maximumScreenshotBytes?: number +} + +type ResolvedTarget = { + backendNodeId: number + bounds: { x: number; y: number; width: number; height: number } +} + +function stringValue(value: CdpAxValue | undefined): string { + return typeof value?.value === 'string' + ? value.value.slice(0, 2_000) + : value?.value === undefined + ? '' + : String(value.value).slice(0, 2_000) +} + +function propertyBoolean( + node: CdpAxNode, + name: string +): boolean | undefined { + const value = node.properties?.find((property) => property.name === name)?.value + ?.value + return typeof value === 'boolean' ? value : undefined +} + +function isProtectedAxNode(node: CdpAxNode): boolean { + const role = stringValue(node.role).toLowerCase() + const properties = new Map( + node.properties?.map((property) => [ + property.name, + property.value?.value + ]) + ) + return ( + role === 'password' || + properties.get('hidden') === true || + properties.get('protected') === true || + properties.get('valuetext') === '••••••••' + ) +} + +function delayAbortable( + milliseconds: number, + signal: AbortSignal +): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(finish, milliseconds) + function finish(): void { + signal.removeEventListener('abort', abort) + resolve() + } + function abort(): void { + clearTimeout(timer) + reject(signal.reason) + } + signal.addEventListener('abort', abort, { once: true }) + if (signal.aborted) { + abort() + } + }) +} + +function jsonStringBytes(value: string): number { + let bytes = 2 + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if ( + code === 0x08 || + code === 0x09 || + code === 0x0a || + code === 0x0c || + code === 0x0d || + code === 0x22 || + code === 0x5c + ) { + bytes += 2 + } else if (code < 0x20) { + bytes += 6 + } else if (code < 0x80) { + bytes += 1 + } else if (code < 0x800) { + bytes += 2 + } else if ( + code >= 0xd800 && + code <= 0xdbff && + value.charCodeAt(index + 1) >= 0xdc00 && + value.charCodeAt(index + 1) <= 0xdfff + ) { + bytes += 4 + index += 1 + } else if (code >= 0xd800 && code <= 0xdfff) { + bytes += 6 + } else { + bytes += 3 + } + } + return bytes +} + +function exceedsJsonByteLimit(value: unknown, maximumBytes: number): boolean { + let bytes = 0 + const stack = [value] + const seen = new WeakSet() + const add = (amount: number): boolean => { + bytes += amount + return bytes > maximumBytes + } + + while (stack.length > 0) { + const current = stack.pop() + if (current === null) { + if (add(4)) return true + } else if (typeof current === 'string') { + if (add(jsonStringBytes(current))) return true + } else if (typeof current === 'number') { + if (add(Number.isFinite(current) ? String(current).length : 4)) { + return true + } + } else if (typeof current === 'boolean') { + if (add(current ? 4 : 5)) return true + } else if (Array.isArray(current)) { + if (seen.has(current) || add(current.length > 0 ? current.length + 1 : 2)) { + return true + } + seen.add(current) + for (let index = current.length - 1; index >= 0; index -= 1) { + stack.push(current[index]) + } + } else if (typeof current === 'object') { + if (seen.has(current)) return true + seen.add(current) + const entries = Object.entries(current).filter( + ([, entryValue]) => entryValue !== undefined + ) + if (add(entries.length > 0 ? entries.length + 1 : 2)) return true + for (let index = entries.length - 1; index >= 0; index -= 1) { + const [key, entryValue] = entries[index]! + if (add(jsonStringBytes(key) + 1)) return true + stack.push(entryValue) + } + } else { + return true + } + } + return false +} + +export class CdpBrowserDriver { + private readonly debugger: BrowserDebugger + private readonly timeoutMs: number + private readonly maximumAxNodes: number + private readonly maximumAxDepth: number + private readonly maximumAxBytes: number + private readonly maximumSnapshotBytes: number + private readonly maximumScreenshotBytes: number + private readonly refSecret = randomBytes(16) + private readonly refs = new Map() + private readonly listeners: Array<{ + target: { off(event: string, listener: BrowserEventListener): unknown } + event: string + listener: BrowserEventListener + }> = [] + private generation = 0 + private disposed = false + + constructor( + private readonly webContents: BrowserWebContents, + options: CdpBrowserDriverOptions = {} + ) { + this.debugger = webContents.debugger + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS + this.maximumAxNodes = options.maximumAxNodes ?? MAX_AX_NODES + this.maximumAxDepth = options.maximumAxDepth ?? MAX_AX_DEPTH + this.maximumAxBytes = options.maximumAxBytes ?? MAX_AX_BYTES + this.maximumSnapshotBytes = + options.maximumSnapshotBytes ?? MAX_SNAPSHOT_BYTES + this.maximumScreenshotBytes = + options.maximumScreenshotBytes ?? MAX_SCREENSHOT_BYTES + this.listen( + webContents, + 'did-start-navigation', + ( + _event: unknown, + _url: string, + _isInPlace: boolean, + isMainFrame: boolean | undefined + ) => { + if (isMainFrame !== false) { + this.invalidate() + } + } + ) + this.listen( + webContents, + 'will-redirect', + ( + _event: unknown, + _url: string, + _isInPlace: boolean, + isMainFrame: boolean | undefined + ) => { + if (isMainFrame !== false) { + this.invalidate() + } + } + ) + this.listen(webContents, 'render-process-gone', () => this.invalidate()) + this.listen(this.debugger, 'detach', () => this.invalidate()) + } + + private listen( + target: { + on(event: string, listener: BrowserEventListener): unknown + off(event: string, listener: BrowserEventListener): unknown + }, + event: string, + listener: BrowserEventListener + ): void { + target.on(event, listener) + this.listeners.push({ target, event, listener }) + } + + private invalidate(): void { + this.generation += 1 + this.refs.clear() + } + + private command( + method: string, + parameters: Record | undefined, + signal: AbortSignal + ): Promise { + if (this.disposed || !this.debugger.isAttached()) { + return Promise.reject(new Error('浏览器调试连接不可用')) + } + signal.throwIfAborted() + const timeout = AbortSignal.timeout(this.timeoutMs) + const effectiveSignal = AbortSignal.any([signal, timeout]) + return new Promise((resolve, reject) => { + const abort = (): void => { + reject( + signal.aborted + ? signal.reason + : new Error(`浏览器操作超时（${this.timeoutMs}ms）`) + ) + } + effectiveSignal.addEventListener('abort', abort, { once: true }) + void this.debugger + .sendCommand(method, parameters) + .then((result) => { + effectiveSignal.removeEventListener('abort', abort) + if (effectiveSignal.aborted) { + abort() + } else { + resolve(result as T) + } + }) + .catch((error: unknown) => { + effectiveSignal.removeEventListener('abort', abort) + reject(new Error(`浏览器命令失败：${method}`, { cause: error })) + }) + }) + } + + async navigate(url: string, signal: AbortSignal): Promise<{ url: string }> { + this.invalidate() + const result = await this.command<{ + errorText?: string + }>('Page.navigate', { url }, signal) + if (result.errorText) { + throw new Error(`浏览器导航失败：${result.errorText.slice(0, 200)}`) + } + await this.waitForDocument(signal) + return { url: this.webContents.getURL() || url } + } + + private async waitForDocument(signal: AbortSignal): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + const result = await this.command<{ + result?: { value?: string } + }>( + 'Runtime.evaluate', + { + expression: 'document.readyState', + returnByValue: true, + awaitPromise: false + }, + signal + ) + if ( + result.result?.value === 'interactive' || + result.result?.value === 'complete' + ) { + return + } + await delayAbortable(50, signal) + } + throw new Error('浏览器页面未在安全期限内就绪') + } + + private refFor(backendNodeId: number): string { + return `b_${createHash('sha256') + .update(this.refSecret) + .update(String(this.generation)) + .update(':') + .update(String(backendNodeId)) + .digest('base64url') + .slice(0, 18)}` + } + + async snapshot(signal: AbortSignal): Promise { + this.invalidate() + const response = await this.command<{ nodes?: CdpAxNode[] }>( + 'Accessibility.getFullAXTree', + { depth: this.maximumAxDepth }, + signal + ) + if (exceedsJsonByteLimit(response, this.maximumAxBytes)) { + throw new Error('浏览器可访问性树超过安全限制') + } + const allNodes = response.nodes ?? [] + const limited = allNodes.slice(0, this.maximumAxNodes) + const knownDepth = new Map() + const output: BrowserSnapshotNode[] = [] + for (const node of limited) { + const parentDepth = node.parentId + ? knownDepth.get(node.parentId) + : -1 + const depth = (parentDepth ?? this.maximumAxDepth) + 1 + if (node.nodeId) { + knownDepth.set(node.nodeId, depth) + } + if ( + depth > this.maximumAxDepth || + node.ignored || + !node.backendDOMNodeId + ) { + continue + } + const role = stringValue(node.role) || 'unknown' + const ref = this.refFor(node.backendDOMNodeId) + const protectedNode = isProtectedAxNode(node) + this.refs.set(ref, { + backendNodeId: node.backendDOMNodeId, + generation: this.generation, + role, + protected: protectedNode + }) + const item: BrowserSnapshotNode = { + ref, + role, + name: stringValue(node.name), + disabled: propertyBoolean(node, 'disabled'), + focused: propertyBoolean(node, 'focused'), + editable: propertyBoolean(node, 'editable') + } + const value = stringValue(node.value) + const redactedValue = + protectedNode || + item.editable === true || + ['combobox', 'searchbox', 'spinbutton', 'textbox'].includes( + role.toLowerCase() + ) + if (value && !redactedValue) { + item.value = value + } + output.push(item) + } + const document = await this.command<{ + result?: { value?: { title?: unknown; url?: unknown } } + }>( + 'Runtime.evaluate', + { + expression: '({title: document.title, url: location.href})', + returnByValue: true, + awaitPromise: false + }, + signal + ) + const title = + typeof document.result?.value?.title === 'string' + ? document.result.value.title.slice(0, 500) + : '' + const url = + typeof document.result?.value?.url === 'string' + ? document.result.value.url.slice(0, 8_192) + : this.webContents.getURL() + const snapshot = { + url, + title, + nodes: output, + truncated: allNodes.length > limited.length + } + if (Buffer.byteLength(JSON.stringify(snapshot)) > this.maximumSnapshotBytes) { + this.refs.clear() + throw new Error('浏览器快照超过安全限制') + } + return snapshot + } + + private async resolveTarget( + ref: string, + action: 'click' | 'type' | 'select', + signal: AbortSignal + ): Promise { + const binding = this.refs.get(ref) + if (!binding || binding.generation !== this.generation) { + throw new BrowserStaleReferenceError() + } + const described = await this.command<{ + node?: Record & { + attributes?: unknown + nodeName?: unknown + backendNodeId?: unknown + } + }>( + 'DOM.describeNode', + { + backendNodeId: binding.backendNodeId, + depth: 0, + pierce: false + }, + signal + ) + const node = described.node + if (!node || node.backendNodeId !== binding.backendNodeId) { + throw new Error('浏览器元素状态已改变，请重新获取快照') + } + const attributes = Array.isArray(node.attributes) + ? node.attributes.filter( + (value): value is string => typeof value === 'string' + ) + : [] + const attributeMap = new Map() + for (let index = 0; index + 1 < attributes.length; index += 2) { + attributeMap.set( + (attributes[index] ?? '').toLowerCase(), + attributes[index + 1] ?? '' + ) + } + const nodeName = + typeof node.nodeName === 'string' ? node.nodeName.toLowerCase() : '' + const inputType = (attributeMap.get('type') ?? '').toLowerCase() + const blocked = + binding.protected || + attributeMap.has('hidden') || + attributeMap.has('disabled') || + attributeMap.has('inert') || + attributeMap.has('readonly') || + attributeMap.get('aria-hidden') === 'true' || + attributeMap.get('aria-disabled') === 'true' || + inputType === 'hidden' || + inputType === 'password' || + inputType === 'file' + if (blocked) { + throw new Error('浏览器拒绝操作受保护、隐藏或禁用字段') + } + if ( + action === 'type' && + !( + nodeName === 'textarea' || + nodeName === 'input' || + attributeMap.get('contenteditable') === 'true' + ) + ) { + throw new Error('浏览器目标不是可编辑字段') + } + if (action === 'select' && nodeName !== 'select') { + throw new Error('浏览器目标不是选择控件') + } + const model = await this.command<{ + model?: { + content?: number[] + border?: number[] + } + }>( + 'DOM.getBoxModel', + { backendNodeId: binding.backendNodeId }, + signal + ) + const quad = model.model?.content ?? model.model?.border + if (!quad || quad.length !== 8 || quad.some((value) => !Number.isFinite(value))) { + throw new Error('浏览器元素不可见或没有有效边界') + } + const xs = [quad[0] ?? 0, quad[2] ?? 0, quad[4] ?? 0, quad[6] ?? 0] + const ys = [quad[1] ?? 0, quad[3] ?? 0, quad[5] ?? 0, quad[7] ?? 0] + const x = Math.min(...xs) + const y = Math.min(...ys) + const width = Math.max(...xs) - x + const height = Math.max(...ys) - y + const metrics = await this.command<{ + cssVisualViewport?: { clientWidth?: number; clientHeight?: number } + layoutViewport?: { clientWidth?: number; clientHeight?: number } + }>('Page.getLayoutMetrics', undefined, signal) + const viewport = metrics.cssVisualViewport ?? metrics.layoutViewport + const viewportWidth = viewport?.clientWidth ?? 0 + const viewportHeight = viewport?.clientHeight ?? 0 + if ( + width < 1 || + height < 1 || + x < 0 || + y < 0 || + x + width > viewportWidth || + y + height > viewportHeight + ) { + throw new Error('浏览器元素超出当前可见页面边界') + } + return { + backendNodeId: binding.backendNodeId, + bounds: { x, y, width, height } + } + } + + async click(ref: string, signal: AbortSignal): Promise { + const target = await this.resolveTarget(ref, 'click', signal) + const x = target.bounds.x + target.bounds.width / 2 + const y = target.bounds.y + target.bounds.height / 2 + await this.command( + 'Input.dispatchMouseEvent', + { type: 'mouseMoved', x, y }, + signal + ) + await this.command( + 'Input.dispatchMouseEvent', + { type: 'mousePressed', x, y, button: 'left', clickCount: 1 }, + signal + ) + await this.command( + 'Input.dispatchMouseEvent', + { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 }, + signal + ) + } + + async type( + ref: string, + text: string, + signal: AbortSignal + ): Promise { + if (!text || text.length > MAX_INPUT_LENGTH) { + throw new Error('浏览器输入内容为空或超过安全限制') + } + const target = await this.resolveTarget(ref, 'type', signal) + const x = target.bounds.x + target.bounds.width / 2 + const y = target.bounds.y + target.bounds.height / 2 + await this.command( + 'Input.dispatchMouseEvent', + { type: 'mousePressed', x, y, button: 'left', clickCount: 1 }, + signal + ) + await this.command( + 'Input.dispatchMouseEvent', + { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 }, + signal + ) + await this.command('Input.insertText', { text }, signal) + } + + async select( + ref: string, + value: string, + signal: AbortSignal + ): Promise { + if (!value || value.length > MAX_SELECT_LENGTH) { + throw new Error('浏览器选择值为空或超过安全限制') + } + const target = await this.resolveTarget(ref, 'select', signal) + const resolved = await this.command<{ + object?: { objectId?: string } + }>( + 'DOM.resolveNode', + { backendNodeId: target.backendNodeId }, + signal + ) + const objectId = resolved.object?.objectId + if (!objectId) { + throw new Error('浏览器选择控件已失效') + } + try { + const result = await this.command<{ + exceptionDetails?: unknown + result?: { + value?: { selected?: unknown; value?: unknown } + } + }>( + 'Runtime.callFunctionOn', + { + objectId, + functionDeclaration: SELECT_OPTION_FUNCTION, + arguments: [{ value }], + returnByValue: true, + awaitPromise: false, + silent: true + }, + signal + ) + if ( + result.exceptionDetails || + result.result?.value?.selected !== true || + result.result.value.value !== value + ) { + throw new Error('浏览器未找到完全匹配的选择项') + } + } finally { + await this.command( + 'Runtime.releaseObject', + { objectId }, + new AbortController().signal + ).catch(() => undefined) + } + } + + async getBackTarget(signal: AbortSignal): Promise { + const history = await this.command<{ + currentIndex?: number + entries?: Array<{ id?: number; url?: string }> + }>('Page.getNavigationHistory', undefined, signal) + const index = history.currentIndex ?? -1 + const entry = history.entries?.[index - 1] + if ( + index < 1 || + typeof entry?.id !== 'number' || + typeof entry.url !== 'string' || + entry.url.length === 0 || + entry.url.length > 8_192 + ) { + throw new Error('浏览器没有可返回的页面') + } + return { entryId: entry.id, url: entry.url } + } + + async backTo( + target: BrowserHistoryTarget, + signal: AbortSignal + ): Promise<{ url: string }> { + const current = await this.getBackTarget(signal) + if (current.entryId !== target.entryId || current.url !== target.url) { + throw new Error('浏览器历史记录已改变，请重试') + } + this.invalidate() + await this.command( + 'Page.navigateToHistoryEntry', + { entryId: target.entryId }, + signal + ) + await this.waitForDocument(signal) + return { url: this.webContents.getURL() } + } + + async back(signal: AbortSignal): Promise<{ url: string }> { + return this.backTo(await this.getBackTarget(signal), signal) + } + + async screenshot(signal: AbortSignal): Promise { + const result = await this.command<{ data?: string }>( + 'Page.captureScreenshot', + { + format: 'png', + fromSurface: true, + captureBeyondViewport: false + }, + signal + ) + if ( + typeof result.data !== 'string' || + result.data.length === 0 || + result.data.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test( + result.data + ) + ) { + throw new Error('浏览器返回了无效截图') + } + const data = Buffer.from(result.data, 'base64') + if ( + data.byteLength > this.maximumScreenshotBytes || + data.byteLength < 8 || + !data.subarray(0, 8).equals( + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + ) || + data.toString('base64') !== result.data + ) { + throw new Error('浏览器截图无效或超过安全限制') + } + return { type: 'image', mimeType: 'image/png', data: result.data } + } + + dispose(): void { + if (this.disposed) { + return + } + this.disposed = true + this.invalidate() + for (const { target, event, listener } of this.listeners.splice(0)) { + target.off(event, listener) + } + } +} diff --git a/src/main/browser/electron-browser-session.test.ts b/src/main/browser/electron-browser-session.test.ts new file mode 100644 index 0000000..eb2deda --- /dev/null +++ b/src/main/browser/electron-browser-session.test.ts @@ -0,0 +1,361 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' +import { BrowserUrlPolicy } from './browser-url-policy' +import { + ElectronBrowserSession, + type BrowserPartitionSession, + type BrowserWebContents, + type BrowserWindowHandle, + type FilteringProxyLike +} from './electron-browser-session' + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +function createHarness() { + const debuggerEvents = new EventEmitter() + const contentEvents = new EventEmitter() + const partitionEvents = new EventEmitter() + let currentUrl = '' + let openHandler: ((details: { url: string }) => { action: 'deny' }) | undefined + const sendCommand = vi.fn(async () => ({})) + const webContents: BrowserWebContents = { + debugger: { + attach: vi.fn(), + detach: vi.fn(), + isAttached: vi.fn(() => true), + sendCommand, + on: (event, listener) => + debuggerEvents.on( + event, + listener as (...argumentsValue: unknown[]) => void + ), + off: (event, listener) => + debuggerEvents.off( + event, + listener as (...argumentsValue: unknown[]) => void + ) + }, + on: (event, listener) => + contentEvents.on( + event, + listener as (...argumentsValue: unknown[]) => void + ), + off: (event, listener) => + contentEvents.off( + event, + listener as (...argumentsValue: unknown[]) => void + ), + setWindowOpenHandler: vi.fn((handler) => { + openHandler = handler + }), + capturePage: vi.fn(async () => ({ + toPNG: () => + Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a + ]) + })), + getURL: vi.fn(() => currentUrl), + stop: vi.fn(), + destroy: vi.fn(), + isDestroyed: vi.fn(() => false) + } + const window: BrowserWindowHandle = { + webContents, + loadURL: vi.fn(async (url: string) => { + currentUrl = url + }), + destroy: vi.fn(), + isDestroyed: vi.fn(() => false) + } + let permissionCheck: ((...values: unknown[]) => boolean) | undefined + let permissionRequest: + | (( + contents: unknown, + permission: string, + callback: (granted: boolean) => void, + details: unknown + ) => void) + | undefined + let displayMedia: + | (( + request: unknown, + callback: (streams: Record) => void + ) => void) + | undefined + const partition: BrowserPartitionSession = { + setPermissionCheckHandler: vi.fn((handler) => { + permissionCheck = handler + }), + setPermissionRequestHandler: vi.fn((handler) => { + permissionRequest = handler + }), + setDisplayMediaRequestHandler: vi.fn((handler) => { + displayMedia = handler + }), + setProxy: vi.fn(async () => undefined), + on: (event, listener) => + partitionEvents.on( + event, + listener as (...argumentsValue: unknown[]) => void + ), + off: (event, listener) => + partitionEvents.off( + event, + listener as (...argumentsValue: unknown[]) => void + ), + clearData: vi.fn(async () => undefined), + closeAllConnections: vi.fn(async () => undefined) + } + const proxy: FilteringProxyLike = { + start: vi.fn(async () => 'http://127.0.0.1:12345'), + dispose: vi.fn(async () => undefined) + } + const policy = new BrowserUrlPolicy(async () => [ + { address: '93.184.216.34', family: 4 } + ]) + return { + contentEvents, + debuggerEvents, + partitionEvents, + partition, + proxy, + policy, + sendCommand, + webContents, + window, + setCurrentUrl(value: string) { + currentUrl = value + }, + getOpenHandler: () => openHandler, + getPermissionCheck: () => permissionCheck, + getPermissionRequest: () => permissionRequest, + getDisplayMedia: () => displayMedia + } +} + +describe('ElectronBrowserSession', () => { + it('creates an isolated sandboxed partition and denies privileged capabilities', async () => { + const harness = createHarness() + const createWindow = vi.fn(async (options: Record) => { + const preferences = options.webPreferences as Record + expect(preferences).toMatchObject({ + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + webSecurity: true, + allowRunningInsecureContent: false, + devTools: false + }) + return harness.window + }) + const session = await ElectronBrowserSession.create({ + policy: harness.policy, + createPartition: vi.fn(async () => harness.partition), + createWindow, + createProxy: () => harness.proxy + }) + + expect(session.partition).toMatch(/^browser-/u) + expect(harness.partition.setProxy).toHaveBeenCalledWith({ + mode: 'fixed_servers', + proxyRules: 'http://127.0.0.1:12345', + proxyBypassRules: '<-loopback>' + }) + expect(harness.getPermissionCheck()?.()).toBe(false) + const permissionCallback = vi.fn() + harness.getPermissionRequest()?.({}, 'geolocation', permissionCallback, {}) + expect(permissionCallback).toHaveBeenCalledWith(false) + const mediaCallback = vi.fn() + harness.getDisplayMedia()?.({}, mediaCallback) + expect(mediaCallback).toHaveBeenCalledWith({}) + expect(harness.getOpenHandler()?.({ url: 'https://example.com' })).toEqual({ + action: 'deny' + }) + expect(harness.window.loadURL).toHaveBeenCalledWith('about:blank') + expect( + vi.mocked(harness.window.loadURL).mock.invocationCallOrder[0] + ).toBeLessThan( + vi.mocked(harness.webContents.debugger.attach).mock + .invocationCallOrder[0] ?? Number.POSITIVE_INFINITY + ) + expect(harness.webContents.debugger.attach).toHaveBeenCalledWith('1.3') + expect(harness.sendCommand).toHaveBeenCalledWith('Page.enable') + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Page.setInterceptFileChooserDialog', + { enabled: true } + ) + await expect( + session.captureScreenshot(new AbortController().signal) + ).resolves.toEqual({ + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=' + }) + + const downloadEvent = { preventDefault: vi.fn() } + const item = { cancel: vi.fn() } + harness.partitionEvents.emit('will-download', downloadEvent, item) + expect(downloadEvent.preventDefault).toHaveBeenCalled() + expect(item.cancel).toHaveBeenCalled() + harness.debuggerEvents.emit( + 'message', + {}, + 'Page.fileChooserOpened', + {} + ) + await vi.waitFor(() => + expect(harness.sendCommand).toHaveBeenCalledWith( + 'Page.handleFileChooser', + { action: 'cancel' } + ) + ) + await session.dispose() + }) + + it('allows only the explicitly approved top-level origin', async () => { + const harness = createHarness() + const session = await ElectronBrowserSession.create({ + policy: harness.policy, + createPartition: async () => harness.partition, + createWindow: async () => harness.window, + createProxy: () => harness.proxy + }) + const target = await harness.policy.validate( + 'https://example.com/start', + new AbortController().signal + ) + session.approveNavigation(target) + harness.setCurrentUrl('https://example.com/page') + expect(session.getCurrentOrigin()).toBe('https://example.com') + + const sameOriginEvent = { preventDefault: vi.fn() } + harness.contentEvents.emit( + 'will-navigate', + sameOriginEvent, + 'https://example.com/next' + ) + expect(sameOriginEvent.preventDefault).not.toHaveBeenCalled() + const foreignEvent = { preventDefault: vi.fn() } + harness.contentEvents.emit( + 'will-redirect', + foreignEvent, + 'https://attacker.example/' + ) + expect(foreignEvent.preventDefault).toHaveBeenCalled() + + harness.setCurrentUrl('https://attacker.example/') + harness.contentEvents.emit( + 'did-navigate', + {}, + 'https://attacker.example/' + ) + expect(harness.webContents.stop).toHaveBeenCalled() + expect(session.getCurrentOrigin()).toBeUndefined() + await expect( + session.validateRedirect( + 'https://attacker.example/', + new AbortController().signal + ) + ).rejects.toThrow('超出已批准来源') + await session.dispose() + }) + + it('detaches listeners and clears isolated data on idempotent disposal', async () => { + const harness = createHarness() + const session = await ElectronBrowserSession.create({ + policy: harness.policy, + createPartition: async () => harness.partition, + createWindow: async () => harness.window, + createProxy: () => harness.proxy + }) + await session.dispose() + await session.dispose() + + expect(harness.webContents.debugger.detach).toHaveBeenCalledOnce() + expect(harness.window.destroy).toHaveBeenCalledOnce() + expect(harness.partition.closeAllConnections).toHaveBeenCalledOnce() + expect(harness.partition.clearData).toHaveBeenCalledOnce() + expect(harness.proxy.dispose).toHaveBeenCalledOnce() + expect(harness.contentEvents.listenerCount('will-navigate')).toBe(0) + expect(harness.debuggerEvents.listenerCount('message')).toBe(0) + }) + + it('cleans up partially created resources when debugger setup fails', async () => { + const harness = createHarness() + harness.sendCommand.mockRejectedValueOnce(new Error('debugger failed')) + await expect( + ElectronBrowserSession.create({ + policy: harness.policy, + createPartition: async () => harness.partition, + createWindow: async () => harness.window, + createProxy: () => harness.proxy + }) + ).rejects.toThrow('无法创建安全浏览器会话') + expect(harness.window.destroy).toHaveBeenCalled() + expect(harness.partition.clearData).toHaveBeenCalled() + expect(harness.proxy.dispose).toHaveBeenCalled() + }) + + it('reclaims a partition that resolves after setup times out', async () => { + const harness = createHarness() + const partitionGate = deferred() + const creation = ElectronBrowserSession.create({ + policy: harness.policy, + setupTimeoutMs: 5, + createPartition: () => partitionGate.promise, + createWindow: async () => harness.window, + createProxy: () => harness.proxy + }) + + await expect(creation).rejects.toThrow('无法创建安全浏览器会话') + partitionGate.resolve(harness.partition) + await vi.waitFor(() => + expect(harness.partition.clearData).toHaveBeenCalledOnce() + ) + expect(harness.partition.closeAllConnections).toHaveBeenCalledOnce() + }) + + it('destroys a hidden window that resolves after setup times out', async () => { + const harness = createHarness() + const windowGate = deferred() + const creation = ElectronBrowserSession.create({ + policy: harness.policy, + setupTimeoutMs: 5, + createPartition: async () => harness.partition, + createWindow: () => windowGate.promise, + createProxy: () => harness.proxy + }) + + await expect(creation).rejects.toThrow('无法创建安全浏览器会话') + windowGate.resolve(harness.window) + await vi.waitFor(() => expect(harness.window.destroy).toHaveBeenCalledOnce()) + expect(harness.partition.clearData).toHaveBeenCalledOnce() + expect(harness.proxy.dispose).toHaveBeenCalledOnce() + }) + + it('disposes a proxy whose start resolves after setup times out', async () => { + const harness = createHarness() + const proxyGate = deferred() + const proxy: FilteringProxyLike = { + start: vi.fn(() => proxyGate.promise), + dispose: vi.fn(async () => undefined) + } + const creation = ElectronBrowserSession.create({ + policy: harness.policy, + setupTimeoutMs: 5, + createPartition: async () => harness.partition, + createWindow: async () => harness.window, + createProxy: () => proxy + }) + + await expect(creation).rejects.toThrow('无法创建安全浏览器会话') + proxyGate.resolve('http://127.0.0.1:12345') + await vi.waitFor(() => expect(proxy.dispose).toHaveBeenCalledOnce()) + }) +}) diff --git a/src/main/browser/electron-browser-session.ts b/src/main/browser/electron-browser-session.ts new file mode 100644 index 0000000..184c6e8 --- /dev/null +++ b/src/main/browser/electron-browser-session.ts @@ -0,0 +1,531 @@ +import { randomUUID } from 'node:crypto' +import { + BrowserUrlPolicy, + canonicalizeBrowserUrl, + type ValidatedBrowserUrl +} from './browser-url-policy' +import { FilteringProxy } from './filtering-proxy' + +export type BrowserEventListener = (...argumentsValue: never[]) => void + +export type BrowserDebugger = { + attach(protocolVersion?: string): void + detach(): void + isAttached(): boolean + sendCommand( + method: string, + commandParams?: Record + ): Promise + on(event: string, listener: BrowserEventListener): unknown + off(event: string, listener: BrowserEventListener): unknown +} + +export type BrowserWebContents = { + debugger: BrowserDebugger + on(event: string, listener: BrowserEventListener): unknown + off(event: string, listener: BrowserEventListener): unknown + setWindowOpenHandler( + handler: (details: { url: string }) => { action: 'deny' } + ): void + capturePage?(): Promise<{ + toPNG(): Buffer + }> + getURL(): string + stop(): void + close?(options?: { waitForBeforeUnload?: boolean }): void + destroy(): void + isDestroyed(): boolean +} + +export type BrowserWindowHandle = { + webContents: BrowserWebContents + loadURL(url: string): Promise + destroy(): void + isDestroyed(): boolean +} + +export type BrowserPartitionSession = { + setPermissionCheckHandler( + handler: (...argumentsValue: never[]) => boolean + ): void + setPermissionRequestHandler( + handler: ( + webContents: unknown, + permission: string, + callback: (granted: boolean) => void, + details: unknown + ) => void + ): void + setDisplayMediaRequestHandler( + handler: ( + request: unknown, + callback: (streams: Record) => void + ) => void + ): void + setProxy(configuration: { + mode: 'fixed_servers' + proxyRules: string + proxyBypassRules: string + }): Promise + on(event: string, listener: BrowserEventListener): unknown + off(event: string, listener: BrowserEventListener): unknown + clearData(): Promise + closeAllConnections(): Promise +} + +export type FilteringProxyLike = { + start(): Promise + dispose(): Promise +} + +export type ElectronBrowserSessionOptions = { + policy: BrowserUrlPolicy + cleanupTimeoutMs?: number + setupTimeoutMs?: number + createPartition?: (partition: string) => Promise + createWindow?: ( + options: Record + ) => Promise + createProxy?: (policy: BrowserUrlPolicy) => FilteringProxyLike +} + +type Listener = { + target: { off(event: string, listener: BrowserEventListener): unknown } + event: string + listener: BrowserEventListener +} + +async function cleanupIsolatedState( + partitionSession: BrowserPartitionSession | undefined, + proxy: FilteringProxyLike, + timeoutMs: number +): Promise { + const cleanup = Promise.allSettled([ + partitionSession?.closeAllConnections() ?? Promise.resolve(), + partitionSession?.clearData() ?? Promise.resolve(), + proxy.dispose() + ]) + let timer: ReturnType | undefined + const results = await Promise.race([ + cleanup, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error('浏览器隔离数据清理超时')), + timeoutMs + ) + }) + ]).finally(() => { + if (timer) { + clearTimeout(timer) + } + }) + const failure = results.find((result) => result.status === 'rejected') + if (failure?.status === 'rejected') { + throw new Error('浏览器隔离数据清理失败', { cause: failure.reason }) + } +} + +async function boundedSetup( + operation: Promise, + signal: AbortSignal, + timeoutMs: number, + cleanupLateValue?: (value: T) => void | Promise +): Promise { + signal.throwIfAborted() + const timeout = AbortSignal.timeout(timeoutMs) + const effectiveSignal = AbortSignal.any([signal, timeout]) + return new Promise((resolve, reject) => { + const abort = (): void => { + reject( + signal.aborted + ? signal.reason + : new Error(`浏览器会话创建超时（${timeoutMs}ms）`) + ) + } + effectiveSignal.addEventListener('abort', abort, { once: true }) + void operation.then( + (value) => { + effectiveSignal.removeEventListener('abort', abort) + if (effectiveSignal.aborted) { + try { + void Promise.resolve(cleanupLateValue?.(value)).catch( + () => undefined + ) + } catch { + // Cleanup is best-effort after the caller has already timed out. + } + abort() + } else { + resolve(value) + } + }, + (error: unknown) => { + effectiveSignal.removeEventListener('abort', abort) + reject(error) + } + ) + }) +} + +async function defaultCreatePartition( + partition: string +): Promise { + const electron = await import('electron') + return electron.session.fromPartition( + partition + ) as unknown as BrowserPartitionSession +} + +async function defaultCreateWindow( + options: Record +): Promise { + const electron = await import('electron') + return new electron.BrowserWindow(options) as unknown as BrowserWindowHandle +} + +export class ElectronBrowserSession { + readonly partition: string + readonly webContents: BrowserWebContents + private approvedOrigin?: string + private readonly listeners: Listener[] = [] + private disposed = false + + private constructor( + private readonly policy: BrowserUrlPolicy, + private readonly partitionSession: BrowserPartitionSession, + private readonly window: BrowserWindowHandle, + private readonly proxy: FilteringProxyLike, + partition: string, + private readonly cleanupTimeoutMs: number + ) { + this.partition = partition + this.webContents = window.webContents + } + + static async create( + options: ElectronBrowserSessionOptions, + signal: AbortSignal = new AbortController().signal + ): Promise { + const partition = `browser-${randomUUID()}` + const createPartition = options.createPartition ?? defaultCreatePartition + const createWindow = options.createWindow ?? defaultCreateWindow + const cleanupTimeoutMs = options.cleanupTimeoutMs ?? 5_000 + const setupTimeoutMs = options.setupTimeoutMs ?? 15_000 + if (!Number.isSafeInteger(cleanupTimeoutMs) || cleanupTimeoutMs < 1) { + throw new Error('浏览器会话清理期限无效') + } + if (!Number.isSafeInteger(setupTimeoutMs) || setupTimeoutMs < 1) { + throw new Error('浏览器会话创建期限无效') + } + const proxy = + options.createProxy?.(options.policy) ?? + new FilteringProxy({ policy: options.policy }) + let proxyDisposal: Promise | undefined + const managedProxy: FilteringProxyLike = { + start: () => proxy.start(), + dispose: () => { + proxyDisposal ??= proxy.dispose() + return proxyDisposal + } + } + let partitionSession: BrowserPartitionSession | undefined + let window: BrowserWindowHandle | undefined + let result: ElectronBrowserSession | undefined + let setupStage = '启动代理' + try { + const proxyUrl = await boundedSetup( + managedProxy.start(), + signal, + setupTimeoutMs, + async () => managedProxy.dispose() + ) + setupStage = '创建隔离会话' + partitionSession = await boundedSetup( + createPartition(partition), + signal, + setupTimeoutMs, + async (latePartition) => + cleanupIsolatedState( + latePartition, + managedProxy, + cleanupTimeoutMs + ) + ) + partitionSession.setPermissionCheckHandler(() => false) + partitionSession.setPermissionRequestHandler( + (_contents, _permission, callback) => callback(false) + ) + partitionSession.setDisplayMediaRequestHandler( + (_request, callback) => callback({}) + ) + setupStage = '配置网络代理' + await boundedSetup( + partitionSession.setProxy({ + mode: 'fixed_servers', + proxyRules: proxyUrl, + proxyBypassRules: '<-loopback>' + }), + signal, + setupTimeoutMs + ) + setupStage = '创建浏览器窗口' + window = await boundedSetup( + createWindow({ + show: false, + width: 1280, + height: 900, + webPreferences: { + partition, + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + nodeIntegrationInSubFrames: false, + nodeIntegrationInWorker: false, + webSecurity: true, + allowRunningInsecureContent: false, + plugins: false, + devTools: false, + safeDialogs: true + } + }), + signal, + setupTimeoutMs, + (lateWindow) => { + if (!lateWindow.isDestroyed()) { + lateWindow.destroy() + } else if (!lateWindow.webContents.isDestroyed()) { + lateWindow.webContents.destroy() + } + } + ) + setupStage = '加载初始页面' + await boundedSetup( + window.loadURL('about:blank'), + signal, + setupTimeoutMs + ) + result = new ElectronBrowserSession( + options.policy, + partitionSession, + window, + managedProxy, + partition, + cleanupTimeoutMs + ) + setupStage = '初始化浏览器协议' + await boundedSetup(result.initialize(), signal, setupTimeoutMs) + return result + } catch (error) { + if (result) { + await result.dispose().catch(() => undefined) + } else if (window && !window.isDestroyed()) { + window.destroy() + await cleanupIsolatedState( + partitionSession, + managedProxy, + cleanupTimeoutMs + ).catch(() => undefined) + } else { + await cleanupIsolatedState( + partitionSession, + managedProxy, + cleanupTimeoutMs + ).catch(() => undefined) + } + const detail = + error instanceof Error && error.message + ? error.message.slice(0, 160) + : '未知错误' + throw new Error( + `无法创建安全浏览器会话：${setupStage}失败（${detail}）`, + { cause: error } + ) + } + } + + private listen( + target: Listener['target'] & { + on(event: string, listener: BrowserEventListener): unknown + }, + event: string, + listener: BrowserEventListener + ): void { + target.on(event, listener) + this.listeners.push({ target, event, listener }) + } + + private async initialize(): Promise { + const contents = this.webContents + contents.setWindowOpenHandler(() => ({ action: 'deny' })) + this.listen(contents, 'will-navigate', (event: { preventDefault(): void }, details: { url?: string } | string) => { + const url = typeof details === 'string' ? details : details.url + if (!url || !this.isApprovedUrl(url)) { + event.preventDefault() + } + }) + this.listen(contents, 'will-redirect', (event: { preventDefault(): void }, details: { url?: string } | string) => { + const url = typeof details === 'string' ? details : details.url + if (!url || !this.isApprovedUrl(url)) { + event.preventDefault() + } + }) + this.listen(contents, 'login', ( + event: { preventDefault(): void }, + _details: unknown, + _authInfo: unknown, + callback: () => void + ) => { + event.preventDefault() + callback() + }) + this.listen(contents, 'select-client-certificate', ( + event: { preventDefault(): void }, + _url: string, + _certificates: unknown[], + callback: (certificate?: unknown) => void + ) => { + event.preventDefault() + callback() + }) + this.listen(contents, 'did-navigate', (_event: unknown, url: string) => { + if (url && !this.isApprovedUrl(url)) { + contents.stop() + } + }) + this.listen( + this.partitionSession, + 'will-download', + (event: { preventDefault(): void }, item: { cancel?(): void }) => { + event.preventDefault() + item.cancel?.() + } + ) + contents.debugger.attach('1.3') + await contents.debugger.sendCommand('Page.enable') + this.assertOpen() + await contents.debugger.sendCommand('Accessibility.enable') + this.assertOpen() + await contents.debugger.sendCommand('Page.setInterceptFileChooserDialog', { + enabled: true + }) + this.assertOpen() + this.listen( + contents.debugger, + 'message', + (_event: unknown, method: string) => { + if (method === 'Page.fileChooserOpened') { + void contents.debugger + .sendCommand('Page.handleFileChooser', { action: 'cancel' }) + .catch(() => undefined) + } + } + ) + } + + private assertOpen(): void { + if (this.disposed) { + throw new Error('浏览器会话已关闭') + } + } + + private isApprovedUrl(input: string): boolean { + try { + return ( + this.approvedOrigin !== undefined && + canonicalizeBrowserUrl(input).origin === this.approvedOrigin + ) + } catch { + return false + } + } + + approveNavigation(target: ValidatedBrowserUrl): void { + if (this.disposed) { + throw new Error('浏览器会话已关闭') + } + this.approvedOrigin = target.origin + } + + getApprovedOrigin(): string | undefined { + return this.approvedOrigin + } + + getCurrentOrigin(): string | undefined { + const current = this.webContents.getURL() + if (!current) { + return undefined + } + try { + const origin = canonicalizeBrowserUrl(current).origin + return origin === this.approvedOrigin ? origin : undefined + } catch { + return undefined + } + } + + async captureScreenshot( + signal: AbortSignal + ): Promise<{ + type: 'image' + mimeType: 'image/png' + data: string + }> { + this.assertOpen() + if (!this.webContents.capturePage) { + throw new Error('浏览器原生画面捕获不可用') + } + const image = await boundedSetup( + this.webContents.capturePage(), + signal, + 2_000 + ) + this.assertOpen() + const data = image.toPNG() + if ( + data.byteLength < 8 || + data.byteLength > 5 * 1_024 * 1_024 || + !data.subarray(0, 8).equals( + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + ) + ) { + throw new Error('浏览器原生画面无效或过大') + } + return { + type: 'image', + mimeType: 'image/png', + data: data.toString('base64') + } + } + + async validateRedirect(url: string, signal: AbortSignal): Promise { + if (!this.approvedOrigin) { + throw new Error('浏览器没有已批准来源') + } + await this.policy.validateRedirect(url, this.approvedOrigin, signal) + } + + async dispose(): Promise { + if (this.disposed) { + return + } + this.disposed = true + this.approvedOrigin = undefined + for (const { target, event, listener } of this.listeners.splice(0)) { + target.off(event, listener) + } + if (this.webContents.debugger.isAttached()) { + this.webContents.debugger.detach() + } + this.webContents.stop() + if (!this.window.isDestroyed()) { + this.window.destroy() + } else if (!this.webContents.isDestroyed()) { + this.webContents.destroy() + } + await cleanupIsolatedState( + this.partitionSession, + this.proxy, + this.cleanupTimeoutMs + ) + } +} diff --git a/src/main/browser/filtering-proxy.test.ts b/src/main/browser/filtering-proxy.test.ts new file mode 100644 index 0000000..adc36f5 --- /dev/null +++ b/src/main/browser/filtering-proxy.test.ts @@ -0,0 +1,388 @@ +import { + Server, + createServer as createHttpServer, + request as httpRequest +} from 'node:http' +import { connect as netConnect, createServer as createNetServer } from 'node:net' +import type { AddressInfo, NetConnectOpts, Socket } from 'node:net' +import { PassThrough } from 'node:stream' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { BrowserUrlPolicy } from './browser-url-policy' +import { FilteringProxy } from './filtering-proxy' + +const disposals: Array<() => Promise> = [] + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +afterEach(async () => { + await Promise.allSettled(disposals.splice(0).map((dispose) => dispose())) +}) + +function closeServer(server: { + close(callback: (error?: Error) => void): void +}): Promise { + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())) + }) +} + +async function listen(server: { + listen(port: number, host: string, callback: () => void): void + address(): string | AddressInfo | null +}): Promise { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') { + throw new Error('test server did not bind') + } + return address.port +} + +describe('FilteringProxy', () => { + it('contains tunnel socket closure errors but still surfaces listen failures', async () => { + const upstreams: PassThrough[] = [] + const policy = { + validate: vi.fn(async (url: URL) => ({ + url, + origin: url.origin, + addresses: [{ address: '127.0.0.1', family: 4 as const }] + })) + } as unknown as BrowserUrlPolicy + const proxy = new FilteringProxy({ + policy, + connect: () => { + const upstream = new PassThrough() + upstreams.push(upstream) + return upstream as unknown as Socket + } + }) + disposals.push(() => proxy.dispose()) + const handleConnect = ( + proxy as unknown as { + handleConnect( + request: { url: string }, + client: PassThrough, + head: Buffer + ): Promise + } + ).handleConnect.bind(proxy) + + for (const code of ['ECONNABORTED', 'ECONNRESET', 'EPIPE']) { + for (const failingSide of ['client', 'upstream'] as const) { + const client = new PassThrough() + await handleConnect( + { url: 'example.com:443' }, + client, + Buffer.alloc(0) + ) + const upstream = upstreams.at(-1) + expect(upstream).toBeDefined() + upstream?.emit('connect') + const socketError = Object.assign(new Error(`write ${code}`), { + code + }) + + expect(() => + (failingSide === 'client' ? client : upstream)?.emit( + 'error', + socketError + ) + ).not.toThrow() + expect(client.destroyed).toBe(true) + expect(upstream?.destroyed).toBe(true) + } + } + + const listenFailure = Object.assign(new Error('listen denied'), { + code: 'EACCES' + }) + const listen = vi + .spyOn(Server.prototype, 'listen') + .mockImplementation(function (this: Server) { + queueMicrotask(() => this.emit('error', listenFailure)) + return this + } as typeof Server.prototype.listen) + const failedProxy = new FilteringProxy({ policy }) + + await expect(failedProxy.start()).rejects.toMatchObject({ + message: '浏览器过滤代理启动失败', + cause: listenFailure + }) + expect(listen).toHaveBeenCalled() + listen.mockRestore() + }) + + it('binds only to loopback, pins HTTP to the validated address, and strips credentials', async () => { + let receivedAuthorization: string | undefined + const upstream = createHttpServer((request, response) => { + receivedAuthorization = request.headers.authorization + response.end('safe') + }) + const upstreamPort = await listen(upstream) + disposals.push(() => closeServer(upstream)) + const policy = { + validate: vi.fn(async (url: URL) => ({ + url, + origin: url.origin, + addresses: [{ address: '127.0.0.1', family: 4 as const }] + })) + } as unknown as BrowserUrlPolicy + const proxy = new FilteringProxy({ policy }) + disposals.push(() => proxy.dispose()) + const proxyUrl = new URL(await proxy.start()) + expect(proxyUrl.hostname).toBe('127.0.0.1') + + const body = await new Promise((resolve, reject) => { + const request = httpRequest( + { + host: proxyUrl.hostname, + port: proxyUrl.port, + method: 'GET', + path: `http://example.com:${upstreamPort}/resource`, + headers: { + authorization: 'Bearer secret', + 'proxy-authorization': 'Basic secret' + } + }, + (response) => { + let value = '' + response.setEncoding('utf8') + response.on('data', (chunk: string) => { + value += chunk + }) + response.on('end', () => resolve(value)) + } + ) + request.once('error', reject) + request.end() + }) + + expect(body).toBe('safe') + expect(receivedAuthorization).toBeUndefined() + expect(policy.validate).toHaveBeenCalled() + }) + + it('contains aborted upstream HTTP responses', async () => { + const upstream = createHttpServer((_request, response) => { + response.writeHead(200) + response.write('partial') + response.socket?.destroy() + }) + const upstreamPort = await listen(upstream) + disposals.push(() => closeServer(upstream)) + const policy = { + validate: vi.fn(async (url: URL) => ({ + url, + origin: url.origin, + addresses: [{ address: '127.0.0.1', family: 4 as const }] + })) + } as unknown as BrowserUrlPolicy + const proxy = new FilteringProxy({ policy }) + disposals.push(() => proxy.dispose()) + const proxyUrl = new URL(await proxy.start()) + + await new Promise((resolve) => { + const request = httpRequest( + { + host: proxyUrl.hostname, + port: proxyUrl.port, + path: `http://example.com:${upstreamPort}/aborted` + }, + (response) => { + response.resume() + response.once('aborted', resolve) + response.once('error', resolve) + response.once('end', resolve) + } + ) + request.once('error', () => resolve()) + request.end() + }) + }) + + it('does not create an upstream HTTP request after the client disconnects during validation', async () => { + const upstreamRequest = vi.fn() + const upstream = createHttpServer(upstreamRequest) + const upstreamPort = await listen(upstream) + disposals.push(() => closeServer(upstream)) + const validation = deferred<{ + url: URL + origin: string + addresses: Array<{ address: string; family: 4 }> + }>() + const policy = { + validate: vi.fn(() => validation.promise) + } as unknown as BrowserUrlPolicy + const proxy = new FilteringProxy({ policy }) + disposals.push(() => proxy.dispose()) + const proxyUrl = new URL(await proxy.start()) + const request = httpRequest({ + host: proxyUrl.hostname, + port: proxyUrl.port, + path: `http://example.com:${upstreamPort}/cancelled` + }) + request.once('error', () => undefined) + request.end() + await vi.waitFor(() => expect(policy.validate).toHaveBeenCalledOnce()) + + request.destroy() + await new Promise((resolve) => setTimeout(resolve, 20)) + validation.resolve({ + url: new URL(`http://example.com:${upstreamPort}/cancelled`), + origin: `http://example.com:${upstreamPort}`, + addresses: [{ address: '127.0.0.1', family: 4 }] + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(upstreamRequest).not.toHaveBeenCalled() + }) + + it('pins CONNECT TCP destinations while leaving TLS hostname handling to the client', async () => { + const upstream = createNetServer((socket) => socket.pipe(socket)) + const upstreamPort = await listen(upstream) + disposals.push(() => closeServer(upstream)) + const policy = { + validate: vi.fn(async (url: URL) => ({ + url, + origin: url.origin, + addresses: [{ address: '93.184.216.34', family: 4 as const }] + })) + } as unknown as BrowserUrlPolicy + let requestedOptions: NetConnectOpts | undefined + const connect = vi.fn((options: NetConnectOpts): Socket => { + requestedOptions = options + return netConnect({ + host: '127.0.0.1', + port: upstreamPort, + family: 4 + }) + }) + const proxy = new FilteringProxy({ policy, connect }) + disposals.push(() => proxy.dispose()) + const proxyUrl = new URL(await proxy.start()) + + const echoed = await new Promise((resolve, reject) => { + const socket = netConnect({ + host: proxyUrl.hostname, + port: Number(proxyUrl.port) + }) + let response = '' + let tunnelReady = false + socket.setEncoding('utf8') + socket.once('error', reject) + socket.on('data', (chunk: string) => { + response += chunk + if (!tunnelReady && response.includes('\r\n\r\n')) { + tunnelReady = true + response = '' + socket.write('tls-bytes') + } else if (tunnelReady && response.includes('tls-bytes')) { + socket.destroy() + resolve(response) + } + }) + socket.once('connect', () => { + socket.write( + `CONNECT example.com:${upstreamPort} HTTP/1.1\r\nHost: example.com\r\n\r\n` + ) + }) + }) + + expect(echoed).toContain('tls-bytes') + expect(requestedOptions).toEqual({ + host: '93.184.216.34', + port: upstreamPort, + family: 4 + }) + }) + + it('fails closed when validation rejects and bounds active connections', async () => { + const policy = { + validate: vi.fn(async () => { + throw new Error('blocked') + }) + } as unknown as BrowserUrlPolicy + const proxy = new FilteringProxy({ + policy, + maximumConnections: 1 + }) + disposals.push(() => proxy.dispose()) + const proxyUrl = new URL(await proxy.start()) + + const status = await new Promise((resolve, reject) => { + const request = httpRequest( + { + host: proxyUrl.hostname, + port: proxyUrl.port, + path: 'http://example.com/' + }, + (response) => { + response.resume() + response.once('end', () => resolve(response.statusCode)) + } + ) + request.once('error', reject) + request.end() + }) + expect(status).toBe(403) + await proxy.dispose() + await expect(proxy.start()).rejects.toThrow('已关闭') + }) + + it('holds request reservations until slow upstream responses complete', async () => { + const releaseUpstream = deferred() + let upstreamRequests = 0 + const upstream = createHttpServer(async (_request, response) => { + upstreamRequests += 1 + await releaseUpstream.promise + response.end('done') + }) + const upstreamPort = await listen(upstream) + disposals.push(() => closeServer(upstream)) + const policy = { + validate: vi.fn(async (url: URL) => ({ + url, + origin: url.origin, + addresses: [{ address: '127.0.0.1', family: 4 as const }] + })) + } as unknown as BrowserUrlPolicy + const proxy = new FilteringProxy({ policy, maximumConnections: 1 }) + disposals.push(() => proxy.dispose()) + const proxyUrl = new URL(await proxy.start()) + const requestStatus = (): Promise => + new Promise((resolve, reject) => { + const request = httpRequest( + { + host: proxyUrl.hostname, + port: proxyUrl.port, + path: `http://example.com:${upstreamPort}/slow` + }, + (response) => { + response.resume() + response.once('end', () => resolve(response.statusCode)) + } + ) + request.once('error', reject) + request.end() + }) + + const first = requestStatus() + await vi.waitFor(() => expect(upstreamRequests).toBe(1)) + const rejected = await Promise.all( + Array.from({ length: 12 }, () => requestStatus()) + ) + expect(rejected).toEqual(Array.from({ length: 12 }, () => 503)) + expect(upstreamRequests).toBe(1) + + releaseUpstream.resolve() + await expect(first).resolves.toBe(200) + await expect(requestStatus()).resolves.toBe(200) + expect(upstreamRequests).toBe(2) + }) +}) diff --git a/src/main/browser/filtering-proxy.ts b/src/main/browser/filtering-proxy.ts new file mode 100644 index 0000000..c94aef3 --- /dev/null +++ b/src/main/browser/filtering-proxy.ts @@ -0,0 +1,345 @@ +import { createServer as createHttpServer, request as httpRequest } from 'node:http' +import { request as httpsRequest } from 'node:https' +import { connect as netConnect } from 'node:net' +import type { NetConnectOpts, Socket } from 'node:net' +import type { Duplex } from 'node:stream' +import type { IncomingMessage, Server, ServerResponse } from 'node:http' +import { BrowserUrlPolicy, type ValidatedBrowserUrl } from './browser-url-policy' + +export type FilteringProxyOptions = { + policy: BrowserUrlPolicy + maximumConnections?: number + maximumRequestBytes?: number + connect?: (options: NetConnectOpts) => Socket +} + +type ActiveStream = { + destroy(error?: Error): void +} + +function rejectHttp(response: ServerResponse, status = 403): void { + if (!response.headersSent) { + response.writeHead(status, { + connection: 'close', + 'content-type': 'text/plain; charset=utf-8' + }) + } + response.end('Request blocked') +} + +function stripProxyHeaders( + headers: IncomingMessage['headers'] +): Record { + const result = { ...headers } + delete result.authorization + delete result['proxy-authorization'] + delete result['proxy-connection'] + delete result.connection + delete result['keep-alive'] + delete result.te + delete result.trailer + delete result['transfer-encoding'] + delete result.upgrade + return result +} + +export class FilteringProxy { + private readonly policy: BrowserUrlPolicy + private readonly maximumConnections: number + private readonly maximumRequestBytes: number + private readonly connectSocket: (options: NetConnectOpts) => Socket + private readonly controller = new AbortController() + private readonly streams = new Set() + private readonly reservations = new Set() + private server?: Server + private proxyUrl?: string + private disposed = false + + constructor(options: FilteringProxyOptions) { + this.policy = options.policy + this.maximumConnections = options.maximumConnections ?? 32 + this.maximumRequestBytes = options.maximumRequestBytes ?? 1024 * 1024 + this.connectSocket = options.connect ?? netConnect + } + + async start(): Promise { + if (this.proxyUrl) { + return this.proxyUrl + } + if (this.disposed) { + throw new Error('浏览器过滤代理已关闭') + } + const server = createHttpServer((request, response) => { + void this.handleHttp(request, response) + }) + server.on('connect', (request, client, head) => { + void this.handleConnect(request, client, head) + }) + server.on('clientError', (_error, socket) => socket.destroy()) + this.server = server + try { + await new Promise((resolve, reject) => { + const onError = (error: Error): void => reject(error) + server.once('error', onError) + server.listen(0, '127.0.0.1', () => { + server.off('error', onError) + resolve() + }) + }) + } catch (error) { + await this.dispose() + throw new Error('浏览器过滤代理启动失败', { cause: error }) + } + const address = server.address() + if (!address || typeof address === 'string' || address.address !== '127.0.0.1') { + await this.dispose() + throw new Error('浏览器过滤代理未安全绑定到回环地址') + } + this.proxyUrl = `http://127.0.0.1:${address.port}` + return this.proxyUrl + } + + private reserve(stream: ActiveStream): boolean { + if ( + this.disposed || + this.controller.signal.aborted || + this.reservations.size >= this.maximumConnections + ) { + return false + } + this.streams.add(stream) + this.reservations.add(stream) + return true + } + + private releaseStream(stream: ActiveStream): void { + this.streams.delete(stream) + } + + private releaseReservation(stream: ActiveStream): void { + this.reservations.delete(stream) + } + + private async validateAtConnect(url: URL): Promise { + return this.policy.validate(url, this.controller.signal) + } + + private async handleHttp( + incoming: IncomingMessage, + response: ServerResponse + ): Promise { + if (!this.reserve(incoming)) { + rejectHttp(response, 503) + return + } + incoming.once('error', () => response.destroy()) + response.once('error', () => incoming.destroy()) + incoming.once('close', () => this.releaseStream(incoming)) + let responseClosed = false + const releaseReservation = (): void => + this.releaseReservation(incoming) + response.once('finish', releaseReservation) + response.once('close', () => { + responseClosed = true + releaseReservation() + }) + try { + if (!incoming.url) { + rejectHttp(response) + return + } + const target = await this.validateAtConnect(new URL(incoming.url)) + const address = target.addresses[0] + if (!address) { + rejectHttp(response) + return + } + if ( + incoming.destroyed || + response.destroyed || + response.writableEnded || + responseClosed + ) { + return + } + const request = ( + target.url.protocol === 'https:' ? httpsRequest : httpRequest + )( + target.url, + { + method: incoming.method, + headers: { + ...stripProxyHeaders(incoming.headers), + host: target.url.host + }, + lookup: (_hostname, options, callback) => { + if (options.all) { + callback(null, [ + { address: address.address, family: address.family } + ]) + } else { + callback(null, address.address, address.family) + } + }, + signal: this.controller.signal + }, + (upstream) => { + const destroyForward = (): void => { + upstream.destroy() + request.destroy() + if (!response.destroyed) { + response.destroy() + } + } + upstream.once('error', destroyForward) + response.once('error', destroyForward) + response.once('close', () => { + if (!upstream.complete) { + upstream.destroy() + } + }) + response.writeHead( + upstream.statusCode ?? 502, + stripProxyHeaders(upstream.headers) + ) + upstream.pipe(response) + } + ) + this.streams.add(request) + request.once('close', () => this.releaseStream(request)) + request.once('error', () => { + if (response.headersSent) { + response.destroy() + } else if (!response.destroyed) { + rejectHttp(response, 502) + } + }) + incoming.once('aborted', () => request.destroy()) + incoming.once('error', () => request.destroy()) + let bytes = 0 + incoming.on('data', (chunk: Buffer) => { + bytes += chunk.byteLength + if (bytes > this.maximumRequestBytes) { + request.destroy(new Error('浏览器请求超过安全限制')) + incoming.destroy() + } + }) + incoming.pipe(request) + } catch { + rejectHttp(response) + } + } + + private async handleConnect( + request: IncomingMessage, + client: Duplex, + head: Buffer + ): Promise { + if (!this.reserve(client)) { + client.destroy() + return + } + let upstream: Socket | undefined + const destroyUpstream = (): void => { + if (upstream && !upstream.destroyed) { + upstream.destroy() + } + } + const destroyTunnel = (): void => { + destroyUpstream() + if (!client.destroyed) { + client.destroy() + } + } + // A browser can abandon a CONNECT tunnel while validation or a piped + // write is in flight. Socket errors are connection-local; without an + // error listener Node promotes them to an uncaught main-process error. + client.once('error', destroyTunnel) + client.once('close', () => { + this.releaseStream(client) + this.releaseReservation(client) + destroyUpstream() + }) + try { + if (!request.url || request.url.length > 1_000) { + client.destroy() + return + } + const authority = new URL(`https://${request.url}`) + if ( + authority.username || + authority.password || + authority.pathname !== '/' || + authority.search || + authority.hash + ) { + client.destroy() + return + } + const target = await this.validateAtConnect(authority) + const address = target.addresses[0] + if (!address) { + client.destroy() + return + } + const port = authority.port ? Number(authority.port) : 443 + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { + client.destroy() + return + } + if (client.destroyed) { + return + } + const connectedUpstream = this.connectSocket({ + // Pin the TCP destination to the policy-approved address. The CONNECT + // tunnel remains opaque, so Chromium still verifies TLS against the + // original authority hostname rather than this address. + host: address.address, + port, + family: address.family + }) + upstream = connectedUpstream + this.streams.add(connectedUpstream) + const release = (): void => this.releaseStream(connectedUpstream) + connectedUpstream.once('close', release) + connectedUpstream.once('error', destroyTunnel) + if (client.destroyed) { + connectedUpstream.destroy() + return + } + connectedUpstream.once('connect', () => { + if (client.destroyed) { + connectedUpstream.destroy() + return + } + client.write('HTTP/1.1 200 Connection Established\r\n\r\n') + if (head.length > 0) { + connectedUpstream.write(head) + } + connectedUpstream.pipe(client) + client.pipe(connectedUpstream) + }) + } catch { + destroyTunnel() + } + } + + async dispose(): Promise { + if (this.disposed) { + return + } + this.disposed = true + this.controller.abort(new Error('浏览器过滤代理已关闭')) + for (const stream of this.streams) { + stream.destroy() + } + this.streams.clear() + this.reservations.clear() + const server = this.server + this.server = undefined + this.proxyUrl = undefined + if (server) { + await new Promise((resolve) => server.close(() => resolve())) + } + } +} diff --git a/src/main/capabilities/browser-profile-service.test.ts b/src/main/capabilities/browser-profile-service.test.ts new file mode 100644 index 0000000..3872f70 --- /dev/null +++ b/src/main/capabilities/browser-profile-service.test.ts @@ -0,0 +1,173 @@ +import { + mkdtemp, + mkdir, + readFile, + rm, + symlink, + writeFile +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + BrowserProfileService, + FileBrowserProfileStore, + MemoryBrowserProfileStore +} from './browser-profile-service' + +const temporaryDirectories: string[] = [] + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => + rm(directory, { recursive: true, force: true }) + ) + ) +}) + +describe('BrowserProfileService', () => { + it('defaults new profiles to isolated managed mode and migrates version 1', async () => { + const legacyId = 'af6774e4-39e0-4479-b81b-42ec0f85c353' + const store = new MemoryBrowserProfileStore({ + version: 1, + profiles: [{ id: legacyId, name: '旧配置' }] + }) + const migrated = await new BrowserProfileService(store).getSnapshot() + + expect(migrated).toEqual({ + version: 2, + profiles: [ + { + id: legacyId, + name: '旧配置', + mode: 'managed-isolated', + references: [] + } + ], + defaultProfileId: legacyId + }) + + const created = await new BrowserProfileService( + new MemoryBrowserProfileStore() + ).createProfile('隔离浏览器') + expect(created.profiles[0]).toMatchObject({ + name: '隔离浏览器', + mode: 'managed-isolated', + references: [] + }) + expect(created.defaultProfileId).toBe(created.profiles[0]?.id) + }) + + it('persists loaded state only when migration changes it', async () => { + const profileId = 'af6774e4-39e0-4479-b81b-42ec0f85c353' + const currentStore = { + load: vi.fn(async () => ({ + version: 2, + profiles: [ + { + id: profileId, + name: '当前配置', + mode: 'managed-isolated', + references: [] + } + ], + defaultProfileId: profileId + })), + save: vi.fn(async () => undefined) + } + const current = new BrowserProfileService(currentStore) + + await current.getSnapshot() + await current.getSnapshot() + + expect(currentStore.load).toHaveBeenCalledOnce() + expect(currentStore.save).not.toHaveBeenCalled() + + const legacyStore = { + load: vi.fn(async () => ({ + version: 1, + profiles: [{ id: profileId, name: '旧配置' }] + })), + save: vi.fn(async () => undefined) + } + await new BrowserProfileService(legacyStore).getSnapshot() + expect(legacyStore.save).toHaveBeenCalledOnce() + expect(legacyStore.save).toHaveBeenCalledWith( + expect.objectContaining({ version: 2 }) + ) + }) + + it('persists only strict browser metadata without arguments or environment', async () => { + const service = new BrowserProfileService(new MemoryBrowserProfileStore()) + const state = await service.createProfile('浏览器') + const id = state.profiles[0]?.id + if (!id) { + throw new Error('Expected browser profile') + } + + await expect( + service.selectBrowser(id, { + executablePath: resolve(process.execPath), + displayName: 'Selected browser', + source: 'user-selected', + args: ['--remote-debugging-port=1'] + } as never) + ).rejects.toThrow() + await expect( + service.selectBrowser(id, { + executablePath: resolve(process.execPath), + displayName: 'Selected browser', + source: 'user-selected', + env: { TOKEN: 'secret' } + } as never) + ).rejects.toThrow() + }) + + it('blocks deletion while a profile is referenced', async () => { + const service = new BrowserProfileService(new MemoryBrowserProfileStore()) + const created = await service.createProfile('自动化配置') + const id = created.profiles[0]?.id + if (!id) { + throw new Error('Expected browser profile') + } + const reference = { kind: 'automation' as const, id: 'job:daily-check' } + await service.addReference(id, reference) + + await expect(service.deleteProfile(id)).rejects.toThrow('Referenced') + await service.removeReference(id, reference) + await expect(service.deleteProfile(id)).resolves.toMatchObject({ + profiles: [], + defaultProfileId: null + }) + }) + + it('uses atomic files under the owned root and rejects a symlink root', async () => { + const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-browser-store-')) + temporaryDirectories.push(directory) + const root = join(directory, 'owned') + const outsideRoot = join(directory, 'outside') + const outside = join(outsideRoot, 'browser-profiles.json') + await mkdir(outsideRoot) + await writeFile(outside, '{"sentinel":true}', 'utf8') + await symlink(outsideRoot, root, 'junction') + const service = new BrowserProfileService( + new FileBrowserProfileStore(root) + ) + + await expect(service.getSnapshot()).rejects.toThrow('real directory') + await expect(readFile(outside, 'utf8')).resolves.toBe('{"sentinel":true}') + }) + + it('rejects invalid defaults and unknown persisted properties', async () => { + const unknownId = 'b7f29e4c-1c4a-4aa0-ac58-5165451dde07' + const service = new BrowserProfileService( + new MemoryBrowserProfileStore({ + version: 2, + profiles: [], + defaultProfileId: unknownId, + args: ['--unsafe'] + }) + ) + await expect(service.getSnapshot()).rejects.toThrow() + }) +}) diff --git a/src/main/capabilities/browser-profile-service.ts b/src/main/capabilities/browser-profile-service.ts new file mode 100644 index 0000000..d3957b5 --- /dev/null +++ b/src/main/capabilities/browser-profile-service.ts @@ -0,0 +1,424 @@ +import { randomUUID } from 'node:crypto' +import { + lstat, + mkdir, + readFile, + realpath, + rename, + rm, + writeFile +} from 'node:fs/promises' +import { isAbsolute, join, relative, resolve } from 'node:path' +import { z } from 'zod' +import { + browserProfileIdSchema, + browserProfileNameSchema +} from '../../shared/capability-contracts' + +const MAX_PROFILES = 32 +const MAX_REFERENCES = 64 +const MAX_STORE_BYTES = 256 * 1024 + +const boundedPathSchema = z + .string() + .trim() + .min(1) + .max(2_048) + .refine((value) => isAbsolute(value), 'Browser executable path must be absolute') + .refine( + (value) => resolve(value) === value, + 'Browser executable path must be normalized' + ) + +export const browserExecutableMetadataSchema = z + .object({ + executablePath: boundedPathSchema, + displayName: browserProfileNameSchema, + source: z.literal('user-selected') + }) + .strict() + +export const browserProfileReferenceSchema = z + .object({ + kind: z.enum(['capability', 'automation']), + id: z.string().trim().min(1).max(128).regex(/^[a-zA-Z0-9._:-]+$/u) + }) + .strict() + +export const browserProfileSchema = z + .object({ + id: browserProfileIdSchema, + name: browserProfileNameSchema, + mode: z.literal('managed-isolated'), + browser: browserExecutableMetadataSchema.optional(), + references: z.array(browserProfileReferenceSchema).max(MAX_REFERENCES) + }) + .strict() + +export const browserProfileStateSchema = z + .object({ + version: z.literal(2), + profiles: z.array(browserProfileSchema).max(MAX_PROFILES), + defaultProfileId: browserProfileIdSchema.nullable() + }) + .strict() + .superRefine((state, context) => { + const ids = new Set() + for (const [index, profile] of state.profiles.entries()) { + if (ids.has(profile.id)) { + context.addIssue({ + code: 'custom', + path: ['profiles', index, 'id'], + message: 'Browser profile IDs must be unique' + }) + } + ids.add(profile.id) + } + if (state.defaultProfileId && !ids.has(state.defaultProfileId)) { + context.addIssue({ + code: 'custom', + path: ['defaultProfileId'], + message: 'Default browser profile must exist' + }) + } + }) + +const version1ProfileSchema = z + .object({ + id: browserProfileIdSchema, + name: browserProfileNameSchema, + browser: browserExecutableMetadataSchema.optional() + }) + .strict() + +const version1StateSchema = z + .object({ + version: z.literal(1), + profiles: z.array(version1ProfileSchema).max(MAX_PROFILES), + defaultProfileId: browserProfileIdSchema.nullable().optional() + }) + .strict() + +export type BrowserExecutableMetadata = z.infer< + typeof browserExecutableMetadataSchema +> +export type BrowserProfileReference = z.infer< + typeof browserProfileReferenceSchema +> +export type BrowserProfile = z.infer +export type BrowserProfileState = z.infer + +const emptyState = (): BrowserProfileState => ({ + version: 2, + profiles: [], + defaultProfileId: null +}) + +function migrateState( + value: unknown +): { state: BrowserProfileState; migrated: boolean } { + const version = z + .object({ version: z.union([z.literal(1), z.literal(2)]) }) + .passthrough() + .parse(value).version + if (version === 2) { + return { + state: browserProfileStateSchema.parse(value), + migrated: false + } + } + const legacy = version1StateSchema.parse(value) + return { + state: browserProfileStateSchema.parse({ + version: 2, + profiles: legacy.profiles.map((profile) => ({ + ...profile, + mode: 'managed-isolated', + references: [] + })), + defaultProfileId: + legacy.defaultProfileId ?? legacy.profiles[0]?.id ?? null + }), + migrated: true + } +} + +export interface BrowserProfileStore { + load(): Promise + save(state: BrowserProfileState): Promise +} + +export class MemoryBrowserProfileStore implements BrowserProfileStore { + private value: unknown + + constructor(initialValue?: unknown) { + this.value = initialValue + } + + async load(): Promise { + return structuredClone(this.value) + } + + async save(state: BrowserProfileState): Promise { + this.value = structuredClone(state) + } +} + +export class FileBrowserProfileStore implements BrowserProfileStore { + private readonly fileName = 'browser-profiles.json' + + constructor(private readonly ownedRoot: string) { + if (!isAbsolute(ownedRoot)) { + throw new Error('Browser profile storage root must be absolute') + } + } + + private async prepareRoot(): Promise<{ root: string; filePath: string }> { + await mkdir(this.ownedRoot, { recursive: true, mode: 0o700 }) + const rootDetails = await lstat(this.ownedRoot) + if (!rootDetails.isDirectory() || rootDetails.isSymbolicLink()) { + throw new Error('Browser profile storage root must be a real directory') + } + const root = await realpath(this.ownedRoot) + const filePath = join(root, this.fileName) + const fromRoot = relative(root, filePath) + if ( + fromRoot.startsWith('..') || + isAbsolute(fromRoot) || + fromRoot === '' + ) { + throw new Error('Browser profile storage path escapes its owned root') + } + return { root, filePath } + } + + async load(): Promise { + const { filePath } = await this.prepareRoot() + try { + const details = await lstat(filePath) + if (details.isSymbolicLink() || !details.isFile()) { + throw new Error('Browser profile storage file must be a regular file') + } + if (details.size > MAX_STORE_BYTES) { + throw new Error('Browser profile storage file is too large') + } + return JSON.parse(await readFile(filePath, 'utf8')) as unknown + } catch (error) { + if ( + error && + typeof error === 'object' && + 'code' in error && + error.code === 'ENOENT' + ) { + return undefined + } + throw error + } + } + + async save(state: BrowserProfileState): Promise { + const { root, filePath } = await this.prepareRoot() + try { + const targetDetails = await lstat(filePath) + if (targetDetails.isSymbolicLink() || !targetDetails.isFile()) { + throw new Error('Browser profile storage file must be a regular file') + } + } catch (error) { + if ( + !( + error && + typeof error === 'object' && + 'code' in error && + error.code === 'ENOENT' + ) + ) { + throw error + } + } + + const temporaryPath = join(root, `.${this.fileName}.${randomUUID()}.tmp`) + try { + await writeFile( + temporaryPath, + `${JSON.stringify(browserProfileStateSchema.parse(state), null, 2)}\n`, + { encoding: 'utf8', mode: 0o600, flag: 'wx' } + ) + await rename(temporaryPath, filePath) + } finally { + await rm(temporaryPath, { force: true }) + } + } +} + +export class BrowserProfileService { + private state?: BrowserProfileState + private updateQueue: Promise = Promise.resolve() + + constructor(private readonly store: BrowserProfileStore) {} + + private async getState(): Promise { + if (!this.state) { + const loaded = await this.store.load() + const result = + loaded === undefined + ? { state: emptyState(), migrated: false } + : migrateState(loaded) + if (result.migrated) { + await this.store.save(result.state) + } + this.state = result.state + } + return this.state + } + + private async update( + operation: (state: BrowserProfileState) => BrowserProfileState + ): Promise { + let result: BrowserProfileState | undefined + const queued = this.updateQueue.then(async () => { + const next = browserProfileStateSchema.parse( + operation(structuredClone(await this.getState())) + ) + await this.store.save(next) + this.state = next + result = next + }) + this.updateQueue = queued.catch(() => undefined) + await queued + if (!result) { + throw new Error('Browser profile update failed') + } + return structuredClone(result) + } + + async getSnapshot(): Promise { + return structuredClone(await this.getState()) + } + + async createProfile(name: string): Promise { + const parsedName = browserProfileNameSchema.parse(name) + return this.update((state) => { + if (state.profiles.length >= MAX_PROFILES) { + throw new Error('Browser profile limit reached') + } + const profile: BrowserProfile = { + id: randomUUID(), + name: parsedName, + mode: 'managed-isolated', + references: [] + } + state.profiles.push(profile) + state.defaultProfileId ??= profile.id + return state + }) + } + + async renameProfile( + profileId: string, + name: string + ): Promise { + const id = browserProfileIdSchema.parse(profileId) + const parsedName = browserProfileNameSchema.parse(name) + return this.update((state) => { + const profile = state.profiles.find((candidate) => candidate.id === id) + if (!profile) { + throw new Error('Browser profile not found') + } + profile.name = parsedName + return state + }) + } + + async selectBrowser( + profileId: string, + browser?: BrowserExecutableMetadata + ): Promise { + const id = browserProfileIdSchema.parse(profileId) + const parsedBrowser = browserExecutableMetadataSchema + .optional() + .parse(browser) + return this.update((state) => { + const profile = state.profiles.find((candidate) => candidate.id === id) + if (!profile) { + throw new Error('Browser profile not found') + } + profile.browser = parsedBrowser + return state + }) + } + + async setDefaultProfile(profileId: string): Promise { + const id = browserProfileIdSchema.parse(profileId) + return this.update((state) => { + if (!state.profiles.some((profile) => profile.id === id)) { + throw new Error('Browser profile not found') + } + state.defaultProfileId = id + return state + }) + } + + async addReference( + profileId: string, + reference: BrowserProfileReference + ): Promise { + const id = browserProfileIdSchema.parse(profileId) + const parsedReference = browserProfileReferenceSchema.parse(reference) + return this.update((state) => { + const profile = state.profiles.find((candidate) => candidate.id === id) + if (!profile) { + throw new Error('Browser profile not found') + } + if ( + !profile.references.some( + (candidate) => + candidate.kind === parsedReference.kind && + candidate.id === parsedReference.id + ) + ) { + profile.references.push(parsedReference) + } + return state + }) + } + + async removeReference( + profileId: string, + reference: BrowserProfileReference + ): Promise { + const id = browserProfileIdSchema.parse(profileId) + const parsedReference = browserProfileReferenceSchema.parse(reference) + return this.update((state) => { + const profile = state.profiles.find((candidate) => candidate.id === id) + if (!profile) { + throw new Error('Browser profile not found') + } + profile.references = profile.references.filter( + (candidate) => + candidate.kind !== parsedReference.kind || + candidate.id !== parsedReference.id + ) + return state + }) + } + + async deleteProfile(profileId: string): Promise { + const id = browserProfileIdSchema.parse(profileId) + return this.update((state) => { + const profile = state.profiles.find((candidate) => candidate.id === id) + if (!profile) { + throw new Error('Browser profile not found') + } + if (profile.references.length > 0) { + throw new Error('Referenced browser profiles cannot be deleted') + } + state.profiles = state.profiles.filter( + (candidate) => candidate.id !== id + ) + if (state.defaultProfileId === id) { + state.defaultProfileId = state.profiles[0]?.id ?? null + } + return state + }) + } +} diff --git a/src/main/capabilities/capability-diagnostics.test.ts b/src/main/capabilities/capability-diagnostics.test.ts new file mode 100644 index 0000000..4c77b92 --- /dev/null +++ b/src/main/capabilities/capability-diagnostics.test.ts @@ -0,0 +1,188 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { capabilityDiagnosticReportSchema } from '../../shared/capability-contracts' +import { + CapabilityDiagnostics, + type CapabilityDiagnosticCheck +} from './capability-diagnostics' + +const originalDiagnosticSecret = process.env.GOODBUDDY_DIAGNOSTIC_TEST_SECRET + +afterEach(() => { + if (originalDiagnosticSecret === undefined) { + delete process.env.GOODBUDDY_DIAGNOSTIC_TEST_SECRET + } else { + process.env.GOODBUDDY_DIAGNOSTIC_TEST_SECRET = originalDiagnosticSecret + } +}) + +function check( + id: string, + status: 'available' | 'degraded' | 'unavailable', + summary = `${id} result` +): CapabilityDiagnosticCheck { + return { + id, + run: async () => ({ status, summary }) + } +} + +describe('CapabilityDiagnostics', () => { + it('aggregates required checks with unavailable taking precedence', async () => { + const diagnostics = new CapabilityDiagnostics( + [ + check('browser-executable', 'degraded'), + check('managed-profile-root', 'unavailable') + ], + { now: () => new Date('2026-08-05T12:00:00.000Z') } + ) + + await expect( + diagnostics.diagnose({ + capabilityId: 'host-browser-control', + enabled: true, + platform: 'win32', + architecture: 'x64' + }) + ).resolves.toMatchObject({ + status: 'unavailable', + checkedAt: '2026-08-05T12:00:00.000Z', + checks: [ + { id: 'browser-executable', status: 'degraded' }, + { id: 'managed-profile-root', status: 'unavailable' } + ] + }) + }) + + it('reports available, degraded, disabled and unsupported states', async () => { + const available = new CapabilityDiagnostics([ + check('browser-executable', 'available'), + check('managed-profile-root', 'available') + ]) + const degraded = new CapabilityDiagnostics([ + check('browser-executable', 'available'), + check('managed-profile-root', 'degraded') + ]) + const request = { + capabilityId: 'host-browser-control' as const, + enabled: true, + platform: 'linux' as const, + architecture: 'arm64' + } + + await expect(available.diagnose(request)).resolves.toMatchObject({ + status: 'available' + }) + await expect(degraded.diagnose(request)).resolves.toMatchObject({ + status: 'degraded' + }) + await expect( + available.diagnose({ ...request, enabled: false }) + ).resolves.toMatchObject({ status: 'disabled', checks: [] }) + await expect( + available.diagnose({ + ...request, + platform: 'freebsd' + }) + ).resolves.toMatchObject({ + status: 'unavailable', + checks: [{ id: 'platform-support', status: 'unavailable' }] + }) + }) + + it('redacts environment values, credentials and sensitive paths', async () => { + process.env.GOODBUDDY_DIAGNOSTIC_TEST_SECRET = + 'environment-secret-value-12345' + const diagnostics = new CapabilityDiagnostics([ + { + id: 'browser-executable', + run: async () => ({ + status: 'degraded', + summary: + 'token=top-secret-token Bearer abc.def.ghi environment-secret-value-12345 /home/alice/private/browser', + remedy: + 'apiKey=sk-abcdefghijklmnop at C:\\Users\\Alice\\AppData\\browser' + }) + }, + check('managed-profile-root', 'available') + ]) + const report = await diagnostics.diagnose({ + capabilityId: 'host-browser-control', + enabled: true, + platform: 'linux', + architecture: 'x64' + }) + const serialized = JSON.stringify(report) + + expect(capabilityDiagnosticReportSchema.parse(report)).toEqual(report) + expect(serialized).not.toContain('top-secret-token') + expect(serialized).not.toContain('abc.def.ghi') + expect(serialized).not.toContain('environment-secret-value-12345') + expect(serialized).not.toContain('/home/alice') + expect(serialized).not.toContain('C:\\\\Users\\\\Alice') + expect(serialized).toContain('[redacted') + }) + + it('fails closed when a check times out', async () => { + const diagnostics = new CapabilityDiagnostics( + [ + { + id: 'browser-executable', + run: async () => new Promise(() => undefined) + }, + check('managed-profile-root', 'available') + ], + { timeoutMs: 10 } + ) + const report = await diagnostics.diagnose({ + capabilityId: 'host-browser-control', + enabled: true, + platform: 'win32', + architecture: 'x64' + }) + + expect(report.status).toBe('unavailable') + expect(report.checks[0]).toMatchObject({ + id: 'browser-executable', + status: 'unavailable', + summary: '诊断检查超时。' + }) + }) + + it('fails closed promptly when cancelled', async () => { + const controller = new AbortController() + const diagnostics = new CapabilityDiagnostics( + [ + { + id: 'browser-executable', + run: async () => new Promise(() => undefined) + }, + check('managed-profile-root', 'available') + ], + { timeoutMs: 1_000 } + ) + const pending = diagnostics.diagnose({ + capabilityId: 'host-browser-control', + enabled: true, + platform: 'darwin', + architecture: 'arm64', + signal: controller.signal + }) + controller.abort() + + await expect(pending).resolves.toMatchObject({ + status: 'unavailable', + checks: [ + { + id: 'browser-executable', + status: 'unavailable', + summary: '诊断检查已取消。' + }, + { + id: 'managed-profile-root', + status: 'unavailable', + summary: '诊断检查已取消。' + } + ] + }) + }) +}) diff --git a/src/main/capabilities/capability-diagnostics.ts b/src/main/capabilities/capability-diagnostics.ts new file mode 100644 index 0000000..7dbcd6b --- /dev/null +++ b/src/main/capabilities/capability-diagnostics.ts @@ -0,0 +1,262 @@ +import { z } from 'zod' +import { + capabilityDiagnosticCheckSchema, + capabilityDiagnosticCheckStatusSchema, + capabilityDiagnosticReportSchema, + type CapabilityDiagnosticReport, + type ComputerCapabilityId +} from '../../shared/capability-contracts' +export type { CapabilityDiagnosticReport } from '../../shared/capability-contracts' +import { redactSensitiveText } from '../agent/approval-summary' +import { + getComputerCapability, + isComputerCapabilitySupported, + type ComputerCapabilityImplementationKind +} from './computer-capability-catalog' + +const MAX_SUMMARY_LENGTH = 240 +const MAX_REMEDY_LENGTH = 400 +const diagnosticCheckIdSchema = capabilityDiagnosticCheckSchema.shape.id + +export const diagnosticCheckResultSchema = z + .object({ + status: capabilityDiagnosticCheckStatusSchema, + summary: z.string().min(1).max(4_096), + remedy: z.string().max(4_096).optional() + }) + .strict() + +export type DiagnosticCheckResult = z.infer< + typeof diagnosticCheckResultSchema +> + +export type CapabilityDiagnosticCheck = Readonly<{ + id: string + run: (signal: AbortSignal) => Promise +}> + +export type CapabilityDiagnosticRequest = Readonly<{ + capabilityId: ComputerCapabilityId + enabled: boolean + platform: NodeJS.Platform + architecture: string + availableImplementationKinds?: ReadonlySet + signal?: AbortSignal +}> + +function redactText(value: string, maximumLength: number): string { + let redacted = redactSensitiveText(value) + const environmentValues = Object.values(process.env) + .filter((candidate): candidate is string => Boolean(candidate?.length && candidate.length >= 8)) + .sort((left, right) => right.length - left.length) + for (const environmentValue of environmentValues) { + redacted = redacted.split(environmentValue).join('[redacted-env]') + } + redacted = redacted + .replace( + /(?:[a-zA-Z]:\\(?:Users|Documents and Settings)\\|\/(?:Users|home|root|private|var\/folders|tmp)\/)[^\s"'<>]+/gu, + '[redacted-path]' + ) + .replace(/[\r\n\t]+/gu, ' ') + .trim() + return redacted.slice(0, maximumLength) +} + +function unavailableCheck( + id: string, + summary: string, + remedy?: string +): CapabilityDiagnosticReport['checks'][number] { + return { + id, + status: 'unavailable', + summary: redactText(summary, MAX_SUMMARY_LENGTH), + ...(remedy + ? { remedy: redactText(remedy, MAX_REMEDY_LENGTH) } + : {}) + } +} + +export class CapabilityDiagnostics { + private readonly checks: ReadonlyMap + private readonly timeoutMs: number + private readonly now: () => Date + + constructor( + checks: readonly CapabilityDiagnosticCheck[], + options: Readonly<{ timeoutMs?: number; now?: () => Date }> = {} + ) { + const mapped = new Map() + for (const check of checks) { + const id = diagnosticCheckIdSchema.parse(check.id) + if (mapped.has(id)) { + throw new Error(`Duplicate capability diagnostic check: ${id}`) + } + mapped.set(id, Object.freeze({ id, run: check.run })) + } + this.checks = mapped + this.timeoutMs = z + .number() + .int() + .min(10) + .max(30_000) + .parse(options.timeoutMs ?? 5_000) + this.now = options.now ?? (() => new Date()) + } + + private checkedAt(): string { + const value = this.now() + if (Number.isNaN(value.getTime())) { + throw new Error('Diagnostic clock returned an invalid date') + } + return value.toISOString() + } + + private async runCheck( + id: string, + parentSignal?: AbortSignal + ): Promise { + const check = this.checks.get(id) + if (!check) { + return unavailableCheck( + id, + '缺少必需的诊断检查。', + '请重新安装或修复此能力的受管组件。' + ) + } + if (parentSignal?.aborted) { + return unavailableCheck(id, '诊断检查已取消。', '请重试诊断。') + } + + const controller = new AbortController() + let timedOut = false + let cancelled = false + let timeoutHandle: ReturnType | undefined + const timeoutResult = new Promise<'timeout'>((resolveTimeout) => { + timeoutHandle = setTimeout(() => { + timedOut = true + controller.abort(new Error('Diagnostic check timed out')) + resolveTimeout('timeout') + }, this.timeoutMs) + }) + let resolveCancellation: (() => void) | undefined + const cancellation = new Promise<'cancelled'>((resolveCancelled) => { + resolveCancellation = () => resolveCancelled('cancelled') + }) + const abortFromParent = (): void => { + cancelled = true + controller.abort(parentSignal?.reason) + resolveCancellation?.() + } + parentSignal?.addEventListener('abort', abortFromParent, { once: true }) + const operation = Promise.resolve() + .then(() => check.run(controller.signal)) + .then( + (result) => ({ kind: 'result' as const, result }), + (error: unknown) => ({ kind: 'error' as const, error }) + ) + + try { + const outcome = await Promise.race([ + operation, + timeoutResult.then((kind) => ({ kind })), + cancellation.then((kind) => ({ kind })) + ]) + if (outcome.kind === 'timeout' || timedOut) { + return unavailableCheck( + id, + '诊断检查超时。', + '请确认受管组件可响应后重试。' + ) + } + if (outcome.kind === 'cancelled' || cancelled) { + return unavailableCheck(id, '诊断检查已取消。', '请重试诊断。') + } + if (outcome.kind === 'error') { + const message = + outcome.error instanceof Error + ? outcome.error.message + : '未知诊断错误' + return unavailableCheck( + id, + `诊断检查失败：${message}`, + '请修复本地环境后重试。' + ) + } + try { + const result = diagnosticCheckResultSchema.parse(outcome.result) + return { + id, + status: result.status, + summary: redactText(result.summary, MAX_SUMMARY_LENGTH), + ...(result.remedy + ? { remedy: redactText(result.remedy, MAX_REMEDY_LENGTH) } + : {}) + } + } catch { + return unavailableCheck( + id, + '诊断检查返回了无效结果。', + '请重新安装或修复此能力的受管组件。' + ) + } + } finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle) + } + parentSignal?.removeEventListener('abort', abortFromParent) + } + } + + async diagnose( + request: CapabilityDiagnosticRequest + ): Promise { + const capability = getComputerCapability(request.capabilityId) + const checkedAt = this.checkedAt() + if (!request.enabled) { + return capabilityDiagnosticReportSchema.parse({ + capabilityId: capability.id, + status: 'disabled', + checkedAt, + checks: [] + }) + } + if ( + !isComputerCapabilitySupported( + capability, + request.platform, + request.architecture, + request.availableImplementationKinds + ) + ) { + return capabilityDiagnosticReportSchema.parse({ + capabilityId: capability.id, + status: 'unavailable', + checkedAt, + checks: [ + unavailableCheck( + 'platform-support', + '当前操作系统或处理器架构不受支持。', + '请在能力目录列出的受支持平台上使用此能力。' + ) + ] + }) + } + + const checks = [] + for (const id of capability.requiredDiagnostics) { + checks.push(await this.runCheck(id, request.signal)) + } + const status = checks.some((check) => check.status === 'unavailable') + ? 'unavailable' + : checks.some((check) => check.status === 'degraded') + ? 'degraded' + : 'available' + return capabilityDiagnosticReportSchema.parse({ + capabilityId: capability.id, + status, + checkedAt, + checks + }) + } +} diff --git a/src/main/capabilities/capability-service.test.ts b/src/main/capabilities/capability-service.test.ts index 90589d4..52f1794 100644 --- a/src/main/capabilities/capability-service.test.ts +++ b/src/main/capabilities/capability-service.test.ts @@ -1,11 +1,17 @@ import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { CapabilityService, - type CapabilityCipher + type CapabilityCipher, + type CapabilityServiceOptions } from './capability-service' +import { + BrowserProfileService, + MemoryBrowserProfileStore +} from './browser-profile-service' +import { CapabilityDiagnostics } from './capability-diagnostics' const temporaryDirectories: string[] = [] @@ -15,6 +21,33 @@ const cipher: CapabilityCipher = { decrypt: (value) => value.toString().replace(/^encrypted:/u, '') } +class FailingBrowserProfileService extends BrowserProfileService { + failNextAddAfterSave = false + failNextRemoveAfterSave = false + + override async addReference( + ...args: Parameters + ): ReturnType { + const result = await super.addReference(...args) + if (this.failNextAddAfterSave) { + this.failNextAddAfterSave = false + throw new Error('Injected add reference failure') + } + return result + } + + override async removeReference( + ...args: Parameters + ): ReturnType { + const result = await super.removeReference(...args) + if (this.failNextRemoveAfterSave) { + this.failNextRemoveAfterSave = false + throw new Error('Injected remove reference failure') + } + return result + } +} + async function writeSkill( root: string, id: string, @@ -42,7 +75,16 @@ async function writeSkill( ) } -async function createService(): Promise<{ +async function createService( + options: CapabilityServiceOptions = { + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles: new BrowserProfileService( + new MemoryBrowserProfileStore() + ) + } +): Promise<{ directory: string filePath: string builtinRoot: string @@ -64,12 +106,14 @@ async function createService(): Promise<{ filePath, builtinRoot, importedRoot, - cipher + cipher, + options ) } } afterEach(async () => { + delete process.env.GOODBUDDY_CAPABILITY_SERVICE_SECRET await Promise.all( temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }) @@ -78,6 +122,77 @@ afterEach(async () => { }) describe('CapabilityService', () => { + it('memoizes concurrent loads and retries after a failed load', async () => { + const initialStore = { + load: vi.fn(async () => undefined), + save: vi.fn(async () => undefined) + } + const created = await createService({ + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles: new BrowserProfileService(initialStore) + }) + + await Promise.all([ + created.service.getComputerCapabilityStatus('host-browser-control'), + created.service.getComputerCapabilityStatus('host-browser-control'), + created.service.getComputerCapabilityStatus('host-browser-control') + ]) + expect(initialStore.load).toHaveBeenCalledOnce() + + let profileLoads = 0 + const browserProfiles = new BrowserProfileService({ + load: vi.fn(async () => { + profileLoads += 1 + if (profileLoads === 1) { + throw new Error('Injected profile load failure') + } + return undefined + }), + save: vi.fn(async () => undefined) + }) + const retrying = new CapabilityService( + created.filePath, + created.builtinRoot, + created.importedRoot, + cipher, + { + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles + } + ) + + await expect( + Promise.all([ + retrying.getComputerCapabilityStatus('host-browser-control'), + retrying.getComputerCapabilityStatus('host-browser-control') + ]) + ).rejects.toThrow('Injected profile load failure') + await expect( + retrying.getComputerCapabilityStatus('host-browser-control') + ).resolves.toEqual({ enabled: false, supported: true }) + expect(profileLoads).toBe(2) + }) + + it('reports safe enabled and supported capability status', async () => { + const { service } = await createService() + + await expect( + service.getComputerCapabilityStatus('host-browser-control') + ).resolves.toEqual({ enabled: false, supported: true }) + + await service.setComputerCapabilityEnabled( + 'host-browser-control', + true + ) + await expect( + service.getComputerCapabilityStatus('host-browser-control') + ).resolves.toEqual({ enabled: true, supported: true }) + }) + it('discovers built-in skills and persists enablement and assignments', async () => { const { filePath, builtinRoot, importedRoot, service } = await createService() @@ -267,4 +382,375 @@ describe('CapabilityService', () => { await expect(service.getResolvedMcpServers('opencode')).resolves.toEqual([]) await expect(service.getResolvedMcpServers('model')).resolves.toHaveLength(1) }) + + it('migrates v1 to v2 without losing skills, MCP configuration, or encrypted secrets', async () => { + const { filePath, builtinRoot, importedRoot } = await createService() + const credential = Buffer.from( + 'encrypted:{"version":1,"serverId":"d2ef774b-146c-4467-a909-6feb112a9c2c","secret":"preserved-secret"}' + ).toString('base64') + await writeFile( + filePath, + JSON.stringify({ + version: 1, + skills: { + 'document-writing': { + enabled: false, + assignments: ['model'] + } + }, + mcpServers: [ + { + id: 'd2ef774b-146c-4467-a909-6feb112a9c2c', + name: 'Preserved MCP', + description: 'migration', + enabled: true, + assignments: ['model'], + credential: { + formatVersion: 1, + scheme: 'electron-safe-storage', + ciphertextBase64: credential + }, + transport: 'http', + url: 'https://mcp.example.com/mcp' + } + ] + }), + 'utf8' + ) + const service = new CapabilityService( + filePath, + builtinRoot, + importedRoot, + cipher, + { + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles: new BrowserProfileService( + new MemoryBrowserProfileStore() + ) + } + ) + + await expect(service.getSnapshot()).resolves.toMatchObject({ + skills: [ + expect.objectContaining({ + id: 'document-writing', + enabled: false, + assignments: ['model'] + }) + ], + mcpServers: [ + expect.objectContaining({ + name: 'Preserved MCP', + secretConfigured: true + }) + ], + computerCapabilities: [ + expect.objectContaining({ + id: 'host-browser-control', + enabled: false + }), + expect.objectContaining({ + id: 'linux-desktop-control', + enabled: false + }) + ] + }) + const persisted = await readFile(filePath, 'utf8') + expect(persisted).toContain('"version": 2') + expect(persisted).toContain(credential) + expect(persisted).not.toContain('preserved-secret') + }) + + it('gates enablement on the supported platform and architecture', async () => { + const { service } = await createService({ + platform: 'darwin', + architecture: 'arm64', + electronTarget: true, + browserProfiles: new BrowserProfileService( + new MemoryBrowserProfileStore() + ) + }) + + await expect( + service.setComputerCapabilityEnabled( + 'linux-desktop-control', + true + ) + ).rejects.toThrow('不支持') + const snapshot = await service.setComputerCapabilityEnabled( + 'host-browser-control', + true + ) + expect(snapshot.computerCapabilities).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'host-browser-control', + enabled: true + }) + ]) + ) + }) + + it('maintains browser profile references and rejects unknown profiles', async () => { + const { service } = await createService() + await expect( + service.setComputerCapabilityBrowserProfile( + 'host-browser-control', + 'b7f29e4c-1c4a-4aa0-ac58-5165451dde07' + ) + ).rejects.toThrow('不存在') + + const created = await service.createBrowserProfile('隔离工作配置') + const profileId = created.browserProfiles?.profiles[0]?.id + if (!profileId) { + throw new Error('Expected browser profile') + } + await service.setComputerCapabilityBrowserProfile( + 'host-browser-control', + profileId + ) + await expect(service.removeBrowserProfile(profileId)).rejects.toThrow( + 'Referenced' + ) + await service.setComputerCapabilityBrowserProfile( + 'host-browser-control', + null + ) + await expect(service.removeBrowserProfile(profileId)).resolves.toMatchObject( + { + browserProfiles: { profiles: [], defaultProfileId: null } + } + ) + }) + + it('compensates browser profile references when add or capability persistence fails', async () => { + const addProfiles = new FailingBrowserProfileService( + new MemoryBrowserProfileStore() + ) + const addCase = await createService({ + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles: addProfiles + }) + const addCreated = await addCase.service.createBrowserProfile( + '添加失败配置' + ) + const addProfileId = addCreated.browserProfiles?.profiles[0]?.id + if (!addProfileId) { + throw new Error('Expected add-failure browser profile') + } + addProfiles.failNextAddAfterSave = true + + await expect( + addCase.service.setComputerCapabilityBrowserProfile( + 'host-browser-control', + addProfileId + ) + ).rejects.toThrow('Injected add reference failure') + await expect( + addCase.service.removeBrowserProfile(addProfileId) + ).resolves.toMatchObject({ + browserProfiles: { profiles: [], defaultProfileId: null } + }) + + const persistProfiles = new BrowserProfileService( + new MemoryBrowserProfileStore() + ) + const persistCase = await createService({ + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles: persistProfiles + }) + const persistCreated = + await persistCase.service.createBrowserProfile('保存失败配置') + const persistProfileId = + persistCreated.browserProfiles?.profiles[0]?.id + if (!persistProfileId) { + throw new Error('Expected persistence-failure browser profile') + } + await mkdir(persistCase.filePath) + + await expect( + persistCase.service.setComputerCapabilityBrowserProfile( + 'host-browser-control', + persistProfileId + ) + ).rejects.toThrow() + await expect( + persistCase.service.removeBrowserProfile(persistProfileId) + ).resolves.toMatchObject({ + browserProfiles: { profiles: [], defaultProfileId: null } + }) + }) + + it('rolls back capability and profile stores when old-reference removal fails', async () => { + const browserProfiles = new FailingBrowserProfileService( + new MemoryBrowserProfileStore() + ) + const { service } = await createService({ + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles + }) + const first = await service.createBrowserProfile('原配置') + const firstId = first.browserProfiles?.profiles[0]?.id + const second = await service.createBrowserProfile('新配置') + const secondId = second.browserProfiles?.profiles[1]?.id + if (!firstId || !secondId) { + throw new Error('Expected two browser profiles') + } + await service.setComputerCapabilityBrowserProfile( + 'host-browser-control', + firstId + ) + browserProfiles.failNextRemoveAfterSave = true + + await expect( + service.setComputerCapabilityBrowserProfile( + 'host-browser-control', + secondId + ) + ).rejects.toThrow('Injected remove reference failure') + await expect(service.getSnapshot()).resolves.toMatchObject({ + computerCapabilities: expect.arrayContaining([ + expect.objectContaining({ + id: 'host-browser-control', + browserProfileId: firstId + }) + ]) + }) + await expect(service.removeBrowserProfile(secondId)).resolves.toBeDefined() + await expect(service.removeBrowserProfile(firstId)).rejects.toThrow( + 'Referenced' + ) + }) + + it('redacts injected diagnostics and never claims browser availability outside Electron', async () => { + process.env.GOODBUDDY_CAPABILITY_SERVICE_SECRET = + 'service-diagnostic-secret-value' + const diagnostics = new CapabilityDiagnostics([ + { + id: 'browser-executable', + run: async () => ({ + status: 'available', + summary: + 'token=visible-token service-diagnostic-secret-value C:\\Users\\Alice\\browser' + }) + }, + { + id: 'managed-profile-root', + run: async () => ({ + status: 'available', + summary: 'managed storage ready' + }) + } + ]) + const { service } = await createService({ + platform: 'win32', + architecture: 'x64', + electronTarget: true, + browserProfiles: new BrowserProfileService( + new MemoryBrowserProfileStore() + ), + diagnostics + }) + await service.setComputerCapabilityEnabled('host-browser-control', true) + const report = await service.diagnoseComputerCapability( + 'host-browser-control' + ) + expect(report.status).toBe('available') + expect(JSON.stringify(report)).not.toContain('visible-token') + expect(JSON.stringify(report)).not.toContain( + 'service-diagnostic-secret-value' + ) + + const outsideElectron = await createService({ + platform: 'win32', + architecture: 'x64', + electronTarget: false, + browserProfiles: new BrowserProfileService( + new MemoryBrowserProfileStore() + ), + diagnostics + }) + await expect( + outsideElectron.service.setComputerCapabilityEnabled( + 'host-browser-control', + true + ) + ).rejects.toThrow('诊断不可用') + await expect( + outsideElectron.service.getComputerCapabilityStatus( + 'host-browser-control' + ) + ).resolves.toEqual({ enabled: false, supported: true }) + delete process.env.GOODBUDDY_CAPABILITY_SERVICE_SECRET + }) + + it('keeps Linux desktop unavailable without a registered native adapter', async () => { + const { service } = await createService({ + platform: 'linux', + architecture: 'x64', + electronTarget: true, + browserProfiles: new BrowserProfileService( + new MemoryBrowserProfileStore() + ) + }) + + await expect( + service.setComputerCapabilityEnabled( + 'linux-desktop-control', + true + ) + ).rejects.toThrow('不支持') + await expect( + service.getComputerCapabilityStatus('linux-desktop-control') + ).resolves.toEqual({ enabled: false, supported: false }) + }) + + it('allows Linux desktop enablement with available injected diagnostics', async () => { + const diagnostics = new CapabilityDiagnostics( + ['linux-session', 'desktop-driver', 'desktop-permissions'].map( + (id) => ({ + id, + run: async () => ({ + status: 'available' as const, + summary: `${id} ready` + }) + }) + ) + ) + const { service } = await createService({ + platform: 'linux', + architecture: 'arm64', + electronTarget: true, + browserProfiles: new BrowserProfileService( + new MemoryBrowserProfileStore() + ), + diagnostics, + availableComputerCapabilityImplementations: [ + 'managed-browser-driver', + 'managed-linux-desktop-driver' + ] + }) + + await expect( + service.setComputerCapabilityEnabled( + 'linux-desktop-control', + true + ) + ).resolves.toMatchObject({ + computerCapabilities: expect.arrayContaining([ + expect.objectContaining({ + id: 'linux-desktop-control', + enabled: true + }) + ]) + }) + }) }) diff --git a/src/main/capabilities/capability-service.ts b/src/main/capabilities/capability-service.ts index 3df9f82..00e554b 100644 --- a/src/main/capabilities/capability-service.ts +++ b/src/main/capabilities/capability-service.ts @@ -14,19 +14,43 @@ import { basename, dirname, join } from 'node:path' import { parse as parseYaml } from 'yaml' import { z } from 'zod' import { + browserProfileIdSchema, + browserProfileNameSchema, + browserProfilesSummarySchema, + capabilityDiagnosticReportSchema, capabilityAssignmentsSchema, + computerCapabilityConfigSummarySchema, + computerCapabilityIdSchema, mcpServerIdSchema, mcpServerInputSchema, mcpServerSummarySchema, skillIdSchema, skillSummarySchema, type CapabilityAssignments, + type CapabilityDiagnosticReport, type CapabilitySnapshot, + type BrowserProfilesSummary, + type ComputerCapabilityId, type McpServerInput, type McpServerSummary, type RuntimeTarget, type SkillSummary } from '../../shared/capability-contracts' +import { + BrowserProfileService, + FileBrowserProfileStore, + type BrowserProfileState +} from './browser-profile-service' +import { + CapabilityDiagnostics, + type CapabilityDiagnosticCheck +} from './capability-diagnostics' +import { + computerCapabilityCatalog, + getComputerCapability, + isComputerCapabilitySupported, + type ComputerCapabilityImplementationKind +} from './computer-capability-catalog' const MAX_SKILL_FILE_BYTES = 2 * 1024 * 1024 const MAX_SKILL_PACKAGE_BYTES = 10 * 1024 * 1024 @@ -92,7 +116,7 @@ const storedMcpServerSchema = z.discriminatedUnion('transport', [ .strict() ]) -const storedCapabilitiesSchema = z +const storedCapabilitiesV1Schema = z .object({ version: z.literal(1), skills: z.record(skillIdSchema, skillStateSchema), @@ -100,6 +124,28 @@ const storedCapabilitiesSchema = z }) .strict() +const computerCapabilityStateSchema = z + .object({ + enabled: z.boolean(), + browserProfileId: browserProfileIdSchema.nullable() + }) + .strict() + +const storedCapabilitiesSchema = z + .object({ + version: z.literal(2), + skills: z.record(skillIdSchema, skillStateSchema), + mcpServers: z.array(storedMcpServerSchema).max(64), + computerCapabilities: z + .object({ + 'host-browser-control': computerCapabilityStateSchema, + 'linux-desktop-control': computerCapabilityStateSchema + }) + .strict() + }) + .strict() + +type StoredCapabilitiesV1 = z.infer type StoredCapabilities = z.infer type StoredMcpServer = z.infer @@ -121,6 +167,37 @@ export type ResolvedMcpServer = McpServerSummary & { secret?: string } +export type CapabilityServiceOptions = Readonly<{ + platform?: NodeJS.Platform + architecture?: string + electronTarget?: boolean + browserProfiles?: BrowserProfileService + diagnostics?: CapabilityDiagnostics + availableComputerCapabilityImplementations?: readonly ComputerCapabilityImplementationKind[] +}> + +function defaultComputerCapabilityStates(): StoredCapabilities['computerCapabilities'] { + return { + 'host-browser-control': { + enabled: false, + browserProfileId: null + }, + 'linux-desktop-control': { + enabled: false, + browserProfileId: null + } + } +} + +function emptyStoredCapabilities(): StoredCapabilities { + return { + version: 2, + skills: {}, + mcpServers: [], + computerCapabilities: defaultComputerCapabilityStates() + } +} + function defaultSkillState(): z.infer { return { enabled: true, @@ -235,24 +312,109 @@ async function copySkillPackage( export class CapabilityService { private state?: StoredCapabilities + private loadPromise?: Promise private updateQueue: Promise = Promise.resolve() + private readonly platform: NodeJS.Platform + private readonly architecture: string + private readonly electronTarget: boolean + private readonly browserProfiles: BrowserProfileService + private readonly diagnostics: CapabilityDiagnostics + private readonly availableComputerCapabilityImplementations: ReadonlySet constructor( private readonly filePath: string, private readonly builtinSkillsRoot: string, private readonly importedSkillsRoot: string, - private readonly cipher: CapabilityCipher - ) {} - - private async load(): Promise { - if (this.state) { - return this.state - } - let loaded: StoredCapabilities - try { - loaded = storedCapabilitiesSchema.parse( - JSON.parse(await readFile(this.filePath, 'utf8')) + private readonly cipher: CapabilityCipher, + options: CapabilityServiceOptions = {} + ) { + this.platform = options.platform ?? process.platform + this.architecture = options.architecture ?? process.arch + this.electronTarget = + options.electronTarget ?? Boolean(process.versions.electron) + this.availableComputerCapabilityImplementations = new Set( + options.availableComputerCapabilityImplementations ?? [ + 'managed-browser-driver' + ] + ) + this.browserProfiles = + options.browserProfiles ?? + new BrowserProfileService( + new FileBrowserProfileStore( + join(dirname(this.filePath), 'browser-profiles') + ) ) + const checks: CapabilityDiagnosticCheck[] = [ + { + id: 'browser-executable', + run: async () => + this.electronTarget + ? { + status: 'available', + summary: 'GoodBuddy 的受管 Electron 浏览器核心可用。' + } + : { + status: 'unavailable', + summary: '当前进程不是受支持的 Electron 桌面目标。', + remedy: '请从 GoodBuddy 桌面应用运行此诊断。' + } + }, + { + id: 'managed-profile-root', + run: async () => { + await this.browserProfiles.getSnapshot() + return { + status: 'available', + summary: '隔离的托管浏览器配置存储可用。' + } + } + } + ] + this.diagnostics = + options.diagnostics ?? new CapabilityDiagnostics(checks) + } + + private load(): Promise { + if (this.state) { + return Promise.resolve(this.state) + } + if (this.loadPromise) { + return this.loadPromise + } + const pending = this.loadUncached() + const tracked = pending.catch((error: unknown) => { + if (this.loadPromise === tracked) { + this.loadPromise = undefined + this.state = undefined + } + throw error + }) + this.loadPromise = tracked + return tracked + } + + private async loadUncached(): Promise { + let loaded: StoredCapabilities + let shouldPersist = false + try { + const raw = JSON.parse(await readFile(this.filePath, 'utf8')) as unknown + const version = z + .object({ version: z.union([z.literal(1), z.literal(2)]) }) + .passthrough() + .parse(raw).version + if (version === 1) { + const legacy: StoredCapabilitiesV1 = + storedCapabilitiesV1Schema.parse(raw) + loaded = { + version: 2, + skills: legacy.skills, + mcpServers: legacy.mcpServers, + computerCapabilities: defaultComputerCapabilityStates() + } + shouldPersist = true + } else { + loaded = storedCapabilitiesSchema.parse(raw) + } } catch (error) { if ( error && @@ -260,13 +422,13 @@ export class CapabilityService { 'code' in error && error.code === 'ENOENT' ) { - loaded = { version: 1, skills: {}, mcpServers: [] } + loaded = emptyStoredCapabilities() } else { await rename( this.filePath, `${this.filePath}.corrupt-${Date.now()}` ).catch(() => undefined) - loaded = { version: 1, skills: {}, mcpServers: [] } + loaded = emptyStoredCapabilities() } } const migrateMcpAssignments = loaded.mcpServers.some((server) => @@ -284,12 +446,30 @@ export class CapabilityService { } : loaded this.state = storedCapabilitiesSchema.parse(migrated) - if (migrateMcpAssignments) { + await this.validateBrowserProfileReferences(this.state) + if (shouldPersist || migrateMcpAssignments) { await this.persist(this.state) } return this.state } + private async validateBrowserProfileReferences( + state: StoredCapabilities + ): Promise { + const profiles = await this.browserProfiles.getSnapshot() + const profileIds = new Set(profiles.profiles.map((profile) => profile.id)) + for (const capability of computerCapabilityCatalog) { + const profileId = + state.computerCapabilities[capability.id].browserProfileId + if (profileId && !profileIds.has(profileId)) { + throw new Error('电脑控制能力引用了不存在的浏览器配置') + } + if (capability.id !== 'host-browser-control' && profileId) { + throw new Error('此电脑控制能力不支持浏览器配置') + } + } + } + private queue(operation: () => Promise): Promise { const result = this.updateQueue.then(operation) this.updateQueue = result.then( @@ -339,9 +519,10 @@ export class CapabilityService { } async getSnapshot(): Promise { - const [state, catalog] = await Promise.all([ + const [state, catalog, browserProfileState] = await Promise.all([ this.load(), - this.getSkillCatalog() + this.getSkillCatalog(), + this.browserProfiles.getSnapshot() ]) return { skills: catalog @@ -358,10 +539,284 @@ export class CapabilityService { ), mcpServers: state.mcpServers.map((server) => this.toMcpSummary(server) + ), + computerCapabilities: computerCapabilityCatalog.map((capability) => + computerCapabilityConfigSummarySchema.parse({ + id: capability.id, + name: capability.name, + description: capability.description, + enabled: state.computerCapabilities[capability.id].enabled, + supported: isComputerCapabilitySupported( + capability, + this.platform, + this.architecture, + this.availableComputerCapabilityImplementations + ), + browserProfileId: + state.computerCapabilities[capability.id].browserProfileId, + riskSummary: capability.riskSummary + }) + ), + browserProfiles: this.toBrowserProfilesSummary(browserProfileState) + } + } + + async getComputerCapabilityStatus( + capabilityId: ComputerCapabilityId + ): Promise<{ enabled: boolean; supported: boolean }> { + const id = computerCapabilityIdSchema.parse(capabilityId) + const capability = getComputerCapability(id) + const state = await this.load() + return { + enabled: state.computerCapabilities[id].enabled, + supported: isComputerCapabilitySupported( + capability, + this.platform, + this.architecture, + this.availableComputerCapabilityImplementations ) } } + private toBrowserProfilesSummary( + state: BrowserProfileState + ): BrowserProfilesSummary { + return browserProfilesSummarySchema.parse({ + profiles: state.profiles.map((profile) => ({ + id: profile.id, + name: profile.name, + mode: profile.mode + })), + defaultProfileId: state.defaultProfileId + }) + } + + setComputerCapabilityEnabled( + capabilityId: ComputerCapabilityId, + enabled: boolean + ): Promise { + return this.queue(async () => { + const id = computerCapabilityIdSchema.parse(capabilityId) + const capability = getComputerCapability(id) + if ( + enabled && + !isComputerCapabilitySupported( + capability, + this.platform, + this.architecture, + this.availableComputerCapabilityImplementations + ) + ) { + throw new Error('当前操作系统或处理器架构不支持此能力') + } + if (enabled) { + const report = await this.diagnoseComputerCapabilityState(id, true) + if (report.status === 'unavailable') { + throw new Error('能力诊断不可用，未启用此能力') + } + } + const state = await this.load() + await this.persist({ + ...state, + computerCapabilities: { + ...state.computerCapabilities, + [id]: { + ...state.computerCapabilities[id], + enabled + } + } + }) + return this.getSnapshot() + }) + } + + setComputerCapabilityBrowserProfile( + capabilityId: ComputerCapabilityId, + browserProfileId: string | null + ): Promise { + return this.queue(async () => { + const id = computerCapabilityIdSchema.parse(capabilityId) + const profileId = browserProfileIdSchema.nullable().parse( + browserProfileId + ) + if (id !== 'host-browser-control' && profileId) { + throw new Error('此电脑控制能力不支持浏览器配置') + } + const profiles = await this.browserProfiles.getSnapshot() + if ( + profileId && + !profiles.profiles.some((profile) => profile.id === profileId) + ) { + throw new Error('浏览器配置不存在') + } + const state = await this.load() + const previousProfileId = + state.computerCapabilities[id].browserProfileId + if (profileId === previousProfileId) { + return this.getSnapshot() + } + const reference = { kind: 'capability' as const, id } + if (profileId) { + try { + await this.browserProfiles.addReference(profileId, reference) + } catch (error) { + try { + await this.browserProfiles.removeReference(profileId, reference) + } catch (compensationError) { + throw new AggregateError( + [error, compensationError], + '浏览器配置引用添加失败，且补偿清理未完成', + { cause: compensationError } + ) + } + throw error + } + } + const nextState: StoredCapabilities = { + ...state, + computerCapabilities: { + ...state.computerCapabilities, + [id]: { + ...state.computerCapabilities[id], + browserProfileId: profileId + } + } + } + try { + await this.persist(nextState) + } catch (error) { + if (profileId) { + try { + await this.browserProfiles.removeReference(profileId, reference) + } catch (compensationError) { + throw new AggregateError( + [error, compensationError], + '电脑控制配置保存失败，且新增引用补偿清理未完成', + { cause: compensationError } + ) + } + } + throw error + } + if (previousProfileId) { + try { + await this.browserProfiles.removeReference( + previousProfileId, + reference + ) + } catch (error) { + try { + await this.browserProfiles.addReference( + previousProfileId, + reference + ) + await this.persist(state) + if (profileId) { + await this.browserProfiles.removeReference( + profileId, + reference + ) + } + } catch (compensationError) { + throw new AggregateError( + [error, compensationError], + '旧浏览器配置引用移除失败，且回滚未完成', + { cause: compensationError } + ) + } + throw error + } + } + return this.getSnapshot() + }) + } + + async diagnoseComputerCapability( + capabilityId: ComputerCapabilityId + ): Promise { + const id = computerCapabilityIdSchema.parse(capabilityId) + const state = await this.load() + return this.diagnoseComputerCapabilityState( + id, + state.computerCapabilities[id].enabled + ) + } + + private async diagnoseComputerCapabilityState( + id: ComputerCapabilityId, + enabled: boolean + ): Promise { + if ( + id === 'host-browser-control' && + enabled && + !this.electronTarget + ) { + return capabilityDiagnosticReportSchema.parse({ + capabilityId: id, + status: 'unavailable', + checkedAt: new Date().toISOString(), + checks: [ + { + id: 'electron-target', + status: 'unavailable', + summary: '当前进程不是受支持的 Electron 桌面目标。', + remedy: '请从 GoodBuddy 桌面应用运行此诊断。' + } + ] + }) + } + return capabilityDiagnosticReportSchema.parse( + await this.diagnostics.diagnose({ + capabilityId: id, + enabled, + platform: this.platform, + architecture: this.architecture, + availableImplementationKinds: + this.availableComputerCapabilityImplementations + }) + ) + } + + createBrowserProfile(name: string): Promise { + return this.queue(async () => { + await this.browserProfiles.createProfile( + browserProfileNameSchema.parse(name) + ) + return this.getSnapshot() + }) + } + + renameBrowserProfile( + profileId: string, + name: string + ): Promise { + return this.queue(async () => { + await this.browserProfiles.renameProfile( + browserProfileIdSchema.parse(profileId), + browserProfileNameSchema.parse(name) + ) + return this.getSnapshot() + }) + } + + setDefaultBrowserProfile(profileId: string): Promise { + return this.queue(async () => { + await this.browserProfiles.setDefaultProfile( + browserProfileIdSchema.parse(profileId) + ) + return this.getSnapshot() + }) + } + + removeBrowserProfile(profileId: string): Promise { + return this.queue(async () => { + await this.browserProfiles.deleteProfile( + browserProfileIdSchema.parse(profileId) + ) + return this.getSnapshot() + }) + } + importSkill(sourcePath: string): Promise { return this.queue(async () => { const canonicalSource = await realpath(sourcePath) diff --git a/src/main/capabilities/computer-capability-catalog.test.ts b/src/main/capabilities/computer-capability-catalog.test.ts new file mode 100644 index 0000000..b7fe3cb --- /dev/null +++ b/src/main/capabilities/computer-capability-catalog.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import { + computerCapabilityCatalog, + getComputerCapability, + isComputerCapabilitySupported +} from './computer-capability-catalog' + +describe('computer capability catalog', () => { + it('defines immutable, disabled curated capabilities without launch overrides', () => { + expect(computerCapabilityCatalog.map(({ id }) => id)).toEqual([ + 'host-browser-control', + 'linux-desktop-control' + ]) + for (const capability of computerCapabilityCatalog) { + expect(capability.enabledByDefault).toBe(false) + expect(capability.requiredDiagnostics.length).toBeGreaterThan(0) + expect(capability).not.toHaveProperty('executable') + expect(capability).not.toHaveProperty('args') + expect(capability).not.toHaveProperty('env') + } + expect(Object.isFrozen(computerCapabilityCatalog)).toBe(true) + }) + + it('rejects unsupported platform and architecture combinations', () => { + const browser = getComputerCapability('host-browser-control') + const desktop = getComputerCapability('linux-desktop-control') + + expect(isComputerCapabilitySupported(browser, 'win32', 'x64')).toBe(true) + expect(isComputerCapabilitySupported(browser, 'linux', 'arm64')).toBe(true) + expect(isComputerCapabilitySupported(browser, 'win32', 'ia32')).toBe(false) + expect(isComputerCapabilitySupported(browser, 'freebsd', 'x64')).toBe(false) + expect(isComputerCapabilitySupported(desktop, 'darwin', 'arm64')).toBe( + false + ) + expect(isComputerCapabilitySupported(desktop, 'linux', 'x64')).toBe(false) + expect( + isComputerCapabilitySupported( + desktop, + 'linux', + 'x64', + new Set(['managed-linux-desktop-driver']) + ) + ).toBe(true) + expect(desktop.description).toContain('技术预览') + expect(desktop.riskSummary).toContain('尚未') + }) +}) diff --git a/src/main/capabilities/computer-capability-catalog.ts b/src/main/capabilities/computer-capability-catalog.ts new file mode 100644 index 0000000..91eb4a7 --- /dev/null +++ b/src/main/capabilities/computer-capability-catalog.ts @@ -0,0 +1,111 @@ +import { + computerCapabilityIdSchema, + type ComputerCapabilityId +} from '../../shared/capability-contracts' + +export const computerCapabilityIds = computerCapabilityIdSchema.options +export type { ComputerCapabilityId } +export type ComputerCapabilityPlatform = 'win32' | 'darwin' | 'linux' +export type ComputerCapabilityArchitecture = 'x64' | 'arm64' +export type ComputerCapabilityImplementationKind = + | 'managed-browser-driver' + | 'managed-linux-desktop-driver' + +const PRODUCTION_IMPLEMENTATIONS: ReadonlySet = + new Set(['managed-browser-driver']) + +export type ComputerCapabilityCatalogEntry = Readonly<{ + id: ComputerCapabilityId + name: string + description: string + enabledByDefault: false + implementationKind: ComputerCapabilityImplementationKind + supportedTargets: readonly Readonly<{ + platform: ComputerCapabilityPlatform + architectures: readonly ComputerCapabilityArchitecture[] + }>[] + riskSummary: string + requiredDiagnostics: readonly string[] +}> + +const ARCHITECTURES = Object.freeze(['x64', 'arm64'] as const) + +export const computerCapabilityCatalog: readonly ComputerCapabilityCatalogEntry[] = + Object.freeze([ + Object.freeze({ + id: 'host-browser-control', + name: '浏览器控制', + description: + '使用临时隔离会话执行网页操作；命名配置当前仅保存未来托管隔离所需的元数据。', + enabledByDefault: false, + implementationKind: 'managed-browser-driver', + supportedTargets: Object.freeze([ + Object.freeze({ + platform: 'win32', + architectures: ARCHITECTURES + }), + Object.freeze({ + platform: 'darwin', + architectures: ARCHITECTURES + }), + Object.freeze({ + platform: 'linux', + architectures: ARCHITECTURES + }) + ]), + riskSummary: + '可读取网页内容并代表用户操作网站；当前执行不会复用命名配置，仍必须保持临时隔离和审批策略。', + requiredDiagnostics: Object.freeze([ + 'browser-executable', + 'managed-profile-root' + ]) + }), + Object.freeze({ + id: 'linux-desktop-control', + name: 'Linux 桌面控制', + description: + '技术预览：保留 Linux 桌面控制核心与诊断，注册真实原生适配器后才可启用。', + enabledByDefault: false, + implementationKind: 'managed-linux-desktop-driver', + supportedTargets: Object.freeze([ + Object.freeze({ + platform: 'linux', + architectures: ARCHITECTURES + }) + ]), + riskSummary: + '真实 D-Bus、PipeWire、libei 或 XTest 适配器尚未随产品提供；注册适配器后仍须保持审批、超时和审计边界。', + requiredDiagnostics: Object.freeze([ + 'linux-session', + 'desktop-driver', + 'desktop-permissions' + ]) + }) + ] satisfies ComputerCapabilityCatalogEntry[]) + +export function getComputerCapability( + id: ComputerCapabilityId +): ComputerCapabilityCatalogEntry { + const capability = computerCapabilityCatalog.find((entry) => entry.id === id) + if (!capability) { + throw new Error(`Unknown computer capability: ${id}`) + } + return capability +} + +export function isComputerCapabilitySupported( + capability: ComputerCapabilityCatalogEntry, + platform: NodeJS.Platform, + architecture: string, + availableImplementations: ReadonlySet = + PRODUCTION_IMPLEMENTATIONS +): boolean { + return ( + availableImplementations.has(capability.implementationKind) && + capability.supportedTargets.some( + (target) => + target.platform === platform && + target.architectures.some((candidate) => candidate === architecture) + ) + ) +} diff --git a/src/main/capabilities/curated-mcp-launch.test.ts b/src/main/capabilities/curated-mcp-launch.test.ts new file mode 100644 index 0000000..6ff646e --- /dev/null +++ b/src/main/capabilities/curated-mcp-launch.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it, vi } from 'vitest' +import type { ResolvedMcpServer } from './capability-service' +import { + createCuratedMcpLaunch, + type CuratedMcpFileSystem, + type CuratedMcpPathMetadata +} from './curated-mcp-launch' + +const transportMocks = vi.hoisted(() => ({ + stdio: vi.fn(function StdioClientTransport(options: unknown) { + return { kind: 'stdio', options } + }), + http: vi.fn(), + sse: vi.fn() +})) + +vi.mock('@modelcontextprotocol/sdk/client/stdio.js', () => ({ + StdioClientTransport: transportMocks.stdio +})) +vi.mock('@modelcontextprotocol/sdk/client/streamableHttp.js', () => ({ + StreamableHTTPClientTransport: transportMocks.http +})) +vi.mock('@modelcontextprotocol/sdk/client/sse.js', () => ({ + SSEClientTransport: transportMocks.sse +})) + +import { createMcpTransport } from './mcp-client-transport' + +const root = process.platform === 'win32' ? 'C:\\GoodBuddy' : '/opt/goodbuddy' +const executable = + process.platform === 'win32' + ? `${root}\\helpers\\curated.exe` + : `${root}/helpers/curated` +const cwd = + process.platform === 'win32' + ? `${root}\\helpers` + : `${root}/helpers` + +const metadata = ( + canonicalPath: string, + kind: 'directory' | 'file', + overrides: Partial = {} +): CuratedMcpPathMetadata => ({ + canonicalPath, + uid: 1000, + mode: kind === 'directory' ? 0o40755 : 0o100755, + isDirectory: kind === 'directory', + isFile: kind === 'file', + isSymbolicLink: false, + ...overrides +}) + +const fileSystem = ( + overrides: Readonly>> = {} +): CuratedMcpFileSystem => ({ + inspect: vi.fn(async (path: string) => { + const kind = path === executable ? 'file' : 'directory' + return metadata(path, kind, overrides[path]) + }) +}) + +describe('curated MCP launches', () => { + it('keeps raw custom stdio on the SDK default environment', () => { + createMcpTransport({ + transport: 'stdio', + command: 'custom-mcp', + args: ['--serve'], + env: { DISPLAY: ':0', TOKEN: 'secret' }, + cwd + } as unknown as ResolvedMcpServer) + + expect(transportMocks.stdio).toHaveBeenLastCalledWith({ + command: 'custom-mcp', + args: ['--serve'], + stderr: 'ignore', + maxBufferSize: 2 * 1024 * 1024 + }) + }) + + it('passes only validated values from an opaque curated descriptor', async () => { + const validateLinuxDesktopEnvironment = vi.fn(async () => ({ + DISPLAY: ':1', + WAYLAND_DISPLAY: 'wayland-1', + XDG_RUNTIME_DIR: '/run/user/1000' + })) + const descriptor = await createCuratedMcpLaunch( + { + executable, + args: ['--stdio'], + cwd, + ownedRoots: [root], + ownerUid: 1000, + allowedEnvironmentNames: ['LANG'], + environment: { LANG: 'zh_CN.UTF-8' }, + linuxDesktopEnvironment: { + source: {}, + uid: 1000, + fileSystem: { inspect: vi.fn() } + } + }, + { + fileSystem: fileSystem(), + validateLinuxDesktopEnvironment + } + ) + + createMcpTransport(descriptor) + + expect(validateLinuxDesktopEnvironment).toHaveBeenCalledOnce() + expect(transportMocks.stdio).toHaveBeenLastCalledWith({ + command: executable, + args: ['--stdio'], + cwd, + env: { + LANG: 'zh_CN.UTF-8', + DISPLAY: ':1', + WAYLAND_DISPLAY: 'wayland-1', + XDG_RUNTIME_DIR: '/run/user/1000' + }, + stderr: 'ignore', + maxBufferSize: 2 * 1024 * 1024 + }) + }) + + it('rejects serialized attempts to spoof a curated descriptor', () => { + expect(() => + createMcpTransport({ + transport: 'curated-stdio', + command: executable, + args: [], + cwd, + env: { DISPLAY: ':0' } + } as unknown as ResolvedMcpServer) + ).toThrow('无效的精选 MCP 启动描述') + }) + + it.each([ + ['symlink executable', { [executable]: { isSymbolicLink: true } }], + [ + 'non-canonical executable', + { [executable]: { canonicalPath: `${executable}.real` } } + ], + ['wrong owner', { [cwd]: { uid: 2000 } }], + ['writable root', { [root]: { mode: 0o40777 } }] + ])('rejects an unsafe path: %s', async (_name, overrides) => { + await expect( + createCuratedMcpLaunch( + { + executable, + cwd, + ownedRoots: [root], + ownerUid: 1000 + }, + { fileSystem: fileSystem(overrides) } + ) + ).rejects.toThrow(/Unsafe curated MCP/u) + }) + + it.each([ + 'LD_PRELOAD', + 'NODE_OPTIONS', + 'GTK_MODULES', + 'QT_PLUGIN_PATH', + 'ELECTRON_RUN_AS_NODE', + 'CHROME_EXTRA_ARGS', + 'HTTPS_PROXY', + 'SERVICE_TOKEN' + ])('rejects unsafe environment name %s', async (name) => { + await expect( + createCuratedMcpLaunch( + { + executable, + cwd, + ownedRoots: [root], + ownerUid: 1000, + allowedEnvironmentNames: [name], + environment: { [name]: 'unsafe' } + }, + { fileSystem: fileSystem() } + ) + ).rejects.toThrow(/Unsafe curated MCP environment/u) + }) +}) diff --git a/src/main/capabilities/curated-mcp-launch.ts b/src/main/capabilities/curated-mcp-launch.ts new file mode 100644 index 0000000..32060ac --- /dev/null +++ b/src/main/capabilities/curated-mcp-launch.ts @@ -0,0 +1,276 @@ +import { lstat, realpath, stat } from 'node:fs/promises' +import { isAbsolute } from 'node:path' +import { + buildDesktopHelperEnvironment, + type SecurePathMetadata, + type SessionEnvironmentOptions +} from '../linux-desktop/session-environment' +import { isPathInside } from '../workspace-file-access' + +const MAX_ARGUMENTS = 64 +const MAX_ARGUMENT_LENGTH = 4_096 +const MAX_ENVIRONMENT_ENTRIES = 64 +const MAX_ENVIRONMENT_VALUE_LENGTH = 8_192 +const DESKTOP_ENVIRONMENT_NAMES = new Set([ + 'DISPLAY', + 'WAYLAND_DISPLAY', + 'XDG_RUNTIME_DIR', + 'DBUS_SESSION_BUS_ADDRESS', + 'XAUTHORITY', + 'NO_AT_BRIDGE' +]) +declare const curatedMcpLaunchBrand: unique symbol + +export type CuratedMcpPathMetadata = Readonly + +export interface CuratedMcpFileSystem { + inspect(path: string): Promise +} + +export type CuratedMcpLaunchOptions = Readonly<{ + executable: string + args?: readonly string[] + cwd: string + ownedRoots: readonly string[] + ownerUid: number + environment?: Readonly> + allowedEnvironmentNames?: readonly string[] + linuxDesktopEnvironment?: SessionEnvironmentOptions +}> + +export type CuratedMcpLaunchDescriptor = Readonly<{ + readonly [curatedMcpLaunchBrand]: true + readonly transport: 'curated-stdio' + readonly command: string + readonly args: readonly string[] + readonly cwd: string + readonly env: Readonly> +}> + +export type CuratedMcpLaunchDependencies = Readonly<{ + fileSystem?: CuratedMcpFileSystem + validateLinuxDesktopEnvironment?: ( + options: SessionEnvironmentOptions + ) => Promise +}> + +const descriptors = new WeakSet() + +const defaultFileSystem: CuratedMcpFileSystem = { + async inspect(path) { + const [linkMetadata, canonicalPath] = await Promise.all([ + lstat(path), + realpath(path) + ]) + const metadata = await stat(canonicalPath) + return { + canonicalPath, + uid: metadata.uid, + mode: metadata.mode, + isDirectory: metadata.isDirectory(), + isFile: metadata.isFile(), + isSymbolicLink: linkMetadata.isSymbolicLink() + } + } +} + +const hasControlCharacters = (value: string): boolean => + [...value].some((character) => { + const code = character.charCodeAt(0) + return code < 32 || code === 127 + }) + +const isSafePathText = (value: string): boolean => + value.length >= 1 && + value.length <= MAX_ENVIRONMENT_VALUE_LENGTH && + !hasControlCharacters(value) + +const assertOwnedCanonicalPath = ( + requestedPath: string, + metadata: CuratedMcpPathMetadata, + ownerUid: number, + expectedKind: 'directory' | 'file' +): void => { + if ( + !isAbsolute(requestedPath) || + metadata.canonicalPath !== requestedPath || + metadata.isSymbolicLink || + metadata.uid !== ownerUid || + (metadata.mode & 0o022) !== 0 || + (expectedKind === 'directory' + ? !metadata.isDirectory + : !metadata.isFile) + ) { + throw new Error(`Unsafe curated MCP ${expectedKind} path`) + } +} + +const isRejectedEnvironmentName = (name: string): boolean => { + const upperName = name.toUpperCase() + return ( + upperName.startsWith('LD_') || + upperName === 'NODE_OPTIONS' || + upperName === 'GTK_MODULES' || + upperName === 'QT_PLUGIN_PATH' || + upperName.startsWith('ELECTRON_') || + upperName.startsWith('CHROME_') || + upperName.startsWith('CHROMIUM_') || + upperName === 'HTTP_PROXY' || + upperName === 'HTTPS_PROXY' || + upperName === 'ALL_PROXY' || + upperName === 'NO_PROXY' || + /(?:CREDENTIAL|PASSWORD|PASSWD|SECRET|TOKEN|API_KEY|ACCESS_KEY|PRIVATE_KEY|AUTHORIZATION|COOKIE)/u.test( + upperName + ) + ) +} + +const copyEnvironment = ( + source: Readonly>, + allowedNames: ReadonlySet, + target: Record +): void => { + for (const [name, value] of Object.entries(source)) { + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name) || + !allowedNames.has(name) || + isRejectedEnvironmentName(name) || + value.length > MAX_ENVIRONMENT_VALUE_LENGTH || + hasControlCharacters(value) + ) { + throw new Error(`Unsafe curated MCP environment variable: ${name}`) + } + target[name] = value + } +} + +export async function createCuratedMcpLaunch( + options: CuratedMcpLaunchOptions, + dependencies: CuratedMcpLaunchDependencies = {} +): Promise { + if ( + options.ownedRoots.length === 0 || + options.ownedRoots.some( + (root) => !isAbsolute(root) || !isSafePathText(root) + ) + ) { + throw new Error('Curated MCP requires an absolute allowlisted root') + } + if ( + !isAbsolute(options.executable) || + !isAbsolute(options.cwd) || + !isSafePathText(options.executable) || + !isSafePathText(options.cwd) || + (options.args?.length ?? 0) > MAX_ARGUMENTS || + options.args?.some( + (argument) => + argument.length > MAX_ARGUMENT_LENGTH || + hasControlCharacters(argument) + ) + ) { + throw new Error('Invalid curated MCP launch parameters') + } + + const fileSystem = dependencies.fileSystem ?? defaultFileSystem + const rootMetadata = await Promise.all( + options.ownedRoots.map((root) => fileSystem.inspect(root)) + ) + rootMetadata.forEach((metadata, index) => { + assertOwnedCanonicalPath( + options.ownedRoots[index]!, + metadata, + options.ownerUid, + 'directory' + ) + }) + + const [executableMetadata, cwdMetadata] = await Promise.all([ + fileSystem.inspect(options.executable), + fileSystem.inspect(options.cwd) + ]) + assertOwnedCanonicalPath( + options.executable, + executableMetadata, + options.ownerUid, + 'file' + ) + assertOwnedCanonicalPath( + options.cwd, + cwdMetadata, + options.ownerUid, + 'directory' + ) + if ( + (process.platform !== 'win32' && + (executableMetadata.mode & 0o111) === 0) || + !rootMetadata.some((root) => + isPathInside(root.canonicalPath, executableMetadata.canonicalPath) + ) || + !rootMetadata.some((root) => + isPathInside(root.canonicalPath, cwdMetadata.canonicalPath) + ) + ) { + throw new Error('Curated MCP paths are outside the owned allowlist') + } + + const allowedNames = new Set(options.allowedEnvironmentNames ?? []) + if ( + (options.allowedEnvironmentNames?.length ?? 0) > + MAX_ENVIRONMENT_ENTRIES || + allowedNames.size > MAX_ENVIRONMENT_ENTRIES || + [...allowedNames].some( + (name) => + isRejectedEnvironmentName(name) || + DESKTOP_ENVIRONMENT_NAMES.has(name.toUpperCase()) + ) + ) { + throw new Error('Unsafe curated MCP environment allowlist') + } + const environment: Record = {} + copyEnvironment(options.environment ?? {}, allowedNames, environment) + + if (options.linuxDesktopEnvironment) { + const validate = + dependencies.validateLinuxDesktopEnvironment ?? + buildDesktopHelperEnvironment + const desktopEnvironment = await validate( + options.linuxDesktopEnvironment + ) + copyEnvironment( + desktopEnvironment as Record, + new Set([ + 'PATH', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + 'DISPLAY', + 'WAYLAND_DISPLAY', + 'XDG_RUNTIME_DIR', + 'DBUS_SESSION_BUS_ADDRESS', + 'XAUTHORITY', + 'NO_AT_BRIDGE' + ]), + environment + ) + } + + const descriptor = Object.freeze({ + transport: 'curated-stdio' as const, + command: executableMetadata.canonicalPath, + args: Object.freeze([...(options.args ?? [])]), + cwd: cwdMetadata.canonicalPath, + env: Object.freeze({ ...environment }) + }) as CuratedMcpLaunchDescriptor + descriptors.add(descriptor) + return descriptor +} + +export function isCuratedMcpLaunchDescriptor( + value: unknown +): value is CuratedMcpLaunchDescriptor { + return ( + typeof value === 'object' && + value !== null && + descriptors.has(value) + ) +} diff --git a/src/main/capabilities/mcp-client-transport.ts b/src/main/capabilities/mcp-client-transport.ts index 8a2c0f4..7fccd3d 100644 --- a/src/main/capabilities/mcp-client-transport.ts +++ b/src/main/capabilities/mcp-client-transport.ts @@ -6,6 +6,10 @@ import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js' import type { ResolvedMcpServer } from './capability-service' +import { + isCuratedMcpLaunchDescriptor, + type CuratedMcpLaunchDescriptor +} from './curated-mcp-launch' function validateRemoteUrl(value: string): URL { const url = new URL(value) @@ -34,8 +38,23 @@ function createRestrictedFetch(origin: string): FetchLike { } export function createMcpTransport( - server: ResolvedMcpServer + server: ResolvedMcpServer | CuratedMcpLaunchDescriptor ): Transport { + if (isCuratedMcpLaunchDescriptor(server)) { + return new StdioClientTransport({ + command: server.command, + args: [...server.args], + cwd: server.cwd, + env: { ...server.env }, + stderr: 'ignore', + maxBufferSize: 2 * 1024 * 1024 + }) + } + + if ((server as { transport?: string }).transport === 'curated-stdio') { + throw new Error('无效的精选 MCP 启动描述') + } + if (server.transport === 'stdio') { return new StdioClientTransport({ command: server.command, diff --git a/src/main/capabilities/mcp-tester.test.ts b/src/main/capabilities/mcp-tester.test.ts index 144b8c8..1a4bf4d 100644 --- a/src/main/capabilities/mcp-tester.test.ts +++ b/src/main/capabilities/mcp-tester.test.ts @@ -141,4 +141,36 @@ describe('testMcpServer', () => { ) expect(mocks.client.close).toHaveBeenCalledOnce() }) + + it('accepts cancellation, closes the client, and hides abort details', async () => { + mocks.client.connect.mockImplementation( + async ( + _transport: unknown, + options: { signal: AbortSignal } + ) => + new Promise((_resolve, reject) => { + options.signal.addEventListener( + 'abort', + () => reject(new Error('sensitive abort reason')), + { once: true } + ) + }) + ) + const controller = new AbortController() + const pending = testMcpServer( + { + ...common, + transport: 'stdio', + command: 'node', + args: ['server.js'] + } satisfies ResolvedMcpServer, + controller.signal + ) + + controller.abort(new Error('caller private context')) + + await expect(pending).rejects.toThrow('MCP 连接测试已取消') + expect(mocks.client.listTools).not.toHaveBeenCalled() + expect(mocks.client.close).toHaveBeenCalledOnce() + }) }) diff --git a/src/main/capabilities/mcp-tester.ts b/src/main/capabilities/mcp-tester.ts index 7b8a2d7..589fe91 100644 --- a/src/main/capabilities/mcp-tester.ts +++ b/src/main/capabilities/mcp-tester.ts @@ -3,10 +3,12 @@ import type { McpServerTestResult } from '../../shared/capability-contracts' import type { ResolvedMcpServer } from './capability-service' import { createMcpTransport } from './mcp-client-transport' -const MCP_TEST_TIMEOUT_MS = 12_000 +const MCP_TEST_TOTAL_TIMEOUT_MS = 12_000 +const MCP_TEST_INACTIVITY_TIMEOUT_MS = 8_000 export async function testMcpServer( - server: ResolvedMcpServer + server: ResolvedMcpServer, + signal?: AbortSignal ): Promise { const client = new Client({ name: 'goodbuddy', @@ -14,19 +16,45 @@ export async function testMcpServer( }) const transport = createMcpTransport(server) const controller = new AbortController() + let timedOut = false + const abortFromCaller = (): void => { + controller.abort() + } + signal?.addEventListener('abort', abortFromCaller, { once: true }) + if (signal?.aborted) { + abortFromCaller() + } const timeout = setTimeout(() => { + timedOut = true controller.abort(new Error('MCP 连接测试超时')) - }, MCP_TEST_TIMEOUT_MS) + }, MCP_TEST_TOTAL_TIMEOUT_MS) + const runWithInactivityLimit = async ( + operation: () => Promise + ): Promise => { + const inactivityTimeout = setTimeout(() => { + timedOut = true + controller.abort(new Error('MCP 连接测试超时')) + }, MCP_TEST_INACTIVITY_TIMEOUT_MS) + try { + return await operation() + } finally { + clearTimeout(inactivityTimeout) + } + } try { - await client.connect(transport, { - timeout: MCP_TEST_TIMEOUT_MS, - signal: controller.signal - }) - const result = await client.listTools(undefined, { - timeout: MCP_TEST_TIMEOUT_MS, - signal: controller.signal - }) + await runWithInactivityLimit(() => + client.connect(transport, { + timeout: MCP_TEST_INACTIVITY_TIMEOUT_MS, + signal: controller.signal + }) + ) + const result = await runWithInactivityLimit(() => + client.listTools(undefined, { + timeout: MCP_TEST_INACTIVITY_TIMEOUT_MS, + signal: controller.signal + }) + ) const version = client.getServerVersion() return { serverName: version?.name.slice(0, 120), @@ -38,7 +66,10 @@ export async function testMcpServer( })) } } catch (error) { - if (controller.signal.aborted) { + if (signal?.aborted && !timedOut) { + throw new Error('MCP 连接测试已取消', { cause: error }) + } + if (timedOut) { throw new Error('MCP 连接测试超时', { cause: error }) } throw new Error( @@ -50,6 +81,7 @@ export async function testMcpServer( ) } finally { clearTimeout(timeout) + signal?.removeEventListener('abort', abortFromCaller) await client.close().catch(() => undefined) } } diff --git a/src/main/computer-control/approval.ts b/src/main/computer-control/approval.ts new file mode 100644 index 0000000..e6c32a8 --- /dev/null +++ b/src/main/computer-control/approval.ts @@ -0,0 +1,13 @@ +import type { + ComputerControlApprovalRequest, + ComputerControlApprovalResult +} from '../../shared/computer-control-contracts' + +export const COMPUTER_CONTROL_APPROVAL_DEADLINE_MS = 120_000 + +export interface ComputerControlApprovalProvider { + request( + request: ComputerControlApprovalRequest, + signal: AbortSignal + ): Promise +} diff --git a/src/main/computer-control/audit.ts b/src/main/computer-control/audit.ts new file mode 100644 index 0000000..f33b6a4 --- /dev/null +++ b/src/main/computer-control/audit.ts @@ -0,0 +1,65 @@ +import { createHash } from 'node:crypto' +import type { + ComputerControlErrorCode, + ComputerControlRisk +} from '../../shared/computer-control-contracts' + +export type ComputerControlAuditEvent = { + timestamp: number + taskId: string + conversationId: string + leaseId: string + commandId: string + action: + | 'observe' + | 'activate' + | 'replace_text' + | 'select_option' + | 'scroll' + risk: ComputerControlRisk + outcome: 'completed' | 'denied' | 'failed' | 'outcome_unknown' + errorCode?: ComputerControlErrorCode + textLength?: number + textDigest?: string +} + +export interface ComputerControlAuditSink { + write(event: ComputerControlAuditEvent): void | Promise +} + +export const digestComputerControlText = (text: string): string => + createHash('sha256').update(text, 'utf8').digest('hex') + +export class InMemoryComputerControlAudit + implements ComputerControlAuditSink +{ + private readonly records: ComputerControlAuditEvent[] = [] + + constructor(private readonly maximumRecords = 1_000) { + if ( + !Number.isInteger(maximumRecords) || + maximumRecords < 1 || + maximumRecords > 10_000 + ) { + throw new Error('Invalid computer control audit capacity') + } + } + + write(event: ComputerControlAuditEvent): void { + const bounded: ComputerControlAuditEvent = { + ...event, + taskId: event.taskId.slice(0, 128), + conversationId: event.conversationId.slice(0, 128), + leaseId: event.leaseId.slice(0, 160), + commandId: event.commandId.slice(0, 160) + } + this.records.push(Object.freeze(bounded)) + if (this.records.length > this.maximumRecords) { + this.records.splice(0, this.records.length - this.maximumRecords) + } + } + + entries(): readonly ComputerControlAuditEvent[] { + return this.records.map((record) => ({ ...record })) + } +} diff --git a/src/main/computer-control/broker.test.ts b/src/main/computer-control/broker.test.ts new file mode 100644 index 0000000..5a8c589 --- /dev/null +++ b/src/main/computer-control/broker.test.ts @@ -0,0 +1,896 @@ +import { describe, expect, it, vi } from 'vitest' +import type { + ComputerControlAction, + ComputerControlApprovalRequest, + ComputerControlApprovalResult, + ComputerControlCommandResult, + ComputerControlObservation +} from '../../shared/computer-control-contracts' +import { InMemoryComputerControlAudit } from './audit' +import type { ComputerControlApprovalProvider } from './approval' +import { + ComputerControlBroker, + type ComputerControlBrokerOptions +} from './broker' +import type { + ComputerControlDriver, + DriverElement, + DriverObservation, + DriverWindowIdentity, + NativeElementIdentity +} from './driver' +import type { ComputerControlLeaseBinding } from './lease-store' + +const binding: ComputerControlLeaseBinding = { + taskId: 'task-1', + conversationId: 'conversation-1', + pid: 42, + processStartTime: 100, + windowIdentity: 'window-1' +} + +const commandId = (suffix: string): string => + `command_identifier_${suffix.padStart(6, '0')}` + +class FakeDriver implements ComputerControlDriver { + readonly available = true + readonly injections: ComputerControlAction[] = [] + releaseCount = 0 + activeInjections = 0 + maximumActiveInjections = 0 + foreground: DriverWindowIdentity = { + pid: binding.pid, + processStartTime: binding.processStartTime, + windowIdentity: binding.windowIdentity + } + injectGate?: Promise + foregroundGate?: Promise + + constructor(readonly elements: DriverElement[]) {} + + async observe(): Promise { + await this.foregroundGate + return { + window: { ...this.foreground }, + windowTitle: 'Test window', + elements: this.elements.map((element) => ({ ...element })) + } + } + + async getForegroundWindow(): Promise { + await this.foregroundGate + return { ...this.foreground } + } + + async resolveElement( + nativeIdentity: NativeElementIdentity + ): Promise { + const element = this.elements.find( + (candidate) => candidate.nativeIdentity === nativeIdentity + ) + return element ? { ...element } : undefined + } + + async focusElement( + nativeIdentity: NativeElementIdentity + ): Promise { + const element = this.elements.find( + (candidate) => candidate.nativeIdentity === nativeIdentity + ) + if (!element || !element.enabled) { + return false + } + element.focused = true + return true + } + + async inject( + nativeIdentity: NativeElementIdentity, + action: ComputerControlAction + ): Promise { + void nativeIdentity + this.injections.push(action) + this.activeInjections += 1 + this.maximumActiveInjections = Math.max( + this.maximumActiveInjections, + this.activeInjections + ) + try { + await this.injectGate + } finally { + this.activeInjections -= 1 + } + } + + async releaseInjectedInput(): Promise { + this.releaseCount += 1 + } +} + +class FakeApproval implements ComputerControlApprovalProvider { + readonly requests: ComputerControlApprovalRequest[] = [] + handler?: ( + request: ComputerControlApprovalRequest, + signal: AbortSignal + ) => Promise + + async request( + request: ComputerControlApprovalRequest, + signal: AbortSignal + ): Promise { + this.requests.push(request) + if (this.handler) { + return this.handler(request, signal) + } + return { + approvalId: request.approvalId, + decision: 'approve_once' + } + } +} + +const makeElement = ( + role: DriverElement['role'], + targetKind: DriverElement['targetKind'] = 'standard', + name = 'Target' +): DriverElement => ({ + nativeIdentity: {}, + role, + name, + enabled: true, + focused: false, + targetKind +}) + +const makeHarness = ( + elements: DriverElement[], + options: { + now?: () => number + driverDeadlineMs?: number + approvalDeadlineMs?: number + audit?: ComputerControlBrokerOptions['audit'] + fallbackAudit?: ComputerControlBrokerOptions['fallbackAudit'] + } = {} +) => { + let idSequence = 0 + const driver = new FakeDriver(elements) + const approval = new FakeApproval() + const audit = new InMemoryComputerControlAudit() + const broker = new ComputerControlBroker({ + driver, + approval, + audit: options.audit ?? audit, + fallbackAudit: options.fallbackAudit, + now: options.now, + driverDeadlineMs: options.driverDeadlineMs, + approvalDeadlineMs: options.approvalDeadlineMs, + createId: () => + `generated_identifier_${String(++idSequence).padStart(6, '0')}` + }) + return { broker, driver, approval, audit } +} + +const observe = async ( + broker: ComputerControlBroker, + leaseId: string, + suffix: string, + context = binding +): Promise => { + const result = await broker.execute( + { + kind: 'observe', + commandId: commandId(suffix), + leaseId + }, + context, + new AbortController().signal + ) + if (result.status !== 'observed') { + throw new Error(`Observation failed: ${JSON.stringify(result)}`) + } + return result.observation +} + +const expectErrorCode = ( + result: ComputerControlCommandResult, + code: string +): void => { + expect(result).toMatchObject({ + status: 'error', + error: { code } + }) +} + +describe('ComputerControlBroker', () => { + it('rejects stale and consumed refs while preserving command idempotency', async () => { + let now = 1_000 + const { broker, driver } = makeHarness([makeElement('link')], { + now: () => now + }) + const lease = broker.createLease(binding) + const staleObservation = await observe( + broker, + lease.leaseId, + 'stale-observe' + ) + now += 3_000 + const stale = await broker.execute( + { + kind: 'act', + commandId: commandId('stale-act'), + leaseId: lease.leaseId, + observationId: staleObservation.observationId, + revision: staleObservation.revision, + action: { + kind: 'activate', + elementRef: staleObservation.elements[0]?.ref + } + }, + binding, + new AbortController().signal + ) + expectErrorCode(stale, 'observation_stale') + + const fresh = await observe( + broker, + lease.leaseId, + 'fresh-observe' + ) + const action = { + kind: 'act' as const, + commandId: commandId('fresh-act'), + leaseId: lease.leaseId, + observationId: fresh.observationId, + revision: fresh.revision, + action: { + kind: 'activate' as const, + elementRef: fresh.elements[0]?.ref + } + } + const first = await broker.execute( + action, + binding, + new AbortController().signal + ) + expect(first.status).toBe('completed') + await expect( + broker.execute( + action, + binding, + new AbortController().signal + ) + ).resolves.toEqual(first) + expect(driver.injections).toHaveLength(1) + + const consumed = await broker.execute( + { ...action, commandId: commandId('consumed') }, + binding, + new AbortController().signal + ) + expectErrorCode(consumed, 'observation_consumed') + }) + + it('rejects command ID reuse with different content', async () => { + const { broker } = makeHarness([makeElement('link')]) + const lease = broker.createLease(binding) + await observe(broker, lease.leaseId, 'same-id') + const conflict = await broker.execute( + { + kind: 'observe', + commandId: commandId('same-id'), + leaseId: 'different_lease_identifier' + }, + binding, + new AbortController().signal + ) + expectErrorCode(conflict, 'command_id_conflict') + }) + + it.each([ + 'protected', + 'password', + 'otp', + 'payment', + 'account_security', + 'privilege', + 'os_permission' + ] as const)('forbids %s targets before approval or injection', async (targetKind) => { + const { broker, driver, approval } = makeHarness([ + makeElement('textbox', targetKind) + ]) + const lease = broker.createLease(binding) + const observation = await observe( + broker, + lease.leaseId, + `forbidden-${targetKind}` + ) + expect(observation.elements[0]).toMatchObject({ + risk: 'forbidden', + blocked: true + }) + const result = await broker.execute( + { + kind: 'act', + commandId: commandId(`act-${targetKind}`), + leaseId: lease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'replace_text', + elementRef: observation.elements[0]?.ref, + text: 'never injected' + } + }, + binding, + new AbortController().signal + ) + expectErrorCode(result, 'forbidden') + expect(approval.requests).toHaveLength(0) + expect(driver.injections).toHaveLength(0) + }) + + it('requires a distinct once approval for every input and commit', async () => { + const { broker, approval } = makeHarness([ + makeElement('textbox', 'standard', 'Editor'), + makeElement('button', 'standard', 'Submit') + ]) + const lease = broker.createLease(binding) + const inputObservation = await observe( + broker, + lease.leaseId, + 'input-observe' + ) + const input = await broker.execute( + { + kind: 'act', + commandId: commandId('input-act'), + leaseId: lease.leaseId, + observationId: inputObservation.observationId, + revision: inputObservation.revision, + action: { + kind: 'replace_text', + elementRef: inputObservation.elements[0]?.ref, + text: 'hello' + } + }, + binding, + new AbortController().signal + ) + expect(input).toMatchObject({ status: 'completed', risk: 'input' }) + + const commitObservation = await observe( + broker, + lease.leaseId, + 'commit-observe' + ) + const commit = await broker.execute( + { + kind: 'act', + commandId: commandId('commit-act'), + leaseId: lease.leaseId, + observationId: commitObservation.observationId, + revision: commitObservation.revision, + action: { + kind: 'activate', + elementRef: commitObservation.elements[1]?.ref + } + }, + binding, + new AbortController().signal + ) + expect(commit).toMatchObject({ + status: 'completed', + risk: 'commit' + }) + expect(approval.requests.map((request) => request.risk)).toEqual([ + 'input', + 'commit' + ]) + expect( + approval.requests.every( + (request) => !('text' in request) + ) + ).toBe(true) + }) + + it('handles approval denial, timeout, and cancellation fail-closed', async () => { + vi.useFakeTimers() + try { + const deniedHarness = makeHarness([makeElement('textbox')]) + deniedHarness.approval.handler = async (request) => ({ + approvalId: request.approvalId, + decision: 'deny' + }) + const deniedLease = deniedHarness.broker.createLease(binding) + const deniedObservation = await observe( + deniedHarness.broker, + deniedLease.leaseId, + 'denied-observe' + ) + const denied = await deniedHarness.broker.execute( + { + kind: 'act', + commandId: commandId('denied-act'), + leaseId: deniedLease.leaseId, + observationId: deniedObservation.observationId, + revision: deniedObservation.revision, + action: { + kind: 'replace_text', + elementRef: deniedObservation.elements[0]?.ref, + text: 'denied' + } + }, + binding, + new AbortController().signal + ) + expectErrorCode(denied, 'approval_denied') + + const timeoutHarness = makeHarness([makeElement('textbox')], { + approvalDeadlineMs: 20 + }) + timeoutHarness.approval.handler = () => + new Promise(() => undefined) + const timeoutLease = timeoutHarness.broker.createLease(binding) + const timeoutObservation = await observe( + timeoutHarness.broker, + timeoutLease.leaseId, + 'timeout-observe' + ) + const timeoutPromise = timeoutHarness.broker.execute( + { + kind: 'act', + commandId: commandId('timeout-act'), + leaseId: timeoutLease.leaseId, + observationId: timeoutObservation.observationId, + revision: timeoutObservation.revision, + action: { + kind: 'replace_text', + elementRef: timeoutObservation.elements[0]?.ref, + text: 'timeout' + } + }, + binding, + new AbortController().signal + ) + await vi.advanceTimersByTimeAsync(20) + expectErrorCode(await timeoutPromise, 'approval_timeout') + + const cancelledHarness = makeHarness([makeElement('textbox')]) + cancelledHarness.approval.handler = () => + new Promise(() => undefined) + const cancelledLease = + cancelledHarness.broker.createLease(binding) + const cancelledObservation = await observe( + cancelledHarness.broker, + cancelledLease.leaseId, + 'cancel-observe' + ) + const controller = new AbortController() + const cancelledPromise = cancelledHarness.broker.execute( + { + kind: 'act', + commandId: commandId('cancel-act'), + leaseId: cancelledLease.leaseId, + observationId: cancelledObservation.observationId, + revision: cancelledObservation.revision, + action: { + kind: 'replace_text', + elementRef: cancelledObservation.elements[0]?.ref, + text: 'cancelled' + } + }, + binding, + controller.signal + ) + await vi.waitFor(() => { + expect(cancelledHarness.approval.requests).toHaveLength(1) + }) + controller.abort() + expectErrorCode(await cancelledPromise, 'cancelled') + expect( + cancelledHarness.broker.leases.peek(cancelledLease.leaseId) + ).toBeUndefined() + expect(cancelledHarness.driver.releaseCount).toBeGreaterThan(0) + } finally { + vi.useRealTimers() + } + }) + + it('reports outcome_unknown when cancellation races injection', async () => { + let releaseInjection: (() => void) | undefined + const { broker, driver } = makeHarness([makeElement('link')]) + driver.injectGate = new Promise((resolve) => { + releaseInjection = resolve + }) + const lease = broker.createLease(binding) + const observation = await observe( + broker, + lease.leaseId, + 'race-observe' + ) + const controller = new AbortController() + const resultPromise = broker.execute( + { + kind: 'act', + commandId: commandId('race-act'), + leaseId: lease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'activate', + elementRef: observation.elements[0]?.ref + } + }, + binding, + controller.signal + ) + await vi.waitFor(() => { + expect(driver.injections).toHaveLength(1) + }) + controller.abort() + expectErrorCode(await resultPromise, 'outcome_unknown') + releaseInjection?.() + expect(driver.releaseCount).toBeGreaterThan(0) + }) + + it('caches outcome_unknown when injection times out before late completion', async () => { + vi.useFakeTimers() + try { + let releaseInjection: (() => void) | undefined + const { broker, driver, audit } = makeHarness( + [makeElement('link')], + { driverDeadlineMs: 20 } + ) + const lease = broker.createLease(binding) + const observation = await observe( + broker, + lease.leaseId, + 'injection-timeout-observe' + ) + driver.injectGate = new Promise((resolve) => { + releaseInjection = resolve + }) + const command = { + kind: 'act' as const, + commandId: commandId('injection-timeout-act'), + leaseId: lease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'activate' as const, + elementRef: observation.elements[0]?.ref + } + } + + const firstPromise = broker.execute( + command, + binding, + new AbortController().signal + ) + await vi.waitFor(() => { + expect(driver.injections).toHaveLength(1) + }) + await vi.advanceTimersByTimeAsync(20) + const first = await firstPromise + expectErrorCode(first, 'outcome_unknown') + + await expect( + broker.execute( + command, + binding, + new AbortController().signal + ) + ).resolves.toEqual(first) + expect(driver.injections).toHaveLength(1) + expect(audit.entries().at(-1)).toMatchObject({ + outcome: 'outcome_unknown', + errorCode: 'outcome_unknown' + }) + + releaseInjection?.() + await Promise.resolve() + expect(driver.injections).toHaveLength(1) + } finally { + vi.useRealTimers() + } + }) + + it('treats an uncertain injection driver failure as outcome_unknown', async () => { + const { broker, driver, audit } = makeHarness([makeElement('link')]) + const lease = broker.createLease(binding) + const observation = await observe( + broker, + lease.leaseId, + 'injection-failure-observe' + ) + driver.inject = vi.fn(async () => { + throw new Error('driver disconnected during injection') + }) + const command = { + kind: 'act' as const, + commandId: commandId('injection-failure-act'), + leaseId: lease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'activate' as const, + elementRef: observation.elements[0]?.ref + } + } + + const first = await broker.execute( + command, + binding, + new AbortController().signal + ) + expectErrorCode(first, 'outcome_unknown') + await expect( + broker.execute( + command, + binding, + new AbortController().signal + ) + ).resolves.toEqual(first) + expect(driver.inject).toHaveBeenCalledOnce() + expect(audit.entries().at(-1)).toMatchObject({ + outcome: 'outcome_unknown', + errorCode: 'outcome_unknown' + }) + }) + + it('isolates throwing primary and fallback audit sinks from cached results', async () => { + const primaryWrite = vi.fn(async () => { + throw new Error('primary audit unavailable') + }) + const fallbackWrite = vi.fn(async () => { + throw new Error('fallback audit unavailable') + }) + const { broker, driver } = makeHarness([makeElement('link')], { + audit: { write: primaryWrite }, + fallbackAudit: { write: fallbackWrite } + }) + const lease = broker.createLease(binding) + const observation = await observe( + broker, + lease.leaseId, + 'throwing-audit-observe' + ) + const command = { + kind: 'act' as const, + commandId: commandId('throwing-audit-act'), + leaseId: lease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'activate' as const, + elementRef: observation.elements[0]?.ref + } + } + + const first = await broker.execute( + command, + binding, + new AbortController().signal + ) + expect(first).toMatchObject({ status: 'completed' }) + await expect( + broker.execute( + command, + binding, + new AbortController().signal + ) + ).resolves.toEqual(first) + expect(driver.injections).toHaveLength(1) + expect(primaryWrite).toHaveBeenCalledTimes(2) + expect(fallbackWrite).toHaveBeenCalledTimes(2) + }) + + it('isolates throwing audit sinks while caching a failed result', async () => { + const primaryWrite = vi.fn(async () => { + throw new Error('primary audit unavailable') + }) + const fallbackWrite = vi.fn(async () => { + throw new Error('fallback audit unavailable') + }) + const { broker, driver } = makeHarness( + [makeElement('textbox', 'protected')], + { + audit: { write: primaryWrite }, + fallbackAudit: { write: fallbackWrite } + } + ) + const lease = broker.createLease(binding) + const observation = await observe( + broker, + lease.leaseId, + 'throwing-failure-audit-observe' + ) + const command = { + kind: 'act' as const, + commandId: commandId('throwing-failure-audit-act'), + leaseId: lease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'replace_text' as const, + elementRef: observation.elements[0]?.ref, + text: 'not injected' + } + } + + const first = await broker.execute( + command, + binding, + new AbortController().signal + ) + expectErrorCode(first, 'forbidden') + await expect( + broker.execute( + command, + binding, + new AbortController().signal + ) + ).resolves.toEqual(first) + expect(driver.injections).toHaveLength(0) + expect(primaryWrite).toHaveBeenCalledTimes(2) + expect(fallbackWrite).toHaveBeenCalledTimes(2) + }) + + it('serializes actions for leases belonging to the same task', async () => { + let releaseFirst: (() => void) | undefined + const { broker, driver } = makeHarness([makeElement('link')]) + driver.injectGate = new Promise((resolve) => { + releaseFirst = resolve + }) + const firstLease = broker.createLease(binding) + const secondLease = broker.createLease(binding) + const firstObservation = await observe( + broker, + firstLease.leaseId, + 'serial-observe-1' + ) + const secondObservation = await observe( + broker, + secondLease.leaseId, + 'serial-observe-2' + ) + const actionFor = ( + leaseId: string, + observation: ComputerControlObservation, + suffix: string + ) => + broker.execute( + { + kind: 'act', + commandId: commandId(suffix), + leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'activate', + elementRef: observation.elements[0]?.ref + } + }, + binding, + new AbortController().signal + ) + const first = actionFor( + firstLease.leaseId, + firstObservation, + 'serial-act-1' + ) + await vi.waitFor(() => { + expect(driver.injections).toHaveLength(1) + }) + const second = actionFor( + secondLease.leaseId, + secondObservation, + 'serial-act-2' + ) + await Promise.resolve() + expect(driver.injections).toHaveLength(1) + releaseFirst?.() + driver.injectGate = undefined + await expect(Promise.all([first, second])).resolves.toEqual([ + expect.objectContaining({ status: 'completed' }), + expect.objectContaining({ status: 'completed' }) + ]) + expect(driver.maximumActiveInjections).toBe(1) + }) + + it('revokes on foreground identity mismatch and enforces driver deadlines', async () => { + vi.useFakeTimers() + try { + const mismatchHarness = makeHarness([makeElement('link')]) + const mismatchLease = + mismatchHarness.broker.createLease(binding) + const observation = await observe( + mismatchHarness.broker, + mismatchLease.leaseId, + 'mismatch-observe' + ) + mismatchHarness.driver.foreground = { + ...mismatchHarness.driver.foreground, + pid: 99 + } + const mismatch = await mismatchHarness.broker.execute( + { + kind: 'act', + commandId: commandId('mismatch-act'), + leaseId: mismatchLease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'activate', + elementRef: observation.elements[0]?.ref + } + }, + binding, + new AbortController().signal + ) + expectErrorCode(mismatch, 'window_not_foreground') + expect( + mismatchHarness.broker.leases.peek(mismatchLease.leaseId) + ).toBeUndefined() + + const timeoutHarness = makeHarness([makeElement('link')], { + driverDeadlineMs: 20 + }) + timeoutHarness.driver.foregroundGate = new Promise( + () => undefined + ) + const timeoutLease = timeoutHarness.broker.createLease(binding) + const timeoutPromise = timeoutHarness.broker.execute( + { + kind: 'observe', + commandId: commandId('driver-timeout'), + leaseId: timeoutLease.leaseId + }, + binding, + new AbortController().signal + ) + await vi.advanceTimersByTimeAsync(20) + expectErrorCode(await timeoutPromise, 'driver_timeout') + } finally { + vi.useRealTimers() + } + }) + + it('audits typed text only as length and digest', async () => { + const { broker, approval, audit } = makeHarness([ + makeElement('textbox', 'standard', 'Editor') + ]) + const lease = broker.createLease(binding) + const observation = await observe( + broker, + lease.leaseId, + 'audit-observe' + ) + const secretText = 'private typed content' + const result = await broker.execute( + { + kind: 'act', + commandId: commandId('audit-act'), + leaseId: lease.leaseId, + observationId: observation.observationId, + revision: observation.revision, + action: { + kind: 'replace_text', + elementRef: observation.elements[0]?.ref, + text: secretText + } + }, + binding, + new AbortController().signal + ) + expect(result.status).toBe('completed') + const entry = audit.entries().at(-1) + expect(entry).toMatchObject({ + action: 'replace_text', + textLength: secretText.length + }) + expect(entry?.textDigest).toMatch(/^[a-f0-9]{64}$/) + expect(JSON.stringify(audit.entries())).not.toContain(secretText) + expect(JSON.stringify(approval.requests)).not.toContain(secretText) + }) +}) diff --git a/src/main/computer-control/broker.ts b/src/main/computer-control/broker.ts new file mode 100644 index 0000000..99f2b39 --- /dev/null +++ b/src/main/computer-control/broker.ts @@ -0,0 +1,729 @@ +import { randomUUID } from 'node:crypto' +import { + computerControlApprovalResultSchema, + computerControlRuntimeCommandSchema, + type ComputerControlAction, + type ComputerControlCommandResult, + type ComputerControlRisk, + type ComputerControlRuntimeCommand +} from '../../shared/computer-control-contracts' +import { + digestComputerControlText, + type ComputerControlAuditEvent, + type ComputerControlAuditSink +} from './audit' +import { + COMPUTER_CONTROL_APPROVAL_DEADLINE_MS, + type ComputerControlApprovalProvider +} from './approval' +import { runWithDeadline } from './deadline' +import { + COMPUTER_CONTROL_DRIVER_DEADLINE_MS, + type ComputerControlDriver, + type DriverElement, + type DriverWindowIdentity +} from './driver' +import { + cancellationFailure, + ComputerControlFailure +} from './errors' +import { + ComputerControlLeaseStore, + type ComputerControlLease, + type ComputerControlLeaseBinding +} from './lease-store' +import { ComputerControlPerceptionStore } from './perception-store' +import { + classifyComputerControlAction, + maximumRisk +} from './risk-policy' + +const INVALID_COMMAND_ID = 'invalid_command_000000' +const MAX_CACHED_COMMANDS = 1_000 + +type BrokerExecutionContext = ComputerControlLeaseBinding + +type InFlightCommand = { + fingerprint: string + result: Promise +} + +export type ComputerControlBrokerOptions = { + driver: ComputerControlDriver + approval: ComputerControlApprovalProvider + audit: ComputerControlAuditSink + fallbackAudit?: ComputerControlAuditSink + leases?: ComputerControlLeaseStore + perceptions?: ComputerControlPerceptionStore + driverDeadlineMs?: number + approvalDeadlineMs?: number + classify?: ( + action: ComputerControlAction, + element: DriverElement + ) => ComputerControlRisk + now?: () => number + createId?: () => string +} + +export class ComputerControlBroker { + readonly leases: ComputerControlLeaseStore + readonly perceptions: ComputerControlPerceptionStore + + private readonly driver: ComputerControlDriver + private readonly approval: ComputerControlApprovalProvider + private readonly audit: ComputerControlAuditSink + private readonly fallbackAudit: ComputerControlAuditSink | undefined + private readonly driverDeadlineMs: number + private readonly approvalDeadlineMs: number + private readonly classify?: ComputerControlBrokerOptions['classify'] + private readonly now: () => number + private readonly createId: () => string + private readonly queues = new Map>() + private readonly completed = new Map< + string, + { fingerprint: string; result: ComputerControlCommandResult } + >() + private readonly inFlight = new Map() + + constructor(options: ComputerControlBrokerOptions) { + this.driver = options.driver + this.approval = options.approval + this.audit = options.audit + this.fallbackAudit = options.fallbackAudit + this.now = options.now ?? Date.now + this.createId = options.createId ?? randomUUID + this.leases = + options.leases ?? + new ComputerControlLeaseStore(this.now, this.createId) + this.perceptions = + options.perceptions ?? + new ComputerControlPerceptionStore(this.now, this.createId) + this.driverDeadlineMs = + options.driverDeadlineMs ?? COMPUTER_CONTROL_DRIVER_DEADLINE_MS + this.approvalDeadlineMs = + options.approvalDeadlineMs ?? + COMPUTER_CONTROL_APPROVAL_DEADLINE_MS + this.classify = options.classify + } + + createLease( + binding: ComputerControlLeaseBinding + ): ComputerControlLease { + return this.leases.create(binding) + } + + revoke(leaseId: string): void { + this.leases.revoke(leaseId) + this.perceptions.revokeLease(leaseId) + void this.releaseInjectedInput() + } + + revokeTask(taskId: string): void { + const revokedLeaseIds = this.leases.revokeTask(taskId) + for (const leaseId of revokedLeaseIds) { + this.perceptions.revokeLease(leaseId) + } + void this.releaseInjectedInput() + } + + async execute( + untrustedCommand: unknown, + context: BrokerExecutionContext, + signal: AbortSignal + ): Promise { + const parsed = + computerControlRuntimeCommandSchema.safeParse(untrustedCommand) + if (!parsed.success) { + return this.errorResult( + INVALID_COMMAND_ID, + new ComputerControlFailure( + 'invalid_request', + 'Invalid computer control command' + ) + ) + } + const command = parsed.data + const fingerprint = JSON.stringify({ command, context }) + + const cached = this.completed.get(command.commandId) + if (cached) { + return cached.fingerprint === fingerprint + ? cached.result + : this.errorResult( + command.commandId, + new ComputerControlFailure( + 'command_id_conflict', + 'Computer control command ID was reused' + ) + ) + } + const active = this.inFlight.get(command.commandId) + if (active) { + return active.fingerprint === fingerprint + ? active.result + : this.errorResult( + command.commandId, + new ComputerControlFailure( + 'command_id_conflict', + 'Computer control command ID was reused' + ) + ) + } + + const result = this.serialize(context.taskId, () => + this.executeSerialized(command, context, signal) + ) + this.inFlight.set(command.commandId, { fingerprint, result }) + result + .then((settled) => { + this.completed.set(command.commandId, { + fingerprint, + result: settled + }) + while (this.completed.size > MAX_CACHED_COMMANDS) { + const oldest = this.completed.keys().next().value + if (oldest === undefined) { + break + } + this.completed.delete(oldest) + } + }) + .finally(() => { + this.inFlight.delete(command.commandId) + }) + .catch(() => { + // executeSerialized always returns a contract result. + }) + return result + } + + private async executeSerialized( + command: ComputerControlRuntimeCommand, + context: BrokerExecutionContext, + signal: AbortSignal + ): Promise { + let injectionStarted = false + const cancel = (): void => { + this.revoke(command.leaseId) + } + signal.addEventListener('abort', cancel, { once: true }) + + try { + if (signal.aborted) { + cancel() + throw cancellationFailure() + } + const lease = this.leases.validate(command.leaseId, context) + if (!this.driver.available) { + throw new ComputerControlFailure( + 'driver_unavailable', + 'Computer control driver is unavailable' + ) + } + + if (command.kind === 'observe') { + const result = await this.observe(command, lease, signal) + await this.writeAudit({ + ...this.auditBase(command, lease), + action: 'observe', + risk: 'observe', + outcome: 'completed' + }) + return result + } + + const result = await this.act( + command, + lease, + signal, + () => { + injectionStarted = true + } + ) + return result + } catch (error) { + const failure = + signal.aborted && injectionStarted + ? new ComputerControlFailure( + 'outcome_unknown', + 'Cancellation raced input injection' + ) + : this.normalizeFailure(error) + if (command.kind === 'act') { + await this.writeFailureAudit(command, context, failure) + } + return this.errorResult(command.commandId, failure) + } finally { + signal.removeEventListener('abort', cancel) + if (signal.aborted) { + await this.releaseInjectedInput() + } + } + } + + private async observe( + command: Extract, + lease: ComputerControlLease, + signal: AbortSignal + ): Promise { + const observation = await this.driverCall( + (driverSignal) => this.driver.observe(driverSignal), + signal + ) + this.assertWindowMatches(lease, observation.window) + const contract = this.perceptions.create( + lease.leaseId, + observation, + (element) => + this.classifyRisk( + { + kind: 'activate', + elementRef: 'opaque_identifier_placeholder' + }, + element + ) + ) + return { + status: 'observed', + commandId: command.commandId, + observation: contract + } + } + + private async act( + command: Extract, + lease: ComputerControlLease, + signal: AbortSignal, + markInjectionStarted: () => void + ): Promise { + const stored = this.perceptions.resolve( + lease.leaseId, + command.observationId, + command.revision, + command.action.elementRef + ) + let resolved = await this.validateTarget( + lease, + stored.driverElement, + stored.nativeIdentity, + signal + ) + let risk = this.classifyRisk(command.action, resolved) + if (risk === 'forbidden') { + throw new ComputerControlFailure( + 'forbidden', + 'Computer control target is protected' + ) + } + + if (risk === 'input' || risk === 'commit') { + await this.requestApproval(command, lease, resolved, risk, signal) + } + + resolved = await this.validateTarget( + lease, + stored.driverElement, + stored.nativeIdentity, + signal + ) + const finalRisk = this.classifyRisk(command.action, resolved) + if (finalRisk === 'forbidden') { + throw new ComputerControlFailure( + 'forbidden', + 'Computer control target is protected' + ) + } + const elevatedRisk = maximumRisk(risk, finalRisk) + if (elevatedRisk !== risk) { + if (elevatedRisk === 'input' || elevatedRisk === 'commit') { + await this.requestApproval( + command, + lease, + resolved, + elevatedRisk, + signal + ) + resolved = await this.validateTarget( + lease, + stored.driverElement, + stored.nativeIdentity, + signal + ) + } + risk = elevatedRisk + } + const injectionRisk = this.classifyRisk(command.action, resolved) + if (injectionRisk === 'forbidden') { + throw new ComputerControlFailure( + 'forbidden', + 'Computer control target is protected' + ) + } + const finalElevatedRisk = maximumRisk(risk, injectionRisk) + if (finalElevatedRisk !== risk) { + throw new ComputerControlFailure( + 'approval_denied', + 'Computer control risk changed after approval' + ) + } + risk = finalElevatedRisk + + this.perceptions.consume(command.observationId) + markInjectionStarted() + try { + await this.driverCall( + (driverSignal) => + this.driver.inject( + stored.nativeIdentity, + command.action, + driverSignal + ), + signal + ) + } catch { + throw new ComputerControlFailure( + 'outcome_unknown', + signal.aborted + ? 'Cancellation raced input injection' + : 'Input injection outcome could not be verified' + ) + } + if (signal.aborted) { + throw new ComputerControlFailure( + 'outcome_unknown', + 'Cancellation raced input injection' + ) + } + + await this.writeAudit({ + ...this.auditBase(command, lease), + action: command.action.kind, + risk, + outcome: 'completed', + ...this.redactedTextMetadata(command.action) + }) + if (risk === 'forbidden') { + throw new ComputerControlFailure( + 'forbidden', + 'Computer control target is protected' + ) + } + return { + status: 'completed', + commandId: command.commandId, + risk + } + } + + private async requestApproval( + command: Extract, + lease: ComputerControlLease, + element: DriverElement, + risk: 'input' | 'commit', + signal: AbortSignal + ): Promise { + const approvalId = this.createId() + const response = await runWithDeadline( + (approvalSignal) => + this.approval.request( + { + approvalId, + leaseId: lease.leaseId, + commandId: command.commandId, + risk, + action: + command.action.kind === 'scroll' + ? 'activate' + : command.action.kind, + targetName: + this.boundedDisplayText(element.name, 256) || + element.role, + ...(command.action.kind === 'replace_text' + ? { textLength: command.action.text.length } + : {}) + }, + approvalSignal + ), + signal, + this.approvalDeadlineMs, + 'approval_timeout' + ) + const parsed = computerControlApprovalResultSchema.safeParse(response) + if (!parsed.success || parsed.data.approvalId !== approvalId) { + throw new ComputerControlFailure( + 'approval_denied', + 'Computer control approval response was invalid' + ) + } + if (parsed.data.decision !== 'approve_once') { + throw new ComputerControlFailure( + 'approval_denied', + 'Computer control approval was denied' + ) + } + } + + private async validateTarget( + lease: ComputerControlLease, + original: DriverElement, + nativeIdentity: DriverElement['nativeIdentity'], + signal: AbortSignal + ): Promise { + const foreground = await this.driverCall( + (driverSignal) => + this.driver.getForegroundWindow(driverSignal), + signal + ) + this.assertWindowMatches(lease, foreground) + const resolved = await this.driverCall( + (driverSignal) => + this.driver.resolveElement(nativeIdentity, driverSignal), + signal + ) + this.assertElementIdentity(original, resolved, nativeIdentity) + if (!resolved.enabled) { + throw new ComputerControlFailure( + 'focus_failed', + 'Computer control element is disabled' + ) + } + const focused = await this.driverCall( + (driverSignal) => + this.driver.focusElement(nativeIdentity, driverSignal), + signal + ) + if (!focused) { + throw new ComputerControlFailure( + 'focus_failed', + 'Computer control element could not be focused', + true + ) + } + const verified = await this.driverCall( + (driverSignal) => + this.driver.resolveElement(nativeIdentity, driverSignal), + signal + ) + this.assertElementIdentity(original, verified, nativeIdentity) + if (!verified.focused) { + throw new ComputerControlFailure( + 'focus_failed', + 'Computer control element focus was not verified', + true + ) + } + const foregroundAfterFocus = await this.driverCall( + (driverSignal) => + this.driver.getForegroundWindow(driverSignal), + signal + ) + this.assertWindowMatches(lease, foregroundAfterFocus) + return verified + } + + private assertElementIdentity( + original: DriverElement, + resolved: DriverElement | undefined, + nativeIdentity: DriverElement['nativeIdentity'] + ): asserts resolved is DriverElement { + if ( + !resolved || + resolved.nativeIdentity !== nativeIdentity || + resolved.role !== original.role || + resolved.name !== original.name + ) { + throw new ComputerControlFailure( + 'element_identity_changed', + 'Computer control element identity changed', + true + ) + } + } + + private assertWindowMatches( + lease: ComputerControlLease, + window: DriverWindowIdentity + ): void { + if ( + lease.pid !== window.pid || + lease.processStartTime !== window.processStartTime || + lease.windowIdentity !== window.windowIdentity + ) { + this.revoke(lease.leaseId) + throw new ComputerControlFailure( + 'window_not_foreground', + 'Leased window is not the foreground window', + true + ) + } + } + + private classifyRisk( + action: ComputerControlAction, + element: DriverElement + ): ComputerControlRisk { + const baseline = classifyComputerControlAction(action, element) + const additional = this.classify?.(action, element) + return maximumRisk(baseline, additional) + } + + private driverCall( + operation: (signal: AbortSignal) => Promise, + signal: AbortSignal + ): Promise { + return runWithDeadline( + operation, + signal, + this.driverDeadlineMs, + 'driver_timeout' + ) + } + + private serialize( + key: string, + operation: () => Promise + ): Promise { + const previous = this.queues.get(key) ?? Promise.resolve() + const result = previous.then(operation, operation) + const tail = result.then( + () => undefined, + () => undefined + ) + this.queues.set(key, tail) + tail.finally(() => { + if (this.queues.get(key) === tail) { + this.queues.delete(key) + } + }) + return result + } + + private auditBase( + command: ComputerControlRuntimeCommand, + context: BrokerExecutionContext + ): Pick< + ComputerControlAuditEvent, + | 'timestamp' + | 'taskId' + | 'conversationId' + | 'leaseId' + | 'commandId' + > { + return { + timestamp: this.now(), + taskId: context.taskId, + conversationId: context.conversationId, + leaseId: command.leaseId, + commandId: command.commandId + } + } + + private redactedTextMetadata( + action: ComputerControlAction + ): Pick< + ComputerControlAuditEvent, + 'textLength' | 'textDigest' + > { + return action.kind === 'replace_text' + ? { + textLength: action.text.length, + textDigest: digestComputerControlText(action.text) + } + : {} + } + + private async writeFailureAudit( + command: Extract, + context: BrokerExecutionContext, + failure: ComputerControlFailure + ): Promise { + const risk: ComputerControlRisk = + failure.code === 'forbidden' + ? 'forbidden' + : command.action.kind === 'replace_text' || + command.action.kind === 'select_option' + ? 'input' + : command.action.kind === 'activate' + ? 'commit' + : 'navigate' + await this.writeAudit({ + ...this.auditBase(command, context), + action: command.action.kind, + risk, + outcome: + failure.code === 'outcome_unknown' + ? 'outcome_unknown' + : failure.code === 'approval_denied' || + failure.code === 'approval_timeout' + ? 'denied' + : 'failed', + errorCode: failure.code, + ...this.redactedTextMetadata(command.action) + }) + } + + private async writeAudit(event: ComputerControlAuditEvent): Promise { + try { + await this.audit.write(event) + return + } catch { + // Audit persistence must not alter the command's cached result. + } + if (!this.fallbackAudit || this.fallbackAudit === this.audit) { + return + } + try { + await this.fallbackAudit.write(event) + } catch { + // The fallback is also best effort and isolated from execution. + } + } + + private normalizeFailure(error: unknown): ComputerControlFailure { + if (error instanceof ComputerControlFailure) { + return error + } + return new ComputerControlFailure( + 'internal_error', + 'Computer control failed safely' + ) + } + + private boundedDisplayText( + value: string, + maximumLength: number + ): string { + return [...value] + .filter((character) => { + const code = character.charCodeAt(0) + return code > 31 && code !== 127 + }) + .join('') + .trim() + .slice(0, maximumLength) + } + + private errorResult( + commandId: string, + failure: ComputerControlFailure + ): ComputerControlCommandResult { + return { + status: 'error', + commandId, + error: failure.toContract() + } + } + + private async releaseInjectedInput(): Promise { + try { + await runWithDeadline( + (signal) => this.driver.releaseInjectedInput(signal), + new AbortController().signal, + this.driverDeadlineMs, + 'driver_timeout' + ) + } catch { + // Cancellation cleanup is best effort and must not mask the result. + } + } +} diff --git a/src/main/computer-control/database-audit.test.ts b/src/main/computer-control/database-audit.test.ts new file mode 100644 index 0000000..317a6d4 --- /dev/null +++ b/src/main/computer-control/database-audit.test.ts @@ -0,0 +1,161 @@ +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import { afterEach, describe, expect, it } from 'vitest' +import { AssistantDatabase } from '../assistant/assistant-database' +import type { ComputerControlAuditEvent } from './audit' +import { DatabaseComputerControlAuditSink } from './database-audit' + +const temporaryDirectories: string[] = [] + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => + rm(directory, { recursive: true, force: true }) + ) + ) +}) + +async function createDatabase(): Promise<{ + database: AssistantDatabase + databasePath: string +}> { + const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-control-audit-')) + temporaryDirectories.push(directory) + const databasePath = join(directory, 'assistant.sqlite') + const database = new AssistantDatabase(databasePath) + database.initialize('C:\\Workspace') + return { database, databasePath } +} + +const createTask = (database: AssistantDatabase, taskId: string): void => { + database.createTask({ + id: taskId, + title: '计算机控制审计', + instructions: '验证持久审计', + workMode: 'execute' + }) +} + +describe('DatabaseComputerControlAuditSink', () => { + it('persists redacted events idempotently across close and reopen', async () => { + const { database, databasePath } = await createDatabase() + const taskId = '00000000-0000-4000-8000-000000000401' + const typedText = 'secret typed content 不得持久化' + const textDigest = createHash('sha256') + .update(typedText, 'utf8') + .digest('hex') + createTask(database, taskId) + const sink = new DatabaseComputerControlAuditSink(database) + const event: ComputerControlAuditEvent = { + timestamp: 1_775_000_000_000, + taskId, + conversationId: 'conversation-control-0001', + leaseId: 'lease_control_000000001', + commandId: 'command_control_0000001', + action: 'replace_text', + risk: 'input', + outcome: 'completed', + textLength: typedText.length, + textDigest + } + + sink.write(event) + sink.write({ ...event, timestamp: event.timestamp + 1 }) + expect(database.listRecentComputerControlAudit()).toEqual([event]) + database.close() + + const reopened = new AssistantDatabase(databasePath) + reopened.initialize('C:\\Workspace') + expect(reopened.listRecentComputerControlAudit(1)).toEqual([event]) + reopened.close() + + const raw = new DatabaseSync(databasePath) + const auditRows = raw + .prepare('SELECT * FROM computer_control_actions') + .all() + const taskEvents = raw + .prepare( + `SELECT payload_json FROM task_events + WHERE task_id = ? AND kind = 'computer_control'` + ) + .all(taskId) as Array<{ payload_json: string }> + expect(auditRows).toHaveLength(1) + expect(taskEvents).toHaveLength(1) + expect(JSON.stringify(auditRows)).not.toContain(typedText) + expect(JSON.stringify(taskEvents)).not.toContain(typedText) + expect(taskEvents[0]?.payload_json).toContain(textDigest) + raw.close() + expect((await readFile(databasePath)).toString('utf8')).not.toContain( + typedText + ) + }) + + it('enforces task foreign keys and cascades task audit deletion', async () => { + const { database } = await createDatabase() + const taskId = '00000000-0000-4000-8000-000000000402' + const base: ComputerControlAuditEvent = { + timestamp: 1_775_000_000_000, + taskId, + conversationId: 'conversation-control-0002', + leaseId: 'lease_control_000000002', + commandId: 'command_control_0000002', + action: 'observe', + risk: 'observe', + outcome: 'completed' + } + + expect(() => database.persistComputerControlAudit(base)).toThrow() + createTask(database, taskId) + database.persistComputerControlAudit(base) + database.clearAssistantData() + expect(database.listRecentComputerControlAudit()).toEqual([]) + database.close() + }) + + it('rejects invalid audit fields before persistence', async () => { + const { database } = await createDatabase() + const taskId = '00000000-0000-4000-8000-000000000403' + createTask(database, taskId) + const valid: ComputerControlAuditEvent = { + timestamp: 1_775_000_000_000, + taskId, + conversationId: 'conversation-control-0003', + leaseId: 'lease_control_000000003', + commandId: 'command_control_0000003', + action: 'observe', + risk: 'observe', + outcome: 'completed' + } + const invalidEvents: unknown[] = [ + { ...valid, timestamp: Number.NaN }, + { ...valid, taskId: '' }, + { ...valid, conversationId: 'x'.repeat(129) }, + { ...valid, leaseId: 'short' }, + { ...valid, action: 'type_secret' }, + { ...valid, risk: 'unsafe' }, + { ...valid, outcome: 'failed' }, + { + ...valid, + action: 'replace_text', + risk: 'input', + textLength: 10, + textDigest: 'not-a-sha256' + }, + { ...valid, textLength: 1, textDigest: 'a'.repeat(64) } + ] + + for (const invalid of invalidEvents) { + expect(() => + database.persistComputerControlAudit( + invalid as ComputerControlAuditEvent + ) + ).toThrow() + } + expect(() => database.listRecentComputerControlAudit(0)).toThrow() + expect(database.listRecentComputerControlAudit()).toEqual([]) + database.close() + }) +}) diff --git a/src/main/computer-control/database-audit.ts b/src/main/computer-control/database-audit.ts new file mode 100644 index 0000000..5dfe581 --- /dev/null +++ b/src/main/computer-control/database-audit.ts @@ -0,0 +1,15 @@ +import type { AssistantDatabase } from '../assistant/assistant-database' +import type { + ComputerControlAuditEvent, + ComputerControlAuditSink +} from './audit' + +export class DatabaseComputerControlAuditSink + implements ComputerControlAuditSink +{ + constructor(private readonly database: AssistantDatabase) {} + + write(event: ComputerControlAuditEvent): void { + this.database.persistComputerControlAudit(event) + } +} diff --git a/src/main/computer-control/deadline.ts b/src/main/computer-control/deadline.ts new file mode 100644 index 0000000..84ee029 --- /dev/null +++ b/src/main/computer-control/deadline.ts @@ -0,0 +1,49 @@ +import { cancellationFailure, ComputerControlFailure } from './errors' + +export const runWithDeadline = async ( + operation: (signal: AbortSignal) => Promise, + parentSignal: AbortSignal, + deadlineMs: number, + timeoutCode: 'driver_timeout' | 'approval_timeout' +): Promise => { + if (parentSignal.aborted) { + throw cancellationFailure() + } + + const controller = new AbortController() + let timedOut = false + const cancel = (): void => { + controller.abort(parentSignal.reason) + } + parentSignal.addEventListener('abort', cancel, { once: true }) + const timeout = setTimeout(() => { + timedOut = true + controller.abort(new Error('Computer control deadline exceeded')) + }, deadlineMs) + + try { + return await Promise.race([ + operation(controller.signal), + new Promise((_resolve, reject) => { + controller.signal.addEventListener( + 'abort', + () => { + reject( + timedOut + ? new ComputerControlFailure( + timeoutCode, + 'Computer control request timed out', + true + ) + : cancellationFailure() + ) + }, + { once: true } + ) + }) + ]) + } finally { + clearTimeout(timeout) + parentSignal.removeEventListener('abort', cancel) + } +} diff --git a/src/main/computer-control/driver.ts b/src/main/computer-control/driver.ts new file mode 100644 index 0000000..75ed0a6 --- /dev/null +++ b/src/main/computer-control/driver.ts @@ -0,0 +1,89 @@ +import type { + ComputerControlAction, + ComputerControlElementRole +} from '../../shared/computer-control-contracts' + +export const COMPUTER_CONTROL_DRIVER_DEADLINE_MS = 2_000 + +export type NativeElementIdentity = object + +export type ComputerControlTargetKind = + | 'standard' + | 'protected' + | 'password' + | 'otp' + | 'payment' + | 'account_security' + | 'privilege' + | 'os_permission' + +export type DriverWindowIdentity = { + pid: number + processStartTime: number + windowIdentity: string +} + +export type DriverElement = { + nativeIdentity: NativeElementIdentity + role: ComputerControlElementRole + name: string + enabled: boolean + focused: boolean + targetKind: ComputerControlTargetKind +} + +export type DriverObservation = { + window: DriverWindowIdentity + windowTitle: string + elements: DriverElement[] +} + +export interface ComputerControlDriver { + readonly available: boolean + observe(signal: AbortSignal): Promise + getForegroundWindow(signal: AbortSignal): Promise + resolveElement( + nativeIdentity: NativeElementIdentity, + signal: AbortSignal + ): Promise + focusElement( + nativeIdentity: NativeElementIdentity, + signal: AbortSignal + ): Promise + inject( + nativeIdentity: NativeElementIdentity, + action: ComputerControlAction, + signal: AbortSignal + ): Promise + releaseInjectedInput(signal: AbortSignal): Promise +} + +export class UnavailableComputerControlDriver + implements ComputerControlDriver +{ + readonly available = false + + async observe(): Promise { + throw new Error('Computer control driver is unavailable') + } + + async getForegroundWindow(): Promise { + throw new Error('Computer control driver is unavailable') + } + + async resolveElement(): Promise { + throw new Error('Computer control driver is unavailable') + } + + async focusElement(): Promise { + throw new Error('Computer control driver is unavailable') + } + + async inject(): Promise { + throw new Error('Computer control driver is unavailable') + } + + async releaseInjectedInput(): Promise { + // There cannot be injected input when no driver exists. + } +} diff --git a/src/main/computer-control/errors.ts b/src/main/computer-control/errors.ts new file mode 100644 index 0000000..5c738f8 --- /dev/null +++ b/src/main/computer-control/errors.ts @@ -0,0 +1,31 @@ +import type { + ComputerControlError, + ComputerControlErrorCode +} from '../../shared/computer-control-contracts' + +export class ComputerControlFailure extends Error { + readonly code: ComputerControlErrorCode + readonly retryable: boolean + + constructor( + code: ComputerControlErrorCode, + message: string, + retryable = false + ) { + super(message) + this.name = 'ComputerControlFailure' + this.code = code + this.retryable = retryable + } + + toContract(): ComputerControlError { + return { + code: this.code, + message: this.message.slice(0, 256) || 'Computer control failed', + retryable: this.retryable + } + } +} + +export const cancellationFailure = (): ComputerControlFailure => + new ComputerControlFailure('cancelled', 'Computer control was cancelled') diff --git a/src/main/computer-control/lease-store.test.ts b/src/main/computer-control/lease-store.test.ts new file mode 100644 index 0000000..c157524 --- /dev/null +++ b/src/main/computer-control/lease-store.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'vitest' +import { + COMPUTER_CONTROL_LEASE_ABSOLUTE_MS, + COMPUTER_CONTROL_LEASE_IDLE_MS, + ComputerControlLeaseStore, + type ComputerControlLeaseBinding +} from './lease-store' + +const binding: ComputerControlLeaseBinding = { + taskId: 'task-1', + conversationId: 'conversation-1', + pid: 42, + processStartTime: 100, + windowIdentity: 'window-1' +} + +describe('ComputerControlLeaseStore', () => { + it('enforces idle and absolute expiry', () => { + let now = 1_000 + let sequence = 0 + const store = new ComputerControlLeaseStore( + () => now, + () => `lease_identifier_${++sequence}` + ) + const idleLease = store.create(binding) + now += COMPUTER_CONTROL_LEASE_IDLE_MS + expect(() => + store.validate(idleLease.leaseId, binding) + ).toThrow('expired') + + now = 1_000 + const absoluteLease = store.create(binding) + for (let elapsed = 60_000; elapsed < COMPUTER_CONTROL_LEASE_ABSOLUTE_MS; elapsed += 60_000) { + now = 1_000 + elapsed + if (elapsed < COMPUTER_CONTROL_LEASE_ABSOLUTE_MS) { + store.validate(absoluteLease.leaseId, binding) + } + } + now = 1_000 + COMPUTER_CONTROL_LEASE_ABSOLUTE_MS + expect(() => + store.validate(absoluteLease.leaseId, binding) + ).toThrow('expired') + }) + + it('revokes and fails on task, conversation, PID, process, or window mismatch', () => { + const variants: ComputerControlLeaseBinding[] = [ + { ...binding, taskId: 'task-2' }, + { ...binding, conversationId: 'conversation-2' }, + { ...binding, pid: 43 }, + { ...binding, processStartTime: 101 }, + { ...binding, windowIdentity: 'window-2' } + ] + let sequence = 0 + const store = new ComputerControlLeaseStore( + () => 1_000, + () => `lease_identifier_${++sequence}` + ) + + for (const mismatch of variants) { + const lease = store.create(binding) + expect(() => + store.validate(lease.leaseId, mismatch) + ).toThrow('binding changed') + expect(store.peek(lease.leaseId)).toBeUndefined() + } + }) + + it('supports explicit lease and task revocation', () => { + let sequence = 0 + const store = new ComputerControlLeaseStore( + () => 1_000, + () => `lease_identifier_${++sequence}` + ) + const first = store.create(binding) + const second = store.create({ ...binding, windowIdentity: 'window-2' }) + const other = store.create({ ...binding, taskId: 'task-2' }) + + store.revoke(first.leaseId) + expect(store.peek(first.leaseId)).toBeUndefined() + store.revokeTask(binding.taskId) + expect(store.peek(second.leaseId)).toBeUndefined() + expect(store.peek(other.leaseId)).toBeDefined() + }) + + it('prunes all expired leases during create and validate', () => { + let now = 1_000 + let sequence = 0 + const store = new ComputerControlLeaseStore( + () => now, + () => `lease_identifier_${++sequence}` + ) + const expiredBeforeCreate = store.create(binding) + now += COMPUTER_CONTROL_LEASE_IDLE_MS + const current = store.create(binding) + expect(store.peek(expiredBeforeCreate.leaseId)).toBeUndefined() + + const expiresBeforeValidate = store.create({ + ...binding, + taskId: 'task-2' + }) + now += COMPUTER_CONTROL_LEASE_IDLE_MS - 1 + store.validate(current.leaseId, binding) + now += 1 + const newest = store.create({ + ...binding, + taskId: 'task-3' + }) + store.validate(newest.leaseId, { + ...binding, + taskId: 'task-3' + }) + expect(store.peek(expiresBeforeValidate.leaseId)).toBeUndefined() + }) + + it('enforces a validated hard lease capacity by evicting oldest entries', () => { + let sequence = 0 + const store = new ComputerControlLeaseStore( + () => 1_000, + () => `lease_identifier_${++sequence}`, + { maximumLeases: 2 } + ) + const first = store.create(binding) + const second = store.create({ + ...binding, + windowIdentity: 'window-2' + }) + const third = store.create({ + ...binding, + windowIdentity: 'window-3' + }) + + expect(store.peek(first.leaseId)).toBeUndefined() + expect(store.peek(second.leaseId)).toBeDefined() + expect(store.peek(third.leaseId)).toBeDefined() + expect( + () => + new ComputerControlLeaseStore(Date.now, undefined, { + maximumLeases: 0 + }) + ).toThrow('capacity') + }) +}) diff --git a/src/main/computer-control/lease-store.ts b/src/main/computer-control/lease-store.ts new file mode 100644 index 0000000..d9100fc --- /dev/null +++ b/src/main/computer-control/lease-store.ts @@ -0,0 +1,145 @@ +import { randomUUID } from 'node:crypto' +import { ComputerControlFailure } from './errors' +import type { DriverWindowIdentity } from './driver' + +export const COMPUTER_CONTROL_LEASE_IDLE_MS = 5 * 60 * 1_000 +export const COMPUTER_CONTROL_LEASE_ABSOLUTE_MS = 10 * 60 * 1_000 +export const COMPUTER_CONTROL_MAXIMUM_LEASES = 1_000 + +export type ComputerControlLeaseBinding = DriverWindowIdentity & { + taskId: string + conversationId: string +} + +export type ComputerControlLease = ComputerControlLeaseBinding & { + leaseId: string + createdAt: number + lastUsedAt: number +} + +export type ComputerControlLeaseStoreOptions = { + maximumLeases?: number +} + +export class ComputerControlLeaseStore { + private readonly leases = new Map() + private readonly maximumLeases: number + + constructor( + private readonly now: () => number = Date.now, + private readonly createId: () => string = randomUUID, + options: ComputerControlLeaseStoreOptions = {} + ) { + this.maximumLeases = + options.maximumLeases ?? COMPUTER_CONTROL_MAXIMUM_LEASES + if ( + !Number.isSafeInteger(this.maximumLeases) || + this.maximumLeases < 1 || + this.maximumLeases > 10_000 + ) { + throw new Error('Invalid computer control lease capacity') + } + } + + create(binding: ComputerControlLeaseBinding): ComputerControlLease { + const timestamp = this.now() + this.pruneExpired(timestamp) + while (this.leases.size >= this.maximumLeases) { + const oldest = this.leases.keys().next().value + if (oldest === undefined) { + break + } + this.leases.delete(oldest) + } + const lease: ComputerControlLease = { + ...binding, + leaseId: this.createId(), + createdAt: timestamp, + lastUsedAt: timestamp + } + this.leases.set(lease.leaseId, lease) + return { ...lease } + } + + peek(leaseId: string): ComputerControlLease | undefined { + const lease = this.leases.get(leaseId) + return lease ? { ...lease } : undefined + } + + validate( + leaseId: string, + binding: ComputerControlLeaseBinding + ): ComputerControlLease { + const timestamp = this.now() + const requestedLeaseExpired = this.isExpired( + this.leases.get(leaseId), + timestamp + ) + this.pruneExpired(timestamp) + const lease = this.leases.get(leaseId) + if (!lease) { + if (requestedLeaseExpired) { + throw new ComputerControlFailure( + 'lease_expired', + 'Computer control lease expired' + ) + } + throw new ComputerControlFailure( + 'lease_not_found', + 'Computer control lease was not found' + ) + } + + if ( + lease.taskId !== binding.taskId || + lease.conversationId !== binding.conversationId || + lease.pid !== binding.pid || + lease.processStartTime !== binding.processStartTime || + lease.windowIdentity !== binding.windowIdentity + ) { + this.leases.delete(leaseId) + throw new ComputerControlFailure( + 'lease_mismatch', + 'Computer control lease binding changed' + ) + } + + lease.lastUsedAt = timestamp + return { ...lease } + } + + revoke(leaseId: string): void { + this.leases.delete(leaseId) + } + + revokeTask(taskId: string): string[] { + const revoked: string[] = [] + for (const [leaseId, lease] of this.leases) { + if (lease.taskId === taskId) { + this.leases.delete(leaseId) + revoked.push(leaseId) + } + } + return revoked + } + + private pruneExpired(timestamp: number): void { + for (const [leaseId, lease] of this.leases) { + if (this.isExpired(lease, timestamp)) { + this.leases.delete(leaseId) + } + } + } + + private isExpired( + lease: ComputerControlLease | undefined, + timestamp: number + ): boolean { + return ( + lease !== undefined && + (timestamp - lease.lastUsedAt >= COMPUTER_CONTROL_LEASE_IDLE_MS || + timestamp - lease.createdAt >= + COMPUTER_CONTROL_LEASE_ABSOLUTE_MS) + ) + } +} diff --git a/src/main/computer-control/perception-store.ts b/src/main/computer-control/perception-store.ts new file mode 100644 index 0000000..9392950 --- /dev/null +++ b/src/main/computer-control/perception-store.ts @@ -0,0 +1,191 @@ +import { + createHmac, + randomBytes, + randomUUID +} from 'node:crypto' +import type { + ComputerControlObservation, + ComputerControlRisk +} from '../../shared/computer-control-contracts' +import type { + DriverElement, + DriverObservation, + NativeElementIdentity +} from './driver' +import { ComputerControlFailure } from './errors' + +export const COMPUTER_CONTROL_OBSERVATION_FRESH_MS = 3_000 +export const COMPUTER_CONTROL_MAX_ELEMENTS = 200 + +type StoredElement = { + nativeIdentity: NativeElementIdentity + driverElement: DriverElement + risk: ComputerControlRisk +} + +type StoredObservation = { + contract: ComputerControlObservation + elements: Map + consumed: boolean +} + +const boundedDisplayText = ( + value: string, + maximumLength: number, + fallback = '' +): string => { + const clean = [...value] + .filter((character) => { + const code = character.charCodeAt(0) + return code > 31 && code !== 127 + }) + .join('') + .trim() + .slice(0, maximumLength) + return clean || fallback +} + +export class ComputerControlPerceptionStore { + private readonly observations = new Map() + private readonly revisions = new Map() + + constructor( + private readonly now: () => number = Date.now, + private readonly createId: () => string = randomUUID, + private readonly hmacKey: Buffer = randomBytes(32) + ) {} + + create( + leaseId: string, + observation: DriverObservation, + classify: (element: DriverElement) => ComputerControlRisk + ): ComputerControlObservation { + if (observation.elements.length > COMPUTER_CONTROL_MAX_ELEMENTS) { + throw new ComputerControlFailure( + 'internal_error', + 'Driver observation exceeded the element limit' + ) + } + + const revision = (this.revisions.get(leaseId) ?? 0) + 1 + this.revisions.set(leaseId, revision) + this.deleteLeaseObservations(leaseId) + + const observationId = this.createId() + const storedElements = new Map() + const elements = observation.elements.map((element, index) => { + const risk = classify(element) + const ref = createHmac('sha256', this.hmacKey) + .update(`${observationId}\u0000${revision}\u0000${index}`) + .digest('base64url') + storedElements.set(ref, { + nativeIdentity: element.nativeIdentity, + driverElement: element, + risk + }) + return { + ref, + role: element.role, + name: boundedDisplayText(element.name, 256, element.role), + enabled: element.enabled, + focused: element.focused, + risk, + blocked: risk === 'forbidden' + } + }) + + const contract: ComputerControlObservation = { + observationId, + leaseId, + revision, + capturedAt: this.now(), + windowTitle: boundedDisplayText( + observation.windowTitle, + 256 + ), + elements + } + this.observations.set(observationId, { + contract, + elements: storedElements, + consumed: false + }) + return structuredClone(contract) + } + + resolve( + leaseId: string, + observationId: string, + revision: number, + elementRef: string + ): StoredElement { + const observation = this.observations.get(observationId) + if (!observation || observation.contract.leaseId !== leaseId) { + throw new ComputerControlFailure( + 'observation_not_found', + 'Computer control observation was not found' + ) + } + if (observation.contract.revision !== revision) { + throw new ComputerControlFailure( + 'observation_stale', + 'Computer control observation revision changed', + true + ) + } + if ( + this.now() - observation.contract.capturedAt >= + COMPUTER_CONTROL_OBSERVATION_FRESH_MS + ) { + throw new ComputerControlFailure( + 'observation_stale', + 'Computer control observation is stale', + true + ) + } + if (observation.consumed) { + throw new ComputerControlFailure( + 'observation_consumed', + 'Computer control observation was already used' + ) + } + const element = observation.elements.get(elementRef) + if (!element) { + throw new ComputerControlFailure( + 'element_not_found', + 'Computer control element was not found' + ) + } + return element + } + + consume(observationId: string): void { + const observation = this.observations.get(observationId) + if (!observation) { + throw new ComputerControlFailure( + 'observation_not_found', + 'Computer control observation was not found' + ) + } + if (observation.consumed) { + throw new ComputerControlFailure( + 'observation_consumed', + 'Computer control observation was already used' + ) + } + observation.consumed = true + } + + revokeLease(leaseId: string): void { + this.deleteLeaseObservations(leaseId) + this.revisions.delete(leaseId) + } + + private deleteLeaseObservations(leaseId: string): void { + for (const [observationId, observation] of this.observations) { + if (observation.contract.leaseId === leaseId) { + this.observations.delete(observationId) + } + } + } +} diff --git a/src/main/computer-control/risk-policy.test.ts b/src/main/computer-control/risk-policy.test.ts new file mode 100644 index 0000000..545697d --- /dev/null +++ b/src/main/computer-control/risk-policy.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import type { DriverElement } from './driver' +import { + classifyComputerControlAction, + maximumRisk +} from './risk-policy' + +const identity = {} +const element: DriverElement = { + nativeIdentity: identity, + role: 'link', + name: 'Next page', + enabled: true, + focused: true, + targetKind: 'standard' +} + +describe('computer control risk policy', () => { + it('classifies semantic actions and lets classification only raise risk', () => { + expect( + classifyComputerControlAction( + { kind: 'activate', elementRef: 'opaque_identifier_123456' }, + element + ) + ).toBe('navigate') + expect( + maximumRisk('commit', 'observe') + ).toBe('commit') + expect(maximumRisk('input', 'commit')).toBe('commit') + }) + + it.each([ + 'protected', + 'password', + 'otp', + 'payment', + 'account_security', + 'privilege', + 'os_permission' + ] as const)('forbids %s targets', (targetKind) => { + expect( + classifyComputerControlAction( + { kind: 'activate', elementRef: 'opaque_identifier_123456' }, + { ...element, targetKind } + ) + ).toBe('forbidden') + }) +}) diff --git a/src/main/computer-control/risk-policy.ts b/src/main/computer-control/risk-policy.ts new file mode 100644 index 0000000..7440172 --- /dev/null +++ b/src/main/computer-control/risk-policy.ts @@ -0,0 +1,79 @@ +import type { + ComputerControlAction, + ComputerControlElementRole, + ComputerControlRisk +} from '../../shared/computer-control-contracts' +import type { + ComputerControlTargetKind, + DriverElement +} from './driver' + +const RISK_ORDER: Record = { + observe: 0, + navigate: 1, + input: 2, + commit: 3, + forbidden: 4 +} + +const FORBIDDEN_TARGETS: ReadonlySet = + new Set([ + 'protected', + 'password', + 'otp', + 'payment', + 'account_security', + 'privilege', + 'os_permission' + ]) + +const activationRisk = ( + role: ComputerControlElementRole +): ComputerControlRisk => { + switch (role) { + case 'button': + case 'checkbox': + case 'radio': + return 'commit' + case 'textbox': + case 'combobox': + case 'scrollarea': + return 'observe' + default: + return 'navigate' + } +} + +export const maximumRisk = ( + baseline: ComputerControlRisk, + classification?: ComputerControlRisk +): ComputerControlRisk => + classification && + RISK_ORDER[classification] > RISK_ORDER[baseline] + ? classification + : baseline + +export const classifyComputerControlAction = ( + action: ComputerControlAction, + element: DriverElement, + driverClassification?: ComputerControlRisk +): ComputerControlRisk => { + if (FORBIDDEN_TARGETS.has(element.targetKind)) { + return 'forbidden' + } + + let baseline: ComputerControlRisk + switch (action.kind) { + case 'replace_text': + case 'select_option': + baseline = 'input' + break + case 'scroll': + baseline = 'navigate' + break + case 'activate': + baseline = activationRisk(element.role) + break + } + return maximumRisk(baseline, driverClassification) +} diff --git a/src/main/index.ts b/src/main/index.ts index cd2614e..06439f1 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -37,6 +37,7 @@ import type { ContinueHostLauncher } from './agent/continue-host-adapter' import { resolvePortableUserDataPath } from './portable-user-data' +import { BrowserService } from './browser/browser-service' const shortcut = 'CommandOrControl+Shift+Space' const portableUserDataPath = resolvePortableUserDataPath({ @@ -63,6 +64,7 @@ let removeIpcHandlers: (() => Promise) | undefined let runtime: AgentRuntimeController | undefined let knowledgeService: KnowledgeService | undefined let assistantDatabase: AssistantDatabase | undefined +let browserService: BrowserService | undefined function createEmbeddingProvider( settings: ResolvedRuntimeSettings @@ -227,6 +229,7 @@ if (hasSingleInstanceLock) { join(app.getPath('userData'), 'skills', 'imported'), secureCipher ) + browserService = new BrowserService() const bundledRuntimePaths = resolveBundledRuntimePaths({ appPath: app.getAppPath(), resourcesPath: process.resourcesPath, @@ -261,15 +264,21 @@ if (hasSingleInstanceLock) { : useOpenCode ? ('opencode' as const) : ('model' as const) - const [skillInstructions, mcpServers] = await Promise.all([ - capabilityService.getSkillInstructions( - target, - target === 'continue' ? 12_000 : 48_000 - ), - target === 'model' - ? capabilityService.getResolvedMcpServers('model') - : Promise.resolve([]) - ]) + const [skillInstructions, mcpServers, browserCapability] = + await Promise.all([ + capabilityService.getSkillInstructions( + target, + target === 'continue' ? 12_000 : 48_000 + ), + target === 'model' + ? capabilityService.getResolvedMcpServers('model') + : Promise.resolve([]), + target === 'model' + ? capabilityService.getComputerCapabilityStatus( + 'host-browser-control' + ) + : Promise.resolve(undefined) + ]) return createAgentRuntime(defaultWorkspace, settings, { skillInstructions, mcpServers, @@ -278,7 +287,11 @@ if (hasSingleInstanceLock) { 'continue-host' ), bundledRuntimePaths, - continueHostLauncher: launchContinueHost + continueHostLauncher: launchContinueHost, + browserService: + browserCapability?.enabled && browserCapability.supported + ? browserService + : undefined }) } runtime = new AgentRuntimeController( @@ -316,7 +329,11 @@ if (hasSingleInstanceLock) { await createConfiguredRuntime() ) } - } + }, + async () => { + await browserService?.clearSessions() + }, + browserService ) loadMainWindow(mainWindow) @@ -354,7 +371,8 @@ app.on('before-quit', (event) => { tray?.destroy() await Promise.allSettled([ runtime?.dispose(), - knowledgeService?.dispose() + knowledgeService?.dispose(), + browserService?.dispose() ]) } finally { assistantDatabase?.close() diff --git a/src/main/ipc.test.ts b/src/main/ipc.test.ts index 55be1ff..e9b1096 100644 --- a/src/main/ipc.test.ts +++ b/src/main/ipc.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { ipcChannels } from '../shared/ipc-channels' +import type { BrowserLiveState } from '../shared/contracts' import { registerIpcHandlers } from './ipc' type InvokeHandler = (event: unknown, input?: unknown) => unknown @@ -20,6 +21,131 @@ const electronMocks = vi.hoisted(() => { } }) +describe('registerIpcHandlers computer capabilities', () => { + afterEach(() => { + electronMocks.handlers.clear() + vi.clearAllMocks() + }) + + it('validates computer capability requests and restricts them to the trusted renderer', async () => { + const webContents = { + mainFrame: { url: 'file:///goodbuddy/index.html' }, + getURL: vi.fn(() => 'file:///goodbuddy/index.html'), + send: vi.fn() + } + const window = { + webContents, + isDestroyed: vi.fn(() => false), + isMaximized: vi.fn(() => false), + on: vi.fn(), + removeListener: vi.fn() + } + const snapshot = { + skills: [], + mcpServers: [], + computerCapabilities: [], + browserProfiles: { profiles: [], defaultProfileId: null } + } + const capabilityService = { + setComputerCapabilityEnabled: vi.fn(async () => snapshot), + createBrowserProfile: vi.fn(async () => snapshot), + diagnoseComputerCapability: vi.fn(async () => ({ + capabilityId: 'host-browser-control', + status: 'disabled', + checkedAt: '2026-08-05T00:00:00.000Z', + checks: [] + })) + } + const onRuntimeSettingsChanged = vi.fn(async () => {}) + const releaseConversation = vi.fn(async () => {}) + let browserStateListener: + | ((state: BrowserLiveState) => void) + | undefined + const dispose = registerIpcHandlers( + window as never, + { capability: 'text' } as never, + 'CommandOrControl+Shift+Space', + {} as never, + capabilityService as never, + { clear: vi.fn() } as never, + {} as never, + { claimDueSchedules: vi.fn(() => []) } as never, + { clear: vi.fn() } as never, + {} as never, + onRuntimeSettingsChanged, + undefined, + { + releaseConversation, + onState: (listener) => { + browserStateListener = listener + return vi.fn() + } + } + ) + const event = { + sender: webContents, + senderFrame: webContents.mainFrame + } + + await expect( + electronMocks.handlers.get( + ipcChannels.capabilitiesToggleComputer + )?.(event, { + capabilityId: 'host-browser-control', + enabled: true + }) + ).resolves.toEqual(snapshot) + expect( + capabilityService.setComputerCapabilityEnabled + ).toHaveBeenCalledWith('host-browser-control', true) + expect(onRuntimeSettingsChanged).toHaveBeenCalledOnce() + + browserStateListener?.({ + conversationId: 'browser-conversation', + status: 'ready', + updatedAt: 1 + }) + expect(webContents.send).toHaveBeenCalledWith( + ipcChannels.browserState, + expect.objectContaining({ + conversationId: 'browser-conversation', + status: 'ready' + }) + ) + await expect( + electronMocks.handlers.get(ipcChannels.browserStop)?.(event, { + conversationId: 'browser-conversation' + }) + ).resolves.toBeUndefined() + expect(releaseConversation).toHaveBeenCalledWith( + 'browser-conversation' + ) + + expect(() => + electronMocks.handlers.get( + ipcChannels.capabilitiesCreateBrowserProfile + )?.(event, { + name: '工作配置', + executable: 'C:\\unsafe.exe' + }) + ).toThrow() + expect(capabilityService.createBrowserProfile).not.toHaveBeenCalled() + + expect(() => + electronMocks.handlers.get( + ipcChannels.capabilitiesDiagnoseComputer + )?.( + { + sender: {}, + senderFrame: webContents.mainFrame + }, + 'host-browser-control' + ) + ).toThrow('拒绝来自未知窗口的 IPC 请求') + await dispose() + }) +}) + vi.mock('electron', () => ({ app: { getName: vi.fn(() => 'GoodBuddy'), @@ -273,14 +399,19 @@ describe('registerIpcHandlers agent terminal state', () => { vi.clearAllMocks() }) - function createHarness(runtime: Record) { + function createHarness( + runtime: Record, + onBeforeClearLocalData?: () => Promise, + toolApproval: 'always' | 'policy' = 'always' + ) { const assistantDatabase = { claimDueSchedules: vi.fn(() => []), createTask: vi.fn(), appendTaskEvent: vi.fn(), updateTaskStatus: vi.fn(), createTextArtifact: vi.fn(), - upsertModelUsageCall: vi.fn() + upsertModelUsageCall: vi.fn(), + clearAssistantData: vi.fn() } const webContents = { mainFrame: { url: 'file:///goodbuddy/index.html' }, @@ -309,7 +440,7 @@ describe('registerIpcHandlers agent terminal state', () => { 'CommandOrControl+Shift+Space', { getResolvedSettings: vi.fn(async () => ({ - toolApproval: 'always' + toolApproval })) } as never, {} as never, @@ -318,12 +449,16 @@ describe('registerIpcHandlers agent terminal state', () => { assistantDatabase as never, approvalBroker as never, {} as never, - vi.fn(async () => {}) + vi.fn(async () => {}), + onBeforeClearLocalData ) return { approvalBroker, assistantDatabase, dispose, + clearHandler: electronMocks.handlers.get( + ipcChannels.appClearLocalData + ), handler: electronMocks.handlers.get(ipcChannels.agentRun), webContents } @@ -336,6 +471,62 @@ describe('registerIpcHandlers agent terminal state', () => { senderFrame: webContents.mainFrame }) + it('aborts active work and clears browser sessions before assistant data', async () => { + const lifecycle: string[] = [] + let markStarted!: () => void + const started = new Promise((resolve) => { + markStarted = resolve + }) + const runtime = { + capability: 'chat', + requiresToolApproval: false, + supportsToolExecution: true, + getStatus: vi.fn(), + dispose: vi.fn(), + async *run( + _request: unknown, + signal: AbortSignal + ): AsyncGenerator { + markStarted() + await new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + lifecycle.push('aborted') + reject(signal.reason) + }, + { once: true } + ) + }) + yield undefined as never + } + } + const harness = createHarness(runtime, async () => { + lifecycle.push('browser-cleared') + }) + vi.mocked( + harness.assistantDatabase.clearAssistantData + ).mockImplementation(() => { + lifecycle.push('assistant-cleared') + }) + harness.handler?.(trustedEvent(harness.webContents), { + requestId: '3f496642-f47d-4e0a-8944-a32c77b0d6ef', + conversationId: 'conversation-clear', + prompt: 'keep running', + workMode: 'execute' + }) + await started + + await harness.clearHandler?.(trustedEvent(harness.webContents)) + + expect(lifecycle).toEqual([ + 'aborted', + 'browser-cleared', + 'assistant-cleared' + ]) + await harness.dispose() + }) + it('marks a request failed when a tool fails before runtime done', async () => { const runtime = { capability: 'chat', @@ -386,6 +577,51 @@ describe('registerIpcHandlers agent terminal state', () => { await harness.dispose() }) + it('allows a completed request after a recoverable tool failure', async () => { + const runtime = { + capability: 'chat', + requiresToolApproval: false, + supportsToolExecution: true, + getStatus: vi.fn(), + dispose: vi.fn(), + async *run(request: { requestId: string }) { + yield { + requestId: request.requestId, + type: 'tool', + callId: 'call-recoverable', + name: '浏览器输入', + state: 'recoverable', + summary: '直连模型工具需要刷新后重试：浏览器输入' + } + yield { requestId: request.requestId, type: 'done' } + } + } + const harness = createHarness(runtime) + const requestId = '3f496642-f47d-4e0a-8944-a32c77b0d6ef' + + harness.handler?.(trustedEvent(harness.webContents), { + requestId, + conversationId: 'conversation-1', + prompt: 'retry browser input', + workMode: 'execute' + }) + + await vi.waitFor(() => + expect(harness.assistantDatabase.updateTaskStatus).toHaveBeenCalledWith( + requestId, + 'completed' + ) + ) + expect( + harness.assistantDatabase.updateTaskStatus + ).not.toHaveBeenCalledWith( + requestId, + 'failed', + expect.any(String) + ) + await harness.dispose() + }) + it.each(['opencode', 'continue'] as const)( 'normalizes interactive %s requests to Execute without GoodBuddy approval', async (runtimeId) => { @@ -438,6 +674,55 @@ describe('registerIpcHandlers agent terminal state', () => { } ) + it.each(['model', 'opencode'] as const)( + 'normalizes legacy interactive Plan requests to Ask for %s', + async (runtimeId) => { + let receivedRequest: + | { requestId: string; prompt: string; workMode?: string } + | undefined + const runtime = { + runtimeId, + capability: 'chat', + requiresToolApproval: false, + supportsToolExecution: true, + getStatus: vi.fn(), + dispose: vi.fn(), + async *run(request: { + requestId: string + prompt: string + workMode?: string + }) { + receivedRequest = request + yield { requestId: request.requestId, type: 'done' } + } + } + const harness = createHarness(runtime) + const requestId = '3f496642-f47d-4e0a-8944-a32c77b0d6ef' + + harness.handler?.(trustedEvent(harness.webContents), { + requestId, + conversationId: 'conversation-1', + prompt: 'draft a plan', + workMode: 'plan' + }) + + await vi.waitFor(() => + expect( + harness.assistantDatabase.updateTaskStatus + ).toHaveBeenCalledWith(requestId, 'completed') + ) + expect(receivedRequest?.workMode).toBe('ask') + expect(receivedRequest?.prompt).toContain('Work mode: Ask.') + expect(receivedRequest?.prompt).not.toContain('Work mode: Plan.') + expect( + harness.assistantDatabase.createTask + ).toHaveBeenCalledWith( + expect.objectContaining({ id: requestId, workMode: 'ask' }) + ) + await harness.dispose() + } + ) + it('rejects Execute before creating a task on an unsupported runtime', async () => { const runtime = { capability: 'chat', @@ -461,7 +746,7 @@ describe('registerIpcHandlers agent terminal state', () => { await harness.dispose() }) - it('routes direct-model tool calls through the GoodBuddy approval broker', async () => { + it('authorizes direct-model Execute tools without approval events or broker prompts', async () => { let receivedAuthorize: | (( request: { @@ -471,6 +756,7 @@ describe('registerIpcHandlers agent terminal state', () => { } ) => Promise) | undefined + let decision: string | undefined const runtime = { runtimeId: 'model', capability: 'chat', @@ -484,7 +770,7 @@ describe('registerIpcHandlers agent terminal state', () => { authorize: typeof receivedAuthorize ) { receivedAuthorize = authorize - await authorize?.({ + decision = await authorize?.({ scopeKey: 'model:builtin:workspace_read_text', title: '允许读取工作区文本？', description: '读取 README.md' @@ -501,7 +787,6 @@ describe('registerIpcHandlers agent terminal state', () => { } } const harness = createHarness(runtime) - harness.approvalBroker.request.mockResolvedValue('once') const requestId = '3f496642-f47d-4e0a-8944-a32c77b0d6ef' harness.handler?.(trustedEvent(harness.webContents), { @@ -517,14 +802,66 @@ describe('registerIpcHandlers agent terminal state', () => { ).toHaveBeenCalledWith(requestId, 'completed') ) expect(receivedAuthorize).toEqual(expect.any(Function)) - expect(harness.approvalBroker.request).toHaveBeenCalledWith( - expect.objectContaining({ - requestId, - conversationId: 'conversation-1', - scopeKey: 'model:builtin:workspace_read_text' - }), - expect.any(AbortSignal), - expect.any(Function) + expect(decision).toBe('once') + expect(harness.approvalBroker.request).not.toHaveBeenCalled() + expect( + harness.assistantDatabase.updateTaskStatus + ).not.toHaveBeenCalledWith(requestId, 'waiting_approval') + expect(harness.webContents.send).not.toHaveBeenCalledWith( + ipcChannels.agentEvent, + expect.objectContaining({ type: 'approval' }) + ) + await harness.dispose() + }) + + it('denies direct-model Execute tools when the deny-all policy is selected', async () => { + let decision: string | undefined + const runtime = { + runtimeId: 'model', + capability: 'chat', + requiresToolApproval: false, + supportsToolExecution: true, + getStatus: vi.fn(), + dispose: vi.fn(), + async *run( + request: { requestId: string }, + _signal: AbortSignal, + authorize: ( + request: { + scopeKey: string + title: string + description: string + } + ) => Promise + ) { + decision = await authorize({ + scopeKey: 'model:builtin:workspace_read_text', + title: '允许读取工作区文本？', + description: '读取 README.md' + }) + yield { requestId: request.requestId, type: 'done' } + } + } + const harness = createHarness(runtime, undefined, 'policy') + const requestId = '3f496642-f47d-4e0a-8944-a32c77b0d6ef' + + harness.handler?.(trustedEvent(harness.webContents), { + requestId, + conversationId: 'conversation-1', + prompt: '读取文件', + workMode: 'execute' + }) + + await vi.waitFor(() => + expect( + harness.assistantDatabase.updateTaskStatus + ).toHaveBeenCalledWith(requestId, 'completed') + ) + expect(decision).toBe('deny') + expect(harness.approvalBroker.request).not.toHaveBeenCalled() + expect(harness.webContents.send).not.toHaveBeenCalledWith( + ipcChannels.agentEvent, + expect.objectContaining({ type: 'approval' }) ) await harness.dispose() }) diff --git a/src/main/ipc.ts b/src/main/ipc.ts index 9a49e4d..948f2f3 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -12,6 +12,7 @@ import { z } from 'zod' import { approvalDecisionSchema, agentRequestSchema, + browserStopRequestSchema, knowledgeCreateSchema, knowledgeEntityUpdateSchema, knowledgeIdSchema, @@ -26,23 +27,32 @@ import { type AgentRuntimeDetection, type AgentEvent, type AppInfo, + type BrowserLiveState, type KnowledgeSnapshot, type RuntimeSettings } from '../shared/contracts' import { ipcChannels } from '../shared/ipc-channels' import { + browserProfileCreateInputSchema, + browserProfileRenameInputSchema, + browserProfileSelectionInputSchema, + computerCapabilityConfigInputSchema, + computerCapabilityIdSchema, + computerCapabilityToggleInputSchema, mcpServerIdSchema, mcpServerInputSchema, skillAssignmentsInputSchema, skillIdSchema, skillToggleInputSchema, type CapabilitySnapshot, + type CapabilityDiagnosticReport, type McpServerTestResult } from '../shared/capability-contracts' import { assistantIdSchema, conversationSnapshotsSchema, memoryCreateSchema, + normalizeInteractiveWorkMode, projectCreateSchema, scheduleCreateSchema, expertCreateSchema, @@ -131,6 +141,12 @@ const taskStatusRequestSchema = z status: z.enum(['completed', 'cancelled']) }) .strict() +const expertUpdateRequestSchema = z + .object({ + expertId: assistantIdSchema, + input: expertCreateSchema + }) + .strict() const imageMimeTypes: Record< string, @@ -346,7 +362,12 @@ export function registerIpcHandlers( assistantDatabase: AssistantDatabase, approvalBroker: ToolApprovalBroker, bundledRuntimePaths: BundledRuntimePaths, - onRuntimeSettingsChanged: () => Promise + onRuntimeSettingsChanged: () => Promise, + onBeforeClearLocalData?: () => Promise, + browserControl?: { + releaseConversation(conversationId: string): Promise + onState(listener: (state: BrowserLiveState) => void): () => void + } ): () => Promise { const activeRequests = new Map() const heartbeatControllers = new Set() @@ -364,6 +385,7 @@ export function registerIpcHandlers( const channels = Object.values(ipcChannels).filter( (channel) => channel !== ipcChannels.agentEvent && + channel !== ipcChannels.browserState && channel !== ipcChannels.conversationNew && channel !== ipcChannels.settingsOpen && channel !== ipcChannels.windowMaximizedChanged @@ -383,6 +405,11 @@ export function registerIpcHandlers( } window.on('maximize', notifyMaximizedChanged) window.on('unmaximize', notifyMaximizedChanged) + const removeBrowserStateListener = browserControl?.onState((state) => { + if (!window.isDestroyed()) { + window.webContents.send(ipcChannels.browserState, state) + } + }) const abortActiveRequests = (reason: string): void => { for (const controller of activeRequests.values()) { @@ -936,6 +963,7 @@ export function registerIpcHandlers( heartbeatControllers.clear() approvalBroker.clear() await Promise.allSettled([...activeExecutions]) + await onBeforeClearLocalData?.() assistantDatabase.clearAssistantData() } finally { executionPaused = false @@ -947,18 +975,27 @@ export function registerIpcHandlers( return runtime.getStatus() }) + ipcMain.handle(ipcChannels.browserStop, async (event, input: unknown) => { + assertTrustedSender(event, window) + const request = browserStopRequestSchema.parse(input) + await browserControl?.releaseConversation(request.conversationId) + }) + ipcMain.handle(ipcChannels.agentRun, (event, input: unknown) => { assertTrustedSender(event, window) if (executionPaused || shuttingDown) { throw new Error('本地数据维护期间暂不接受新任务') } const parsedInput = agentRequestSchema.parse(input) + const normalizedWorkMode = normalizeInteractiveWorkMode( + parsedInput.workMode + ) const agentRuntimeSelected = isAgentRuntime(runtime) const parsedRequest = { ...parsedInput, - workMode: agentRuntimeSelected + workMode: agentRuntimeSelected && parsedInput.workMode !== 'plan' ? ('execute' as const) - : (parsedInput.workMode ?? ('ask' as const)) + : normalizedWorkMode } if ( parsedRequest.workMode === 'execute' && @@ -977,13 +1014,11 @@ export function registerIpcHandlers( ? '' : enrichedRequest.workMode === 'ask' ? 'Work mode: Ask. Do not call tools or make changes. Answer using only the explicitly supplied context.' - : enrichedRequest.workMode === 'plan' - ? 'Work mode: Plan. Do not call tools or make changes. Produce a concrete reviewable plan and wait for user confirmation.' - : enrichedRequest.workMode === 'execute' - ? agentRuntimeSelected - ? 'Work mode: Execute. Follow the user request. Agent Runtime tool calls execute without GoodBuddy approval and must remain visible in runtime activity.' - : 'Work mode: Execute. Follow the approved request; all tool actions remain subject to GoodBuddy permission controls.' - : '' + : enrichedRequest.workMode === 'execute' + ? agentRuntimeSelected + ? 'Work mode: Execute. Follow the user request. Agent Runtime tool calls execute without GoodBuddy approval and must remain visible in runtime activity.' + : 'Work mode: Execute. Follow the approved request. Enabled direct-model tools are authorized for this interactive run and must remain visible in runtime activity.' + : '' const expertInstruction = enrichedRequest.expertId && !imageGeneration ? `Selected expert role:\n${ @@ -1024,41 +1059,17 @@ export function registerIpcHandlers( Extract >() try { - const authorize: RuntimeAuthorizer = async (approvalRequest) => { - assistantDatabase.updateTaskStatus( - request.requestId, - 'waiting_approval' - ) - const settings = await settingsStore.getResolvedSettings() - try { - return await approvalBroker.request( - { - ...approvalRequest, - policy: - settings.toolApproval === 'policy' - ? 'policy' - : undefined, - requestId: request.requestId, - conversationId: request.conversationId - }, - controller.signal, - (approvalEvent) => { - if (!window.isDestroyed()) { - window.webContents.send( - ipcChannels.agentEvent, - approvalEvent - ) - } - } - ) - } finally { - if (!controller.signal.aborted) { - assistantDatabase.updateTaskStatus( - request.requestId, - 'running' - ) - } - } + controller.signal.throwIfAborted() + const executeToolPolicy = + request.workMode === 'execute' && !agentRuntimeSelected + ? (await settingsStore.getResolvedSettings()).toolApproval + : 'policy' + const authorize: RuntimeAuthorizer = async () => { + controller.signal.throwIfAborted() + return request.workMode === 'execute' && + executeToolPolicy !== 'policy' + ? 'once' + : 'deny' } const eventStream = request.teamMode ? runExpertTeam(request, controller.signal) @@ -1105,7 +1116,9 @@ export function registerIpcHandlers( } if (publicEvent.type === 'done') { const unsuccessfulTool = [...toolStates.values()].find( - (tool) => tool.state !== 'completed' + (tool) => + tool.state !== 'completed' && + tool.state !== 'recoverable' ) if (unsuccessfulTool) { throw new Error( @@ -1623,6 +1636,17 @@ export function registerIpcHandlers( return assistantDatabase.createExpert(expertCreateSchema.parse(input)) }) + ipcMain.handle(ipcChannels.expertsUpdate, (event, input: unknown) => { + assertTrustedSender(event, window) + const value = expertUpdateRequestSchema.parse(input) + return assistantDatabase.updateExpert(value.expertId, value.input) + }) + + ipcMain.handle(ipcChannels.expertsRemove, (event, input: unknown) => { + assertTrustedSender(event, window) + assistantDatabase.removeExpert(assistantIdSchema.parse(input)) + }) + ipcMain.handle( ipcChannels.capabilitiesSnapshot, (event): Promise => { @@ -1721,6 +1745,88 @@ export function registerIpcHandlers( } ) + ipcMain.handle( + ipcChannels.capabilitiesToggleComputer, + (event, input: unknown): Promise => { + assertTrustedSender(event, window) + const value = computerCapabilityToggleInputSchema.parse(input) + return refreshCapabilities( + capabilityService.setComputerCapabilityEnabled( + value.capabilityId, + value.enabled + ) + ) + } + ) + + ipcMain.handle( + ipcChannels.capabilitiesConfigureComputer, + (event, input: unknown): Promise => { + assertTrustedSender(event, window) + const value = computerCapabilityConfigInputSchema.parse(input) + return refreshCapabilities( + capabilityService.setComputerCapabilityBrowserProfile( + value.capabilityId, + value.browserProfileId + ) + ) + } + ) + + ipcMain.handle( + ipcChannels.capabilitiesDiagnoseComputer, + (event, input: unknown): Promise => { + assertTrustedSender(event, window) + return capabilityService.diagnoseComputerCapability( + computerCapabilityIdSchema.parse(input) + ) + } + ) + + ipcMain.handle( + ipcChannels.capabilitiesCreateBrowserProfile, + (event, input: unknown): Promise => { + assertTrustedSender(event, window) + const value = browserProfileCreateInputSchema.parse(input) + return refreshCapabilities( + capabilityService.createBrowserProfile(value.name) + ) + } + ) + + ipcMain.handle( + ipcChannels.capabilitiesRenameBrowserProfile, + (event, input: unknown): Promise => { + assertTrustedSender(event, window) + const value = browserProfileRenameInputSchema.parse(input) + return refreshCapabilities( + capabilityService.renameBrowserProfile(value.profileId, value.name) + ) + } + ) + + ipcMain.handle( + ipcChannels.capabilitiesDefaultBrowserProfile, + (event, input: unknown): Promise => { + assertTrustedSender(event, window) + const value = browserProfileSelectionInputSchema.parse(input) + return refreshCapabilities( + capabilityService.setDefaultBrowserProfile(value.profileId) + ) + } + ) + + ipcMain.handle( + ipcChannels.capabilitiesRemoveBrowserProfile, + (event, input: unknown): Promise => { + assertTrustedSender(event, window) + const value = browserProfileSelectionInputSchema.parse(input) + return refreshCapabilities( + capabilityService.removeBrowserProfile(value.profileId) + ) + } + ) + ipcMain.handle(ipcChannels.contextSelectFiles, (event) => { assertTrustedSender(event, window) return contextManager.selectFiles(window) @@ -2033,6 +2139,7 @@ export function registerIpcHandlers( return async () => { shuttingDown = true + removeBrowserStateListener?.() clearInterval(scheduleInterval) remoteDelegation?.stop() abortActiveRequests('应用正在退出') diff --git a/src/main/linux-desktop/atspi-semantic-core.test.ts b/src/main/linux-desktop/atspi-semantic-core.test.ts new file mode 100644 index 0000000..29455fd --- /dev/null +++ b/src/main/linux-desktop/atspi-semantic-core.test.ts @@ -0,0 +1,280 @@ +import { describe, expect, it, vi } from 'vitest' +import { + AtspiSemanticCore, + type AtspiRawNode, + type AtspiTransport +} from './atspi-semantic-core' + +const transportFor = ( + nodes: Record, + children: Record +): AtspiTransport => ({ + readNode: vi.fn(async (reference) => { + const node = nodes[reference] + if (!node) { + throw new Error('missing node') + } + return node + }), + listChildren: vi.fn(async (reference) => children[reference] ?? []), + invoke: vi.fn(async () => true), + setText: vi.fn(async () => true), + select: vi.fn(async () => true), + focus: vi.fn(async () => true) +}) + +const node = ( + nativeReference: string, + overrides: Partial = {} +): AtspiRawNode => ({ + nativeReference, + owner: 'private-owner', + window: 'private-window', + role: 'push button', + name: 'Action', + states: ['enabled', 'enabled'], + actions: ['Click'], + geometry: { x: -10, y: 20, width: 100, height: 30 }, + ...overrides +}) + +describe('AtspiSemanticCore', () => { + it('traverses a cyclic tree safely and returns only normalized opaque data', async () => { + const transport = transportFor( + { + '/raw/root': node('/raw/root', { + role: 'Application', + name: ' Demo\u0000 app ' + }), + '/raw/child': node('/raw/child') + }, + { + '/raw/root': ['/raw/child'], + '/raw/child': ['/raw/root'] + } + ) + let token = 0 + const core = new AtspiSemanticCore(transport, { + createToken: () => `opaque-${++token}` + }) + const tree = await core.snapshot('/raw/root', new AbortController().signal) + const serialized = JSON.stringify(tree) + + expect(tree.truncated).toBe(true) + expect(tree.root).toMatchObject({ + ref: 'opaque-1', + role: 'application', + name: 'Demo app', + states: ['enabled'], + children: [ + { + ref: 'opaque-2', + role: 'push-button', + actions: ['click'], + children: [] + } + ] + }) + expect(serialized).not.toContain('/raw/') + expect(serialized).not.toContain('private-owner') + expect(serialized).not.toContain('private-window') + }) + + it('bounds node count, depth, children, text, and invalid geometry', async () => { + const transport = transportFor( + { + root: node('root', { + text: 'abcdef', + geometry: { x: 0, y: 0, width: -1, height: 10 } + }), + one: node('one'), + two: node('two') + }, + { root: ['one', 'two'] } + ) + let token = 0 + const core = new AtspiSemanticCore(transport, { + createToken: () => `ref-${++token}`, + maximumNodes: 2, + maximumChildrenPerNode: 1, + maximumTextLength: 3 + }) + const tree = await core.snapshot('root', new AbortController().signal) + + expect(tree.truncated).toBe(true) + expect(tree.root.text).toBe('abc') + expect(tree.root.geometry).toBeUndefined() + expect(tree.root.children).toHaveLength(1) + }) + + it('redacts and refuses protected and password elements', async () => { + const transport = transportFor( + { + password: node('password', { + role: 'password text', + name: 'bank password', + text: 'hunter2', + value: 123, + password: true + }) + }, + {} + ) + const core = new AtspiSemanticCore(transport, { + createToken: () => 'protected-ref' + }) + const tree = await core.snapshot( + 'password', + new AbortController().signal + ) + + expect(tree.root).toMatchObject({ + ref: 'protected-ref', + name: '受保护内容', + actions: [], + protected: true + }) + expect(tree.root.text).toBeUndefined() + expect(tree.root.value).toBeUndefined() + await expect( + core.focus('protected-ref', new AbortController().signal) + ).rejects.toThrow('Protected') + expect(transport.focus).not.toHaveBeenCalled() + }) + + it('routes semantic operations and rejects stale or unsupported references', async () => { + let now = 10 + let token = 0 + const transport = transportFor( + { + first: node('first', { owner: 'owner-a', window: 'window-a' }), + second: node('second', { owner: 'owner-b', window: 'window-b' }) + }, + {} + ) + const core = new AtspiSemanticCore(transport, { + now: () => now, + referenceTtlMs: 50, + createToken: () => `ref-${++token}` + }) + await core.snapshot('first', new AbortController().signal) + await expect( + core.invoke('ref-1', 'CLICK', new AbortController().signal) + ).resolves.toBe(true) + await expect( + core.setText('ref-1', 'new text', new AbortController().signal) + ).resolves.toBe(true) + await expect( + core.select('ref-1', new AbortController().signal) + ).resolves.toBe(true) + await expect( + core.invoke('ref-1', 'delete', new AbortController().signal) + ).rejects.toThrow('unavailable') + + core.invalidateWindow('owner-a', 'window-a') + await expect( + core.focus('ref-1', new AbortController().signal) + ).rejects.toThrow('stale') + + await core.snapshot('second', new AbortController().signal) + core.invalidateOwner('owner-b') + await expect( + core.focus('ref-2', new AbortController().signal) + ).rejects.toThrow('stale') + + await core.snapshot('first', new AbortController().signal) + now = 60 + await expect( + core.focus('ref-3', new AbortController().signal) + ).rejects.toThrow('stale') + + now = 10 + await core.snapshot('first', new AbortController().signal) + core.invalidateRegistryOwner() + await expect( + core.focus('ref-4', new AbortController().signal) + ).rejects.toThrow('stale') + }) + + it('invalidates every reference from snapshot N when snapshot N+1 starts', async () => { + let token = 0 + const core = new AtspiSemanticCore( + transportFor( + { + first: node('first'), + second: node('second') + }, + {} + ), + { createToken: () => `generation-ref-${++token}` } + ) + + await core.snapshot('first', new AbortController().signal) + await core.snapshot('second', new AbortController().signal) + + await expect( + core.focus('generation-ref-1', new AbortController().signal) + ).rejects.toThrow('stale') + await expect( + core.focus('generation-ref-2', new AbortController().signal) + ).resolves.toBe(true) + }) + + it('keeps the opaque reference registry hard bounded across snapshots', async () => { + let token = 0 + const core = new AtspiSemanticCore( + transportFor( + { + root: node('root'), + child: node('child'), + extra: node('extra') + }, + { root: ['child', 'extra'] } + ), + { + maximumReferences: 2, + createToken: () => `bounded-ref-${++token}` + } + ) + + for (let index = 0; index < 50; index += 1) { + const tree = await core.snapshot( + 'root', + new AbortController().signal + ) + expect(tree.truncated).toBe(true) + expect( + ( + core as unknown as { + references: Map + } + ).references.size + ).toBeLessThanOrEqual(2) + } + }) + + it('does not surface raw transport errors or accept duplicate opaque tokens', async () => { + const failedTransport = transportFor({ root: node('root') }, {}) + failedTransport.readNode = vi.fn(async () => { + throw new Error('private bus path /org/a11y/atspi/accessible/123') + }) + const failedCore = new AtspiSemanticCore(failedTransport) + await expect( + failedCore.snapshot('root', new AbortController().signal) + ).rejects.toThrow('could not be read') + await expect( + failedCore.snapshot('root', new AbortController().signal) + ).rejects.not.toThrow('/org/a11y/') + + const duplicateCore = new AtspiSemanticCore( + transportFor( + { root: node('root'), child: node('child') }, + { root: ['child'] } + ), + { createToken: () => 'duplicate-token' } + ) + await expect( + duplicateCore.snapshot('root', new AbortController().signal) + ).rejects.toThrow('reference creation failed') + }) +}) diff --git a/src/main/linux-desktop/atspi-semantic-core.ts b/src/main/linux-desktop/atspi-semantic-core.ts new file mode 100644 index 0000000..6c0564a --- /dev/null +++ b/src/main/linux-desktop/atspi-semantic-core.ts @@ -0,0 +1,442 @@ +import { randomBytes } from 'node:crypto' + +export type AtspiNativeReference = string + +export type AtspiRect = { + x: number + y: number + width: number + height: number +} + +export type AtspiRawNode = { + nativeReference: AtspiNativeReference + owner: string + window: string + role?: string + name?: string + states?: string[] + actions?: string[] + text?: string + value?: number + geometry?: AtspiRect + protected?: boolean + password?: boolean +} + +export interface AtspiTransport { + readNode( + reference: AtspiNativeReference, + signal: AbortSignal + ): Promise + listChildren( + reference: AtspiNativeReference, + signal: AbortSignal + ): Promise + invoke( + reference: AtspiNativeReference, + action: string, + signal: AbortSignal + ): Promise + setText( + reference: AtspiNativeReference, + text: string, + signal: AbortSignal + ): Promise + select( + reference: AtspiNativeReference, + signal: AbortSignal + ): Promise + focus( + reference: AtspiNativeReference, + signal: AbortSignal + ): Promise +} + +export type SemanticElement = { + ref: string + role: string + name: string + states: string[] + actions: string[] + text?: string + value?: number + geometry?: AtspiRect + protected: boolean + children: SemanticElement[] +} + +export type SemanticTree = { + generation: number + truncated: boolean + root: SemanticElement +} + +export type AtspiSemanticCoreOptions = { + now?: () => number + createToken?: () => string + referenceTtlMs?: number + maximumNodes?: number + maximumDepth?: number + maximumChildrenPerNode?: number + maximumTextLength?: number + maximumReferences?: number +} + +type StoredReference = { + nativeReference: AtspiNativeReference + owner: string + window: string + generation: number + expiresAt: number + protected: boolean + actions: Set +} + +const cleanText = ( + value: string | undefined, + maximumLength: number, + fallback = '' +): string => { + if (!value) { + return fallback + } + const clean = [...value] + .filter((character) => { + const code = character.charCodeAt(0) + return code >= 32 && code !== 127 + }) + .join('') + .trim() + return clean.slice(0, maximumLength) || fallback +} + +const normalizeIdentifier = ( + value: string | undefined, + fallback: string +): string => { + const normalized = cleanText(value, 64, fallback) + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + return normalized || fallback +} + +const normalizeGeometry = (geometry: AtspiRect | undefined): AtspiRect | undefined => { + if ( + !geometry || + !Number.isFinite(geometry.x) || + !Number.isFinite(geometry.y) || + !Number.isFinite(geometry.width) || + !Number.isFinite(geometry.height) || + geometry.width < 0 || + geometry.height < 0 + ) { + return undefined + } + return { + x: geometry.x, + y: geometry.y, + width: geometry.width, + height: geometry.height + } +} + +const sensitiveRoles = new Set([ + 'password-text', + 'password', + 'secret', + 'credential' +]) + +export class AtspiSemanticCore { + private readonly now: () => number + private readonly createToken: () => string + private readonly referenceTtlMs: number + private readonly maximumNodes: number + private readonly maximumDepth: number + private readonly maximumChildrenPerNode: number + private readonly maximumTextLength: number + private readonly maximumReferences: number + private readonly references = new Map() + private generation = 0 + + constructor( + private readonly transport: AtspiTransport, + options: AtspiSemanticCoreOptions = {} + ) { + this.now = options.now ?? Date.now + this.createToken = + options.createToken ?? (() => randomBytes(24).toString('base64url')) + this.referenceTtlMs = options.referenceTtlMs ?? 3_000 + this.maximumNodes = options.maximumNodes ?? 500 + this.maximumDepth = options.maximumDepth ?? 24 + this.maximumChildrenPerNode = options.maximumChildrenPerNode ?? 100 + this.maximumTextLength = options.maximumTextLength ?? 4_096 + this.maximumReferences = + options.maximumReferences ?? this.maximumNodes + if ( + !Number.isSafeInteger(this.maximumReferences) || + this.maximumReferences < 1 || + this.maximumReferences > 10_000 + ) { + throw new Error('Invalid semantic reference capacity') + } + } + + async snapshot( + rootReference: AtspiNativeReference, + signal: AbortSignal + ): Promise { + this.generation += 1 + const snapshotGeneration = this.generation + this.pruneReferences() + const visited = new Set() + let remaining = Math.min( + this.maximumNodes, + this.maximumReferences + ) + let truncated = false + + const visit = async ( + reference: AtspiNativeReference, + depth: number + ): Promise => { + if ( + signal.aborted || + visited.has(reference) || + depth > this.maximumDepth || + remaining <= 0 + ) { + truncated = true + return undefined + } + visited.add(reference) + remaining -= 1 + let raw: AtspiRawNode + try { + raw = await this.transport.readNode(reference, signal) + } catch { + throw new Error('Accessibility element could not be read') + } + if (snapshotGeneration !== this.generation) { + throw new Error('Accessibility snapshot was superseded') + } + const role = normalizeIdentifier(raw.role, 'unknown') + const isProtected = + raw.protected === true || + raw.password === true || + sensitiveRoles.has(role) + const actions = [ + ...new Set( + (raw.actions ?? []).map((action) => + normalizeIdentifier(action, 'action') + ) + ) + ].slice(0, 32) + const states = [ + ...new Set( + (raw.states ?? []).map((state) => + normalizeIdentifier(state, 'state') + ) + ) + ].slice(0, 64) + const opaqueReference = this.createToken() + if ( + !/^[A-Za-z0-9_-]{5,256}$/.test(opaqueReference) || + this.references.has(opaqueReference) + ) { + throw new Error('Opaque accessibility reference creation failed') + } + while (this.references.size >= this.maximumReferences) { + const oldest = this.references.keys().next().value + if (oldest === undefined) { + break + } + this.references.delete(oldest) + truncated = true + } + this.references.set(opaqueReference, { + nativeReference: raw.nativeReference, + owner: raw.owner, + window: raw.window, + generation: snapshotGeneration, + expiresAt: this.now() + this.referenceTtlMs, + protected: isProtected, + actions: new Set(actions) + }) + const element: SemanticElement = { + ref: opaqueReference, + role, + name: isProtected + ? '受保护内容' + : cleanText(raw.name, 256, role), + states, + actions: isProtected ? [] : actions, + protected: isProtected, + children: [] + } + if (!isProtected) { + const text = cleanText(raw.text, this.maximumTextLength) + if (text) { + element.text = text + } + if (raw.value !== undefined && Number.isFinite(raw.value)) { + element.value = raw.value + } + } + const geometry = normalizeGeometry(raw.geometry) + if (geometry) { + element.geometry = geometry + } + + if (depth === this.maximumDepth || remaining <= 0) { + truncated = true + return element + } + let children: AtspiNativeReference[] + try { + children = await this.transport.listChildren(reference, signal) + } catch { + throw new Error('Accessibility children could not be read') + } + if (snapshotGeneration !== this.generation) { + throw new Error('Accessibility snapshot was superseded') + } + if (children.length > this.maximumChildrenPerNode) { + truncated = true + } + for (const child of children.slice(0, this.maximumChildrenPerNode)) { + const normalized = await visit(child, depth + 1) + if (normalized) { + element.children.push(normalized) + } + } + return element + } + + const root = await visit(rootReference, 0) + if (snapshotGeneration !== this.generation) { + throw new Error('Accessibility snapshot was superseded') + } + if (!root) { + throw new Error('Accessibility tree root is unavailable') + } + return { + generation: snapshotGeneration, + truncated, + root + } + } + + invalidateRegistryOwner(): void { + this.generation += 1 + this.references.clear() + } + + invalidateOwner(owner: string): void { + for (const [reference, stored] of this.references) { + if (stored.owner === owner) { + this.references.delete(reference) + } + } + } + + invalidateWindow(owner: string, window: string): void { + for (const [reference, stored] of this.references) { + if (stored.owner === owner && stored.window === window) { + this.references.delete(reference) + } + } + } + + async invoke( + reference: string, + action: string, + signal: AbortSignal + ): Promise { + const stored = this.resolve(reference) + const normalizedAction = normalizeIdentifier(action, 'action') + if (!stored.actions.has(normalizedAction)) { + throw new Error('Semantic action is unavailable') + } + try { + return await this.transport.invoke( + stored.nativeReference, + normalizedAction, + signal + ) + } catch { + throw new Error('Semantic action failed') + } + } + + async setText( + reference: string, + text: string, + signal: AbortSignal + ): Promise { + const stored = this.resolve(reference) + if (text.length > this.maximumTextLength) { + throw new Error('Text exceeds the semantic input limit') + } + try { + return await this.transport.setText(stored.nativeReference, text, signal) + } catch { + throw new Error('Semantic text update failed') + } + } + + async select( + reference: string, + signal: AbortSignal + ): Promise { + const stored = this.resolve(reference) + try { + return await this.transport.select(stored.nativeReference, signal) + } catch { + throw new Error('Semantic selection failed') + } + } + + async focus( + reference: string, + signal: AbortSignal + ): Promise { + const stored = this.resolve(reference) + try { + return await this.transport.focus(stored.nativeReference, signal) + } catch { + throw new Error('Semantic focus failed') + } + } + + private resolve(reference: string): StoredReference { + this.pruneReferences() + const stored = this.references.get(reference) + if ( + !stored || + stored.generation !== this.generation || + this.now() >= stored.expiresAt + ) { + this.references.delete(reference) + throw new Error('Semantic element is stale') + } + if (stored.protected) { + throw new Error('Protected semantic element cannot be controlled') + } + return stored + } + + private pruneReferences(): void { + const timestamp = this.now() + for (const [reference, stored] of this.references) { + if ( + stored.generation !== this.generation || + timestamp >= stored.expiresAt + ) { + this.references.delete(reference) + } + } + } +} diff --git a/src/main/linux-desktop/capability-probe.test.ts b/src/main/linux-desktop/capability-probe.test.ts new file mode 100644 index 0000000..6483e03 --- /dev/null +++ b/src/main/linux-desktop/capability-probe.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest' +import { + probeLinuxDesktopCapabilities, + type LinuxCapabilityProbes +} from './capability-probe' + +const probes = ( + overrides: Partial = {} +): LinuxCapabilityProbes => ({ + sessionBus: async () => true, + accessibilityBus: async () => ({ + busOwner: true, + registryOwner: true + }), + portalVersions: async () => ({ screenCast: 4, remoteDesktop: 2 }), + pipeWire: async () => true, + eis: async () => true, + x11: async () => false, + xTest: async () => false, + ydotool: async () => false, + electronCapture: async () => true, + ...overrides +}) + +describe('probeLinuxDesktopCapabilities', () => { + it('reports concrete protocol checks without using desktop labels', async () => { + const result = await probeLinuxDesktopCapabilities(probes(), { + XDG_CURRENT_DESKTOP: 'GNOME:Treeland' + }) + + expect(result.overall).toBe('supported') + expect(result.sessionLabels).toEqual(['GNOME', 'Treeland']) + expect(result.checks).toContainEqual({ + capability: 'portal-remote-desktop', + status: 'supported', + diagnostic: 'RemoteDesktop portal version 2 responded' + }) + expect(result.checks.find((check) => check.capability === 'xtest')?.status) + .toBe('unavailable') + }) + + it('does not infer availability from a known desktop product name', async () => { + const unavailable = probes({ + sessionBus: async () => false, + accessibilityBus: async () => ({ + busOwner: false, + registryOwner: false + }), + portalVersions: async () => ({}), + pipeWire: async () => false, + eis: async () => false, + electronCapture: async () => false + }) + const result = await probeLinuxDesktopCapabilities(unavailable, { + XDG_CURRENT_DESKTOP: 'KDE' + }) + + expect(result.overall).toBe('unavailable') + expect(result.sessionLabels).toEqual(['KDE']) + expect(result.checks.every((check) => check.status === 'unavailable')).toBe( + true + ) + }) + + it('contains bounded diagnostics rather than thrown details or environment values', async () => { + const result = await probeLinuxDesktopCapabilities( + probes({ + sessionBus: async () => { + throw new Error( + 'address=unix:path=/run/user/1000/bus token=very-secret' + ) + }, + ydotool: async () => true, + accessibilityBus: async () => ({ + busOwner: true, + registryOwner: false + }) + }), + { DBUS_SESSION_BUS_ADDRESS: 'private-address' } + ) + const diagnostics = result.checks + .map((check) => check.diagnostic) + .join(' ') + + expect(diagnostics).not.toContain('/run/') + expect(diagnostics).not.toContain('secret') + expect(diagnostics).not.toContain('private-address') + expect( + result.checks.find((check) => check.capability === 'accessibility') + ?.status + ).toBe('degraded') + expect( + result.checks.find((check) => check.capability === 'ydotool')?.status + ).toBe('supported') + }) +}) diff --git a/src/main/linux-desktop/capability-probe.ts b/src/main/linux-desktop/capability-probe.ts new file mode 100644 index 0000000..1523920 --- /dev/null +++ b/src/main/linux-desktop/capability-probe.ts @@ -0,0 +1,237 @@ +export type CapabilityStatus = 'supported' | 'degraded' | 'unavailable' + +export type CapabilityCheck = { + capability: + | 'session-bus' + | 'accessibility' + | 'portal-screen-cast' + | 'portal-remote-desktop' + | 'pipewire' + | 'eis' + | 'x11' + | 'xtest' + | 'ydotool' + | 'electron-capture' + status: CapabilityStatus + diagnostic: string +} + +export type LinuxDesktopCapabilities = { + overall: CapabilityStatus + checks: CapabilityCheck[] + sessionLabels: string[] +} + +export interface LinuxCapabilityProbes { + sessionBus(): Promise + accessibilityBus(): Promise<{ + busOwner: boolean + registryOwner: boolean + }> + portalVersions(): Promise<{ + screenCast?: number + remoteDesktop?: number + }> + pipeWire(): Promise + eis(): Promise + x11(): Promise + xTest(): Promise + ydotool(): Promise + electronCapture(): Promise +} + +const diagnostic = ( + available: boolean, + positive: string, + negative: string +): string => (available ? positive : negative) + +const statusRank: Record = { + unavailable: 0, + degraded: 1, + supported: 2 +} + +const settle = async ( + probe: () => Promise, + fallback: T +): Promise => { + try { + return await probe() + } catch { + return fallback + } +} + +const sessionLabels = (source: NodeJS.ProcessEnv): string[] => { + const text = [ + source.XDG_CURRENT_DESKTOP, + source.XDG_SESSION_DESKTOP, + source.DESKTOP_SESSION + ] + .filter((value): value is string => typeof value === 'string') + .join(':') + .toLowerCase() + const labels = [ + ['gnome', 'GNOME'], + ['kde', 'KDE'], + ['deepin', 'DDE'], + ['dde', 'DDE'], + ['treeland', 'Treeland'], + ['ukui', 'UKUI'] + ] as const + return [ + ...new Set( + labels + .filter(([needle]) => text.includes(needle)) + .map(([, label]) => label) + ) + ] +} + +/** + * Probes protocols and concrete operations. Desktop names are intentionally + * excluded from capability decisions and are returned only as diagnostics. + */ +export async function probeLinuxDesktopCapabilities( + probes: LinuxCapabilityProbes, + environment: NodeJS.ProcessEnv = process.env +): Promise { + const [ + bus, + accessibility, + portals, + pipeWire, + eis, + x11, + xTest, + ydotool, + electronCapture + ] = await Promise.all([ + settle(() => probes.sessionBus(), false), + settle(() => probes.accessibilityBus(), { + busOwner: false, + registryOwner: false + }), + settle(() => probes.portalVersions(), {}), + settle(() => probes.pipeWire(), false), + settle(() => probes.eis(), false), + settle(() => probes.x11(), false), + settle(() => probes.xTest(), false), + settle(() => probes.ydotool(), false), + settle(() => probes.electronCapture(), false) + ]) + + const screenCast = portals.screenCast ?? 0 + const remoteDesktop = portals.remoteDesktop ?? 0 + const checks: CapabilityCheck[] = [ + { + capability: 'session-bus', + status: bus ? 'supported' : 'unavailable', + diagnostic: diagnostic( + bus, + 'Session bus responded', + 'Session bus did not respond' + ) + }, + { + capability: 'accessibility', + status: + accessibility.busOwner && accessibility.registryOwner + ? 'supported' + : accessibility.busOwner + ? 'degraded' + : 'unavailable', + diagnostic: + accessibility.busOwner && accessibility.registryOwner + ? 'Accessibility bus and registry responded' + : accessibility.busOwner + ? 'Accessibility bus responded but registry is unavailable' + : 'Accessibility bus is unavailable' + }, + { + capability: 'portal-screen-cast', + status: screenCast > 0 ? 'supported' : 'unavailable', + diagnostic: + screenCast > 0 + ? `ScreenCast portal version ${screenCast} responded` + : 'ScreenCast portal is unavailable' + }, + { + capability: 'portal-remote-desktop', + status: remoteDesktop > 0 ? 'supported' : 'unavailable', + diagnostic: + remoteDesktop > 0 + ? `RemoteDesktop portal version ${remoteDesktop} responded` + : 'RemoteDesktop portal is unavailable' + }, + { + capability: 'pipewire', + status: pipeWire ? 'supported' : 'unavailable', + diagnostic: diagnostic( + pipeWire, + 'PipeWire connection succeeded', + 'PipeWire connection failed' + ) + }, + { + capability: 'eis', + status: eis ? 'supported' : 'unavailable', + diagnostic: diagnostic( + eis, + 'EIS helper handshake succeeded', + 'EIS helper handshake failed' + ) + }, + { + capability: 'x11', + status: x11 ? 'supported' : 'unavailable', + diagnostic: diagnostic( + x11, + 'X11 connection succeeded', + 'X11 connection failed' + ) + }, + { + capability: 'xtest', + status: x11 && xTest ? 'supported' : 'unavailable', + diagnostic: + x11 && xTest + ? 'XTest extension query succeeded' + : 'XTest is unavailable on the proven display connection' + }, + { + capability: 'ydotool', + status: ydotool ? 'supported' : 'unavailable', + diagnostic: ydotool + ? 'Optional ydotool helper passed its direct probe' + : 'Optional ydotool helper is unavailable' + }, + { + capability: 'electron-capture', + status: electronCapture ? 'supported' : 'unavailable', + diagnostic: diagnostic( + electronCapture, + 'Electron one-shot capture succeeded', + 'Electron one-shot capture failed' + ) + } + ] + + const usable = checks.filter( + (check) => + check.capability !== 'session-bus' && + check.capability !== 'ydotool' + ) + const best = usable.reduce( + (current, check) => + statusRank[check.status] > statusRank[current] ? check.status : current, + 'unavailable' + ) + + return { + overall: best, + checks, + sessionLabels: sessionLabels(environment) + } +} diff --git a/src/main/linux-desktop/coordinate-mapper.test.ts b/src/main/linux-desktop/coordinate-mapper.test.ts new file mode 100644 index 0000000..999f48d --- /dev/null +++ b/src/main/linux-desktop/coordinate-mapper.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from 'vitest' +import { + CoordinateMapper, + type CapturedFrameLayout, + type DesktopLayout +} from './coordinate-mapper' + +const layout: DesktopLayout = { + revision: 7, + capturedAt: 1_000, + displays: [ + { + id: 'left', + logicalBounds: { x: -1024, y: -100, width: 1024, height: 768 }, + scale: 1.25, + rotation: 0 + }, + { + id: 'rotated', + logicalBounds: { x: 0, y: 0, width: 800, height: 600 }, + scale: 2, + rotation: 90 + } + ] +} + +const frame = ( + overrides: Partial = {} +): CapturedFrameLayout => ({ + frameId: 'frame-1', + displayId: 'left', + sourceBounds: { x: -1024, y: -100, width: 1024, height: 768 }, + width: 1280, + height: 960, + layoutRevision: 7, + capturedAt: 1_001, + expiresAt: 2_000, + ...overrides +}) + +describe('CoordinateMapper', () => { + it('maps negative origins and fractional display scale', () => { + const mapper = new CoordinateMapper(() => 1_100) + mapper.updateLayout(layout) + + expect(mapper.mapPoint({ x: -512, y: 284 }, frame())).toEqual({ + frameId: 'frame-1', + x: 640, + y: 480 + }) + }) + + it('maps rotated displays into their physical frame orientation', () => { + const mapper = new CoordinateMapper(() => 1_100) + mapper.updateLayout(layout) + const rotated = frame({ + frameId: 'rotated-frame', + displayId: 'rotated', + sourceBounds: { x: 0, y: 0, width: 800, height: 600 }, + width: 1200, + height: 1600 + }) + + expect(mapper.mapPoint({ x: 400, y: 300 }, rotated)).toEqual({ + frameId: 'rotated-frame', + x: 600, + y: 800 + }) + expect(mapper.mapPoint({ x: 0, y: 0 }, rotated)).toEqual({ + frameId: 'rotated-frame', + x: 1199, + y: 0 + }) + }) + + it('maps bounded rectangles without crossing frame edges', () => { + const mapper = new CoordinateMapper(() => 1_100) + mapper.updateLayout(layout) + + expect( + mapper.mapRect( + { x: -1024, y: -100, width: 1024, height: 768 }, + frame() + ) + ).toEqual({ x: 0, y: 0, width: 1280, height: 960 }) + }) + + it.each([ + frame({ layoutRevision: 6 }), + frame({ expiresAt: 1_100 }), + frame({ width: 1270 }), + frame({ + sourceBounds: { x: -2000, y: -100, width: 1024, height: 768 } + }) + ])('rejects stale or inconsistent captured frame layout', (captured) => { + const mapper = new CoordinateMapper(() => 1_100) + mapper.updateLayout(layout) + expect(() => mapper.mapPoint({ x: -512, y: 284 }, captured)).toThrow() + }) + + it('rejects stale and non-monotonic desktop layouts', () => { + const mapper = new CoordinateMapper(() => 7_000) + mapper.updateLayout(layout) + expect(() => mapper.mapPoint({ x: -512, y: 284 }, frame())).toThrow( + 'stale' + ) + expect(() => mapper.updateLayout(layout)).toThrow('revision is stale') + }) +}) diff --git a/src/main/linux-desktop/coordinate-mapper.ts b/src/main/linux-desktop/coordinate-mapper.ts new file mode 100644 index 0000000..46d3581 --- /dev/null +++ b/src/main/linux-desktop/coordinate-mapper.ts @@ -0,0 +1,212 @@ +export type DesktopPoint = { + x: number + y: number +} + +export type DesktopRect = DesktopPoint & { + width: number + height: number +} + +export type DisplayRotation = 0 | 90 | 180 | 270 + +export type DisplayLayout = { + id: string + logicalBounds: DesktopRect + scale: number + rotation: DisplayRotation +} + +export type DesktopLayout = { + revision: number + capturedAt: number + displays: DisplayLayout[] +} + +export type CapturedFrameLayout = { + frameId: string + displayId: string + sourceBounds: DesktopRect + width: number + height: number + layoutRevision: number + capturedAt: number + expiresAt: number +} + +export type FramePoint = { + frameId: string + x: number + y: number +} + +const finiteRect = (rect: DesktopRect): boolean => + Number.isFinite(rect.x) && + Number.isFinite(rect.y) && + Number.isFinite(rect.width) && + Number.isFinite(rect.height) && + rect.width > 0 && + rect.height > 0 + +const containsPoint = (rect: DesktopRect, point: DesktopPoint): boolean => + point.x >= rect.x && + point.y >= rect.y && + point.x < rect.x + rect.width && + point.y < rect.y + rect.height + +const containsRect = (outer: DesktopRect, inner: DesktopRect): boolean => + inner.x >= outer.x && + inner.y >= outer.y && + inner.x + inner.width <= outer.x + outer.width && + inner.y + inner.height <= outer.y + outer.height + +export class CoordinateMapper { + private layout: DesktopLayout | undefined + + constructor( + private readonly now: () => number = Date.now, + private readonly maximumLayoutAgeMs = 5_000 + ) {} + + updateLayout(layout: DesktopLayout): void { + if ( + !Number.isSafeInteger(layout.revision) || + layout.revision < 1 || + !Number.isFinite(layout.capturedAt) || + layout.displays.length === 0 || + layout.displays.some( + (display) => + !display.id || + !finiteRect(display.logicalBounds) || + !Number.isFinite(display.scale) || + display.scale <= 0 || + display.scale > 8 + ) || + new Set(layout.displays.map((display) => display.id)).size !== + layout.displays.length + ) { + throw new Error('Invalid desktop layout') + } + if (this.layout && layout.revision <= this.layout.revision) { + throw new Error('Desktop layout revision is stale') + } + this.layout = structuredClone(layout) + } + + mapPoint( + point: DesktopPoint, + frame: CapturedFrameLayout + ): FramePoint { + const { display } = this.validateFrame(frame) + if ( + !Number.isFinite(point.x) || + !Number.isFinite(point.y) || + !containsPoint(frame.sourceBounds, point) + ) { + throw new Error('Desktop point is outside the captured frame') + } + + const u = (point.x - frame.sourceBounds.x) / frame.sourceBounds.width + const v = (point.y - frame.sourceBounds.y) / frame.sourceBounds.height + const rotated = this.rotate(u, v, display.rotation) + return { + frameId: frame.frameId, + x: Math.min(frame.width - 1, Math.max(0, Math.floor(rotated.x * frame.width))), + y: Math.min( + frame.height - 1, + Math.max(0, Math.floor(rotated.y * frame.height)) + ) + } + } + + mapRect(rect: DesktopRect, frame: CapturedFrameLayout): DesktopRect { + if (!finiteRect(rect) || !containsRect(frame.sourceBounds, rect)) { + throw new Error('Desktop rectangle is outside the captured frame') + } + const right = rect.x + rect.width + const bottom = rect.y + rect.height + const xInset = Math.min(rect.width / 2, Math.max(1e-9, rect.width * 1e-9)) + const yInset = Math.min( + rect.height / 2, + Math.max(1e-9, rect.height * 1e-9) + ) + const points = [ + this.mapPoint({ x: rect.x, y: rect.y }, frame), + this.mapPoint({ x: right - xInset, y: rect.y }, frame), + this.mapPoint({ x: rect.x, y: bottom - yInset }, frame), + this.mapPoint( + { x: right - xInset, y: bottom - yInset }, + frame + ) + ] + const xs = points.map((point) => point.x) + const ys = points.map((point) => point.y) + const x = Math.min(...xs) + const y = Math.min(...ys) + return { + x, + y, + width: Math.max(...xs) - x + 1, + height: Math.max(...ys) - y + 1 + } + } + + private validateFrame(frame: CapturedFrameLayout): { + display: DisplayLayout + } { + const layout = this.layout + if (!layout) { + throw new Error('Desktop layout is unavailable') + } + if ( + this.now() - layout.capturedAt >= this.maximumLayoutAgeMs || + frame.layoutRevision !== layout.revision || + this.now() >= frame.expiresAt || + frame.capturedAt < layout.capturedAt + ) { + throw new Error('Desktop layout or captured frame is stale') + } + const display = layout.displays.find( + (candidate) => candidate.id === frame.displayId + ) + if ( + !display || + !frame.frameId || + !finiteRect(frame.sourceBounds) || + !containsRect(display.logicalBounds, frame.sourceBounds) || + !Number.isSafeInteger(frame.width) || + frame.width < 1 || + !Number.isSafeInteger(frame.height) || + frame.height < 1 + ) { + throw new Error('Invalid captured frame layout') + } + const rotated = display.rotation === 90 || display.rotation === 270 + const expectedWidth = + (rotated ? frame.sourceBounds.height : frame.sourceBounds.width) * + display.scale + const expectedHeight = + (rotated ? frame.sourceBounds.width : frame.sourceBounds.height) * + display.scale + if ( + Math.abs(frame.width - expectedWidth) > 1 || + Math.abs(frame.height - expectedHeight) > 1 + ) { + throw new Error('Captured frame scale does not match the display') + } + return { display } + } + + private rotate(u: number, v: number, rotation: DisplayRotation): DesktopPoint { + switch (rotation) { + case 0: + return { x: u, y: v } + case 90: + return { x: 1 - v, y: u } + case 180: + return { x: 1 - u, y: 1 - v } + case 270: + return { x: v, y: 1 - u } + } + } +} diff --git a/src/main/linux-desktop/input-router.test.ts b/src/main/linux-desktop/input-router.test.ts new file mode 100644 index 0000000..710ec61 --- /dev/null +++ b/src/main/linux-desktop/input-router.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it, vi } from 'vitest' +import { + LinuxDesktopInputRouter, + type InjectedInputBackend, + type SemanticInput, + type YdotoolInputBackend +} from './input-router' + +const semantic = (result: boolean): SemanticInput => ({ + perform: vi.fn(async () => result) +}) + +const backend = (): InjectedInputBackend => ({ + inject: vi.fn(async () => undefined), + releasePressedInput: vi.fn(async () => undefined) +}) + +describe('LinuxDesktopInputRouter', () => { + it('always attempts the semantic action before native injection', async () => { + const portal = backend() + const router = new LinuxDesktopInputRouter({ + semantic: semantic(true), + portal, + portalConsentActive: () => true, + provenX11: () => false + }) + + await expect( + router.route( + { type: 'text', text: 'hello' }, + new AbortController().signal + ) + ).resolves.toEqual({ route: 'semantic' }) + expect(portal.inject).not.toHaveBeenCalled() + }) + + it('uses portal input only while user consent is active', async () => { + let consent = false + const portal = backend() + const router = new LinuxDesktopInputRouter({ + semantic: semantic(false), + portal, + portalConsentActive: () => consent, + provenX11: () => false + }) + + await expect( + router.route( + { type: 'pointer-move', x: 1, y: 2 }, + new AbortController().signal, + { backend: 'portal' } + ) + ).rejects.toThrow('active user consent') + consent = true + await expect( + router.route( + { type: 'pointer-move', x: 1, y: 2 }, + new AbortController().signal + ) + ).resolves.toEqual({ route: 'portal' }) + }) + + it('uses XTest only for a proven X11 connection', async () => { + let proven = false + const xTest = backend() + const router = new LinuxDesktopInputRouter({ + semantic: semantic(false), + portalConsentActive: () => false, + xTest, + provenX11: () => proven + }) + await expect( + router.route( + { type: 'key', key: 'Enter', pressed: true }, + new AbortController().signal, + { backend: 'xtest' } + ) + ).rejects.toThrow('proven X11') + + proven = true + await expect( + router.route( + { type: 'key', key: 'Enter', pressed: true }, + new AbortController().signal + ) + ).resolves.toEqual({ route: 'xtest' }) + }) + + it('never silently selects ydotool and requires its exact fixed executable', async () => { + const ydotool: YdotoolInputBackend = { + ...backend(), + executablePath: '/usr/bin/ydotool' + } + const router = new LinuxDesktopInputRouter({ + semantic: semantic(false), + portalConsentActive: () => false, + provenX11: () => false, + ydotool, + ydotoolOptIn: true, + fixedYdotoolExecutable: '/usr/bin/ydotool' + }) + + await expect( + router.route( + { type: 'scroll', deltaX: 0, deltaY: 1 }, + new AbortController().signal + ) + ).rejects.toThrow('No consented') + await expect( + router.route( + { type: 'scroll', deltaX: 0, deltaY: 1 }, + new AbortController().signal, + { backend: 'ydotool' } + ) + ).resolves.toEqual({ route: 'ydotool' }) + + expect( + () => + new LinuxDesktopInputRouter({ + semantic: semantic(false), + portalConsentActive: () => false, + provenX11: () => false, + ydotoolOptIn: true, + fixedYdotoolExecutable: 'ydotool' + }) + ).toThrow('absolute fixed executable') + }) + + it('releases pressed input best-effort when an operation is aborted', async () => { + const controller = new AbortController() + const portal = backend() + portal.inject = vi.fn( + async (_action, signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('aborted')), { + once: true + }) + }) + ) + portal.releasePressedInput = vi.fn(async () => { + throw new Error('release failed') + }) + const router = new LinuxDesktopInputRouter({ + semantic: semantic(false), + portal, + portalConsentActive: () => true, + provenX11: () => false + }) + const routed = router.route( + { type: 'pointer-button', button: 1, pressed: true }, + controller.signal + ) + await Promise.resolve() + controller.abort() + + await expect(routed).rejects.toThrow() + expect(portal.releasePressedInput).toHaveBeenCalled() + }) + + it('releases partially pressed input after a non-abort injection failure', async () => { + const portal = backend() + portal.inject = vi.fn(async () => { + throw new Error('partial native injection') + }) + portal.releasePressedInput = vi.fn(async () => { + throw new Error('best-effort release failed') + }) + const router = new LinuxDesktopInputRouter({ + semantic: semantic(false), + portal, + portalConsentActive: () => true, + provenX11: () => false + }) + + await expect( + router.route( + { type: 'key', key: 'Control_L', pressed: true }, + new AbortController().signal + ) + ).rejects.toThrow('partial native injection') + expect(portal.releasePressedInput).toHaveBeenCalledOnce() + }) + + it.each([ + { type: 'pointer-move', x: Number.NaN, y: 0 } as const, + { type: 'pointer-button', button: 0, pressed: true } as const, + { type: 'scroll', deltaX: 0, deltaY: Number.POSITIVE_INFINITY } as const, + { type: 'key', key: '\n', pressed: true } as const, + { type: 'text', text: '\u0000secret' } as const + ])('rejects malformed actions before any backend sees them', async (action) => { + const semanticInput = semantic(false) + const portal = backend() + const router = new LinuxDesktopInputRouter({ + semantic: semanticInput, + portal, + portalConsentActive: () => true, + provenX11: () => false + }) + + await expect( + router.route(action, new AbortController().signal) + ).rejects.toThrow('Invalid') + expect(semanticInput.perform).not.toHaveBeenCalled() + expect(portal.inject).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/linux-desktop/input-router.ts b/src/main/linux-desktop/input-router.ts new file mode 100644 index 0000000..2f8f082 --- /dev/null +++ b/src/main/linux-desktop/input-router.ts @@ -0,0 +1,194 @@ +import { isAbsolute } from 'node:path' + +export type DesktopInputAction = + | { type: 'pointer-move'; x: number; y: number } + | { type: 'pointer-button'; button: number; pressed: boolean } + | { type: 'scroll'; deltaX: number; deltaY: number } + | { type: 'key'; key: string; pressed: boolean } + | { type: 'text'; text: string } + +export type InputBackendName = 'portal' | 'xtest' | 'ydotool' + +export interface SemanticInput { + perform(action: DesktopInputAction, signal: AbortSignal): Promise +} + +export interface InjectedInputBackend { + inject(action: DesktopInputAction, signal: AbortSignal): Promise + releasePressedInput(): Promise +} + +export interface YdotoolInputBackend extends InjectedInputBackend { + readonly executablePath: string +} + +export type InputRouterOptions = { + semantic: SemanticInput + portal?: InjectedInputBackend + portalConsentActive: () => boolean + xTest?: InjectedInputBackend + provenX11: () => boolean + ydotool?: YdotoolInputBackend + ydotoolOptIn?: boolean + fixedYdotoolExecutable?: string +} + +export type RouteInputOptions = { + backend?: 'auto' | InputBackendName +} + +export type RoutedInputResult = { + route: 'semantic' | InputBackendName +} + +const hasControlCharacters = (value: string): boolean => + [...value].some((character) => { + const code = character.charCodeAt(0) + return code < 32 || code === 127 + }) + +const validateAction = (action: DesktopInputAction): void => { + switch (action.type) { + case 'pointer-move': + if (!Number.isFinite(action.x) || !Number.isFinite(action.y)) { + throw new Error('Invalid pointer coordinates') + } + return + case 'pointer-button': + if ( + !Number.isSafeInteger(action.button) || + action.button < 1 || + action.button > 32 + ) { + throw new Error('Invalid pointer button') + } + return + case 'scroll': + if ( + !Number.isFinite(action.deltaX) || + !Number.isFinite(action.deltaY) || + Math.abs(action.deltaX) > 100_000 || + Math.abs(action.deltaY) > 100_000 + ) { + throw new Error('Invalid scroll delta') + } + return + case 'key': + if ( + action.key.length < 1 || + action.key.length > 64 || + hasControlCharacters(action.key) + ) { + throw new Error('Invalid key identifier') + } + return + case 'text': + if ( + action.text.length > 4_096 || + action.text.includes('\u0000') + ) { + throw new Error('Invalid text input') + } + } +} + +export class LinuxDesktopInputRouter { + private readonly usedBackends = new Set() + + constructor(private readonly options: InputRouterOptions) { + if ( + options.ydotoolOptIn && + (!options.fixedYdotoolExecutable || + !isAbsolute(options.fixedYdotoolExecutable)) + ) { + throw new Error('ydotool requires an absolute fixed executable path') + } + } + + async route( + action: DesktopInputAction, + signal: AbortSignal, + routeOptions: RouteInputOptions = {} + ): Promise { + validateAction(action) + if (signal.aborted) { + throw signal.reason + } + if (await this.options.semantic.perform(action, signal)) { + return { route: 'semantic' } + } + + const selected = this.selectBackend(routeOptions.backend ?? 'auto') + this.usedBackends.add(selected.backend) + let released = false + let releasePromise: Promise | undefined + const releaseOnAbort = (): void => { + if (!released) { + released = true + releasePromise = this.releasePressedInput() + } + } + signal.addEventListener('abort', releaseOnAbort, { once: true }) + try { + await selected.backend.inject(action, signal) + if (signal.aborted) { + releaseOnAbort() + throw signal.reason + } + return { route: selected.name } + } catch (error) { + await (releasePromise ?? this.releasePressedInput()) + throw error + } finally { + signal.removeEventListener('abort', releaseOnAbort) + } + } + + async releasePressedInput(): Promise { + const backends = [...this.usedBackends] + this.usedBackends.clear() + await Promise.allSettled( + backends.map((backend) => backend.releasePressedInput()) + ) + } + + private selectBackend(preference: 'auto' | InputBackendName): { + name: InputBackendName + backend: InjectedInputBackend + } { + if (preference === 'portal') { + if (!this.options.portal || !this.options.portalConsentActive()) { + throw new Error('Portal input requires active user consent') + } + return { name: 'portal', backend: this.options.portal } + } + if (preference === 'xtest') { + if (!this.options.xTest || !this.options.provenX11()) { + throw new Error('XTest requires a proven X11 connection') + } + return { name: 'xtest', backend: this.options.xTest } + } + if (preference === 'ydotool') { + const fixed = this.options.fixedYdotoolExecutable + if ( + !this.options.ydotoolOptIn || + !fixed || + !this.options.ydotool || + this.options.ydotool.executablePath !== fixed + ) { + throw new Error( + 'ydotool requires explicit opt-in and the fixed executable' + ) + } + return { name: 'ydotool', backend: this.options.ydotool } + } + + if (this.options.portal && this.options.portalConsentActive()) { + return { name: 'portal', backend: this.options.portal } + } + if (this.options.xTest && this.options.provenX11()) { + return { name: 'xtest', backend: this.options.xTest } + } + throw new Error('No consented or proven input backend is available') + } +} diff --git a/src/main/linux-desktop/portal-session.test.ts b/src/main/linux-desktop/portal-session.test.ts new file mode 100644 index 0000000..c371895 --- /dev/null +++ b/src/main/linux-desktop/portal-session.test.ts @@ -0,0 +1,366 @@ +import { + afterEach, + describe, + expect, + it, + vi, + type Mock +} from 'vitest' +import { + PortalDesktopSession, + PortalSessionError, + validatePipeWireFrameMetadata, + type ClosableResource, + type PortalResponse, + type PortalTransport +} from './portal-session' + +type MockPortal = { + transport: PortalTransport + pipeWireClose: Mock<() => void> + eisClose: Mock<() => void> +} + +const mockPortal = ( + responseOverrides: Record> = {} +): MockPortal => { + const pipeWireClose = vi.fn<() => void>() + const eisClose = vi.fn<() => void>() + const resource = (close: Mock<() => void>): ClosableResource => ({ + close: () => close() + }) + const transport: PortalTransport = { + createSession: vi.fn(async () => ({ requestHandle: 'create' })), + selectDevices: vi.fn(async () => ({ requestHandle: 'devices' })), + selectSources: vi.fn(async () => ({ requestHandle: 'sources' })), + start: vi.fn(async () => ({ requestHandle: 'start' })), + waitForResponse: vi.fn(async (requestHandle) => ({ + requestHandle, + response: 0, + results: + requestHandle === 'create' + ? { session_handle: 'private-session' } + : {}, + ...responseOverrides[requestHandle] + })), + openPipeWireRemote: vi.fn(async () => resource(pipeWireClose)), + connectEis: vi.fn(async () => resource(eisClose)), + closeRequest: vi.fn(async () => undefined), + closeSession: vi.fn(async () => undefined) + } + return { transport, pipeWireClose, eisClose } +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('PortalDesktopSession', () => { + it('performs the portal protocol in order and cleans every resource on stop', async () => { + const mock = mockPortal() + const session = new PortalDesktopSession(mock.transport) + + await session.open( + { devices: true, sources: true, parentWindow: 'window-token' }, + new AbortController().signal + ) + + expect(session.state).toBe('active') + expect(session.hasActiveConsent).toBe(true) + expect(mock.transport.selectDevices).toHaveBeenCalledWith( + 'private-session', + expect.any(AbortSignal) + ) + expect(mock.transport.selectSources).toHaveBeenCalled() + expect(mock.transport.start).toHaveBeenCalledWith( + 'private-session', + 'window-token', + expect.any(AbortSignal) + ) + + await session.stop() + + expect(session.state).toBe('stopped') + expect(session.hasActiveConsent).toBe(false) + expect(mock.pipeWireClose).toHaveBeenCalledOnce() + expect(mock.eisClose).toHaveBeenCalledOnce() + expect(mock.transport.closeRequest).toHaveBeenCalledTimes(4) + expect(mock.transport.closeSession).toHaveBeenCalledWith('private-session') + }) + + it.each([ + [1, 'cancelled'], + [2, 'denied'] + ] as const)('handles portal consent response %s as %s', async (code, reason) => { + const mock = mockPortal({ + sources: { response: code } + }) + const session = new PortalDesktopSession(mock.transport) + + await expect( + session.open( + { devices: false, sources: true }, + new AbortController().signal + ) + ).rejects.toMatchObject({ reason }) + expect(session.state).toBe('failed') + expect(mock.transport.closeRequest).toHaveBeenCalledWith('sources') + expect(mock.transport.closeSession).toHaveBeenCalledWith('private-session') + expect(mock.transport.openPipeWireRemote).not.toHaveBeenCalled() + }) + + it('rejects mismatched response handles and closes the request', async () => { + const mock = mockPortal({ + create: { requestHandle: 'unrelated-response' } + }) + const session = new PortalDesktopSession(mock.transport) + + await expect( + session.open( + { devices: true, sources: false }, + new AbortController().signal + ) + ).rejects.toMatchObject({ reason: 'protocol' }) + expect(mock.transport.closeRequest).toHaveBeenCalledWith('create') + }) + + it('aborts a pending request on timeout and performs cleanup', async () => { + vi.useFakeTimers() + const mock = mockPortal() + mock.transport.waitForResponse = vi.fn( + async (_handle, signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { + once: true + }) + }) + ) + const session = new PortalDesktopSession(mock.transport) + const opening = session.open( + { devices: true, sources: false, timeoutMs: 10 }, + new AbortController().signal + ) + const rejection = expect(opening).rejects.toMatchObject({ + reason: 'timeout' + }) + + await vi.advanceTimersByTimeAsync(10) + + await rejection + expect(mock.transport.closeRequest).toHaveBeenCalledWith('create') + }) + + it('cleans active descriptors and revokes consent on portal owner loss', async () => { + const mock = mockPortal() + const session = new PortalDesktopSession(mock.transport) + await session.open( + { devices: true, sources: true }, + new AbortController().signal + ) + + await session.portalOwnerLost() + + expect(session.state).toBe('failed') + expect(session.hasActiveConsent).toBe(false) + expect(mock.pipeWireClose).toHaveBeenCalledOnce() + expect(mock.eisClose).toHaveBeenCalledOnce() + expect(mock.transport.closeSession).toHaveBeenCalledOnce() + }) + + it('stops an opening session without leaving requests or consent active', async () => { + const mock = mockPortal() + mock.transport.waitForResponse = vi.fn( + async (_handle, signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { + once: true + }) + }) + ) + const session = new PortalDesktopSession(mock.transport) + const opening = session.open( + { devices: true, sources: false }, + new AbortController().signal + ) + const rejection = expect(opening).rejects.toMatchObject({ + reason: 'aborted' + }) + await Promise.resolve() + + await session.stop() + await rejection + + expect(session.state).toBe('stopped') + expect(session.hasActiveConsent).toBe(false) + expect(mock.transport.closeRequest).toHaveBeenCalledWith('create') + }) + + it('prevents a late stopped open from mutating a reopened session', async () => { + const mock = mockPortal() + let resolveFirst: + | ((request: { requestHandle: string }) => void) + | undefined + const firstCreate = new Promise<{ requestHandle: string }>((resolve) => { + resolveFirst = resolve + }) + let createCount = 0 + mock.transport.createSession = vi.fn(async () => { + createCount += 1 + if (createCount === 1) { + return firstCreate + } + return { requestHandle: 'create-second' } + }) + mock.transport.waitForResponse = vi.fn(async (requestHandle) => ({ + requestHandle, + response: 0, + results: + requestHandle === 'create-second' + ? { session_handle: 'session-second' } + : {} + })) + const session = new PortalDesktopSession(mock.transport) + const firstOpening = session.open( + { devices: true, sources: false }, + new AbortController().signal + ) + const firstRejection = expect(firstOpening).rejects.toMatchObject({ + reason: 'aborted' + }) + await vi.waitFor(() => { + expect(mock.transport.createSession).toHaveBeenCalledOnce() + }) + + await session.stop() + await firstRejection + await session.open( + { devices: true, sources: false }, + new AbortController().signal + ) + expect(session.state).toBe('active') + + resolveFirst?.({ requestHandle: 'late-first-create' }) + await vi.waitFor(() => { + expect(mock.transport.closeRequest).toHaveBeenCalledWith( + 'late-first-create' + ) + }) + expect(session.state).toBe('active') + expect(session.hasActiveConsent).toBe(true) + expect(mock.transport.closeSession).not.toHaveBeenCalledWith( + 'session-second' + ) + }) + + it.each(['stop', 'owner-loss'] as const)( + 'bounds hung cleanup during %s', + async (lifecycle) => { + const mock = mockPortal() + const never = () => new Promise(() => undefined) + mock.transport.closeRequest = vi.fn(never) + mock.transport.closeSession = vi.fn(never) + mock.transport.openPipeWireRemote = vi.fn(async () => ({ + close: never + })) + const session = new PortalDesktopSession(mock.transport, { + cleanupTimeoutMs: 10 + }) + await session.open( + { devices: false, sources: true }, + new AbortController().signal + ) + vi.useFakeTimers() + + const cleanup = + lifecycle === 'stop' + ? session.stop() + : session.portalOwnerLost() + await vi.advanceTimersByTimeAsync(10) + await expect(cleanup).resolves.toBeUndefined() + expect(session.state).toBe( + lifecycle === 'stop' ? 'stopped' : 'failed' + ) + } + ) + + it('bounds hung cleanup after a failed open', async () => { + vi.useFakeTimers() + const mock = mockPortal({ + sources: { response: 2 } + }) + const never = () => new Promise(() => undefined) + mock.transport.closeRequest = vi.fn(never) + mock.transport.closeSession = vi.fn(never) + const session = new PortalDesktopSession(mock.transport, { + cleanupTimeoutMs: 10 + }) + const opening = session.open( + { devices: false, sources: true }, + new AbortController().signal + ) + const rejection = expect(opening).rejects.toMatchObject({ + reason: 'denied' + }) + + await vi.waitFor(() => { + expect(mock.transport.closeSession).toHaveBeenCalled() + }) + await vi.advanceTimersByTimeAsync(10) + await rejection + expect(session.state).toBe('failed') + }) + + it('requires at least one consented portal capability', async () => { + const session = new PortalDesktopSession(mockPortal().transport) + await expect( + session.open( + { devices: false, sources: false }, + new AbortController().signal + ) + ).rejects.toBeInstanceOf(PortalSessionError) + await expect( + session.open( + { devices: true, sources: false, timeoutMs: 0 }, + new AbortController().signal + ) + ).rejects.toMatchObject({ reason: 'protocol' }) + }) +}) + +describe('validatePipeWireFrameMetadata', () => { + const valid = { + width: 1920, + height: 1080, + stride: 7680, + planes: [{ offset: 0, stride: 7680, bytes: 8_294_400 }], + byteLength: 8_294_400, + fpsNumerator: 60, + fpsDenominator: 1 + } + + it('accepts bounded, internally consistent frame metadata', () => { + expect(validatePipeWireFrameMetadata(valid)).toEqual(valid) + }) + + it.each([ + { ...valid, width: 0 }, + { ...valid, height: 20_000 }, + { ...valid, stride: 100 }, + { ...valid, fpsDenominator: 0 }, + { ...valid, fpsNumerator: 241 }, + { ...valid, byteLength: 10 }, + { + ...valid, + planes: [{ offset: 1, stride: 7680, bytes: 8_294_399 }] + }, + { + ...valid, + planes: [ + { offset: 0, stride: 7680, bytes: 100 }, + { offset: 50, stride: 960, bytes: 8_294_350 } + ] + } + ])('rejects unsafe or inconsistent metadata', (metadata) => { + expect(() => validatePipeWireFrameMetadata(metadata)).toThrow() + }) +}) diff --git a/src/main/linux-desktop/portal-session.ts b/src/main/linux-desktop/portal-session.ts new file mode 100644 index 0000000..be08d1f --- /dev/null +++ b/src/main/linux-desktop/portal-session.ts @@ -0,0 +1,564 @@ +export type PortalSessionState = + | 'idle' + | 'creating' + | 'selecting' + | 'starting' + | 'active' + | 'stopping' + | 'stopped' + | 'failed' + +export type PortalRequest = { + requestHandle: string +} + +export type PortalResponse = { + requestHandle: string + response: number + results: Readonly> +} + +export interface ClosableResource { + close(): void | Promise +} + +export interface PortalTransport { + createSession(signal: AbortSignal): Promise + selectDevices( + sessionHandle: string, + signal: AbortSignal + ): Promise + selectSources( + sessionHandle: string, + signal: AbortSignal + ): Promise + start( + sessionHandle: string, + parentWindow: string, + signal: AbortSignal + ): Promise + waitForResponse( + requestHandle: string, + signal: AbortSignal + ): Promise + openPipeWireRemote( + sessionHandle: string, + signal: AbortSignal + ): Promise + connectEis?( + sessionHandle: string, + signal: AbortSignal + ): Promise + closeRequest(requestHandle: string): Promise + closeSession(sessionHandle: string): Promise +} + +export type PortalSessionOptions = { + devices: boolean + sources: boolean + parentWindow?: string + timeoutMs?: number +} + +export type PortalDesktopSessionOptions = { + cleanupTimeoutMs?: number +} + +export class PortalSessionError extends Error { + constructor( + readonly reason: + | 'cancelled' + | 'denied' + | 'protocol' + | 'timeout' + | 'aborted' + | 'owner-lost', + message: string + ) { + super(message) + this.name = 'PortalSessionError' + } +} + +const getSessionHandle = (response: PortalResponse): string => { + const handle = response.results.session_handle + if (typeof handle !== 'string' || handle.length === 0) { + throw new PortalSessionError( + 'protocol', + 'Portal did not return a session handle' + ) + } + return handle +} + +const throwForResponse = (response: PortalResponse): void => { + if (response.response === 0) { + return + } + if (response.response === 1) { + throw new PortalSessionError('cancelled', 'Portal consent was cancelled') + } + if (response.response === 2) { + throw new PortalSessionError('denied', 'Portal consent was denied') + } + throw new PortalSessionError('protocol', 'Portal returned an unknown response') +} + +export class PortalDesktopSession { + private stateValue: PortalSessionState = 'idle' + private generation = 0 + private operation: PortalOperation | undefined + private openingPromise: Promise | undefined + private readonly cleanupTimeoutMs: number + + constructor( + private readonly transport: PortalTransport, + options: PortalDesktopSessionOptions = {} + ) { + this.cleanupTimeoutMs = options.cleanupTimeoutMs ?? 1_000 + if ( + !Number.isSafeInteger(this.cleanupTimeoutMs) || + this.cleanupTimeoutMs < 1 || + this.cleanupTimeoutMs > 30_000 + ) { + throw new Error('Invalid portal cleanup timeout') + } + } + + get state(): PortalSessionState { + return this.stateValue + } + + get hasActiveConsent(): boolean { + return ( + this.stateValue === 'active' && + this.operation !== undefined && + !this.operation.ownerLost + ) + } + + async open( + options: PortalSessionOptions, + outerSignal: AbortSignal + ): Promise { + if (this.stateValue !== 'idle' && this.stateValue !== 'stopped') { + throw new PortalSessionError('protocol', 'Portal session is already open') + } + if (!options.devices && !options.sources) { + throw new PortalSessionError( + 'protocol', + 'Portal session must request a concrete capability' + ) + } + const timeoutMs = options.timeoutMs ?? 30_000 + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > 120_000 + ) { + throw new PortalSessionError('protocol', 'Invalid portal timeout') + } + + this.stateValue = 'creating' + const operation: PortalOperation = { + generation: ++this.generation, + controller: new AbortController(), + requestHandles: new Set(), + resources: new Set(), + ownerLost: false, + stopRequested: false + } + this.operation = operation + const opening = this.openOperation( + operation, + options, + outerSignal, + timeoutMs + ) + this.openingPromise = opening + opening + .finally(() => { + if (this.openingPromise === opening) { + this.openingPromise = undefined + } + }) + .catch(() => { + // The caller receives the original open rejection. + }) + return opening + } + + private async openOperation( + operation: PortalOperation, + options: PortalSessionOptions, + outerSignal: AbortSignal, + timeoutMs: number + ): Promise { + const controller = operation.controller + const abort = (): void => controller.abort(outerSignal.reason) + outerSignal.addEventListener('abort', abort, { once: true }) + if (outerSignal.aborted) { + abort() + } + const timer = setTimeout( + () => controller.abort(new PortalSessionError('timeout', 'Portal timed out')), + timeoutMs + ) + + try { + const created = await this.request( + operation, + () => this.transport.createSession(controller.signal), + controller.signal + ) + operation.sessionHandle = getSessionHandle(created) + + this.setOwnedState(operation, 'selecting') + if (options.devices) { + await this.request( + operation, + () => + this.transport.selectDevices( + this.requireSessionHandle(operation), + controller.signal + ), + controller.signal + ) + } + if (options.sources) { + await this.request( + operation, + () => + this.transport.selectSources( + this.requireSessionHandle(operation), + controller.signal + ), + controller.signal + ) + } + + this.setOwnedState(operation, 'starting') + await this.request( + operation, + () => + this.transport.start( + this.requireSessionHandle(operation), + options.parentWindow ?? '', + controller.signal + ), + controller.signal + ) + + if (options.sources) { + operation.resources.add( + await this.acquireResource( + this.transport.openPipeWireRemote( + this.requireSessionHandle(operation), + controller.signal + ), + controller.signal + ) + ) + } + if (options.devices && this.transport.connectEis) { + operation.resources.add( + await this.acquireResource( + this.transport.connectEis( + this.requireSessionHandle(operation), + controller.signal + ), + controller.signal + ) + ) + } + if (controller.signal.aborted) { + throw controller.signal.reason + } + this.setOwnedState(operation, 'active') + } catch (error) { + await this.cleanup(operation) + this.setOwnedState( + operation, + operation.stopRequested ? 'stopped' : 'failed' + ) + if (operation.ownerLost) { + throw new PortalSessionError('owner-lost', 'Portal owner disappeared') + } + if (controller.signal.aborted) { + const reason = controller.signal.reason + if (reason instanceof PortalSessionError) { + throw reason + } + throw new PortalSessionError('aborted', 'Portal session was aborted') + } + throw error + } finally { + clearTimeout(timer) + outerSignal.removeEventListener('abort', abort) + } + } + + async stop(): Promise { + if (this.stateValue === 'stopped' || this.stateValue === 'idle') { + this.stateValue = 'stopped' + return + } + this.stateValue = 'stopping' + const operation = this.operation + if (!operation) { + this.stateValue = 'stopped' + return + } + operation.stopRequested = true + operation.controller.abort( + new PortalSessionError('aborted', 'Portal session stopped') + ) + await this.cleanup(operation) + await this.awaitOpening(operation) + if (this.operation === operation) { + this.operation = undefined + this.stateValue = 'stopped' + } + } + + async portalOwnerLost(): Promise { + const operation = this.operation + if (!operation) { + this.stateValue = 'failed' + return + } + operation.ownerLost = true + operation.controller.abort( + new PortalSessionError('owner-lost', 'Portal owner disappeared') + ) + await this.cleanup(operation) + await this.awaitOpening(operation) + if (this.operation === operation) { + this.operation = undefined + this.stateValue = 'failed' + } + } + + private async request( + operation: PortalOperation, + initiate: () => Promise, + signal: AbortSignal + ): Promise { + const initiating = Promise.resolve().then(initiate) + let request: PortalRequest + try { + request = await this.abortable(initiating, signal) + } catch (error) { + void initiating + .then((lateRequest) => + this.runCleanup([ + () => this.transport.closeRequest(lateRequest.requestHandle) + ]) + ) + .catch(() => undefined) + throw error + } + operation.requestHandles.add(request.requestHandle) + const response = await this.abortable( + this.transport.waitForResponse(request.requestHandle, signal), + signal + ) + if (response.requestHandle !== request.requestHandle) { + throw new PortalSessionError( + 'protocol', + 'Portal response handle did not match its request' + ) + } + throwForResponse(response) + return response + } + + private requireSessionHandle(operation: PortalOperation): string { + if (!operation.sessionHandle) { + throw new PortalSessionError( + 'protocol', + 'Portal session handle is unavailable' + ) + } + return operation.sessionHandle + } + + private async cleanup(operation: PortalOperation): Promise { + if (operation.cleanupPromise) { + return operation.cleanupPromise + } + const resources = [...operation.resources] + const requests = [...operation.requestHandles] + const sessionHandle = operation.sessionHandle + operation.resources.clear() + operation.requestHandles.clear() + operation.sessionHandle = undefined + operation.cleanupPromise = this.runCleanup([ + ...resources.map((resource) => () => resource.close()), + ...requests.map( + (handle) => () => this.transport.closeRequest(handle) + ), + ...(sessionHandle + ? [() => this.transport.closeSession(sessionHandle)] + : []) + ]) + return operation.cleanupPromise + } + + private async acquireResource( + resourcePromise: Promise, + signal: AbortSignal + ): Promise { + try { + return await this.abortable(resourcePromise, signal) + } catch (error) { + void resourcePromise + .then((resource) => + this.runCleanup([() => resource.close()]) + ) + .catch(() => undefined) + throw error + } + } + + private abortable( + promise: Promise, + signal: AbortSignal + ): Promise { + if (signal.aborted) { + promise.catch(() => undefined) + return Promise.reject(signal.reason) + } + return new Promise((resolve, reject) => { + const abort = (): void => { + promise.catch(() => undefined) + reject(signal.reason) + } + signal.addEventListener('abort', abort, { once: true }) + promise.then( + (value) => { + signal.removeEventListener('abort', abort) + resolve(value) + }, + (error: unknown) => { + signal.removeEventListener('abort', abort) + reject(error) + } + ) + }) + } + + private async runCleanup( + operations: Array<() => void | Promise> + ): Promise { + const cleanup = Promise.allSettled( + operations.map((operation) => + Promise.resolve().then(operation) + ) + ).then(() => undefined) + let timer: ReturnType | undefined + const timeout = new Promise((resolve) => { + timer = setTimeout(resolve, this.cleanupTimeoutMs) + }) + await Promise.race([cleanup, timeout]) + if (timer !== undefined) { + clearTimeout(timer) + } + } + + private async awaitOpening(operation: PortalOperation): Promise { + const opening = this.openingPromise + if (!opening || this.operation !== operation) { + return + } + await opening.catch(() => undefined) + } + + private setOwnedState( + operation: PortalOperation, + state: PortalSessionState + ): void { + if ( + this.operation === operation && + this.operation.generation === operation.generation + ) { + this.stateValue = state + } + } +} + +type PortalOperation = { + generation: number + controller: AbortController + sessionHandle?: string + requestHandles: Set + resources: Set + ownerLost: boolean + stopRequested: boolean + cleanupPromise?: Promise +} + +export type PipeWirePlaneMetadata = { + offset: number + stride: number + bytes: number +} + +export type PipeWireFrameMetadata = { + width: number + height: number + stride: number + planes: PipeWirePlaneMetadata[] + byteLength: number + fpsNumerator: number + fpsDenominator: number +} + +const isPositiveInteger = (value: number): boolean => + Number.isSafeInteger(value) && value > 0 + +export function validatePipeWireFrameMetadata( + metadata: PipeWireFrameMetadata +): PipeWireFrameMetadata { + if ( + !isPositiveInteger(metadata.width) || + !isPositiveInteger(metadata.height) || + metadata.width > 16_384 || + metadata.height > 16_384 || + !isPositiveInteger(metadata.stride) || + metadata.stride < metadata.width || + metadata.stride > 1_048_576 || + !isPositiveInteger(metadata.byteLength) || + metadata.byteLength > 256 * 1024 * 1024 || + !isPositiveInteger(metadata.fpsNumerator) || + !isPositiveInteger(metadata.fpsDenominator) || + metadata.fpsNumerator / metadata.fpsDenominator > 240 || + metadata.planes.length < 1 || + metadata.planes.length > 4 + ) { + throw new Error('Invalid PipeWire frame metadata') + } + let previousEnd = 0 + for (const [index, plane] of metadata.planes.entries()) { + if ( + !Number.isSafeInteger(plane.offset) || + plane.offset < 0 || + !isPositiveInteger(plane.stride) || + plane.stride > 1_048_576 || + !isPositiveInteger(plane.bytes) || + (index === 0 && plane.offset !== 0) || + plane.offset < previousEnd || + plane.offset + plane.bytes > metadata.byteLength || + (index === 0 && + (plane.stride !== metadata.stride || + plane.bytes < metadata.stride * metadata.height)) + ) { + throw new Error('Invalid PipeWire plane metadata') + } + previousEnd = plane.offset + plane.bytes + } + if (previousEnd !== metadata.byteLength) { + throw new Error('PipeWire plane metadata does not cover the frame buffer') + } + return structuredClone(metadata) +} diff --git a/src/main/linux-desktop/session-environment.test.ts b/src/main/linux-desktop/session-environment.test.ts new file mode 100644 index 0000000..5e4d367 --- /dev/null +++ b/src/main/linux-desktop/session-environment.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest' +import { + buildDesktopHelperEnvironment, + type SecurePathMetadata, + type SessionEnvironmentFileSystem +} from './session-environment' + +const safeMetadata = ( + canonicalPath: string, + kind: 'file' | 'directory' +): SecurePathMetadata => ({ + canonicalPath, + uid: 1000, + mode: kind === 'file' ? 0o600 : 0o700, + isDirectory: kind === 'directory', + isFile: kind === 'file', + isSymbolicLink: false +}) + +const fileSystem = ( + overrides: Partial = {} +): SessionEnvironmentFileSystem => ({ + inspect: async (path) => ({ + ...safeMetadata( + path, + path.endsWith('.Xauthority') ? 'file' : 'directory' + ), + ...overrides + }) +}) + +describe('buildDesktopHelperEnvironment', () => { + it('constructs a minimal fresh helper environment', async () => { + const environment = await buildDesktopHelperEnvironment({ + uid: 1000, + fileSystem: fileSystem(), + source: { + PATH: '/usr/bin:/bin', + LANG: 'zh_CN.UTF-8', + DISPLAY: ':0.0', + WAYLAND_DISPLAY: 'wayland-1', + XDG_RUNTIME_DIR: '/run/user/1000', + DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus', + XAUTHORITY: '/home/user/.Xauthority', + NO_AT_BRIDGE: '1', + LD_PRELOAD: '/tmp/inject.so', + NODE_OPTIONS: '--require /tmp/inject.js', + GTK_MODULES: 'inject', + QT_PLUGIN_PATH: '/tmp/plugins', + HTTPS_PROXY: 'http://proxy.invalid', + API_KEY: 'secret' + } + }) + + expect(environment).toEqual({ + PATH: '/usr/bin:/bin', + LANG: 'zh_CN.UTF-8', + DISPLAY: ':0.0', + WAYLAND_DISPLAY: 'wayland-1', + XDG_RUNTIME_DIR: '/run/user/1000', + DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus', + XAUTHORITY: '/home/user/.Xauthority', + NO_AT_BRIDGE: '1' + }) + }) + + it.each([ + ['symbolic link', { isSymbolicLink: true }], + ['world writable', { mode: 0o707 }], + ['group writable', { mode: 0o720 }], + ['wrong owner', { uid: 1001 }], + ['non-canonical path', { canonicalPath: '/different' }] + ])('rejects an unsafe %s runtime path', async (_name, override) => { + await expect( + buildDesktopHelperEnvironment({ + uid: 1000, + fileSystem: fileSystem(override), + source: { XDG_RUNTIME_DIR: '/run/user/1000' } + }) + ).rejects.toThrow('Unsafe directory path') + }) + + it.each([ + { DISPLAY: 'host:abc' }, + { WAYLAND_DISPLAY: '../wayland-0', XDG_RUNTIME_DIR: '/run/user/1000' }, + { WAYLAND_DISPLAY: 'wayland-0' }, + { DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus,bad' }, + { DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus;tcp:' }, + { LANG: 'en_US.UTF-8\nINJECTED=1' }, + { PATH: '/usr/bin:relative/bin' }, + { PATH: '/usr/bin::/bin' } + ])('rejects malformed desktop environment input', async (source) => { + await expect( + buildDesktopHelperEnvironment({ + uid: 1000, + fileSystem: fileSystem(), + source + }) + ).rejects.toThrow() + }) + + it('does not forward values that merely resemble safe opt-ins', async () => { + await expect( + buildDesktopHelperEnvironment({ + uid: 1000, + fileSystem: fileSystem(), + source: { NO_AT_BRIDGE: '0', LD_LIBRARY_PATH: '/tmp' } + }) + ).resolves.toEqual({}) + }) +}) diff --git a/src/main/linux-desktop/session-environment.ts b/src/main/linux-desktop/session-environment.ts new file mode 100644 index 0000000..6416c36 --- /dev/null +++ b/src/main/linux-desktop/session-environment.ts @@ -0,0 +1,198 @@ +import { isAbsolute, basename, posix } from 'node:path' + +export type SecurePathMetadata = { + canonicalPath: string + uid: number + mode: number + isDirectory: boolean + isFile: boolean + isSymbolicLink: boolean +} + +export interface SessionEnvironmentFileSystem { + inspect(path: string): Promise +} + +export type SessionEnvironmentOptions = { + source?: NodeJS.ProcessEnv + uid: number + fileSystem: SessionEnvironmentFileSystem +} + +const DISPLAY_PATTERN = /^(?:(?:[A-Za-z0-9._-]+)?):\d+(?:\.\d+)?$/ +const WAYLAND_BASENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/ +const LOCALE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}$/ +const DBUS_VALUE_PATTERN = /^(?:[A-Za-z0-9._~:/@+-]|%[0-9A-Fa-f]{2})+$/ + +const hasControlCharacters = (value: string): boolean => + [...value].some((character) => { + const code = character.charCodeAt(0) + return code < 32 || code === 127 + }) + +const isSafePathText = (value: string): boolean => + value.length >= 1 && + value.length <= 8192 && + !hasControlCharacters(value) + +const assertSafePath = async ( + path: string, + expectedKind: 'directory' | 'file', + options: SessionEnvironmentOptions +): Promise => { + if (!isAbsolute(path) || !isSafePathText(path)) { + throw new Error(`Invalid ${expectedKind} path`) + } + const metadata = await options.fileSystem.inspect(path) + if ( + metadata.isSymbolicLink || + metadata.canonicalPath !== path || + metadata.uid !== options.uid || + (metadata.mode & 0o022) !== 0 || + (expectedKind === 'directory' + ? !metadata.isDirectory + : !metadata.isFile) + ) { + throw new Error(`Unsafe ${expectedKind} path`) + } + return path +} + +const validateBusAddress = (address: string): string => { + if ( + address.length === 0 || + address.length > 4096 || + hasControlCharacters(address) + ) { + throw new Error('Invalid session bus address') + } + const alternatives = address.split(';') + if (alternatives.some((entry) => entry.length === 0)) { + throw new Error('Invalid session bus address') + } + for (const entry of alternatives) { + const separator = entry.indexOf(':') + if (separator <= 0 || !/^[a-z][a-z0-9_-]*$/.test(entry.slice(0, separator))) { + throw new Error('Invalid session bus address') + } + const properties = entry.slice(separator + 1).split(',') + if (properties.some((property) => property.length === 0)) { + throw new Error('Invalid session bus address') + } + const seen = new Set() + for (const property of properties) { + const equals = property.indexOf('=') + const key = property.slice(0, equals) + const value = property.slice(equals + 1) + if ( + equals <= 0 || + !/^[a-z][a-z0-9_-]*$/.test(key) || + !DBUS_VALUE_PATTERN.test(value) || + seen.has(key) + ) { + throw new Error('Invalid session bus address') + } + seen.add(key) + } + } + return address +} + +const copySafeValue = ( + target: NodeJS.ProcessEnv, + source: NodeJS.ProcessEnv, + name: string, + pattern: RegExp +): void => { + const value = source[name] + if (value !== undefined) { + if (!pattern.test(value)) { + throw new Error(`Invalid ${name}`) + } + target[name] = value + } +} + +const copySafePath = ( + target: NodeJS.ProcessEnv, + source: NodeJS.ProcessEnv +): void => { + const value = source.PATH + if (value === undefined) { + return + } + const entries = value.split(':') + if ( + !isSafePathText(value) || + entries.length === 0 || + entries.some((entry) => !posix.isAbsolute(entry)) + ) { + throw new Error('Invalid PATH') + } + target.PATH = value +} + +/** + * Builds a new environment rather than filtering in place. Native desktop + * helpers must never inherit loader hooks, provider credentials, or proxies. + */ +export async function buildDesktopHelperEnvironment( + options: SessionEnvironmentOptions +): Promise { + const source = options.source ?? process.env + const environment: NodeJS.ProcessEnv = {} + + copySafePath(environment, source) + copySafeValue(environment, source, 'LANG', LOCALE_PATTERN) + copySafeValue(environment, source, 'LC_ALL', LOCALE_PATTERN) + copySafeValue(environment, source, 'LC_CTYPE', LOCALE_PATTERN) + + const display = source.DISPLAY + if (display !== undefined) { + if (!DISPLAY_PATTERN.test(display)) { + throw new Error('Invalid DISPLAY') + } + environment.DISPLAY = display + } + + const runtimeDirectory = source.XDG_RUNTIME_DIR + if (runtimeDirectory !== undefined) { + environment.XDG_RUNTIME_DIR = await assertSafePath( + runtimeDirectory, + 'directory', + options + ) + } + + const waylandDisplay = source.WAYLAND_DISPLAY + if (waylandDisplay !== undefined) { + if ( + basename(waylandDisplay) !== waylandDisplay || + !WAYLAND_BASENAME_PATTERN.test(waylandDisplay) || + runtimeDirectory === undefined + ) { + throw new Error('Invalid WAYLAND_DISPLAY') + } + environment.WAYLAND_DISPLAY = waylandDisplay + } + + const busAddress = source.DBUS_SESSION_BUS_ADDRESS + if (busAddress !== undefined) { + environment.DBUS_SESSION_BUS_ADDRESS = validateBusAddress(busAddress) + } + + const authority = source.XAUTHORITY + if (authority !== undefined) { + environment.XAUTHORITY = await assertSafePath( + authority, + 'file', + options + ) + } + + if (source.NO_AT_BRIDGE === '1') { + environment.NO_AT_BRIDGE = '1' + } + + return environment +} diff --git a/src/preload/index.ts b/src/preload/index.ts index 80f9380..e8ec3e0 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -6,6 +6,7 @@ import { type AgentRuntimeDetection, type AgentRuntimeStatus, type AppInfo, + type BrowserLiveState, type ContextAttachment, type DesktopApi, type KnowledgeLibrary, @@ -17,7 +18,11 @@ import { } from '../shared/contracts' import { ipcChannels } from '../shared/ipc-channels' import type { + BrowserProfileCreateInput, + BrowserProfileRenameInput, + CapabilityDiagnosticReport, CapabilitySnapshot, + ComputerCapabilityId, McpServerTestResult } from '../shared/capability-contracts' import type { @@ -40,7 +45,8 @@ import type { ScheduleCreateInput, HeartbeatCreateInput, HeartbeatUpdateInput, - ExpertCreateInput + ExpertCreateInput, + ExpertUpdateInput } from '../shared/assistant-contracts' const desktopApi: DesktopApi = { @@ -116,6 +122,23 @@ const desktopApi: DesktopApi = { return () => ipcRenderer.removeListener(ipcChannels.agentEvent, handler) } }, + browser: { + stop: async (conversationId: string) => { + await ipcRenderer.invoke( + ipcChannels.browserStop, + { conversationId } + ) + }, + onState: (listener) => { + const handler = ( + _event: Electron.IpcRendererEvent, + payload: BrowserLiveState + ): void => listener(payload) + ipcRenderer.on(ipcChannels.browserState, handler) + return () => + ipcRenderer.removeListener(ipcChannels.browserState, handler) + } + }, settings: { getRuntime: () => ipcRenderer.invoke( @@ -329,7 +352,15 @@ const desktopApi: DesktopApi = { ipcRenderer.invoke( ipcChannels.expertsCreate, input - ) as Promise + ) as Promise, + update: (expertId: string, input: ExpertUpdateInput) => + ipcRenderer.invoke(ipcChannels.expertsUpdate, { + expertId, + input + }) as Promise, + remove: async (expertId: string) => { + await ipcRenderer.invoke(ipcChannels.expertsRemove, expertId) + } }, capabilities: { getSnapshot: () => @@ -369,7 +400,46 @@ const desktopApi: DesktopApi = { ipcRenderer.invoke( ipcChannels.capabilitiesTestMcp, serverId - ) as Promise + ) as Promise, + setComputerCapabilityEnabled: ( + capabilityId: ComputerCapabilityId, + enabled: boolean + ) => + ipcRenderer.invoke(ipcChannels.capabilitiesToggleComputer, { + capabilityId, + enabled + }) as Promise, + setComputerCapabilityBrowserProfile: ( + capabilityId: ComputerCapabilityId, + browserProfileId: string | null + ) => + ipcRenderer.invoke(ipcChannels.capabilitiesConfigureComputer, { + capabilityId, + browserProfileId + }) as Promise, + diagnoseComputerCapability: (capabilityId: ComputerCapabilityId) => + ipcRenderer.invoke( + ipcChannels.capabilitiesDiagnoseComputer, + capabilityId + ) as Promise, + createBrowserProfile: (input: BrowserProfileCreateInput) => + ipcRenderer.invoke( + ipcChannels.capabilitiesCreateBrowserProfile, + input + ) as Promise, + renameBrowserProfile: (input: BrowserProfileRenameInput) => + ipcRenderer.invoke( + ipcChannels.capabilitiesRenameBrowserProfile, + input + ) as Promise, + setDefaultBrowserProfile: (profileId: string) => + ipcRenderer.invoke(ipcChannels.capabilitiesDefaultBrowserProfile, { + profileId + }) as Promise, + removeBrowserProfile: (profileId: string) => + ipcRenderer.invoke(ipcChannels.capabilitiesRemoveBrowserProfile, { + profileId + }) as Promise }, context: { selectFiles: () => diff --git a/src/preload/preload-sandbox.test.ts b/src/preload/preload-sandbox.test.ts index 93015c9..905f7d3 100644 --- a/src/preload/preload-sandbox.test.ts +++ b/src/preload/preload-sandbox.test.ts @@ -11,4 +11,29 @@ describe('sandboxed preload', () => { expect(source).not.toMatch(/\bfrom\s+['"]node:/u) expect(source).not.toMatch(/\brequire\(\s*['"]node:/u) }) + + it('does not load runtime schema libraries in the Electron sandbox', () => { + const source = readFileSync( + join(process.cwd(), 'src', 'preload', 'index.ts'), + 'utf8' + ) + expect(source).not.toMatch(/\b\w+Schema\b/u) + expect(source).not.toMatch(/\bfrom\s+['"]zod['"]/u) + }) + + it('exposes only explicit computer capability and managed profile IPC methods', () => { + const source = readFileSync( + join(process.cwd(), 'src', 'preload', 'index.ts'), + 'utf8' + ) + expect(source).toContain('setComputerCapabilityEnabled:') + expect(source).toContain('diagnoseComputerCapability:') + expect(source).toContain('createBrowserProfile:') + expect(source).toContain('renameBrowserProfile:') + expect(source).toContain('setDefaultBrowserProfile:') + expect(source).toContain('removeBrowserProfile:') + expect(source).not.toMatch( + /(?:setComputerCapability|BrowserProfile).{0,80}(?:executablePath|command|env|args)/su + ) + }) }) diff --git a/src/renderer/src/App.test.tsx b/src/renderer/src/App.test.tsx index f8a8cd4..9806be7 100644 --- a/src/renderer/src/App.test.tsx +++ b/src/renderer/src/App.test.tsx @@ -8,10 +8,15 @@ import { waitFor } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentEvent, DesktopApi } from '../../shared/contracts' +import type { + AgentEvent, + BrowserLiveState, + DesktopApi +} from '../../shared/contracts' import App from './App' let agentListener: ((event: AgentEvent) => void) | undefined +let browserListener: ((state: BrowserLiveState) => void) | undefined let newConversationListener: (() => void) | undefined let maximizedChangedListener: ((maximized: boolean) => void) | undefined const removeMaximizedChangedListener = vi.fn() @@ -75,6 +80,15 @@ const api: DesktopApi = { } }) }, + browser: { + stop: vi.fn(async () => {}), + onState: vi.fn((listener) => { + browserListener = listener + return () => { + browserListener = undefined + } + }) + }, settings: { getRuntime: vi.fn(async () => ({ provider: 'auto', @@ -319,7 +333,15 @@ const api: DesktopApi = { enabled: true, createdAt: '2026-07-31T00:00:00.000Z', updatedAt: '2026-07-31T00:00:00.000Z' - })) + })), + update: vi.fn(async (expertId, input) => ({ + ...input, + id: expertId, + enabled: true, + createdAt: '2026-07-31T00:00:00.000Z', + updatedAt: '2026-08-04T00:00:00.000Z' + })), + remove: vi.fn(async () => {}) }, capabilities: { getSnapshot: vi.fn(async () => ({ @@ -413,6 +435,7 @@ describe('App', () => { document.documentElement.style.colorScheme = '' vi.clearAllMocks() newConversationListener = undefined + browserListener = undefined maximizedChangedListener = undefined vi.mocked(api.agent.getStatus).mockResolvedValue({ id: 'model', @@ -452,6 +475,22 @@ describe('App', () => { expect(removeMaximizedChangedListener).toHaveBeenCalledOnce() }) + it('keeps rendering when an older preload has no browser bridge', async () => { + Object.defineProperty(window, 'goodbuddy', { + configurable: true, + value: { + ...api, + browser: undefined + } + }) + + render() + + expect( + await screen.findByLabelText('向 GoodBuddy 提问') + ).toBeInTheDocument() + }) + it('keeps conversation actions in the conversation list', async () => { const { container } = render() const topbar = container.querySelector('.topbar') @@ -938,6 +977,55 @@ describe('App', () => { expect(within(stats).getByText('345')).toBeInTheDocument() }) + it('offers only Ask and Execute in visible work mode controls', async () => { + render() + + const mode = await screen.findByLabelText('工作模式') + expect( + within(mode) + .getAllByRole('option') + .map((option) => option.textContent) + ).toEqual(['Ask · 只读问答', 'Execute · 受控执行']) + + fireEvent.click(screen.getByLabelText('新建项目')) + const dialog = screen.getByRole('dialog', { name: '新建项目' }) + const defaultMode = within(dialog).getByRole('combobox', { + name: '默认模式' + }) + expect( + within(defaultMode) + .getAllByRole('option') + .map((option) => option.textContent) + ).toEqual(['Ask · 只读问答', 'Execute · 受控执行']) + expect(screen.queryByRole('option', { name: /Plan/u })).toBeNull() + }) + + it('normalizes a legacy Plan project default to Ask', async () => { + vi.mocked(api.projects.list).mockResolvedValueOnce([ + { + ...project, + defaultWorkMode: 'plan' + } + ]) + render() + + const mode = await screen.findByLabelText('工作模式') + expect(mode).toHaveValue('ask') + fireEvent.change(screen.getByLabelText('向 GoodBuddy 提问'), { + target: { value: '制定发布方案' } + }) + fireEvent.click(screen.getByLabelText('发送')) + + await waitFor(() => + expect(run).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: '制定发布方案', + workMode: 'ask' + }) + ) + ) + }) + it.each([ ['opencode', 'OpenCode'], ['continue', 'Continue CLI'] @@ -1349,6 +1437,47 @@ describe('App', () => { expect(sidebar).not.toHaveClass('assistant-sidebar--open') }) + it('opens the live browser tab for the active conversation and can stop it', async () => { + render() + + fireEvent.change(screen.getByLabelText('向 GoodBuddy 提问'), { + target: { value: '打开示例网页' } + }) + fireEvent.click(await screen.findByLabelText('发送')) + await waitFor(() => expect(run).toHaveBeenCalledOnce()) + const conversationId = run.mock.calls[0]?.[0].conversationId + expect(conversationId).toBeTruthy() + + act(() => { + browserListener?.({ + conversationId: conversationId ?? '', + status: 'ready', + url: 'https://example.com/', + frameDataUrl: 'data:image/png;base64,iVBORw0KGgo=', + updatedAt: Date.now() + }) + }) + + expect(screen.getByLabelText('助手工作栏')).toHaveClass( + 'assistant-sidebar--open' + ) + expect( + screen.getByRole('tab', { name: '浏览器' }) + ).toHaveAttribute('aria-selected', 'true') + expect( + screen.getByAltText('Agent 实时浏览器画面') + ).toHaveAttribute( + 'src', + 'data:image/png;base64,iVBORw0KGgo=' + ) + fireEvent.click( + screen.getByRole('button', { name: '停止浏览器' }) + ) + await waitFor(() => + expect(api.browser.stop).toHaveBeenCalledWith(conversationId) + ) + }) + it('opens Smart Heartbeat as a first-class workspace', async () => { render() diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index f7186cf..42a3cd2 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -40,6 +40,7 @@ import type { AgentEvent, AgentRuntimeStatus, AppInfo, + BrowserLiveState, ContextAttachment, KnowledgeSearchReference, KnowledgeSnapshot, @@ -60,9 +61,13 @@ import type { TokenUsageSummary, ConversationSnapshot, ProjectCreateInput, - WorkMode, + InteractiveWorkMode, WorkspaceChanges } from '../../shared/assistant-contracts' +import { + interactiveWorkModes, + normalizeInteractiveWorkMode +} from '../../shared/assistant-contracts' import { ActivityPanel } from './ActivityPanel' import { loadActivityRecords, @@ -110,6 +115,7 @@ type ToolActivity = { | 'running' | 'completed' | 'failed' + | 'recoverable' | 'cancelled' | 'interrupted' summary: string @@ -194,6 +200,7 @@ const toolStateLabels: Record = { running: '进行中', completed: '已完成', failed: '失败', + recoverable: '可重试', cancelled: '已取消', interrupted: '已中断' } @@ -552,7 +559,8 @@ function App(): React.JSX.Element { const workspaceChangesRequestRef = useRef(0) const viewRef = useRef('chat') const heartbeatLoadRequestRef = useRef(0) - const [workMode, setWorkMode] = useState('ask') + const [workMode, setWorkMode] = + useState('ask') const [input, setInput] = useState('') const [voiceListening, setVoiceListening] = useState(false) const [runtime, setRuntime] = useState() @@ -585,6 +593,9 @@ function App(): React.JSX.Element { ) const [assistantSidebarTab, setAssistantSidebarTab] = useState('tasks') + const [browserStates, setBrowserStates] = useState< + Record + >({}) const [view, setView] = useState('chat') const [searchQuery, setSearchQuery] = useState('') const [conversationActionsId, setConversationActionsId] = useState('') @@ -1286,7 +1297,9 @@ function App(): React.JSX.Element { const project = value[0]! setProjects(value) setActiveProjectId(project.id) - setWorkMode(project.defaultWorkMode) + setWorkMode( + normalizeInteractiveWorkMode(project.defaultWorkMode) + ) let nextConversations: Conversation[] = persistedConversations.length > 0 ? persistedConversations @@ -1665,6 +1678,41 @@ function App(): React.JSX.Element { } }, [handleAgentEvent]) + useEffect( + () => { + const browserApi = window.goodbuddy.browser + if (!browserApi) { + return + } + return browserApi.onState((state) => { + setBrowserStates((current) => { + const previous = current[state.conversationId] + return { + ...current, + [state.conversationId]: + state.status === 'stopped' || + state.frameDataUrl || + !previous?.frameDataUrl + ? state + : { + ...state, + frameDataUrl: previous.frameDataUrl + } + } + }) + if ( + state.status !== 'stopped' && + state.conversationId === + conversationNavigationRef.current.activeId + ) { + setAssistantSidebarOpen(true) + setAssistantSidebarTab('browser') + } + }) + }, + [] + ) + useEffect( () => window.goodbuddy.app.onNewConversation(() => { @@ -1689,7 +1737,7 @@ function App(): React.JSX.Element { return } setActiveProjectId(projectId) - setWorkMode(project.defaultWorkMode) + setWorkMode(normalizeInteractiveWorkMode(project.defaultWorkMode)) const conversation = conversations.find( (candidate) => candidate.projectId === projectId ) @@ -1709,7 +1757,7 @@ function App(): React.JSX.Element { const project = await window.goodbuddy.projects.create(input) setProjects((current) => [project, ...current]) setActiveProjectId(project.id) - setWorkMode(project.defaultWorkMode) + setWorkMode(normalizeInteractiveWorkMode(project.defaultWorkMode)) const conversation = createConversation(project.id) setConversations((current) => [conversation, ...current]) setActiveId(conversation.id) @@ -1747,7 +1795,7 @@ function App(): React.JSX.Element { const useHeartbeatTask = (task: AssistantTask): void => { newConversation() - setWorkMode('plan') + setWorkMode('ask') setInput( [ '请根据以下智能心跳建议制定可执行方案：', @@ -1790,6 +1838,17 @@ function App(): React.JSX.Element { if (activeRequest) { void window.goodbuddy.agent.cancel(activeRequest) } + const browserStop = window.goodbuddy.browser?.stop(conversationId) + if (browserStop) { + void browserStop.catch(() => { + setNotice('关闭已删除对话的浏览器失败') + }) + } + setBrowserStates((current) => { + const next = { ...current } + delete next[conversationId] + return next + }) const remaining = conversations.filter( (conversation) => conversation.id !== conversationId ) @@ -2232,7 +2291,9 @@ function App(): React.JSX.Element { ) setActiveProjectId(conversation.projectId) if (project) { - setWorkMode(project.defaultWorkMode) + setWorkMode( + normalizeInteractiveWorkMode(project.defaultWorkMode) + ) } } setActiveId(conversationId) @@ -3129,24 +3190,24 @@ function App(): React.JSX.Element { aria-label="工作模式" disabled={agentRuntimeSelected} onChange={(event) => - setWorkMode(event.target.value as WorkMode) + setWorkMode( + event.target.value as InteractiveWorkMode + ) } value={effectiveWorkMode} > - {Object.entries(workModeLabels).map( - ([value, label]) => ( - - ) - )} + {interactiveWorkModes.map((value) => ( + + ))}
@@ -3288,10 +3349,8 @@ function App(): React.JSX.Element { : agentRuntimeSelected ? `${runtime.label} 固定为 Execute，工具调用不会弹出 GoodBuddy 审批，并会记录到活动。` : effectiveWorkMode === 'ask' - ? 'Ask 模式：只读问答，不会调用工具或修改文件。' - : effectiveWorkMode === 'plan' - ? 'Plan 模式：只读制定计划，不会调用工具或修改文件。' - : 'Execute 模式：可执行工具，调用前请检查参数和权限。')} + ? 'Ask 模式：只读问答，不会调用工具或修改文件。' + : 'Execute 模式：已启用工具自动授权，调用仍会记录到活动。')} {appInfo?.shortcut && ` 快捷唤起：${appInfo.shortcut}`}

@@ -3471,6 +3530,19 @@ function App(): React.JSX.Element { onClearLocalData={clearLocalData} onClose={() => setView('chat')} onCreateHeartbeat={createHeartbeat} + onExpertsChanged={(experts) => { + setAssistantExperts(experts) + if ( + (selectedExpertId === 'team' && experts.length < 2) || + (selectedExpertId && + selectedExpertId !== 'team' && + !experts.some( + (expert) => expert.id === selectedExpertId + )) + ) { + setSelectedExpertId('') + } + }} onRemoveHeartbeat={removeHeartbeat} onRunHeartbeat={runHeartbeat} onSaved={(settings) => { @@ -3511,11 +3583,27 @@ function App(): React.JSX.Element { approvals={pendingSidebarApprovals} artifacts={sidebarArtifacts} attachments={attachments} + browserState={browserStates[activeId]} enabledLibraries={enabledSidebarLibraries} heartbeatEntries={heartbeatEntries} heartbeats={assistantHeartbeats} memories={assistantMemories} onClose={() => setAssistantSidebarOpen(false)} + onStopBrowser={async () => { + if (!activeId) { + return + } + const browserApi = window.goodbuddy.browser + if (!browserApi) { + setNotice('浏览器控制组件尚未加载，请重启 GoodBuddy') + return + } + try { + await browserApi.stop(activeId) + } catch { + setNotice('停止浏览器失败，请重试') + } + }} onOpenHeartbeat={() => setView('heartbeat')} onCreateMemory={async (content) => { const memory = await window.goodbuddy.memory.create({ diff --git a/src/renderer/src/McpSettingsSection.tsx b/src/renderer/src/McpSettingsSection.tsx index 7139bce..07eabda 100644 --- a/src/renderer/src/McpSettingsSection.tsx +++ b/src/renderer/src/McpSettingsSection.tsx @@ -1,15 +1,23 @@ import { + CircleAlert, FlaskConical, + Globe2, + MonitorCog, Network, Pencil, Plus, + RefreshCw, Trash2, + Wrench, X } from 'lucide-react' import { useEffect, useState } from 'react' +import { builtinModelTools } from '../../shared/builtin-model-tools' import type { + CapabilityDiagnosticReport, CapabilityAssignments, CapabilitySnapshot, + ComputerCapabilityId, McpServerInput, McpServerSummary, McpServerTestResult, @@ -23,6 +31,15 @@ const runtimeLabels: Record = { continue: 'Continue' } const configurableMcpTargets: RuntimeTarget[] = ['model'] +const diagnosticStatusLabels: Record< + CapabilityDiagnosticReport['status'], + string +> = { + available: '可用', + degraded: '部分可用', + unavailable: '不可用', + disabled: '未启用' +} type McpEditor = { id?: string @@ -77,6 +94,13 @@ export function McpSettingsSection(): React.JSX.Element { const [testResults, setTestResults] = useState< Record >({}) + const [diagnostics, setDiagnostics] = useState< + Partial> + >({}) + const [newProfileName, setNewProfileName] = useState('') + const [profileNames, setProfileNames] = useState>( + {} + ) useEffect(() => { void window.goodbuddy.capabilities @@ -97,13 +121,51 @@ export function McpSettingsSection(): React.JSX.Element { setSnapshot(await operation()) return true } catch (reason) { - setError(reason instanceof Error ? reason.message : 'MCP 操作失败') + setError(reason instanceof Error ? reason.message : '能力设置操作失败') return false } finally { setBusy(undefined) } } + const diagnose = async ( + capabilityId: ComputerCapabilityId + ): Promise => { + setBusy(`diagnose:${capabilityId}`) + setError(undefined) + try { + const diagnoseCapability = + window.goodbuddy.capabilities.diagnoseComputerCapability + if (!diagnoseCapability) { + throw new Error('当前版本不支持电脑控制能力诊断') + } + const report = await diagnoseCapability(capabilityId) + setDiagnostics((current) => ({ + ...current, + [capabilityId]: report + })) + } catch (reason) { + setError(reason instanceof Error ? reason.message : '能力诊断失败') + } finally { + setBusy(undefined) + } + } + + const createProfile = async (): Promise => { + const name = newProfileName.trim() + if (!name) { + return + } + if ( + await run('profile:create', () => + window.goodbuddy.capabilities.createBrowserProfile?.({ name }) ?? + Promise.reject(new Error('当前版本不支持托管浏览器配置')) + ) + ) { + setNewProfileName('') + } + } + const save = async (): Promise => { if (!editor) { return @@ -178,13 +240,19 @@ export function McpSettingsSection(): React.JSX.Element { }) } + const computerCapabilities = snapshot?.computerCapabilities ?? [] + const browserProfiles = snapshot?.browserProfiles ?? { + profiles: [], + defaultProfileId: null + } + return (
- MCP Servers - 支持 stdio、Streamable HTTP 和兼容 SSE + 工具与 MCP + 查看直连模型内置工具并管理外部 MCP Server
+ {report && ( +
+ + 诊断结果：{diagnosticStatusLabels[report.status]} + + {report.checks.map((check) => ( +

+ {check.summary} + {check.remedy ? ` 处理建议：${check.remedy}` : ''} +

+ ))} +
+ )} +
+ + ) + })} +
+ + +
+
+
+ + 托管浏览器配置 +
+ {browserProfiles.profiles.length} 个 +
+

+ 每个配置使用 GoodBuddy 管理的隔离存储；界面不会接收或显示可执行路径、命令参数与环境变量。 +

+
+ + +
+
+ {browserProfiles.profiles.length === 0 && ( +

尚未创建托管浏览器配置

+ )} + {browserProfiles.profiles.map((profile) => { + const referenced = computerCapabilities.some( + (capability) => + capability.browserProfileId === profile.id + ) + return ( +
+ + + + +
+ ) + })} +
+
+ +
+
+
+ + 直连模型内置工具 +
+ {builtinModelTools.length} 个 +
+
+ {builtinModelTools.map((tool) => ( +
+
+
+ {tool.displayName} + + GoodBuddy 内置 ·{' '} + {tool.access === 'write' ? '写入工具' : '只读工具'} + +
+ 直连模型 +
+

{tool.description}

+ {tool.name} +
+ ))} +
+
+ {editor && (
@@ -381,6 +719,16 @@ export function McpSettingsSection(): React.JSX.Element {
)} +
+
+ + 自定义 MCP Servers（高级） +
+ {snapshot?.mcpServers.length ?? 0} 个 +
+

+ 自定义 stdio MCP 会以受限环境启动，不会获得桌面会话变量。需要电脑控制时请使用上方经过诊断的内置能力。 +

{snapshot?.mcpServers.length === 0 && !editor && (

尚未配置 MCP Server

diff --git a/src/renderer/src/ProjectSwitcher.tsx b/src/renderer/src/ProjectSwitcher.tsx index 59a5f28..ee5c980 100644 --- a/src/renderer/src/ProjectSwitcher.tsx +++ b/src/renderer/src/ProjectSwitcher.tsx @@ -2,9 +2,11 @@ import { Archive, FolderOpen, Plus, X } from 'lucide-react' import { useEffect, useRef, useState } from 'react' import type { AssistantProject, + InteractiveWorkMode, ProjectCreateInput, WorkMode } from '../../shared/assistant-contracts' +import { interactiveWorkModes } from '../../shared/assistant-contracts' type ProjectSwitcherProps = { projects: AssistantProject[] @@ -15,9 +17,8 @@ type ProjectSwitcherProps = { onSelectRoot: () => Promise } -export const workModeLabels: Record = { +export const workModeLabels: Record = { ask: 'Ask · 只读问答', - plan: 'Plan · 先审计划', execute: 'Execute · 受控执行' } @@ -215,9 +216,9 @@ export function ProjectSwitcher({ } value={draft.defaultWorkMode} > - {Object.entries(workModeLabels).map(([value, label]) => ( + {interactiveWorkModes.map((value) => ( ))} diff --git a/src/renderer/src/RightAssistantSidebar.resize.test.tsx b/src/renderer/src/RightAssistantSidebar.resize.test.tsx new file mode 100644 index 0000000..e9deaa1 --- /dev/null +++ b/src/renderer/src/RightAssistantSidebar.resize.test.tsx @@ -0,0 +1,151 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { RightAssistantSidebar } from './RightAssistantSidebar' + +afterEach(cleanup) + +beforeEach(() => { + Object.defineProperty(window, 'innerWidth', { + configurable: true, + value: 1400 + }) +}) + +function renderSidebar(): HTMLElement { + render( + undefined)} + onCreateMemory={vi.fn(async () => undefined)} + onCreateSchedule={vi.fn(async () => undefined)} + onImportArtifacts={vi.fn(async () => undefined)} + onListWorkspaceDirectory={vi.fn(async (path: string) => ({ + path, + entries: [], + truncated: false + }))} + onLoadArtifact={vi.fn(async () => undefined)} + onLoadWorkspaceFile={vi.fn()} + onOpenConversation={vi.fn()} + onOpenHeartbeat={vi.fn()} + onRefreshChanges={vi.fn(async () => undefined)} + onRemoveAttachment={vi.fn()} + onRemoveHeartbeat={vi.fn(async () => undefined)} + onRemoveMemory={vi.fn(async () => undefined)} + onRemoveSchedule={vi.fn(async () => undefined)} + onRespondApproval={vi.fn()} + onRunHeartbeat={vi.fn(async () => undefined)} + onRunSchedule={vi.fn(async () => undefined)} + onSetHeartbeatPaused={vi.fn(async () => undefined)} + onSetMemoryStatus={vi.fn(async () => undefined)} + onStopBrowser={vi.fn(async () => undefined)} + onTabChange={vi.fn()} + open + schedules={[]} + tab="context" + tasks={[]} + /> + ) + + return screen.getByRole('complementary', { + name: '助手工作栏' + }) +} + +describe('RightAssistantSidebar resizing', () => { + it('resizes with pointer capture and clamps the resulting width', () => { + const sidebar = renderSidebar() + const separator = screen.getByRole('separator', { + name: '调整助手工作栏宽度' + }) + const setPointerCapture = vi.fn() + const releasePointerCapture = vi.fn() + Object.defineProperties(separator, { + setPointerCapture: { value: setPointerCapture }, + hasPointerCapture: { value: () => true }, + releasePointerCapture: { value: releasePointerCapture } + }) + + fireEvent.pointerDown(separator, { + button: 0, + clientX: 900, + pointerId: 7 + }) + fireEvent.pointerMove(separator, { + clientX: 600, + pointerId: 7 + }) + + expect(setPointerCapture).toHaveBeenCalledWith(7) + expect(sidebar).toHaveClass('assistant-sidebar--resizing') + expect( + sidebar.style.getPropertyValue('--assistant-sidebar-width') + ).toBe('640px') + + fireEvent.pointerUp(separator, { pointerId: 7 }) + expect(releasePointerCapture).toHaveBeenCalledWith(7) + expect(sidebar).not.toHaveClass('assistant-sidebar--resizing') + }) + + it('supports arrow, Home, and End keyboard resizing', () => { + const sidebar = renderSidebar() + const separator = screen.getByRole('separator', { + name: '调整助手工作栏宽度' + }) + + fireEvent.keyDown(separator, { key: 'ArrowLeft' }) + expect( + sidebar.style.getPropertyValue('--assistant-sidebar-width') + ).toBe('366px') + expect(separator).toHaveAttribute('aria-valuenow', '366') + + fireEvent.keyDown(separator, { key: 'Home' }) + expect( + sidebar.style.getPropertyValue('--assistant-sidebar-width') + ).toBe('300px') + + fireEvent.keyDown(separator, { key: 'End' }) + expect( + sidebar.style.getPropertyValue('--assistant-sidebar-width') + ).toBe('640px') + }) + + it('remains resizable when the sidebar overlays a medium window', () => { + Object.defineProperty(window, 'innerWidth', { + configurable: true, + value: 1024 + }) + const sidebar = renderSidebar() + const separator = screen.getByRole('separator', { + name: '调整助手工作栏宽度' + }) + Object.defineProperties(separator, { + setPointerCapture: { value: vi.fn() }, + hasPointerCapture: { value: () => true }, + releasePointerCapture: { value: vi.fn() } + }) + + expect(separator).toHaveAttribute('tabindex', '0') + fireEvent.pointerDown(separator, { + button: 0, + clientX: 674, + pointerId: 8 + }) + fireEvent.pointerMove(separator, { + clientX: 600, + pointerId: 8 + }) + + expect( + sidebar.style.getPropertyValue('--assistant-sidebar-width') + ).toBe('424px') + }) +}) diff --git a/src/renderer/src/RightAssistantSidebar.tsx b/src/renderer/src/RightAssistantSidebar.tsx index fa1d8f8..ffccd9f 100644 --- a/src/renderer/src/RightAssistantSidebar.tsx +++ b/src/renderer/src/RightAssistantSidebar.tsx @@ -5,6 +5,7 @@ import { FileText, FolderTree, Hourglass, + Monitor, PanelRightClose, PlayCircle, RefreshCw, @@ -13,7 +14,7 @@ import { X, XCircle } from 'lucide-react' -import { useRef, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import type { AssistantMemory, AssistantSchedule, @@ -29,6 +30,7 @@ import type { import { MarkdownRenderer } from './MarkdownRenderer' import type { ApprovalDecision, + BrowserLiveState, ContextAttachment, KnowledgeLibrary } from '../../shared/contracts' @@ -41,6 +43,7 @@ export type AssistantSidebarTab = | 'context' | 'artifacts' | 'changes' + | 'browser' | 'preview' export type SidebarArtifact = { @@ -75,7 +78,9 @@ type RightAssistantSidebarProps = { heartbeatEntries: AssistantHeartbeatEntry[] workspaceChanges?: WorkspaceChanges workspaceProjectId?: string + browserState?: BrowserLiveState onClose: () => void + onStopBrowser: () => Promise onOpenHeartbeat: () => void onOpenConversation: (conversationId: string) => void onImportArtifacts: () => Promise @@ -117,15 +122,44 @@ const tabs: Array<{ { id: 'context', label: '上下文' }, { id: 'artifacts', label: '成果' }, { id: 'changes', label: '更改' }, + { id: 'browser', label: '浏览器' }, { id: 'preview', label: '预览' } ] const emptyChangedFiles: WorkspaceChanges['files'] = [] +const defaultSidebarWidth = 350 +const minimumSidebarWidth = 300 +const maximumSidebarWidth = 640 +const minimumRemainingAppWidth = 520 +const compactSidebarBreakpoint = 720 +const keyboardResizeStep = 16 +const sidebarTimeFormatter = new Intl.DateTimeFormat('zh-CN', { + hour: '2-digit', + minute: '2-digit' +}) + +function getSidebarWidthLimits(viewportWidth: number): { + minimum: number + maximum: number +} { + return { + minimum: minimumSidebarWidth, + maximum: Math.max( + minimumSidebarWidth, + Math.min( + maximumSidebarWidth, + viewportWidth - minimumRemainingAppWidth + ) + ) + } +} + +function clampSidebarWidth(width: number, viewportWidth: number): number { + const limits = getSidebarWidthLimits(viewportWidth) + return Math.min(limits.maximum, Math.max(limits.minimum, width)) +} function formatTime(timestamp: number | string): string { - return new Intl.DateTimeFormat('zh-CN', { - hour: '2-digit', - minute: '2-digit' - }).format(new Date(timestamp)) + return sidebarTimeFormatter.format(new Date(timestamp)) } export function RightAssistantSidebar({ @@ -143,7 +177,9 @@ export function RightAssistantSidebar({ heartbeatEntries, workspaceChanges, workspaceProjectId, + browserState, onClose, + onStopBrowser, onOpenHeartbeat, onOpenConversation, onImportArtifacts, @@ -165,6 +201,12 @@ export function RightAssistantSidebar({ onRespondApproval, onTabChange }: RightAssistantSidebarProps): React.JSX.Element { + const [viewportWidth, setViewportWidth] = useState(window.innerWidth) + const [sidebarWidth, setSidebarWidth] = useState(defaultSidebarWidth) + const [isResizing, setIsResizing] = useState(false) + const sidebarRef = useRef(null) + const liveSidebarWidth = useRef(defaultSidebarWidth) + const resizePointerId = useRef(undefined) const [selectedArtifactId, setSelectedArtifactId] = useState() const [workspacePreview, setWorkspacePreview] = useState< | { @@ -198,12 +240,20 @@ export function RightAssistantSidebar({ const [scheduleRecurrence, setScheduleRecurrence] = useState< ScheduleCreateInput['recurrence'] >('once') - const recentTasks = activities - .filter((activity) => activity.kind === 'request') - .slice(0, 20) - const changes = activities - .filter((activity) => activity.kind === 'tool') - .slice(0, 30) + const recentTasks = useMemo( + () => + activities + .filter((activity) => activity.kind === 'request') + .slice(0, 20), + [activities] + ) + const changes = useMemo( + () => + activities + .filter((activity) => activity.kind === 'tool') + .slice(0, 30), + [activities] + ) const artifactPreview = artifacts.find((artifact) => artifact.id === selectedArtifactId) ?? artifacts[0] @@ -211,6 +261,86 @@ export function RightAssistantSidebar({ workspacePreview?.projectId === workspaceProjectId ? workspacePreview : undefined + const sidebarWidthLimits = getSidebarWidthLimits(viewportWidth) + const canResize = + open && viewportWidth >= compactSidebarBreakpoint + + useEffect(() => { + const handleViewportResize = (): void => { + setViewportWidth(window.innerWidth) + setSidebarWidth((currentWidth) => { + const width = clampSidebarWidth( + currentWidth, + window.innerWidth + ) + liveSidebarWidth.current = width + return width + }) + } + + window.addEventListener('resize', handleViewportResize) + return () => window.removeEventListener('resize', handleViewportResize) + }, []) + + const resizeFromClientX = ( + clientX: number, + commit: boolean + ): void => { + const width = clampSidebarWidth( + window.innerWidth - clientX, + window.innerWidth + ) + liveSidebarWidth.current = width + if (commit) { + setSidebarWidth(width) + return + } + sidebarRef.current?.style.setProperty( + '--assistant-sidebar-width', + `${width}px` + ) + } + + const finishResize = ( + event: React.PointerEvent + ): void => { + if (resizePointerId.current !== event.pointerId) { + return + } + resizePointerId.current = undefined + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId) + } + setSidebarWidth(liveSidebarWidth.current) + setIsResizing(false) + } + + const resizeWithKeyboard = ( + event: React.KeyboardEvent + ): void => { + if (!canResize) { + return + } + const limits = getSidebarWidthLimits(window.innerWidth) + const nextWidth = + event.key === 'Home' + ? limits.minimum + : event.key === 'End' + ? limits.maximum + : event.key === 'ArrowLeft' + ? sidebarWidth + keyboardResizeStep + : event.key === 'ArrowRight' + ? sidebarWidth - keyboardResizeStep + : undefined + + if (nextWidth === undefined) { + return + } + event.preventDefault() + setSidebarWidth( + clampSidebarWidth(nextWidth, window.innerWidth) + ) + } const openWorkspaceFile = (path: string): void => { const requestId = workspacePreviewRequest.current + 1 @@ -276,15 +406,65 @@ export function RightAssistantSidebar({ return (