diff --git a/UI-DESIGN.md b/UI-DESIGN.md index 734ac977..fff5fae5 100644 --- a/UI-DESIGN.md +++ b/UI-DESIGN.md @@ -750,6 +750,7 @@ GoodBuddy 是可调整窗口大小的桌面应用。响应式设计优先保证 - OpenCode 的默认 Agent 使用原生下拉选择；Continue 预设编辑器允许管理名称、说明、启用的 Rules 以及 Prompt 名称、说明和正文，并可展开查看原生 Rules 与启用预设 Rules 的最终合并顺序。持久启停仍使用共享 Switch，添加与删除使用明确按钮和可访问名称。 - MCP 设置按“内置 MCP / 直连模型 / 自定义 MCP / 电脑控制”四个同级 `PageTabs` 组织。联网搜索保留在直连模型；内置浏览器归入内置 MCP，提供持久启停和 Runtime 分配，并明确说明其操作 GoodBuddy 隔离浏览器、不控制客户端已安装的浏览器，开启后可由已分配 Runtime 的 Execute 直接使用，不逐次询问。该开关只控制 Agent 工具访问，关闭时不得禁用浏览器工作栏中的用户手动导航、返回、刷新、停止加载、交互或关闭操作。尚未生效的命名浏览器配置不得显示在界面中，“电脑控制”只显示实际操作客户端电脑的能力。内置 MCP 卡片与 Skills 一样提供持久启停和 Runtime 分配；直连模型、GoodBuddy 管理的 OpenCode 与 Continue 默认选中且可调整，DeepSeek Harness 必须以置灰、未选择和“暂不支持”文案持续显示，不能呈现为可保存的分配。魔法笔记 MCP 的自身启停与平台功能依赖分别显示，依赖未开启时保留用户配置并说明当前不会加载。 - MCP Server 测试结果在同一展开卡片中分组显示 Tools、Prompts 和 Resources 的支持状态、数量与有界元数据；Prompt 参数标明必填项，Resource 只显示 URI、名称、类型和说明，不读取或渲染 Resource 内容。 +- 内置 MCP 中的“图片生成与编辑”是自动管理例外：该行始终可见，标记“内置 · 自动管理”，不提供独立启停、分配保存、移除或凭据编辑。直连 Model、OpenCode、Continue、DeepSeek Harness 的分配只由已保存图片模型的“允许 AI 在会话中调用”设置推导；至少一个开启则全部勾选，没有开启模型则全部取消。使用禁用的原生 Checkbox、Runtime 标签及关联说明，保留可聚焦的“前往图片模型设置”导航；远程适用范围通过 GoodBuddy Agent 说明持续可见。Ask、离线、无工具能力或调用错误不改变勾选。图片模型默认关闭会话调用；默认生图模型独立于普通默认连接，关闭或删除后保留失效选择并提示改选，不自动回退。 ### 13.9 文档解析设置 diff --git a/docs/features/conversation-media-generation/README.md b/docs/features/conversation-media-generation/README.md index 6c8cf429..0146c3fc 100644 --- a/docs/features/conversation-media-generation/README.md +++ b/docs/features/conversation-media-generation/README.md @@ -1,16 +1,18 @@ # 会话媒体生成 -状态：设计草案，尚未实现。日期：2026-09-15。 +状态：设计要求已明确，工作区已有未提交实现，产品验收未完成。日期：2026-09-15。验证范围见[进度记录](./progress.md)。 -用户添加生图或生视频模型后，可以开启“允许 AI 在会话中调用”。支持工具调用的聊天模型通过 GoodBuddy 使用这些模型，图片和视频返回当前会话。切换聊天模型或 Agent Runtime 时，生成模型配置、已提交的生成操作和历史成果各自保持明确的归属。 +用户添加图片模型后，可以开启“允许 AI 在会话中调用”。支持工具调用的聊天模型通过 GoodBuddy 生成图片、编辑上传图片和历史成果。切换聊天模型或 Agent Runtime 后仍可发起生成和编辑；已提交操作固定生成模型，结果保存到原会话。 本目录负责普通 LLM 调用媒体模型的产品行为、调用入口和生命周期。现有直连图片协议及连续修改仍由[图片生成](../image-generation/README.md)维护；本设计不表示现有功能已改变。 -## 首期范围 +## 范围 -首期以图片为主：在直连文本 Model Runtime 中调用已有生图协议，支持文生图及协议已有的参考图编辑。提供模型调用开关、全局默认和本次指定模型，复用图片成果展示。切换聊天模型或离开会话后，已提交生成继续执行，结果保存到原会话。 +图片生成与编辑覆盖直连文本 Model Runtime、本地 OpenCode、Continue、DeepSeek Harness，以及 GoodBuddy Agent 远程路径上的适用 Runtime。复用已有图片协议和各 Runtime 的内部工具或 MCP 注入，由 GoodBuddy 自动管理；用户无需创建 Skill、搭建 MCP server 或重复配置凭据。 -视频、其他 Runtime 的自动调用、会话级生成模型选择、独立生成表单及跨重启查询恢复留到后续。首期只需 `generate_image` 核心工具，Skill 按实际使用需要补充。需求的阶段划分以[产品需求](./prd.md)为准；下文各设计保留的后续规则不作为首期实现要求。 +使用稳定工具 `generate_image` 完成生成与编辑。全局默认和本次指定模型足够，不新增会话级选择器或独立生成表单。无工具能力的聊天模型明确提示限制，可使用已有直连图片工作流。视频不在本设计范围；完整需求以[产品需求](./prd.md)为准。 + +图片能力在现有内置 MCP 列表持续可见，Runtime 分配以只读复选框展示，由图片模型的会话调用设置自动管理；没有开启的模型时全部取消勾选，仍保留该行和模型设置入口。分配规则见[行为规则](./logic-design.md)，展示与说明见[界面设计](./ui-design.md)。 ## 文档导航 @@ -20,14 +22,14 @@ | [用户场景](./user-stories.md) | 用户操作及场景验收 | | [行为规则](./logic-design.md) | 选择优先级、切换规则、状态和失败处理 | | [界面设计](./ui-design.md) | 配置入口、会话操作、结果卡片和反馈 | -| [技术设计](./technical-design.md) | Skill、工具、Main 服务、存储与 Runtime 接入 | +| [技术设计](./technical-design.md) | 工具注入、Main 服务、存储与 Runtime 接入 | | [进度记录](./progress.md) | 当前代码事实、剩余工作与验证证据 | ## 术语与依赖 -聊天模型负责理解请求、调用工具和组织回答；生成模型负责图片或视频输出；Agent Runtime 承载聊天模型及工具执行。Skill 提供能力使用说明，工具提供结构化调用入口，两者不能互相替代。 +聊天模型负责理解请求、调用工具和组织回答；生成模型负责图片生成与编辑；Agent Runtime 承载聊天模型及工具执行。工具提供结构化调用入口，使用说明随工具提供，不要求用户管理 Skill。 -“生成操作”指一次媒体生成及其结果记录，不自动创建任务中心的 Task，也不引入新的通用 Job 调度系统。领域边界遵循 [Task 与 Job 模型](../task-and-job/task-and-job-model.md)。 +“生成操作”指一次图片生成或编辑及其结果记录，不自动创建任务中心的 Task，也不引入新的通用 Job 调度系统。领域边界遵循 [Task 与 Job 模型](../task-and-job/task-and-job-model.md)。 - [模型连接](../model-connections/README.md)：连接、凭据、协议及请求定制。 - [直连文本 Agent](../direct-model-agent/README.md)：文本工具循环与上下文。 diff --git a/docs/features/conversation-media-generation/logic-design.md b/docs/features/conversation-media-generation/logic-design.md index 4ac69205..27bcc900 100644 --- a/docs/features/conversation-media-generation/logic-design.md +++ b/docs/features/conversation-media-generation/logic-design.md @@ -1,77 +1,78 @@ # 行为规则 -状态：设计草案。此文是模型选择、切换与生成状态的权威定义。 - -## 首期适用规则 - -阶段范围以[产品需求](./prd.md)为准。以下 L-1 至 L-5 保留完整目标，首期按这些限定执行： - -- L-1：只在直连文本 Model Runtime 暴露图片工具；未接入的 Runtime 或无工具模型显示限制，沿用已有直连图片工作流，不新增显式生成入口。 -- L-2：新建图片按“本次明确指定 > 全局默认”选择；编辑按“本次明确指定 > 原成果模型 > 全局默认”选择。只有前一项未设置时才查下一项；选定模型失效时提示改选。 -- L-3：提交时固定生成模型、连接参数、素材和原会话。首期没有会话级生成选择；新请求在实际提交时解析默认值。切换到未接入的 Runtime 只影响后续自动调用，不取消已提交操作。 -- L-4：对图片请求复用进行中、已完成、失败和已停止等待的状态表达，保存成功后才完成，不预建视频任务状态机。取消后若仍取得成果，更新原卡片并说明取消未生效。重启后未完成记录显示结果未确认，不查询恢复或自动重提。人工重新生成创建新操作；首期不提供独立保存重试。 -- L-5：图片成果引用、无视觉模型的文字上下文及会话归属规则全部适用。 +状态：设计草案。此文是模型选择、切换与生成状态的权威定义，范围见[产品需求](./prd.md)。L-1 至 L-5 编号保留，以下规则替代原视频、会话级选择及通用查询恢复规则。 ## 能力与调用时点 -L-1（FR-1、FR-3、FR-7、FR-9；US-01、US-08、US-11）：会话自动生成需要模型连接允许调用、对应协议可执行、Runtime 能暴露工具、聊天模型支持工具调用，并且请求处于 Execute。Skill 只描述这些真实能力。无工具能力时，显式生成入口直接调用同一服务；Ask 下生成入口不能绕过只读限制。 +L-1（FR-1、FR-3、FR-7、FR-9；US-01、US-08、US-11、US-15 至 US-19）：自动生成与编辑要求图片连接允许会话调用、协议支持所需操作、聊天模型支持工具调用且当前请求处于 Execute。直连文本 Model Runtime、本地 OpenCode、Continue、DeepSeek Harness 及 GoodBuddy Agent 远程适用 Runtime 均须接入同一图片能力，内部工具或 MCP 注入由 GoodBuddy 管理。 -请求开始和聊天模型、Runtime、配置切换后重新计算可用能力。运行中的聊天回合保留原有工具协议，Main 在真正提交生成前检查最新可用性。这样已关闭的模型即使残留在该回合的工具列表中，也不能接受新操作。 +允许调用的模型目录取自已保存且“允许 AI 在会话中调用”开启的图片模型配置。目录非空时，图片生成与编辑能力的全部支持 Runtime 分配自动勾选；目录为空时全部取消勾选。模型配置是唯一事实来源，不保存独立可变的能力启用或分配设置，用户不能操作分配复选框。 + +| 配置或运行变化 | 分配与目录结果 | +| --- | --- | +| 首次开启或重新开启一个图片模型 | 自动勾选全部支持 Runtime，目录包含该模型 | +| 关闭或删除一个模型，仍有其他开启模型 | 分配保持勾选，从目录移除该模型 | +| 最后一个开启模型被关闭或删除，或从未开启任何模型 | 全部分配取消勾选，目录为空；能力行、说明和模型设置入口仍可见 | +| Runtime 离线、聊天模型无工具能力、Ask、配置或执行错误 | 保持按模型开关推导的勾选和目录，另行显示调用限制或错误 | + +配置保存成功后同步刷新分配展示和允许模型目录；保存失败时保留最后已保存配置推导的状态。分配不表示运行就绪，也不保证每个图片模型支持编辑；协议及实际提交校验仍适用。 + +请求开始和聊天模型、Runtime、配置切换后刷新可用能力；进程复用也须刷新。运行中的聊天回合保留原工具协议，Main 在实际提交前检查最新配置及工作模式，已关闭的连接不能接受新操作。无工具模型说明限制并提示已有直连图片工作流；Ask 不提交生成或编辑，不自动切换模式。Runtime 接入尚未完成属于实施缺口，不能作为减少支持范围的规则。 ## 模型和素材选择 -L-2（FR-2、FR-5；US-02、US-03、US-05、US-07、US-14）：先确定生成类型和操作意图，再按下表选模型。图片与视频的默认值独立保存。 +L-2（FR-2、FR-5；US-02、US-03、US-05、US-07、US-14、US-15）：先确定新建或编辑，再选择图片模型。 | 操作 | 选择优先级，由高到低 | | --- | --- | -| 新建图片或视频 | 用户本次明确指定 > 当前会话明确选择 > 全局默认 | -| 编辑历史成果 | 用户本次明确指定 > 当前会话明确选择 > 原成果模型 > 全局默认 | +| 新建图片 | 用户本次明确指定 > 全局默认 | +| 编辑历史成果 | 用户本次明确指定 > 原成果模型 > 全局默认 | +| 编辑上传图片（无来源模型） | 用户本次明确指定 > 全局默认 | -只有较高优先级未设置时才查看下一项。已选目标被删除、关闭或不支持所需操作时，要求改选，不能静默跳过。没有默认值时由用户选择，可以同时设为默认；名称歧义必须解析到唯一连接后再提交。LLM 不得无依据地填写模型 ID。 +只有较高优先级未设置时才查看下一项。选定模型被删除、关闭或不支持编辑时提示改选，不静默跳过。未设默认时提示本次指定或去设置默认后重新发送；名称歧义先澄清，LLM 不得编造模型 ID。不持久化会话级生成模型选择。 -显式附件或明确点选的成果优先。用户使用“刚才那张”等明确指代且仅有一个匹配对象时才引用历史图；多义指代先澄清。新建不相关图片不附带旧图。图生视频复用素材引用，但按视频模型选择顺序处理，不沿用图片模型。 - -编辑能力不足或原图缺失时保留输入并提示。新工具路径不自动将编辑降为纯文生图；用户可以明确选择按文字重新生成。已有直连路径的降级规则仍见[图片技术设计](../image-generation/technical-design.md)。 +显式附件或点选成果优先。“刚才那张”等指代仅在对象唯一时解析；多义指代先澄清。新建不相关图片不附带旧图。编辑能力不足或原图缺失时保留输入并提示，不把编辑自动降为文生图；用户可以明确要求按文字重新生成。已有直连图片路径的行为仍由[图片技术设计](../image-generation/technical-design.md)定义。 ## 切换规则 -L-3（FR-4；US-03 至 US-06、US-11）：生成服务接受提交时固定所属会话、触发消息、实际生成模型、生成参数与素材引用。用户尚未发送的草稿使用发送时配置；已发送且排队的请求保留会话选择，跟随全局默认的选项在实际提交时解析。该时点以后不再重选模型。 +L-3（FR-4；US-03 至 US-07、US-11、US-15）：Main 接受提交时固定原会话、触发消息、实际图片模型、当次连接参数、生成参数及素材引用。未提交请求在实际提交时解析默认值并校验显式选择；提交后不重选模型。图片请求由 Main 持有，独立于聊天 Runtime 生命周期。 | 变化 | 未提交操作 | 已提交操作与成果 | | --- | --- | --- | -| 切换聊天模型 | 下一回合使用新模型及其可用工具 | 不改变生成模型；原回合按 Runtime 切换机制结束，操作独立继续 | -| 切换 Runtime | 下一回合按新 Runtime 的工具接入能力执行 | 不依赖旧 Runtime 进程；不向新进程补发未配对的旧工具响应 | -| 切换全局默认 | 跟随默认的操作解析新默认 | 继续原模型 | -| 切换会话生成模型 | 后续操作使用新选择 | 继续原模型 | -| 关闭或删除生成连接 | 拒绝该连接的新操作，提示改选 | 已发请求尽力完成；若后续查询所需连接已失效，显示无法继续查询 | -| 更改地址、凭据或请求参数 | 新操作用新配置 | 当前操作不混用新端点；重启后连接已改变则提示无法继续原操作 | -| 离开或重开会话 | 会话选择独立保留 | 只更新所属会话，不能写入当前打开的其他会话 | +| 切换聊天模型 | 新回合使用新模型及可用工具，无工具能力时说明限制 | 图片模型不变，操作独立继续 | +| 切换本地或远程 Runtime | 新回合注入图片工具，可发起生成和编辑 | 不依赖旧进程，不向新进程补发旧工具响应，不重放生成 | +| 更改全局默认或本次指定 | 新操作按 L-2 解析 | 保持提交时模型 | +| 关闭或删除连接 | 拒绝新操作并提示改选 | 已发请求使用当次配置尽力完成 | +| 更改地址、凭据或参数 | 新操作使用新配置 | 当前操作不混用新端点或凭据 | +| 离开或重开会话 | 使用当前请求及全局配置 | 只更新原会话，历史成果仍可引用 | +| 远程断连 | 未被 Main 接收的请求不能声称已提交 | 桌面仍运行时由 Main 继续管理已接收操作 | -切换聊天模型不会自动取消媒体生成。“停止回复”只结束当前文字回合；生成卡片的“取消生成”单独处理已提交操作。切到 Ask 不撤销先前 Execute 已提交的操作，也不允许新增生成。 +“停止回复”只结束文字回合；“取消生成”单独处理图片操作。切到 Ask 不撤销先前 Execute 已提交的操作，也不允许新增生成。运行时切换不自动唤起新的 LLM 请求。 ## 操作状态 -L-4（FR-6、FR-8；US-04、US-09、US-10、US-13）：状态描述应用实际知道的事实。上游没有百分比时，只显示阶段及已等待时间。 +L-4（FR-6、FR-8；US-04、US-09、US-13）：复用现有操作与成果反馈，状态只表达实际知道的事实。没有上游百分比时显示状态及等待时间，不虚构进度。 -| 状态 | 进入条件 | 后续处理 | -| --- | --- | --- | -| 提交中 | 已接受操作，正在发送 | 获得上游响应后生成中或保存中；失败保留原因 | -| 生成中 | 上游已接受、尚无最终成果 | 查询状态或等待响应，结果返回后保存中 | -| 保存中 | 已获得成果，Main 正在校验和保存 | 本地成果及引用保存成功后才完成 | -| 已完成 | 成果可从会话重新读取 | 终态，后续编辑或重试创建新操作 | -| 取消中 | 用户请求取消、等待终止结果 | 上游确认停止后已取消；若已完成则展示成果并说明取消未生效 | -| 已取消 | 本地已停止等待或上游确认取消 | 区分“已停止等待，上游可能继续生成”与“生成已取消” | -| 失败 | 上游明确失败或成果保存失败 | 提供失败原因及适用的重试入口 | -| 状态待确认 | 提交响应丢失、断连或应用重启，无法确认上游结果 | 有上游 ID 时查询；不能查询时提示用户，不自动重新提交 | +| 状态表达 | 条件与处理 | +| --- | --- | +| 进行中 | 正在发送或等待响应；仅有上游确认时才说明已接受生成 | +| 保存中 | 已取得图片，Main 正在校验并保存 | +| 已完成 | 图片及会话引用已保存，可重新读取 | +| 取消中 | 已请求终止，尚无明确结果 | +| 已停止等待／已取消 | 本地中断只说明“已停止等待，上游可能继续生成”；上游确认停止才说明“生成已取消” | +| 失败 | 有明确的请求或保存失败原因，保留输入和错误上下文 | +| 结果未确认 | 响应丢失、超时或重启后无法确认结果，不当作上游失败或取消 | -同一操作的重复状态通知只能更新同一记录。取消后的晚到结果不恢复旧工具回合，不自动开始下一次生成。若仍能取得成果，可在原卡片标明“取消未生效，已生成”并展示。超时不等于上游取消，不承诺退款。 +同一操作的通知更新同一记录。取消后仍取得图片时，在原卡片保存并说明“取消未生效，已生成”，不恢复旧工具回合。超时不等于上游取消，不承诺退款。 -重新生成创建独立操作，沿用旧操作的模型和输入，提交前检查可用性；用户改选模型后将新选择显示在新尝试上。保存失败且上游成果仍可读取时，“重试保存”使用原操作 ID，由失败回到保存中并更新原卡片，不再次提交生成。无法确认上游是否接受的请求，不做自动生成重试。 +人工重新生成创建新操作，沿用原模型与输入并重新校验；改选模型时在新卡片显示实际选择。失败、结果未确认及保存失败均不自动重发收费请求，不设置独立保存重试入口。应用退出后不承诺继续本地执行，重启时未完成记录显示结果未确认，不建立跨重启查询恢复或通用恢复框架。 -L-5（FR-5、FR-8；US-07、US-12）：历史上下文传成果引用、提示词及实际模型信息。只有具备视觉输入能力的聊天模型才接收适用的图片内容。不能以“有成果引用”推断模型已经看过图片。删除会话沿用已有确认流程，并说明活动生成的取消及不能保证上游停止的情况，删除后晚到结果不能重建会话。 +## 历史成果 -## 完整性与待定项 +L-5（FR-5、FR-8；US-07、US-12、US-14、US-15）：历史上下文提供成果引用、提示词及实际图片模型信息。只有具备视觉输入能力的聊天模型才接收适用的图片内容；无视觉模型可根据用户文字将原图引用交给编辑工具，但不能声称看过图片。上传图片和历史成果均使用已有可持久化素材引用。 -已覆盖首次配置、模型选择、历史编辑、模型与 Runtime 切换、只读模式、失败、取消、重启和成果归属。无产品规则需要在本次文档交付前补充确认。 +删除会话沿用已有确认流程，说明活动生成的取消及无法保证上游停止的情况；删除后的晚到结果不能重建会话。 -视频服务商、协议、取消与查询能力、可接受的文件限制属于实施前的技术选型。远程 Runtime 的工具往返需要原型验证；无法接入时按 L-1 提供显式入口，不把工具接入状态写成已支持。跨设备同步、应用退出后的本地执行不在本次范围。 +## 完整性与实施验证 + +规则覆盖配置、新建、上传图与历史图编辑、模型与 Runtime 切换、Ask、无工具或无视觉模型、取消、失败、重启和会话归属。没有待定产品范围；各 Runtime 工具注入、远程往返及取消传播的具体接法需在实施时核验，按[技术设计](./technical-design.md)完成真实路径验收。视频、跨设备同步和应用退出后的本地执行不在范围内。 diff --git a/docs/features/conversation-media-generation/prd.md b/docs/features/conversation-media-generation/prd.md index d5cd3a61..c5dc9db7 100644 --- a/docs/features/conversation-media-generation/prd.md +++ b/docs/features/conversation-media-generation/prd.md @@ -4,40 +4,34 @@ ## 用户问题与范围 -用户希望在文字会话中直接提出“画一张海报”“把背景改成蓝色”“用这张图生成视频”，由聊天助手调用已配置的专用模型。用户不应为了生成媒体而丢失当前聊天入口；切换聊天模型后，也应能继续引用已有成果。 +用户希望在文字会话中直接提出“画一张海报”“把上传图片的背景改成蓝色”“继续修改刚才的图片”，由聊天助手调用已配置的图片模型。切换聊天模型或 Runtime 后，应能继续生成、引用和编辑图片。 -范围包括模型调用开关、默认与会话选择、文生图、支持协议下的参考图编辑、文生视频及图生视频、结果展示和生成过程管理。视频具体服务商及协议待技术选型；接通真实服务前不能宣称视频可用。 +范围包括图片模型调用开关、全局默认与本次指定、文生图、已有协议支持的上传图片和历史图片编辑、成果保存、取消及状态反馈。支持直连文本 Model Runtime、本地 OpenCode、Continue、DeepSeek Harness，以及 GoodBuddy Agent 远程路径上的适用 Runtime；各路径的生成、编辑和切换均属于验收范围。 -不包含模型训练、自动采购服务、无工具能力 LLM 的通用文本指令执行器、批量渲染平台、应用退出后仍由桌面后台执行的承诺。直接选择图片模型的已有工作流继续保留。 +GoodBuddy 自动管理内部工具或 MCP 注入，复用模型连接与凭据。用户无需创建 Skill、搭建 MCP server 或填写第二套凭据。自动调用要求聊天模型支持工具调用；无工具能力时明确说明限制，用户可使用已有直连图片工作流。 + +不包含视频生成、模型训练、自动采购服务、通用文本指令执行器、批量渲染平台或应用退出后继续本地执行的承诺。不新增图片协议、会话级生成模型选择器、独立生成表单或通用恢复框架，也不安排视频交付计划。 ## 功能需求 -### 首期交付 - -首期只交付图片生成，自动调用先接通直连文本 Model Runtime。支持已有图片协议下的文生图、参考图编辑、历史图片引用与成果保存，不新增图片协议。保留模型调用开关、一个全局默认生图模型及用户本次明确指定；编辑默认沿用原成果模型。首期不持久化会话级生成模型选择。 - -切换聊天模型、生成模型或离开会话后，已提交操作仍固定原模型和原会话。切换到尚未接入工具的 Runtime 后，原操作继续，但该 Runtime 暂不能自动发起生图。不支持工具调用时说明限制，用户可使用已有直连图片工作流；首期不增加独立生成表单。 - -运行状态、取消、错误说明及人工重新生成属于首期。应用退出后不继续本地生成；重新打开时将未完成操作标明结果未确认，不自动重新提交。跨重启查询恢复和独立“重试保存”入口后续再做。 - -### 后续范围 - -视频生成与播放、其他本地及远程 Runtime 自动调用、会话级模型选择、独立生成表单、查询恢复及保存重试不纳入首期验收。不为这些能力预建工具、存储字段或界面占位。 - -以下 FR 编号保留完整目标；FR-1 至 FR-8 按上述首期范围验收，FR-9 首期只要求准确说明已接通的 Runtime 和使用限制，其余接入后分别验收。 +保留 FR-1 至 FR-9 编号。原需求中的视频部分移出范围，FR-2 的会话级选择由全局默认和本次指定替代，FR-7 的新增显式入口由已有直连图片工作流替代；下表为当前定义。 | 编号 | 要求 | 产品验收 | | --- | --- | --- | -| FR-1 | 图片、视频模型分别提供会话调用开关；新增和已有连接默认关闭，由用户主动开启 | 保存后成为对应可用能力，关闭后不能再提交新操作；历史成果保留 | -| FR-2 | 支持全局默认、当前会话选择和本次明确指定 | 按[选择规则](./logic-design.md)解析并显示实际模型；不静默改用其他服务 | -| FR-3 | 在支持工具调用的聊天模型下，使用自然语言生成媒体 | 从聊天输入到真实服务输出、保存与会话展示完整可用，活动记录可见 | -| FR-4 | 支持聊天模型、Runtime 及生成模型切换 | 新请求获得当前能力，已提交操作保持原模型和原会话；切换不触发重新生成 | -| FR-5 | 支持历史成果引用与继续编辑 | 重新打开会话后可引用已保存原图；缺失素材或编辑能力不足时有明确处理入口 | -| FR-6 | 长耗时生成可独立显示状态、取消与重试 | 离开会话不取消；状态区分上游生成、结果保存、完成及失败，不虚构进度 | -| FR-7 | 适配工作模式及能力边界 | Ask 不提交生成；不支持工具调用时仍可用 Execute 下的显式生成入口 | -| FR-8 | 图片、视频作为会话成果保存 | 结果可预览和导出，来源模型可追溯；聊天模型不能看图时只接收文字事实及引用 | -| FR-9 | 本地和远程 Runtime 有可核验的支持说明 | 自动调用以真实工具往返验收为准；未接通的 Runtime 显示原因及显式生成入口 | +| FR-1 | 图片模型提供“允许 AI 在会话中调用”开关；新增和已有连接默认关闭，由用户主动开启；模型配置是图片能力启用与 Runtime 分配的唯一事实来源 | 至少一个模型开启时自动勾选全部支持 Runtime；全部关闭或删除时全部取消勾选；关闭或删除其中一个后若仍有开启模型，保持勾选并更新允许调用的模型目录；被关闭模型拒绝新操作，历史成果保留 | +| FR-2 | 支持全局默认和本次明确指定；编辑可沿用原成果模型 | 按[选择规则](./logic-design.md)解析并显示实际模型；失效或有歧义时提示，不静默换服务 | +| FR-3 | 在支持工具调用的聊天模型下，以自然语言生成和编辑图片 | 上传图片、历史图片及文生图均可经真实服务处理，保存后在原会话展示，活动记录可见 | +| FR-4 | 支持聊天模型、Runtime 及生成模型切换 | 切换后可发起新的生成和编辑；已提交操作固定原模型和原会话，不重复提交 | +| FR-5 | 支持上传素材、历史成果引用与连续编辑 | 重开会话及切换 Runtime 后可引用已保存原图；缺失素材或能力不足时保留输入并提示处理方式 | +| FR-6 | 生成操作独立显示状态，支持取消及人工重新生成 | 离开会话不取消；保存成功才完成；不虚构进度，不自动重试收费请求；重启后未完成操作显示结果未确认 | +| FR-7 | 遵循工作模式及模型能力边界 | Ask 不提交生成或编辑；无工具模型说明限制，可使用已有直连图片工作流，无需新增表单 | +| FR-8 | 图片作为会话成果保存 | 可预览、导出和重新打开，来源模型可追溯；无视觉聊天模型只接收文字事实及引用，不编造观察 | +| FR-9 | 本地及 GoodBuddy Agent 远程适用 Runtime 均支持图片工具；GoodBuddy 管理的图片生成与编辑能力及 Runtime 分配持续可见 | 分别完成真实生成、编辑、切换验收；分配复选框只读，不可手动切换；无开启模型时仍显示该行、未勾选状态、自动管理说明及模型设置入口；无独立能力启停、分配、移除或凭据编辑设置，用户无需维护 Skill 或 MCP server | -## 交付标准 +分配表示模型配置允许的调用范围。聊天模型不支持工具、Ask、Runtime 离线、配置错误或执行失败均单独说明，不取消配置分配的勾选；不保存可独立修改的能力启用或 Runtime 分配状态。具体推导与刷新规则由 L-1 定义。 -以[用户场景](./user-stories.md)验收产品路径。生图、视频和各 Runtime 分别记录验证结果，不用某一个模型或 Runtime 的成功代替其他路径。配置连接检查成功不等于生成成功，真实生成成果必须在会话中可见且能重新打开。 +## 产品验收 + +以[用户场景](./user-stories.md)验收。各 Runtime 分别记录真实工具往返、生成、上传图编辑、历史图编辑及切换结果，不能用单一路径成功代替其他路径。配置检查或工具注入成功不等于生成成功，成果必须可在会话中重新打开。 + +Runtime 接入细节和真实 Host 验证属于实施工作，未验证时如实记录未完成，不据此缩减 FR-9 的支持范围。 diff --git a/docs/features/conversation-media-generation/progress.md b/docs/features/conversation-media-generation/progress.md index d57ae865..fe673db0 100644 --- a/docs/features/conversation-media-generation/progress.md +++ b/docs/features/conversation-media-generation/progress.md @@ -2,13 +2,63 @@ ## 2026-09-15 -根据用户“首期以图片为主”的意见收敛设计，已同步需求、场景、规则、界面及技术文档的阶段范围。首期接通直连文本图片工具，保留模型切换、原会话归属及历史图片引用；后续范围见[产品需求](./prd.md)。 +### 真实图片请求与构建修复 -本次只修改设计文档，未检查或改变应用实现，2026-09-14 的代码检查记录保留为当日证据，不表示已重新验证。图片首期仍待实施。 +已使用现有 `gpt-image2` 连接（模型 `sub-gpt-image-2`）经生产 `ImageGenerationService` 发起两次真实图片请求：生成白底蓝色圆形，再上传该成果将圆形改成红色。两次均 HTTP 200，保存并重新解码成功，导出图片目视检查符合要求。请求数为 2，自动重试为 0，用户保存的配置未修改。请求尺寸为 1024×1024，供应商实际返回两张 1254×1254 PNG；未对结果重采样。证据保存在本机临时目录 `image-bounded-oiOPRi/`。此项验证 Main 图片服务与真实供应商，尚不替代真实聊天模型经各 Runtime 自行调用工具及完整界面切换验收。 -文档验证：7 份文档的一致性审阅未发现首期范围冲突；31 个相对 Markdown 链接均可解析。`deai-writing` 扫描阻断项为 0，31 个复核候选为设计状态、能力职责及行为限制，按语境保留。`git diff --check -- docs/features/conversation-media-generation` 通过；目录当前未跟踪，该命令不覆盖未跟踪文件，内容与链接另行审阅。未运行应用测试或真实生成请求。 +用户报告 Renderer 生产构建失败，定位到 `remote-image-tool-contracts.ts` 的 `node:crypto` 被共享契约间接引入浏览器。已将 MCP 名称哈希移至仅 Main 与 Agent 引用的 `remote-image-tool-node.ts`，共享消息编解码使用标准 `TextEncoder`/`TextDecoder`，无新增依赖或 polyfill。修复后 `npm run build`（含类型检查、Main、Preload、Renderer 构建）、`npm run lint` 通过，4 个相关测试文件共 89 项通过。此前仅运行类型检查和测试未发现这一构建问题，不能作为生产构建通过证据。 -## 2026-09-14 +### 实施与集成复验 + +工作区已有模型会话调用开关、默认图片模型、配置派生的四种 Runtime 只读分配、Main 图片服务、上传与历史成果引用、操作状态及取消/重新生成 IPC。本地 Model、OpenCode、Continue、DeepSeek Harness 和受管远程 OpenCode 均已接入同一服务。操作复用消息 metadata 和成果存储，未新增任务表或独立分配设置。 + +本轮修复远程保留会话在图片目录清空后未刷新 MCP，以及 Continue 准备自定义 MCP 失败时未释放本次图片工具令牌的问题。图片卡片更新保持原顺序；取消提示区分请求取消、停止等待和取消未生效后的结果保存。失败卡片提供模型设置与素材重选入口，无工具能力及 Ask 有相应提示。长名称和提示词自动换行，窄窗口及 200% 缩放下恢复按钮可换行。 + +验证证据如下；此前小节中的超时和未验证记录仅代表各次检查时点： + +| 验证 | 结果与边界 | +| --- | --- | +| `npm test` | 364 个文件通过、9 个跳过；4,267 项通过、67 项跳过。后续恢复入口及样式修改另有 269 项定向测试通过，未再运行全量测试 | +| `npm run typecheck`、`npm run lint` | 最后一次 Renderer 修改后通过 | +| 真实 Linux Host | 最终 Agent 构建通过固定主机身份连接，使用实际远程 OpenCode；Ask、Execute 各 1 次真实文本请求均 HTTP 200 并完成，真实图片请求 0 次 | +| Host 图片工具往返 | 最终运行含 7 次受控图片请求、15 次受控文本请求；生成、上传和历史图编辑、目录刷新、Ask 隐藏、并发会话归属、停止回复和断连后 Main 继续保存均通过。图片服务返回固定内联 PNG，不构成真实生图证明 | +| Electron 界面 | 当前生产组件、样式、字体及双语文案，使用隔离 fixture API；初轮 94 项检查，恢复入口完成后 40 项检查通过。覆盖明暗主题、四种窗口尺寸、键盘导航及 200% 缩放；没有修改用户配置 | + +最终 Host 运行退出码为 0，残留自有进程为 0，隔离目录和上传文件已清理。此前受控复验另有 7 次图片请求和 15 次文本请求，因此本轮合计真实文本请求 2 次、真实图片请求 0 次、受控图片请求 14 次、受控文本请求 30 次。此前历史测试记录不计入此合计。 + +本机临时证据位于 `C:/Users/jiang/AppData/Local/Temp/opencode/`：`image-review-full-tests-20260915.stdout.log`、`image-host-final-real-20260915.stdout.log`、`image-host-review-20260915.stdout.log` 及 `image-visual-0915/`。这些是开发检查产物，不随产品发布。 + +当时尚未完成真实图片供应商生成与编辑；后续验证见上方记录。完整 Electron Main/Preload 路径中的自然语言触发及本地/远程双向切换验收仍待完成，FR-3、FR-9 尚未完成全部产品验收。 + +隔离的真实图片验证脚本离线校验通过后，已按用户要求执行上述两次真实请求。该连接的会话调用仅在验证内存中启用，不修改保存配置。此前 `deai-writing` 文档扫描阻断项为 0，25 个候选经语境复核保留产品边界、状态限制和历史证据；`git diff --check` 通过。 + +### Renderer 与设置复查 + +本轮在已有未提交实现上修复图片操作卡片：状态事件原位更新，保留同一消息中多次生成的卡片顺序；取消提示区分正在请求取消、已停止等待和取消未生效但图片已保存。按钮通过 `aria-describedby` 关联就地操作错误，中英文提示同步更新。补充组件回归，并在现有 App 测试中检查离开原会话、结果晚到、返回及切换 Runtime 后继续修改时的卡片顺序。 + +已检查模型设置、RuntimeSettingsStore、CapabilityService 和共享设置契约。现有实现从已保存模型开关派生四种 Runtime 的只读分配，不持久化独立分配；关闭或删除默认图片模型后保留失效默认。相关测试本轮通过，未改动这些生产契约。共享工具目录测试原先仅排除浏览器工具，现补充图片受管能力例外，确认 `generate_image` 保持写操作且不重复进入直连模型设置分组。 + +验证记录： + +- `npx vitest run src/renderer/src/ImageOperationStatus.test.tsx src/renderer/src/App.test.tsx src/renderer/src/SettingsPanel.test.tsx src/main/runtime-settings-store.test.ts src/main/capabilities/capability-service.test.ts`：5 个文件、451 项通过。 +- 修正目录测试后，`npx vitest run src/shared/builtin-model-tools.test.ts src/renderer/src/ImageOperationStatus.test.tsx`：2 个文件、5 项通过。 +- `npm run typecheck`、`npm run lint` 通过。 +- `npm test` 在 120 秒上限被终止；期间报告的共享工具目录测试失败已单独修正并复验，未取得全量测试通过结论。 +- 未调用真实文本或图片服务，请求数均为 0；未执行 Electron 视觉验收或真实 Host 验证。本轮没有修改 `src/main/agent` 或 `src/agent-daemon`。各 Runtime 的真实生成、编辑、取消及切换验收仍待完成。 + +### 此前文档修订记录 + +本日补充：已按最新明确接受的要求修订 FR-1、FR-9、L-1、界面与技术设计，并新增 US-16 至 US-19。只读分配及模型配置派生规则见[行为规则](./logic-design.md)，受管行、说明与无障碍要求见[界面设计](./ui-design.md)。已读取根目录与 features 的 AGENTS、根 UI-DESIGN.md 及本目录七份现有文档，保留此前未提交修订；本次未修改目录外文件或应用代码，相关行为仍待实现与产品验收。 + +本日更正：此前将其他 Runtime 自动调用排除在交付范围之外的记录不符合用户要求，现由本条替代。七份文档已统一为图片生成和编辑，支持范围及验收以[产品需求](./prd.md)和[用户场景](./user-stories.md)为准，不再保留分期规则。视频移出范围，US-10 保留编号并明确标注；其他被替代要求已记录编号变更。 + +本次只修改本目录设计文档，未检查或改变应用实现。2026-09-14 记录仅为当日历史证据，不表示已重新验证。图片工具、Main 请求生命周期、各本地及 GoodBuddy Agent 远程适用 Runtime 的生成、编辑和切换均仍需实施与实际验证；具体注入及桥接核验属于实施工作，不缩减支持范围。 + +本次文档验证：7 份文档的 38 个相对 Markdown 链接均可解析；一致性审阅覆盖模型配置唯一来源、只读分配、空目录保留行、运行状态独立及完整 Runtime 范围，未发现分期或缩减支持范围的现行规则。`deai-writing` 扫描阻断项为 0，25 个复核候选涉及设计状态、职责与必要行为限制，逐条按语境保留，并通读检查结构及前后规则。`git diff --check -- docs/features/conversation-media-generation` 通过；本次七份文件均已被 Git 跟踪，检查覆盖其工作区差异。该命令不覆盖未跟踪文件，不能沿用此前“目录未跟踪”的结论。未运行应用测试或真实生成请求，未声称功能完成或重新验证此前代码。 + +## 2026-09-14（历史记录） + +以下保留当日检查与原设计记录，其中视频、保存重试等旧范围已被 2026-09-15 更正取代，不作为当前实施要求。 当前阶段：设计草案已落盘，功能未实现。本次只新增设计文档与相关入口链接。 diff --git a/docs/features/conversation-media-generation/technical-design.md b/docs/features/conversation-media-generation/technical-design.md index 8afae250..767e1050 100644 --- a/docs/features/conversation-media-generation/technical-design.md +++ b/docs/features/conversation-media-generation/technical-design.md @@ -1,91 +1,81 @@ # 技术设计 -状态：设计草案。拟议字段和工具名用于评审，不表示已存在公开 API。实现 [FR-1 至 FR-9](./prd.md)及[行为规则 L-1 至 L-5](./logic-design.md)。 - -## 首期实现边界 - -首期调用路径为：直连文本 Model Runtime → `generate_image` → Main 图片生成服务 → 现有图片协议 → 原会话 artifact。提取已有图片请求和保存逻辑，Main 持有活动请求，使聊天模型或 Runtime 切换不释放已提交生成。复用现有成果视图与活动反馈。 - -只新增图片会话调用开关和全局默认图片 profile ID；本次指定作为工具参数。生成记录优先放入现有消息 metadata，保存实际模型、输入、状态及成果引用。首期没有会话级生成选择、视频字段、上游任务恢复信息或独立任务表。 - -工具说明随每个新回合提供可调用图片模型的脱敏 ID、名称和已支持能力，提交前重新校验配置。先以工具描述承载使用规则；只有实际调用验证表明说明不足时才补充一份共享 Skill，不把 Skill 管理作为交付依赖。首期不注册 `list_media_models`、视频、查询和取消工具，取消通过会话中的操作入口完成。 - -工具等待遵循现有超时约束；超出回合等待或发生 Runtime 切换后，Main 仍将结果写回原会话。后续回合读取已保存的操作摘要和成果，不补发旧工具响应。应用重启后未完成记录只提示结果未确认，不建立恢复查询或自动重试机制。 - -以下章节描述完整目标中的调用关系和扩展约束。出现视频、会话选择、额外工具、独立生成表单、其他 Runtime 接入和恢复查询时，均属后续设计，不要求首期预建抽象或占位实现。 +状态：工作区实现已接通，真实图片服务与完整产品验收未完成，证据见[进度记录](./progress.md)。内部契约实现 [FR-1 至 FR-9](./prd.md)及[行为规则 L-1 至 L-5](./logic-design.md)，不作为外部公开 API。 ## 调用结构 ```mermaid flowchart TD - Chat[聊天模型与 Agent Runtime] --> Tool[媒体工具入口] - Skill[内置媒体 Skill 使用说明] -.-> Chat - UI[会话显式生成入口] --> Main[Main 媒体生成服务] - Tool --> Main - Main --> Profile[模型连接与凭据存储] - Main --> Adapter[图片或视频协议实现] - Adapter --> Provider[生成服务商] - Main --> Artifact[操作记录与会话成果] - Artifact --> Card[原会话生成卡片] + Direct[直连文本 Model Runtime] --> Tool[generate_image] + Local[本地 OpenCode / Continue / DeepSeek Harness] --> Injection[GoodBuddy 管理的内部 MCP 注入] + Remote[GoodBuddy Agent 远程适用 Runtime] --> Bridge[现有 Agent 通信与工具桥接] + Bridge --> Injection + Injection --> Tool + Tool --> Main[Main 图片服务] + Main --> Profile[现有模型连接与凭据] + Main --> Adapter[现有图片生成与编辑协议] + Main --> Artifact[原会话操作记录与图片成果] ``` -Main 持有生成请求、取消句柄和结果保存职责。服务生命周期独立于聊天 Runtime controller，切换或释放 controller 不释放已提交的媒体操作。Renderer 经明确 preload/IPC 方法调用；IPC 使用共享契约校验并保留现有发送方校验。凭据只在 Main 解密，不进入 Skill、工具参数或工具输出。 +Main 持有图片请求、取消句柄及结果保存职责，生命周期独立于聊天 Runtime controller。切换或释放聊天进程不释放已提交图片操作。Renderer 的取消等操作经明确 preload/IPC 方法进入 Main，沿用共享契约和发送方校验。凭据只在 Main 解密，不写入工具说明、参数、输出或远程 Runtime 配置。 -从 `ModelAgentRuntime` 提取现有图片请求、校验和成果处理中的可复用部分，使直连图片与媒体工具共用协议实现。直连路径保持原有上下文与编辑降级策略；工具路径依据 L-2 传入明确的素材和降级选项，避免协议重用改变产品语义。 +`ImageGenerationService` 复用 `ModelAgentRuntime.generateImage` 的图片请求和校验，在 Main 保存操作及成果。工具路径按 L-2 明确区分新建和编辑，不自动将缺少素材或不支持的编辑降为文生图；已有直连图片路径保持原上下文规则。 -## Skill 与工具 +## 工具与注入 -提供一份内置媒体生成 Skill，说明新建、编辑、图生视频、模型选择、素材引用和结果使用。开启模型配置仅更新能力目录，不为每个连接复制 Skill。能力目录与工具从同一份有效配置生成；Skill 中不保存可过期的模型名单或凭据。 +稳定图片工具 `generate_image` 同时负责生成和编辑。当前输入为 `prompt`、可选 `modelProfileId`、`intent`（`create` 或 `edit`）、`sourceArtifactIds` 和可选 `quality`；类型及编辑素材约束由 `src/shared/image-generation-contracts.ts` 校验。每次请求沿用已有协议的单图生成和尺寸行为，没有新增数量或尺寸参数。原会话、触发消息和工具调用关联由运行上下文绑定，LLM 不能自由指定结果写入位置或任意文件路径。 -拟议工具：`list_media_models` 返回脱敏的可用连接 ID、名称、已支持操作及参数约束；`generate_image`、`generate_video` 提交生成；`get_media_generation` 查询；`cancel_media_generation` 取消。按操作性质接入现有工作模式控制，不因媒体工具增加额外通用审批。 +每个新回合提供可调用图片模型的脱敏 ID、名称及生成、编辑能力，模型切换、配置变化或进程复用时刷新。Main 提交前重新校验工作模式、连接开关、所需能力及素材。优先由工具描述承载模型选择、图片引用和结果使用说明，不把 Skill 管理作为依赖。 -生成输入包括 `prompt`、可选 `modelProfileId`、显式 `operation`、`sourceArtifactIds` 及协议支持的类型化选项。图片数量、尺寸与质量，以及视频时长、分辨率等按已接入协议校验，不能假定服务商完全兼容。`conversationId`、触发消息及请求身份由运行上下文绑定，不能由 LLM 自由指定目标会话。 +直连文本 Model Runtime 使用内置工具契约及分发；外部 Runtime 复用内部 MCP 注入，由 GoodBuddy 自动注册、配置和管理生命周期，远程路径经现有 Agent 工具桥接调用同一服务。内置 MCP 列表中的图片行只展示这项受管能力，不代表直连 Model 也通过 MCP 调用。无需用户创建 Skill、另建 MCP server 或复制凭据，不按模型连接复制工具或 Skill。 -提交后返回操作 ID、实际模型和当前状态；完成结果返回成果 ID 与必要文字信息，不把 Base64 或视频字节塞入工具历史。等待有界，长任务通过查询及卡片更新呈现。原回合仍在执行时，可通过其合法工具响应返回最新结果；回合已结束或 Runtime 已切换时，只更新操作记录与原会话卡片，下一回合读取成果摘要，不自动唤起新 LLM 请求。 +Main 从已保存图片 profile 的会话调用开关推导允许模型目录，并以目录是否非空推导全部支持 Runtime 的分配。Renderer 只读取派生分配；该受管行不接入通用 MCP 的启停、分配、删除或凭据编辑写入接口。模型配置保存后刷新展示及注入目录，进程复用和下一回合重新读取；运行中的回合及已提交操作继续遵循 L-1、L-3。配置错误、Runtime 在线状态、聊天模型工具能力与执行结果独立于派生分配，不因这些状态移除允许目录中的模型或取消勾选。 + +不预建能力发现、查询或取消工具目录。只有实际 Runtime 协议要求独立发现、查询或取消交互，且现有工具描述、响应及 UI 操作不能满足时，才增加所需的最小适配，并记录协议依据；这些适配仍调用同一 Main 图片服务，不形成新的媒体平台。取消默认从会话卡片进入 Main。 + +工具等待遵循现有超时约束，返回操作 ID、实际模型、真实状态及完成后的成果 ID，不把 Base64 塞入工具历史。原回合仍有效时通过合法工具响应返回结果；回合结束或 Runtime 切换后，Main 继续写原会话，下一回合读取操作摘要及成果引用，不补发旧响应、不自动唤起 LLM 或重发图片请求。 ## 配置与持久化 -| 对象 | 最小拟议数据 | 存储职责 | -| --- | --- | --- | -| 模型连接 | 会话调用开关、适用生成能力 | 复用 settings/profile；现有连接缺少开关时解释为关闭 | -| 全局偏好 | 默认图片、视频 profile ID | 复用应用设置；不复制连接及凭据 | -| 会话选择 | 图片、视频分别为跟随默认或显式 profile ID | 持久化到会话设置，独立于聊天 Runtime selection | -| 生成操作 | ID、原会话/消息 ID、工具调用关联、状态、实际模型 ID 与名称、提示词、有效参数、素材引用、时间、错误、成果 ID | 优先扩展现有会话消息 metadata；实施时评估查询需求后决定是否需要独立表 | -| 上游异步信息 | 必要的上游任务 ID、查询所需非敏感参数 | 仅在协议确实支持异步查询时保存；无凭据副本 | -| 成果 | 媒体内容及类型、原操作与素材关联 | 复用 artifact 体系；不放入 Skill 或 Runtime 内存作为唯一副本 | +| 对象 | 最小数据与职责 | +| --- | --- | +| 图片模型连接 | 复用 settings/profile，增加会话调用开关；现有连接缺少开关时解释为关闭 | +| 图片能力与 Runtime 分配 | 从图片模型配置派生，只读展示；不新增独立可变的 capability enabled、assignment 设置或持久化副本 | +| 全局偏好 | 默认图片 profile ID，复用应用设置；本次指定只属于请求，不新增会话级选择 | +| 生成操作 | 优先使用现有消息 metadata，保存操作 ID、原会话/消息、工具关联、状态、实际模型 ID 与名称、提示词、有效参数、素材引用、时间、错误及成果 ID | +| 素材与成果 | 复用附件和 artifact 持久化，保存上传图片、生成图片及编辑来源关系，不能只存于 Runtime 内存 | -已提交操作在内存持有当次解析的连接参数；持久记录只保留来源与恢复查询所必需的非敏感信息。应用重启后通过原连接核对是否仍可查询，不增加凭据历史、通用恢复日志或多阶段提交机制。不能确认时采用 L-4 的“状态待确认”。 +活动请求在 Main 内存持有提交时连接参数，持久记录不复制凭据。应用重启后未完成操作显示结果未确认，不建立跨重启查询恢复、凭据历史、通用恢复日志或自动重试。取消和晚到结果按 L-4 更新同一记录；人工重新生成使用新操作 ID。 -操作不自动成为 Task/Job，已有 `runs` 表也不等于通用后台调度已经存在。数据库变更实施前按[迁移规范](../../development/database-migrations.md)评审已发布数据兼容；本次文档不预先引入新表和迁移。 +不新增独立任务表、视频字段或通用调度器，操作不自动成为 Task/Job。若实施涉及已发布数据，按[迁移规范](../../development/database-migrations.md)处理实际兼容需求，不预建迁移或恢复机制。 -Main 必须在成果持久化成功后发布完成事件，事件按操作 ID 更新而非重复追加。校验素材来自当前会话可访问的附件或成果，继承现有项目、会话隔离，不允许模型填写任意文件路径代替成果引用。 +Main 仅在图片和会话引用持久化成功后发布完成事件。素材必须是当前会话可访问的附件或成果，沿用已有项目和会话边界；删除会话后的晚到结果不重建会话。 -## 图片与视频结果 +## 图片协议 -图片继续遵循[图片技术设计](../image-generation/technical-design.md)：校验有界内联图片，不下载 Provider 返回的图片 URL。现有成果内容字符串上限为 5 MiB，这不是原始二进制文件大小，实施时必须正确处理编码开销和超限提示。 - -视频不沿用内联图片存储。优先复用 `managed_file` 成果和现有文件导出能力，补充媒体类型、播放所需信息及预览。协议选型必须明确提交、查询、取消、结果读取、认证、文件大小上限、超时和过期时间；最终方案用实际服务的证据补入本文。 - -若视频服务只返回下载地址，需在视频协议设计中明确 Main 端的读取来源、认证、重定向及字节限制后实现，不能借用或放宽图片 URL 禁止规则。生成媒体读取不使用应用更新或模型包下载源设置；受管软件包镜像规则也不适用于动态生成的媒体结果。无法保存或播放的上游结果不能标记为交付完成。 +沿用[图片技术设计](../image-generation/technical-design.md)的已有生成、编辑协议及素材限制。接收并校验有界内联图片，不下载 Provider 返回的图片 URL。沿用现有成果存储限制，实施时核对编码开销与超限反馈，不以配置检查成功代替真实生成验收。视频协议、下载、播放及交付计划均不属于本设计。 ## Runtime 接入 -| 路径 | 设计接入点 | 验证要求 | +| 路径 | 设计接入点 | 必须完成的验证 | | --- | --- | --- | -| 直连文本 Model Runtime | `builtin-model-tools.ts` 的工具契约及 `model-tool-provider.ts` 分发 | 自然语言请求实际调用生成模型并返回成果 | -| 本地 OpenCode、Continue、DeepSeek Harness | 复用各 Runtime 已有工具或 MCP 注入机制，转到同一 Main 服务 | 分别验证发现、调用、查询、取消和切换；不能以 Skill 注入成功代替 | -| GoodBuddy Agent 远程路径 | 经现有桌面与 Agent 通信能力将工具请求绑定到原会话，Main 执行生成 | 验证真实 Host 的工具往返、断连和结果归属，凭据不下发远端 | -| 不支持工具调用的聊天模型或尚未接入的 Runtime | Renderer 显式入口经 IPC 调用同一服务 | Execute 可完成生成，Ask 不提交，原会话可查看成果 | +| 直连文本 Model Runtime | 现有内置工具契约及工具分发转入 Main 图片服务 | 自然语言生成、上传图编辑、历史图编辑及切换 | +| 本地 OpenCode | 现有内部 MCP 注入转入同一服务 | 实际工具往返、能力刷新、生成编辑及切换 | +| 本地 Continue | 现有内部 MCP 注入转入同一服务 | 实际工具往返、能力刷新、生成编辑及切换 | +| 本地 DeepSeek Harness | 现有内部 MCP 注入转入同一服务 | 实际工具往返、能力刷新、生成编辑及切换 | +| GoodBuddy Agent 远程适用 Runtime | 复用桌面与 Agent 通信及工具桥接，将请求绑定原会话，由 Main 调用图片模型 | 逐一验证实际支持的远程 Runtime，覆盖真实 Host 工具往返、生成编辑、本地与远程双向切换、断连及成果归属 | +| 无工具能力的聊天模型 | 说明自动调用限制，可使用已有直连图片工作流 | 不解析普通文字伪装工具调用，不新增表单；Ask 不提交 | -Runtime 进程复用时，新的聊天回合仍需刷新能力目录；不假定修改配置必然重启进程。远程桥接是否可直接复用现有工具分发是实施前的验证项，不能由本地成功推断远程支持。远程断连后桌面仍运行时，可继续管理已提交生成；远程 LLM 停止等待不等于媒体取消。 +本地 OpenCode、Continue 和 DeepSeek Harness 经内部 MCP gateway 注入；直连 Model 使用内置工具分发。远程通过 `main-image-tool-session.ts`、Agent 协议和 `image-tool-mcp.ts` 回到同一 Main 服务。现有 GoodBuddy Agent 的受管远程 Runtime 为 OpenCode，项目校验和 Runtime 创建均沿用此范围；共享契约包含其他 Runtime 不表示存在其远程执行路径。保留会话在目录清空时也刷新 MCP，Continue 准备失败时释放本次分配的令牌。工具发现成功或 Skill 注入成功不能代替真实生成与编辑。 -## 实施顺序与验证 +Main 已接收的请求在桌面仍运行时独立于远程连接继续；未确认是否接收的请求不自动重放。聊天回合停止等待与用户取消图片是不同动作，须验证现有取消传播不会误杀 Main 图片请求。 -1. 首期：实现图片调用开关、全局默认及本次指定，提取图片服务并接通直连文本 `generate_image`。完成文生图、已有协议编辑、成果保存、取消及模型切换后的结果交付。 -2. 后续按实际需求接通其余本地 Runtime 与远程工具路径，分别验证真实调用及切换。 -3. 后续评估会话级选择、独立生成入口及恢复能力，确认需要后再增加界面和数据。 -4. 视频单独选型与验收，实际接通服务后再增加视频工具、异步查询和播放能力。 +## 实施与验证 -首期以图片真实产品路径验收，完成后标记“图片首期完成”，后续能力单独记录。针对首期适用的 L-1 至 L-5 编写选择优先级、配置失效、会话隔离、重复结果、取消及晚到结果测试。验证原直连图片行为不回归，检查进程切换不会重复提交。 +实施工作包括配置与选择、Main 图片服务提取、各本地及远程 Runtime 工具适配、成果与状态 UI。它们共同完成本图片功能，不按 Runtime 划分交付范围。 -源代码实现后运行 `npm test`、`npm run typecheck`、`npm run lint`。首期真实生成从直连文本聊天输入进入，覆盖[用户场景](./user-stories.md)中的首期验收；记录实际请求数量、模型、Runtime 和结果。后续显式入口需独立验证。涉及 Agent 的变更按[远程主机验证要求](../remote-host/technical-design.md)在真实测试 Host 验证。当前仅写文档，不发起收费生成调用。 +针对 L-1 至 L-5 验证选择优先级、失效配置、上传与历史素材、无视觉模型、Ask、会话归属、重复通知、取消及晚到结果。检查切换聊天模型、图片默认模型、本地与远程 Runtime 后的新生成和编辑，以及旧操作不重复提交。回归已有直连图片工作流。 + +按 US-16 至 US-19 验证从零到一个开启模型、多个中关闭或删除一个、最后一个关闭或删除、重新开启，以及配置错误和 Runtime 离线时的派生分配、目录刷新、行可见性与只读交互。重开设置和应用后均从模型配置推导，不恢复独立分配值；辅助技术可读取标签、勾选状态和说明，键盘可到达模型设置导航。 + +源代码实现后运行 `npm test`、`npm run typecheck`、`npm run lint`，并逐路径执行[用户场景](./user-stories.md)的真实产品验收，记录模型、Runtime、请求数量和结果。涉及 Agent 的实现按[远程主机验证要求](../remote-host/technical-design.md)在真实测试 Host 验证。实施和集成验证状态由[进度记录](./progress.md)维护。 diff --git a/docs/features/conversation-media-generation/ui-design.md b/docs/features/conversation-media-generation/ui-design.md index ac6e529d..4bab592d 100644 --- a/docs/features/conversation-media-generation/ui-design.md +++ b/docs/features/conversation-media-generation/ui-design.md @@ -1,42 +1,41 @@ # 界面设计 -状态：设计草案。遵循[统一界面设计](../../../UI-DESIGN.md)，实现 [FR-1 至 FR-9](./prd.md)，行为以[行为规则](./logic-design.md)为准。 - -## 首期界面 - -首期只在图片模型配置提供会话调用开关和“设为默认生图模型”，沿用下文共享控件规范。关闭或删除默认模型后保留失效选择并提示改选，不自动改用其他模型。 - -会话以自然语言发起生图，本次指定模型也通过请求表达。不新增会话级模型选择器、独立生成表单或视频入口。未设默认时，助手提示用户本次指定一个可用模型，或到设置中选择默认后重新发送请求；名称有歧义时先澄清。 - -复用图片成果展示与引用入口，显示实际模型、进行状态、结果或错误，并支持取消和人工重新生成。首期不增加视频播放和独立“重试保存”按钮。不支持工具调用或 Runtime 未接通时说明限制，提示可切换到支持的聊天路径或使用已有直连图片工作流；Ask 下说明生成需要 Execute，不自动切换模式。 - -以下为完整目标的界面设计，按[产品需求的阶段划分](./prd.md)实施；首期不预留后续控件。 +状态：工作区已有实现及 Electron 组件验收，完整应用验收见[进度记录](./progress.md)。遵循[统一界面设计](../../../UI-DESIGN.md)，实现 [FR-1 至 FR-9](./prd.md)，行为以[行为规则](./logic-design.md)为准。 ## 模型配置 -在现有模型连接表单中，按已支持的生成能力显示持久开关： +在现有图片模型连接表单提供“允许 AI 在会话中调用”。说明文字：“开启后，支持工具调用的聊天助手可使用此模型生成和编辑图片，编辑能力取决于图片协议。结果保存在发起请求的会话中。”复用共享 Switch，提供 `role="switch"`、键盘操作及可见焦点。 -- 图片模型：“允许 AI 在会话中调用此模型生成图片”。 -- 视频模型：“允许 AI 在会话中调用此模型生成视频”。 +开启后提供“设为默认生图模型”，列表显示可调用和默认标识。关闭或删除默认连接时提示默认选择不可用，保留失效选择供改选，不自动指定其他模型。用户无需创建 Skill、搭建 MCP server 或重复填写 API 凭据；内部注入由 GoodBuddy 管理，不作为配置步骤展示。 -说明文字：“开启后，支持工具调用的聊天助手可使用此模型。生成结果保存在发起请求的会话中。”开关复用共享 Switch，并提供 `role="switch"`、键盘操作及可见焦点。不要求用户维护 Skill 文件或填写第二套 API 凭据。 +## 能力与 Runtime 分配 -开启后提供“设为默认生图模型”或“设为默认视频模型”操作。列表在对应模型旁显示“可在会话中调用”和默认标识。关闭、删除默认连接时提示会话选择将不可用，保留原选择供改选，不自动指定其他模型。 +复用设置中心“MCP / 内置 MCP”的列表与分配布局，增加“图片生成与编辑”行，标记“内置 · 自动管理”。这是 GoodBuddy 管理能力的展示入口，直连 Model 通过内部工具调用，外部 Runtime 通过内部 MCP 接入；不显示为用户另行添加的 MCP 或 Skill，也不混入 Runtime 原生能力清单。 + +该行按 L-1 展示直连文本 Model Runtime、本地 OpenCode、Continue、DeepSeek Harness 及 GoodBuddy Agent 远程适用 Runtime 的只读分配复选框。此图片能力采用用户已接受的自动管理规则，不沿用其他内置 MCP 的手动启停、可调分配或 DeepSeek Harness“暂不支持”状态。没有开启模型时仍保留整行及全部未勾选项，不能因目录为空隐藏。 + +持续显示说明“由图片模型的「允许 AI 在会话中调用」设置自动管理。”，并提供“前往图片模型设置”导航。该行不提供独立启用开关、保存分配、移除、编辑连接或凭据入口。运行就绪、无工具能力、配置或调用错误单独显示，不改变分配勾选。 + +分配复用共享 Checkbox 的方形视觉和原生语义，以 `checked` 表示派生值并使用 `disabled` 禁止鼠标、标签点击或键盘切换；仅在共享控件已有受支持的只读实现时采用该实现，不给原生 Checkbox 添加无效的 `readonly` 或自行添加 `aria-readonly`。每项保留 Runtime 标签和可访问名称，以 `aria-describedby` 关联自动管理说明。说明放在禁用控件之外，保持可读及辅助技术可访问；导航保持正常 Tab 顺序和可见焦点。窄窗口下分配项换行，保留标签、勾选状态、说明和导航。 ## 会话入口 -在现有输入区工具菜单提供“生成图片”“生成视频”，以及对应模型选择。默认项写成“跟随全局设置（模型名称）”；没有默认值则显示“选择生成模型”。选择只影响当前会话，设为全局默认必须使用独立操作。 +在现有聊天输入区以自然语言发起图片生成或编辑，本次指定模型也通过请求表达。复用上传附件和历史成果引用入口，输入区显示素材缩略图或文件名并允许移除。不新增会话级图片模型选择器、独立生成表单或视频入口。 -自然语言调用与显式入口共用成果卡片。显式入口提供提示词和参考素材选择，按所选模型展示适用参数，不呈现供应商请求字段。输入区显示已引用的素材缩略图或文件名，可移除。 +未设默认时，助手提示本次指定可用模型，或到设置选择默认后重新发送；名称歧义先澄清。没有图片连接时提供现有模型设置入口。模型不支持编辑或素材缺失时保留输入，提示更换模型或重新选择素材。 -不支持工具调用时提示“当前聊天模型无法自动调用，可使用生成入口”。Ask 下提示“生成图片或视频需要 Execute 模式”，由用户切换工作模式；不能自动切换。未配置生成模型时给出“添加生成模型”入口。 +直连文本、本地各 Runtime 和 GoodBuddy Agent 远程适用 Runtime 使用一致的图片操作与成果反馈。切换后可继续生成及编辑历史图；工具调用异常时显示具体原因，不宣称已提交。接入未完成的开发状态记录在进度文档，不能作为产品支持范围。 -## 生成卡片 +不支持工具调用时提示“当前聊天模型无法自动调用图片工具，可使用已有直连图片工作流”。Ask 下提示“生成或编辑图片需要 Execute 模式”，由用户切换，不自动切换。无视觉能力时说明无法直接查看图片，不妨碍按用户文字调用编辑工具。 -卡片就近显示媒体类型、实际模型、当前阶段、成果或错误。参数、提示词和来源素材放在可展开详情中；不显示密钥、认证头或供应商私有下载地址。运行活动使用同一操作标识，与卡片关联。 +## 图片卡片 -图片完成后可预览、下载和“继续修改”；存在可用图生视频能力时提供“生成视频”。视频完成后提供带原生播放控制的预览及下载，不自动播放。点击继续修改会把成果加入输入区，用户填写本次修改要求。 +复用现有图片成果展示，卡片就近显示实际模型、操作状态、成果或错误。提示词、适用参数和来源素材放在详情中；不显示密钥、认证头或供应商私有地址。活动记录与卡片使用同一操作标识。 -进行中显示“取消生成”，取消结果按行为规则展示。失败显示有针对性的“重试”“重新选择模型”“重新选择素材”或“重试保存”。卡片错误不重复弹出全局错误通知；模型设置保存等非上下文反馈使用应用通知。 +完成后可预览、下载和“继续修改”。继续修改将成果加入输入区，用户填写修改要求；切换 Runtime 或重开会话后仍可使用。离开会话不取消操作，结果只出现在原会话。 -窄窗口中操作允许换行，媒体宽度不超过消息区；模型长名称可截断并提供完整可访问名称。状态使用文字表达，不能只依靠颜色；阶段变化通过适度的 `aria-live` 提示，避免等待时间逐秒播报。弹出选择器关闭后焦点回到触发控件。 +进行中提供“取消生成”，与“停止回复”分开。取消、晚到结果和结果未确认的文案遵循 L-4。失败提供适用的“重新生成”“重新选择模型”或“重新选择素材”；重新生成由用户明确触发，并说明会发起新的模型请求。没有独立“重试保存”按钮。 + +“重新选择模型”进入已有模型连接设置；改变模型后，用户在新请求中明确指定。“重新选择素材”聚焦输入区并打开已有附件选择器，保留草稿和已有附件；空草稿填入原提示词及模型名称。历史成果仍使用“继续修改”加入引用并聚焦输入区。上述操作不自动发送请求，“重新生成”继续使用原操作的模型和输入。 + +卡片错误不重复弹全局通知；模型设置保存等非上下文反馈使用应用通知。状态使用文字，不能只靠颜色；状态变化通过适度的 `aria-live` 提示，避免逐秒播报等待时间。窄窗口操作可换行，图片不超过消息区宽度，模型长名称截断时保留完整可访问名称。复用的菜单关闭后焦点回到触发控件。 diff --git a/docs/features/conversation-media-generation/user-stories.md b/docs/features/conversation-media-generation/user-stories.md index 6e1a56de..9b2b9cd3 100644 --- a/docs/features/conversation-media-generation/user-stories.md +++ b/docs/features/conversation-media-generation/user-stories.md @@ -1,39 +1,34 @@ # 用户场景 -状态：设计草案。表中 Given、When、Then 分别为前提、操作和预期结果。 +状态：设计草案。Given、When、Then 分别为前提、操作和预期结果。范围以[产品需求](./prd.md)为准，行为以[行为规则](./logic-design.md)为准。 -## 首期验收 +## 场景验收 -原 US 编号保留完整目标。首期在直连文本 Model Runtime 验收 US-01、US-03、US-04、US-07、US-12、US-14，其中切换后的自动编辑要求目标聊天模型支持工具调用。切换到其他 Runtime 时，只验证原生成继续并写回原会话，不要求新 Runtime 能发起生成。 - -其余场景的首期预期如下，以[首期行为规则](./logic-design.md)为准： - -| 场景 | 首期验收 | -| --- | --- | -| US-02 | 未设默认时提示本次指定模型或去设置默认；重新发送后生成，不新增模型选择弹窗 | -| US-05 | 当前操作使用 X，修改全局默认为 Y 后，新建图片使用 Y，旧操作保持 X | -| US-06 | 默认或本次指定的 X 不可用时提示改选，历史图片保留；没有会话级选择 | -| US-08 | 无工具能力时说明限制及已有直连图片工作流；Ask 不提交生成 | -| US-09 | 图片生成可取消并如实显示结果，人工重新生成创建新操作；没有独立保存重试 | -| US-13 | 应用重启后未完成图片操作显示结果未确认，不自动查询或重新提交 | - -US-10 视频生成、US-11 远程自动调用，以及下表中超出上述范围的预期，留到后续验收。 - -## 完整目标场景 +适用的自动调用场景须在直连文本 Model Runtime、本地 OpenCode、Continue、DeepSeek Harness 和 GoodBuddy Agent 远程适用 Runtime 分别验证，前提为 Execute 且聊天模型支持工具调用。切换验收同时检查原操作归属和目标 Runtime 的新生成、编辑能力。 | 编号 | 需求 | Given | When | Then | | --- | --- | --- | --- | --- | -| US-01 | FR-1、FR-3 | 已添加生图模型并开启会话调用，已选默认模型 | 在支持工具的文字模型中要求画海报 | 活动中显示调用，完成后原会话出现图片及实际模型 | -| US-02 | FR-2 | 有两个可调用生图模型，未选默认 | 请求生成图片 | 提供模型选择入口，选择后继续原请求；不会自行按列表顺序提交 | -| US-03 | FR-4、FR-5 | 模型 A 已生成图片 | 切换聊天模型 B，要求修改该图 | B 获得成果引用和可用工具，编辑模型按规则选择 | -| US-04 | FR-4、FR-6 | 一个生成操作已经提交 | 切换 Runtime 并打开其他会话 | 原操作继续，结果只写回原会话；新 Runtime 不重复调用 | -| US-05 | FR-2、FR-4 | 会话跟随全局默认，当前操作使用模型 X | 将默认改成 Y，再发一个新建图片请求 | 旧操作仍用 X，新操作用 Y，卡片显示各自来源 | -| US-06 | FR-4 | 会话明确选择 X | X 被关闭或删除后请求生成 | 显示 X 不可用并允许改选；历史图仍可打开 | -| US-07 | FR-5、FR-8 | 已保存生成图并重新打开会话 | 显式引用原图，指定另一个支持编辑的模型 | 新模型接收原图；成果丢失时要求重新选择素材，不能假装已编辑 | -| US-08 | FR-7 | 切换到不支持工具调用的文字模型 | 点击会话显式生图入口 | Execute 下可生成并回写；Ask 下说明需要切换 Execute，不能发出生成请求 | -| US-09 | FR-6 | 操作正在生成或保存结果 | 点击取消，或失败后选择重试 | 如实显示取消结果；重新生成创建独立操作，重试保存更新原操作，不自动重复收费请求 | -| US-10 | FR-3、FR-8 | 已接通支持图生视频的模型 | 从图片成果选择生成视频 | 使用所选图片，视频完成并保存后可播放、导出，重开会话仍可访问 | -| US-11 | FR-4、FR-9 | 远程 Runtime 已接入媒体工具 | 在远程会话中生成后切换本地 Runtime | 结果与活动属于原会话，凭据留在 Main，后续请求可引用成果 | -| US-12 | FR-5、FR-7 | 新聊天模型没有视觉能力 | 请求分析已生成图片或继续修改 | 说明无法直接查看图片；可根据用户文字调用编辑工具，不编造视觉观察 | -| US-13 | FR-6 | 上游视频返回可查询的任务标识 | 退出并重开应用 | 查询已有任务；无法查询的中断操作明确提示，不自动重新提交 | -| US-14 | FR-5 | 同一会话有多张成果 | 要求“修改第二张”或“新画一张无关图片” | 前者引用指定成果，后者不自动附带上一张图；指代不清时先澄清 | +| US-01 | FR-1、FR-3、FR-9 | 已添加图片模型、开启会话调用并设为默认 | 在文字会话中要求画海报 | 自动调用并显示活动，原会话出现图片及实际模型；无需创建 Skill、MCP server 或重复凭据 | +| US-02 | FR-2 | 有两个可调用图片模型，未选默认 | 请求生成图片 | 提示本次指定模型或到设置选择默认后重新发送；名称歧义先澄清，不按列表顺序提交 | +| US-03 | FR-4、FR-5 | 聊天模型 A 已调用图片模型生成图片 | 切换聊天模型 B，要求修改该图 | B 获得成果引用和可用工具，编辑模型按规则选择 | +| US-04 | FR-4、FR-6、FR-9 | 生成操作已提交 | 切换 Runtime 并打开其他会话，再请求生成或编辑 | 原操作继续且只写回原会话；目标 Runtime 可提交新操作，不重放旧调用 | +| US-05 | FR-2、FR-4 | 当前操作使用图片模型 X | 将全局默认改为 Y，再请求新建图片 | 旧操作仍用 X，新操作用 Y；本次明确指定可覆盖默认 | +| US-06 | FR-1、FR-2、FR-4 | 默认或本次指定模型为 X | X 被关闭或删除后请求生成 | 显示 X 不可用并提示改选，历史图仍可打开，不静默换模型 | +| US-07 | FR-5、FR-8 | 已保存图片并重新打开会话或切换 Runtime | 引用原图，指定另一个支持编辑的模型 | 新模型接收原图；素材缺失时要求重新选择，不能假装已编辑 | +| US-08 | FR-7 | 当前聊天模型不支持工具调用，或处于 Ask | 请求生成或编辑 | 无工具能力时说明可使用已有直连图片工作流；Ask 下不提交并提示需要 Execute，不自动切换模式 | +| US-09 | FR-6 | 操作正在生成或保存，或已经失败 | 取消，或明确要求重新生成 | 如实显示取消结果；人工重新生成创建新操作，不自动重复收费请求，不提供独立保存重试入口 | +| US-11 | FR-4、FR-9 | 使用 GoodBuddy Agent 远程适用 Runtime | 生成及编辑图片，切换本地后继续编辑，再切回远程生成 | 工具往返可用，成果可继续引用；活动归原会话，凭据留在 Main；远程断连不取消 Main 已接收的操作 | +| US-12 | FR-5、FR-7、FR-8 | 聊天模型无视觉能力但支持工具 | 请求分析图片或根据文字修改图片 | 说明无法直接查看图片；可传原图给编辑工具，不编造视觉观察 | +| US-13 | FR-6 | 图片操作尚未取得可保存结果 | 退出并重开应用 | 未完成记录显示结果未确认，不承诺后台继续，不自动查询恢复或重新提交 | +| US-14 | FR-5 | 会话有多张图片 | 要求“修改第二张”或“新画一张无关图片” | 前者引用指定成果，后者不附带旧图；指代不清先澄清 | +| US-15 | FR-3、FR-5、FR-9 | 上传图片并选择支持编辑的图片模型 | 要求修改背景，再切换 Runtime 继续修改成果 | 上传原图及后续成果被正确引用，编辑结果保存在各自发起会话；不降为纯文生图 | +| US-16 | FR-1、FR-9 | 尚无开启会话调用的图片模型 | 查看内置 MCP 列表，再开启一个图片模型并保存 | 图片生成与编辑行一直可见且标记“内置 · 自动管理”；全部支持 Runtime 从未勾选变为勾选，目录包含该模型；重开设置或应用仍按模型配置显示 | +| US-17 | FR-1、FR-9 | 至少两个图片模型开启会话调用 | 关闭或删除其中一个并保存 | 全部支持 Runtime 保持勾选，允许目录移除该模型、保留其他开启模型；新操作不能调用已关闭或删除模型 | +| US-18 | FR-1、FR-9 | 仅剩一个开启会话调用的图片模型 | 关闭或删除该模型，再重新开启或添加并开启一个模型 | 目录为空时全部取消勾选，整行和说明、模型设置导航仍可见；再次开启后自动恢复全部勾选，无需另行分配 | +| US-19 | FR-1、FR-7、FR-9 | 至少一个图片模型开启会话调用 | 尝试用鼠标或键盘切换分配，或切换到无工具聊天模型、Ask、离线 Runtime，或遇到配置或执行错误 | 分配不可手动修改，运行限制与错误不取消勾选；无独立启用、分配保存、移除或凭据编辑操作；自动管理说明持续可读，辅助技术可识别勾选和禁用状态，键盘可通过“前往图片模型设置”导航修改源配置 | + +## 编号变更 + +US-10（图生视频）移出本图片设计范围，编号保留，不安排实施。US-02、US-05、US-06 的会话选择预期由全局默认和本次指定替代；US-08 使用已有直连图片工作流；US-09 取消独立保存重试要求；US-13 的视频查询恢复由图片中断状态说明替代。其余编号保持，US-15 补充上传图片编辑的显式验收。 + +US-16 至 US-19 补充自动管理分配验收，设置展示不依赖 Execute 或当前聊天模型支持工具；自动生成与编辑仍遵循本节的调用前提。 diff --git a/src/agent-daemon/agent-acp-connection.ts b/src/agent-daemon/agent-acp-connection.ts index efb247c6..f0ed1ef7 100644 --- a/src/agent-daemon/agent-acp-connection.ts +++ b/src/agent-daemon/agent-acp-connection.ts @@ -87,12 +87,13 @@ export class AgentAcpConnection { sessionId: string, operationId: string, socketPath?: string, - workMode?: 'ask' | 'execute' + workMode?: 'ask' | 'execute', + imageToolName?: string ): Promise { if (!this.bridgeOrigin) throw new Error('Shared model bridge is not ready') const response = await fetch(`${this.bridgeOrigin}/session`, { method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ sessionId, operationId, ...(socketPath ? { socketPath, workMode } : { release: true }) }), + body: JSON.stringify({ sessionId, operationId, ...(socketPath ? { socketPath, workMode, imageToolName } : { release: true }) }), signal: AbortSignal.timeout(10_000) }) if (!response.ok) throw new Error(`Shared model bridge route failed (${response.status})`) diff --git a/src/agent-daemon/agent-owned-acp-prompt.test.ts b/src/agent-daemon/agent-owned-acp-prompt.test.ts index 2908eef6..639853ee 100644 --- a/src/agent-daemon/agent-owned-acp-prompt.test.ts +++ b/src/agent-daemon/agent-owned-acp-prompt.test.ts @@ -30,23 +30,27 @@ describe('AgentOwnedAcpPrompt', () => { let client!: Client let finish!: (response: PromptResponse) => void const newSession = vi.fn(async () => ({ sessionId: 'session-1' })) + const resumeSession = vi.fn(async () => ({})) + let imageUrl: string | undefined + const mcpServers = () => imageUrl ? [{ type: 'http' as const, name: 'images', url: imageUrl, headers: [] }] : [] const prepareSession = vi.fn(async () => undefined) const completePrompt = vi.fn(async () => undefined) const owner = new AgentOwnedAcpPrompt({ bindingId: 'binding-1', controllerId: 'controller-1', workspaceDirectory: '/workspace', process: new MemoryProcess(), - transcript, prepareSession, completePrompt, + transcript, prepareSession, completePrompt, mcpServers, createConnection: factory => { client = factory() return { initialize: async () => ({ protocolVersion: 1, agentCapabilities: {} }), - newSession, + newSession, resumeSession, prompt: () => new Promise(resolve => { finish = resolve }) } as unknown as ClientSideConnection } }) try { - for (const [index, workMode] of (['execute', 'ask', 'execute'] as const).entries()) { + for (const [index, workMode] of (['execute', 'execute', 'ask', 'execute', 'ask'] as const).entries()) { + imageUrl = workMode === 'execute' && index !== 1 ? `http://127.0.0.1:1234/prompt-${index}` : undefined const operationId = `operation-${index}` transcript.prepare({ bindingId: 'binding-1', operationId, requestId: operationId, @@ -58,6 +62,9 @@ describe('AgentOwnedAcpPrompt', () => { prompt: [{ type: 'text', text: 'Continue with the selected mode' }] }, workMode) expect(prepareSession).toHaveBeenLastCalledWith('session-1', operationId, workMode) + if (index !== 2) { + expect(index === 0 ? newSession : resumeSession).toHaveBeenLastCalledWith(expect.objectContaining({ mcpServers: mcpServers() })) + } const result = await client.requestPermission!({ sessionId: 'session-1', toolCall: { toolCallId: `tool-${index}`, kind: 'execute', title: 'Run a test' }, @@ -74,7 +81,8 @@ describe('AgentOwnedAcpPrompt', () => { .toMatchObject({ state: 'completed' })) } expect(newSession).toHaveBeenCalledOnce() - expect(completePrompt).toHaveBeenCalledTimes(3) + expect(resumeSession).toHaveBeenCalledTimes(3) + expect(completePrompt).toHaveBeenCalledTimes(5) } finally { owner.close() transcript.close() diff --git a/src/agent-daemon/agent-owned-acp-prompt.ts b/src/agent-daemon/agent-owned-acp-prompt.ts index 22258ab6..62928575 100644 --- a/src/agent-daemon/agent-owned-acp-prompt.ts +++ b/src/agent-daemon/agent-owned-acp-prompt.ts @@ -4,6 +4,7 @@ import { PROTOCOL_VERSION, ndJsonStream, type Client, + type McpServer, type PromptResponse, type RequestPermissionRequest, type RequestPermissionResponse, @@ -26,6 +27,7 @@ export type AgentOwnedAcpPromptOptions = { bindingId: string controllerId: string workspaceDirectory: string + mcpServers?: () => McpServer[] expectedModel?: string process: RuntimeAcpProcessOwner transport?: AgentAcpConnection @@ -60,6 +62,7 @@ export class AgentOwnedAcpPrompt { #promptSettled?: Promise #closed = false #initialized = false + #hasMcpServers = false readonly #questions = new Map() @@ -109,6 +112,7 @@ export class AgentOwnedAcpPrompt { ) } + const mcpServers = this.#options.mcpServers?.() ?? [] if (!this.#initialized) { const initialization = await this.#whileProcessAlive( this.#transport.initialize({ @@ -135,7 +139,7 @@ export class AgentOwnedAcpPrompt { this.#connection.loadSession({ sessionId: request.acpSessionId, cwd: this.#options.workspaceDirectory, - mcpServers: [] + mcpServers }) ) } else if ( @@ -145,7 +149,7 @@ export class AgentOwnedAcpPrompt { this.#connection.resumeSession({ sessionId: request.acpSessionId, cwd: this.#options.workspaceDirectory, - mcpServers: [] + mcpServers }) ) } else { @@ -156,7 +160,7 @@ export class AgentOwnedAcpPrompt { const created = await this.#whileProcessAlive( this.#connection.newSession({ cwd: this.#options.workspaceDirectory, - mcpServers: [] + mcpServers }) ) this.#sessionId = created.sessionId @@ -186,7 +190,13 @@ export class AgentOwnedAcpPrompt { request.acpSessionId !== this.#sessionId ) { throw new Error('ACP session identity cannot change within a binding') + } else if (mcpServers.length > 0 || this.#hasMcpServers) { + await this.#whileProcessAlive(this.#connection.resumeSession({ + sessionId: this.#sessionId!, cwd: this.#options.workspaceDirectory, + mcpServers + })) } + this.#hasMcpServers = mcpServers.length > 0 await this.#options.prepareSession?.(this.#sessionId!, request.operationId, workMode) this.#active = { operationId: request.operationId, diff --git a/src/agent-daemon/image-tool-mcp.test.ts b/src/agent-daemon/image-tool-mcp.test.ts new file mode 100644 index 00000000..a078b45e --- /dev/null +++ b/src/agent-daemon/image-tool-mcp.test.ts @@ -0,0 +1,146 @@ +// @vitest-environment node +import { randomUUID } from 'node:crypto' +import { createServer } from 'node:http' +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { AgentImageToolMcp } from './image-tool-mcp' +import { imageOperationSchema } from '../shared/image-generation-contracts' +import { AssistantDatabase } from '../main/assistant/assistant-database' +import { ImageGenerationService } from '../main/agent/image-generation-service' +import { MainImageToolSession } from '../main/remote-agent/main-image-tool-session' +import type { RuntimeProtocolBinaryChannel, RuntimeProtocolBinaryFrame } from '../main/remote-agent/protocol-remote-runtime-channel' + +const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=' +vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64') }) } })) +const cleanup: (() => void | Promise)[] = [] +afterEach(async () => { for (const close of cleanup.splice(0).reverse()) await close() }) + +async function setup() { + const received: { path: string; body: string; authorization?: string }[] = [] + let release: (() => void) | undefined + let hold = false + const provider = createServer((req, res) => { + void (async () => { + let body = '' + for await (const chunk of req) body += chunk + received.push({ path: req.url!, body, authorization: req.headers.authorization }) + if (hold) await new Promise(resolve => { release = resolve }) + res.setHeader('content-type', 'application/json') + res.end(JSON.stringify({ data: [{ b64_json: png }] })) + })() + }) + await new Promise(resolve => provider.listen(0, '127.0.0.1', resolve)) + cleanup.push(() => { release?.(); provider.closeAllConnections(); provider.close() }) + const address = provider.address() as { port: number } + const database = new AssistantDatabase(':memory:') + database.initialize(process.cwd()) + cleanup.push(() => database.close()) + const context = { conversationId: randomUUID(), messageId: randomUUID(), requestId: randomUUID(), workMode: 'execute' as 'ask' | 'execute' } + database.saveLocalConversations([{ header: { id: context.conversationId, title: 'Remote images', updatedAt: 1 }, + messages: [{ id: context.messageId, role: 'assistant', content: '', createdAt: 1, state: 'complete' }] }]) + const profile = { id: randomUUID(), name: 'Controlled image provider', modelName: 'controlled-image', + protocol: 'openai-images-generations', allowConversationInvocation: true, + baseUrl: `http://127.0.0.1:${address.port}/v1`, authentication: 'api-key' as const, apiKey: 'controlled-main-only' } + const service = new ImageGenerationService({ database, getSettings: async () => ({ modelProfiles: [profile], defaultImageModelProfileId: profile.id }) }) + service.initialize() + cleanup.push(() => service.dispose()) + const frames: Uint8Array[] = [] + const connect = async (mode: 'ask' | 'execute' = 'execute') => { + const binding = service.bind({ ...context, requestId: randomUUID(), workMode: mode }) + const queue: RuntimeProtocolBinaryFrame[] = [] + let receiver: ((frame: RuntimeProtocolBinaryFrame) => void) | undefined + let rejectReceive: ((error: unknown) => void) | undefined + const listeners = new Set<() => void>() + let closed = false + const channel: RuntimeProtocolBinaryChannel = { + channelId: randomUUID(), channelEpoch: '1', + send: async payload => { frames.push(payload); adapter.onReply(payload) }, + receive: async () => queue.shift() ?? await new Promise((resolve, reject) => { receiver = resolve; rejectReceive = reject }), + close: () => { if (closed) return; closed = true; rejectReceive?.(new Error('closed')); for (const fn of listeners) fn() }, + onClose: fn => { listeners.add(fn); return () => { listeners.delete(fn) } } + } + const wait = new AbortController() + const main = new MainImageToolSession(channel, binding, wait.signal) + const adapter = new AgentImageToolMcp({ channelId: channel.channelId, channelEpoch: '1', description: await binding.describe() ?? 'Not available' }, async payload => { + if (closed) throw new Error('disconnected') + frames.push(payload) + const frame = { payload, sequence: '1', consume: async () => {} } + if (receiver) { const resolve = receiver; receiver = undefined; resolve(frame) } else queue.push(frame) + }) + await adapter.start() + const client = new Client({ name: 'remote-image-test', version: '1' }) + await client.connect(new StreamableHTTPClientTransport(new URL(adapter.url!))) + cleanup.push(async () => { main.close(); adapter.close(); await client.close() }) + return { client, adapter, wait, main, binding } + } + return { received, context, profile, service, database, connect, frames, + hold: () => { hold = true }, release: () => release?.() } +} + +describe('remote image MCP to Main service', () => { + it('generates, edits persisted uploads and history through a controlled HTTP provider', async () => { + const h = await setup() + const remote = await h.connect() + expect((await remote.client.listTools()).tools[0]!.description).toContain(h.profile.id) + const call = async (args: Record) => { + const result = await remote.client.callTool({ name: 'generate_image', arguments: args }) + expect(result.isError).not.toBe(true) + return imageOperationSchema.parse(JSON.parse((result.content as { text: string }[])[0]!.text)) + } + const first = await call({ intent: 'create', prompt: 'A blue square' }) + const uploaded = h.service.persistUploads(h.context, [{ name: 'source.png', mediaType: 'image/png', data: png }]) + const uploadEdit = await call({ intent: 'edit', prompt: 'Make the uploaded square red', sourceArtifactIds: uploaded }) + const historyEdit = await call({ intent: 'edit', prompt: 'Make the previous result green', sourceArtifactIds: first.artifactIds }) + expect([first, uploadEdit, historyEdit].map(op => op.state)).toEqual(['completed', 'completed', 'completed']) + expect(h.received.map(call => call.path)).toEqual(['/v1/images/generations', '/v1/images/edits', '/v1/images/edits']) + expect(h.received.every(call => call.authorization === 'Bearer controlled-main-only')).toBe(true) + expect(h.received[1]!.body).toContain('name="image"') + expect(h.database.getArtifact(historyEdit.artifactIds[0]!).content).toContain(png) + expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageOperations).toHaveLength(3) + const wire = Buffer.concat(h.frames).toString() + expect(wire).not.toContain(png) + expect(wire).not.toContain('controlled-main-only') + expect(wire).not.toContain(h.profile.baseUrl) + }) + + it.each(['chat cancellation', 'disconnect'])('preserves Main operation after %s without replay', async reason => { + const h = await setup() + h.hold() + const remote = await h.connect() + const waiting = remote.client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } }).catch(() => undefined) + await vi.waitFor(() => expect(h.received).toHaveLength(1)) + if (reason === 'chat cancellation') remote.wait.abort() + else remote.main.close() + remote.adapter.close() + h.release() + await vi.waitFor(() => expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageOperations?.[0]?.state).toBe('completed')) + await waiting + expect(h.received).toHaveLength(1) + }) + + it('rejects Ask calls at the desktop binding and refreshes a new prompt description', async () => { + const h = await setup() + const ask = await h.connect('ask') + const denied = await ask.client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } }) + expect(denied.isError).toBe(true) + expect(h.received).toHaveLength(0) + h.profile.name = 'Changed next prompt' + const execute = await h.connect() + expect((await execute.client.listTools()).tools[0]!.description).toContain('Changed next prompt') + }) + + it('returns unknown delivery without sending a second request for the same MCP call', async () => { + const send = vi.fn(async () => { throw new Error('connection lost after send') }) + const adapter = new AgentImageToolMcp({ channelId: randomUUID(), channelEpoch: '1', description: 'Images' }, send) + await adapter.start() + cleanup.push(() => adapter.close()) + const request = { jsonrpc: '2.0', id: 7, method: 'tools/call', params: { name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } } } + for (let i = 0; i < 2; i++) { + const response = await fetch(adapter.url!, { method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream' }, body: JSON.stringify(request) }) + const result = await response.json() + expect(JSON.stringify(result)).toContain('unknown') + } + expect(send).toHaveBeenCalledOnce() + }) +}) diff --git a/src/agent-daemon/image-tool-mcp.ts b/src/agent-daemon/image-tool-mcp.ts new file mode 100644 index 00000000..e7dd07e8 --- /dev/null +++ b/src/agent-daemon/image-tool-mcp.ts @@ -0,0 +1,121 @@ +import { randomUUID } from 'node:crypto' +import { createServer, type Server } from 'node:http' +import { Server as McpServer } from '@modelcontextprotocol/sdk/server/index.js' +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js' +import { CallToolRequestSchema, ListToolsRequestSchema, type CallToolResult } from '@modelcontextprotocol/sdk/types.js' +import { z } from 'zod' +import { imageToolInputSchema, imageToolName } from '../shared/image-generation-contracts' +import { + decodeRemoteImageToolMessage, encodeRemoteImageToolMessage, + remoteImageToolMaximumBytes, remoteImageToolReplySchema, type remoteImageToolSchema +} from '../shared/remote-image-tool-contracts' + +/** One prompt's local MCP adapter. It never holds provider credentials or image bytes. */ +export class AgentImageToolMcp { + private readonly calls = new Map }>() + private readonly pending = new Map void + reject: (error: Error) => void + }>() + private server?: Server + private closed = false + url?: string + + constructor( + readonly descriptor: z.infer, + private readonly send: (payload: Uint8Array) => Promise + ) {} + + async start(): Promise { + const path = `/${randomUUID()}/mcp` + const schema = z.toJSONSchema(imageToolInputSchema, { target: 'draft-7', io: 'input' }) + Reflect.deleteProperty(schema, '$schema') + this.server = createServer((request, response) => { + void (async () => { + if (this.closed || request.url !== path) { response.writeHead(404).end(); return } + if (request.method !== 'POST') { response.writeHead(405).end(); return } + const chunks: Buffer[] = [] + let size = 0 + for await (const chunk of request) { + const bytes = Buffer.from(chunk) + size += bytes.length + if (size > remoteImageToolMaximumBytes) { response.writeHead(413).end(); return } + chunks.push(bytes) + } + const body: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8')) + const mcp = new McpServer({ name: 'goodbuddy-image', version: '1' }, { capabilities: { tools: {} } }) + mcp.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: [{ + name: imageToolName, description: this.descriptor.description, + inputSchema: schema as { type: 'object' } + }] })) + mcp.setRequestHandler(CallToolRequestSchema, async (call, extra) => { + if (call.params.name !== imageToolName) throw new Error('Unknown image tool') + const input = imageToolInputSchema.parse(call.params.arguments) + // Repeated delivery of an MCP request returns the same outcome, including unknown delivery. + const key = `${typeof extra.requestId}:${extra.requestId}` + let existing = this.calls.get(key) + if (existing && existing.input !== JSON.stringify(input)) throw new Error('Image tool request ID reused with different input') + if (!existing) { + const result = this.call(input).catch((error: unknown) => ({ + isError: true, + content: [{ type: 'text' as const, text: error instanceof Error ? error.message : 'Image tool delivery failed; do not retry automatically' }] + })) + existing = { input: JSON.stringify(input), result } + this.calls.set(key, existing) + } + return await existing.result + }) + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined, enableJsonResponse: true }) + response.once('close', () => { void mcp.close().catch(() => undefined) }) + await mcp.connect(transport) + await transport.handleRequest(request, response, body) + })().catch(() => { + if (!response.headersSent) response.writeHead(400) + response.end() + }) + }) + await new Promise((resolve, reject) => { + this.server!.once('error', reject) + this.server!.listen(0, '127.0.0.1', resolve) + }) + const address = this.server.address() + if (!address || typeof address === 'string') throw new Error('Image MCP endpoint did not start') + this.url = `http://127.0.0.1:${address.port}${path}` + } + + onReply(payload: Uint8Array): void { + const reply = remoteImageToolReplySchema.parse(decodeRemoteImageToolMessage(payload)) + const pending = this.pending.get(reply.callId) + if (!pending) return + if (reply.error || !reply.result) pending.reject(new Error(reply.error ?? 'Image tool response has no result')) + else pending.resolve({ content: [{ type: 'text', text: JSON.stringify(reply.result) }] }) + } + + close(): void { + if (this.closed) return + this.closed = true + for (const pending of this.pending.values()) pending.reject(new Error('Desktop image tool disconnected; delivery may be unknown. Do not retry automatically.')) + this.pending.clear() + this.server?.closeAllConnections() + this.server?.close() + } + + private async call(input: z.infer): Promise { + if (this.closed) throw new Error('Image tool prompt has ended') + const callId = randomUUID() + let timer: NodeJS.Timeout | undefined + const result = new Promise((resolve, reject) => { + this.pending.set(callId, { resolve, reject }) + timer = setTimeout(() => reject(new Error('Image tool wait timed out; delivery may be unknown. Do not retry automatically.')), 5 * 60_000) + timer.unref() + }) + // Register the wait before sending; send failures must not cause a retry. + void this.send(encodeRemoteImageToolMessage({ callId, input })).catch(() => { + this.pending.get(callId)?.reject(new Error('Image tool delivery may be unknown. Do not retry automatically.')) + }) + try { return await result } finally { + clearTimeout(timer) + this.pending.delete(callId) + } + } +} diff --git a/src/agent-daemon/model-bridge-helper.ts b/src/agent-daemon/model-bridge-helper.ts index 4d95e9dd..2fc8f43e 100644 --- a/src/agent-daemon/model-bridge-helper.ts +++ b/src/agent-daemon/model-bridge-helper.ts @@ -133,7 +133,7 @@ const FORWARDED_HEADER_NAMES = [ export class ModelBridgeLoopbackProxy { readonly #exchange: ModelBridgeExchange readonly #sharedSessions: boolean - readonly #sessionRoutes = new Map() + readonly #sessionRoutes = new Map() readonly #routeToken: string readonly #maximumConnections: number readonly #requestTimeoutMs: number @@ -317,12 +317,12 @@ export class ModelBridgeLoopbackProxy { const route = this.#sessionRoutes.get(url.searchParams.get('sessionId') ?? '') if (!route) throw new HttpRequestError(409, 'session-inactive') outgoing.setHeader('content-type', 'application/json') - outgoing.end(JSON.stringify({ operationId: route.operationId, workMode: route.workMode })) + outgoing.end(JSON.stringify({ operationId: route.operationId, workMode: route.workMode, imageToolName: route.imageToolName })) return } if (incoming.method !== 'POST') throw new HttpRequestError(405, 'method-not-allowed') const value = JSON.parse((await readBoundedBody(incoming)).toString('utf8')) as { - sessionId?: string; operationId?: string; socketPath?: string; release?: boolean; workMode?: 'ask' | 'execute' + sessionId?: string; operationId?: string; socketPath?: string; release?: boolean; workMode?: 'ask' | 'execute'; imageToolName?: string } if ( typeof value.sessionId !== 'string' || value.sessionId.length > 128 || @@ -335,7 +335,8 @@ export class ModelBridgeLoopbackProxy { } else { if (value.workMode !== 'ask' && value.workMode !== 'execute') throw new HttpRequestError(400, 'request-invalid') const socketPath = normalizedAbsolutePath(value.socketPath ?? '', 'Model bridge socket') - this.#sessionRoutes.set(value.sessionId, { operationId: value.operationId, socketPath, workMode: value.workMode }) + if (value.imageToolName !== undefined && !/^goodbuddy_image_[a-f0-9]{24}$/.test(value.imageToolName)) throw new HttpRequestError(400, 'request-invalid') + this.#sessionRoutes.set(value.sessionId, { operationId: value.operationId, socketPath, workMode: value.workMode, imageToolName: value.imageToolName }) } outgoing.end('{}') return diff --git a/src/agent-daemon/opencode-subagent-plugin.test.ts b/src/agent-daemon/opencode-subagent-plugin.test.ts index e3b7942c..fc63b3ca 100644 --- a/src/agent-daemon/opencode-subagent-plugin.test.ts +++ b/src/agent-daemon/opencode-subagent-plugin.test.ts @@ -5,6 +5,29 @@ import { describe, expect, it, vi } from 'vitest' import { openCodeSubagentPluginSource } from './opencode-subagent-plugin' describe('OpenCode ACP child event plugin', () => { + it('hides foreign image tools and removes the current image tool in Ask on a reused Session', async () => { + let mode = 'execute' + const imageToolName = `goodbuddy_image_${'a'.repeat(24)}` + const patch = vi.fn() + const factory = runInNewContext( + `(${openCodeSubagentPluginSource('http://127.0.0.1:12345/fixture').replace(/^import .*\n/gmu, '').replace('export default ', '')})`, + { process: { stdout: { write: vi.fn() } }, createServer, randomBytes, + fetch: async () => ({ ok: true, json: async () => ({ operationId: mode, workMode: mode, imageToolName }) }) } + ) + const hooks = await factory({ client: { + _client: { get: vi.fn(), post: vi.fn(), patch }, + session: { get: async () => ({ data: { id: 'root' } }) } + } }) + try { + for (mode of ['execute', 'ask', 'execute']) { + await hooks.event({ event: { type: 'session.status', properties: { sessionID: 'root', status: { type: 'idle' } } } }) + await hooks['chat.message']({ sessionID: 'root' }, { message: { id: mode } }) + const rules = patch.mock.calls.at(-1)![0].body.permission + expect(rules).toContainEqual({ permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' }) + expect(rules.some((rule: { permission: string; action: string }) => rule.permission === `${imageToolName}_*` && rule.action === 'allow')).toBe(mode === 'execute') + } + } finally { await hooks.dispose() } + }) it('attributes child and compaction model rounds to the frozen root operation', async () => { const operations = new Map([['one', 'operation-1'], ['two', 'operation-2']]) const fetchRoute = vi.fn(async (url: string) => { @@ -33,7 +56,10 @@ describe('OpenCode ACP child event plugin', () => { try { await hooks['chat.message']({ sessionID: 'one' }, { message: { id: 'first' } }) expect(patch).toHaveBeenCalledWith({ - url: '/session/one', body: { permission: [{ permission: '*', pattern: '*', action: 'allow' }] }, throwOnError: true + url: '/session/one', body: { permission: [ + { permission: '*', pattern: '*', action: 'allow' }, + { permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' } + ] }, throwOnError: true }) expect(await headers('child', 'child-message')).toEqual({ 'x-goodbuddy-session': 'one', 'x-goodbuddy-operation': 'operation-1' @@ -42,7 +68,8 @@ describe('OpenCode ACP child event plugin', () => { expect(patch).toHaveBeenLastCalledWith({ url: '/session/child', body: { permission: [ { permission: '*', pattern: '*', action: 'allow' }, - { permission: 'task', pattern: '*', action: 'deny' } + { permission: 'task', pattern: '*', action: 'deny' }, + { permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' } ] }, throwOnError: true }) expect(await headers('two', 'compact')).toEqual({ diff --git a/src/agent-daemon/opencode-subagent-plugin.ts b/src/agent-daemon/opencode-subagent-plugin.ts index c075bfda..73a42f5f 100644 --- a/src/agent-daemon/opencode-subagent-plugin.ts +++ b/src/agent-daemon/opencode-subagent-plugin.ts @@ -26,7 +26,7 @@ async function plugin(input?: { }) { const tasks = new Map() const pending = new Map() - const modelMessages = new Map() + const modelMessages = new Map() const modelRoute = async (sessionId: string, messageId: string) => { const key = `${sessionId}\0${messageId}` const previous = modelMessages.get(key) @@ -43,8 +43,8 @@ async function plugin(input?: { } const response = await fetch(`${modules.modelBridgeOrigin}/session?sessionId=${encodeURIComponent(root)}`) if (!response.ok) throw new Error('GoodBuddy model operation is no longer active') - const { operationId, workMode } = await response.json() as { operationId: string; workMode: 'ask' | 'execute' } - const route = { sessionId: root, operationId, workMode } + const { operationId, workMode, imageToolName } = await response.json() as { operationId: string; workMode: 'ask' | 'execute'; imageToolName?: string } + const route = { sessionId: root, operationId, workMode, imageToolName } modelMessages.set(key, route) return route } @@ -131,6 +131,10 @@ async function plugin(input?: { // Keep its explicit subagent restrictions after the request rules. permission.push(...(child.data.permission ?? []).filter(rule => rule.permission !== '*')) } + permission.push({ permission: 'goodbuddy_image_*', pattern: '*', action: 'deny' }) + if (route.workMode === 'execute' && route.imageToolName) { + permission.push({ permission: `${route.imageToolName}_*`, pattern: '*', action: 'allow' }) + } await input!.client._client.patch({ url: `/session/${encodeURIComponent(request.sessionID)}`, body: { permission }, throwOnError: true }) diff --git a/src/agent-daemon/runtime-acp-backend.test.ts b/src/agent-daemon/runtime-acp-backend.test.ts index fe53b753..77b6108d 100644 --- a/src/agent-daemon/runtime-acp-backend.test.ts +++ b/src/agent-daemon/runtime-acp-backend.test.ts @@ -1,4 +1,8 @@ import { describe, expect, it, vi } from 'vitest' +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' +import { decodeRemoteImageToolMessage, encodeRemoteImageToolMessage, remoteImageToolCallSchema } from '../shared/remote-image-tool-contracts' +import type { AgentFrame } from '../shared/agent-protocol' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -309,7 +313,7 @@ function harness(input: { throw new Error('broker close failed') } }) - const blobSink = vi.fn(async () => undefined) + const blobSink = vi.fn<(frame: AgentFrame) => Promise>(async () => undefined) let bridgeDispatch: ModelBridgeBrokerDispatch | undefined const launchModelBridges: unknown[] = [] const resolved = resolvedBundle() @@ -451,6 +455,49 @@ function harness(input: { } describe('RuntimeAcpBackend', () => { + it('binds MCP image calls to authenticated blob framing and closes each prompt endpoint', async () => { + const fixture = harness({ workMode: 'execute' }) + const client = new Client({ name: 'image-backend-test', version: '1' }) + try { + await open(fixture) + const accepted = await invoke(fixture, 'runtime/preparePrompt', fixture.preparation({ + imageTool: { channelId: 'image-channel', channelEpoch: '1', description: 'Image catalog' } + })) as { imageToolUrl: string } + await client.connect(new StreamableHTTPClientTransport(new URL(accepted.imageToolUrl))) + expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image']) + const result = client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } }) + await vi.waitFor(() => expect(fixture.blobSink).toHaveBeenCalledOnce()) + const frame = fixture.blobSink.mock.calls[0]![0] + expect(frame.header).toMatchObject({ channelId: 'image-channel', channelEpoch: '1', sequence: '1', direction: 'agent-to-main' }) + expect(fixture.backend.authorizeBlobFrame(frame, fixture.context.controller)).toBe(true) + expect(fixture.backend.authorizeBlobFrame(frame, { ...fixture.context.controller, connectionId: 'foreign' })).toBe(false) + const call = remoteImageToolCallSchema.parse(decodeRemoteImageToolMessage(frame.payload)) + const payload = encodeRemoteImageToolMessage({ callId: call.callId, error: 'Controlled test failure' }) + const reply = { header: { ...frame.header, direction: 'main-to-agent' as const, payloadLength: payload.byteLength }, payload } + await expect(fixture.backend.onBlobFrame(reply, { ...fixture.context, channelId: 'image-channel', controller: { ...fixture.context.controller, connectionId: 'foreign' } })).rejects.toThrow('authority') + await fixture.backend.onBlobFrame(reply, { ...fixture.context, channelId: 'image-channel' }) + expect(await result).toMatchObject({ isError: true }) + await invoke(fixture, 'runtime/completePrompt', { bindingId: 'binding-1', operationId: 'request-1', requestId: 'request-1' }) + await expect(fetch(accepted.imageToolUrl)).rejects.toThrow() + const next = await invoke(fixture, 'runtime/preparePrompt', fixture.preparation({ + requestId: 'request-2', operationId: 'request-2', promptSequence: 1, + imageTool: { channelId: 'image-channel-2', channelEpoch: '2', description: 'Refreshed image catalog' } + })) as { imageToolUrl: string } + expect(next.imageToolUrl).not.toBe(accepted.imageToolUrl) + } finally { await client.close(); await fixture.backend.dispose() } + }) + + it('rejects image tool preparation in Ask before launching a process or bridge', async () => { + const fixture = harness() + try { + await open(fixture) + await expect(invoke(fixture, 'runtime/preparePrompt', fixture.preparation({ + imageTool: { channelId: 'image-channel', channelEpoch: '1', description: 'Images' } + }))).rejects.toThrow('Execute') + expect(fixture.launches).toHaveLength(0) + expect(fixture.lifecycle).not.toContain('bridge-listen') + } finally { await fixture.backend.dispose() } + }) it.each([false, true])('allows mode changes only on a shared Session (shared=%s)', async (shared) => { const fixture = harness({ agentOwned: true, shareOwnedProcesses: shared, workMode: 'execute' }) try { diff --git a/src/agent-daemon/runtime-acp-backend.ts b/src/agent-daemon/runtime-acp-backend.ts index f923f30f..336b6ce4 100644 --- a/src/agent-daemon/runtime-acp-backend.ts +++ b/src/agent-daemon/runtime-acp-backend.ts @@ -1,4 +1,6 @@ import { z } from 'zod' +import { AgentImageToolMcp } from './image-tool-mcp' +import { remoteImageToolMcpName } from '../shared/remote-image-tool-node' import { remoteQuestionResponseSchema } from '../shared/remote-question-contracts' import { createHash } from 'node:crypto' import { @@ -338,6 +340,7 @@ type BindingState = { ownedPromptStartTimer?: NodeJS.Timeout modelBridgePolicy?: ModelBridgePolicy modelBridgeClient?: ModelBridgeBlobClient + imageTool?: AgentImageToolMcp modelBridgeBroker?: ModelBridgeBrokerServer ownedAcp?: AgentOwnedAcpPrompt workspaceDirectory?: string @@ -558,6 +561,18 @@ export class RuntimeAcpBackend { 'identity' ) } + const imageBinding = [...this.#bindings.values()].find(binding => + binding.controllerId === context.controller.controllerId && + binding.controllerGeneration === context.controller.generation && + binding.connectionId === context.controller.connectionId && + binding.activeOperationId !== undefined && + binding.imageTool?.descriptor.channelId === frame.header.channelId && + binding.imageTool.descriptor.channelEpoch === frame.header.channelEpoch && + context.channelId === frame.header.channelId) + if (imageBinding) { + imageBinding.imageTool!.onReply(frame.payload) + return + } const matches = [...this.#bindings.values()].filter( (binding) => binding.controllerId === context.controller.controllerId && @@ -613,10 +628,10 @@ export class RuntimeAcpBackend { binding.controllerId === controller.controllerId && binding.controllerGeneration === controller.generation && binding.connectionId === controller.connectionId && - binding.modelBridgeClient?.binding.channelId === - frame.header.channelId && - binding.modelBridgeClient.binding.channelEpoch === - frame.header.channelEpoch && + ((binding.modelBridgeClient?.binding.channelId === frame.header.channelId && + binding.modelBridgeClient.binding.channelEpoch === frame.header.channelEpoch) || + (binding.imageTool?.descriptor.channelId === frame.header.channelId && + binding.imageTool.descriptor.channelEpoch === frame.header.channelEpoch)) && frame.header.connectionId === binding.connectionId && frame.header.generation === binding.controllerGeneration ) @@ -1140,11 +1155,37 @@ export class RuntimeAcpBackend { } } } + if (preparation.imageTool && (preparation.workMode !== 'execute' || !this.#options.blobSink)) { + throw new RuntimeAcpBackendError('Image tools require Execute and a desktop transport', 'identity') + } if (requestedModelBridgePolicy !== undefined) { await this.#startModelBridge(binding, preparation, workspace) } const acceptedAt = new Date(this.#now()).toISOString() + binding.imageTool?.close() + binding.imageTool = undefined + if (preparation.imageTool) { + let sequence = 0n + const imageTool = new AgentImageToolMcp(preparation.imageTool, async payload => { + if (binding.activeOperationId !== preparation.operationId || binding.transportState === 'detached') { + throw new Error('Image tool prompt is no longer connected') + } + await this.#options.blobSink!({ + header: { + protocolMajor: AGENT_PROTOCOL_VERSION.major, protocolMinor: AGENT_PROTOCOL_VERSION.minor, + kind: 'blob', direction: 'agent-to-main', + connectionId: binding.connectionId, generation: binding.controllerGeneration, + channelId: preparation.imageTool!.channelId, channelEpoch: preparation.imageTool!.channelEpoch, + sequence: String(++sequence), payloadLength: payload.byteLength + }, payload + }, { bindingId: binding.request.bindingId, controllerId: binding.controllerId, + controllerGeneration: binding.controllerGeneration }) + }) + binding.imageTool = imageTool + await imageTool.start() + } const acceptance = remotePromptOperationAcceptanceSchema.parse({ + ...(binding.imageTool?.url ? { imageToolUrl: binding.imageTool.url } : {}), bindingId: preparation.bindingId, operationId: preparation.operationId, requestId: preparation.requestId, @@ -1448,6 +1489,9 @@ export class RuntimeAcpBackend { bindingId: binding.request.bindingId, controllerId: binding.controllerId, workspaceDirectory: binding.workspaceDirectory, + mcpServers: () => binding.imageTool?.url ? [{ + type: 'http', name: remoteImageToolMcpName(binding.request.bindingId), url: binding.imageTool.url, headers: [] + }] : [], ...(binding.modelBridgePolicy === undefined ? {} : { @@ -1461,7 +1505,8 @@ export class RuntimeAcpBackend { transport: binding.sharedProcess.transport, prepareSession: async (sessionId: string, operationId: string, workMode: 'ask' | 'execute') => { await binding.sharedProcess!.transport.setModelRoute( - sessionId, operationId, binding.modelBridgeBroker!.socketPath, workMode + sessionId, operationId, binding.modelBridgeBroker!.socketPath, workMode, + binding.imageTool ? remoteImageToolMcpName(binding.request.bindingId) : undefined ) } } : {}), @@ -1816,6 +1861,8 @@ export class RuntimeAcpBackend { poisonIfActive: boolean, preserveRuntime = false ): Promise { + binding.imageTool?.close() + binding.imageTool = undefined const broker = binding.modelBridgeBroker const client = binding.modelBridgeClient binding.modelBridgeBroker = undefined @@ -2141,6 +2188,8 @@ export class RuntimeAcpBackend { return } binding.transportState = 'detached' + binding.imageTool?.close() + binding.imageTool = undefined if (binding.modelBridgeClient !== undefined) { await this.#closeModelBridge(binding, false, true).catch( () => undefined diff --git a/src/main/agent/acp-remote-runtime.ts b/src/main/agent/acp-remote-runtime.ts index 5940c3a2..08c71e3b 100644 --- a/src/main/agent/acp-remote-runtime.ts +++ b/src/main/agent/acp-remote-runtime.ts @@ -1,4 +1,5 @@ import { createHash, randomUUID } from 'node:crypto' +import { remoteImageToolMcpName } from '../../shared/remote-image-tool-node' import { remoteQuestionSchema } from '../../shared/remote-question-contracts' import type { AgentQuestionAnswer } from '../../shared/contracts' import { isDeepStrictEqual } from 'node:util' @@ -191,6 +192,7 @@ type SessionRecord = { sessionId: string context: ChannelContext modelBridge?: RemoteModelBridgeSession + imageToolUrl?: string sendHistoryWithNextPrompt: boolean ownedPromptAttached?: boolean } @@ -1183,7 +1185,9 @@ export class AcpRemoteRuntime implements AgentRuntime { raw = await this.awaitOperation( context, '启动远端请求', - context.channel.preparePrompt(preparation), + request.imageToolBinding + ? context.channel.preparePrompt(preparation, request.imageToolBinding, signal) + : context.channel.preparePrompt(preparation), signal ) } catch (error) { @@ -1596,7 +1600,7 @@ export class AcpRemoteRuntime implements AgentRuntime { state.agent.loadSession({ sessionId: binding.acpSessionId, cwd: this.options.workspacePath, - mcpServers: [] + mcpServers: this.imageMcpServers(binding.bindingId, prepared.acceptance.imageToolUrl) }), signal ) @@ -1607,7 +1611,7 @@ export class AcpRemoteRuntime implements AgentRuntime { state.agent.resumeSession({ sessionId: binding.acpSessionId, cwd: this.options.workspacePath, - mcpServers: [] + mcpServers: this.imageMcpServers(binding.bindingId, prepared.acceptance.imageToolUrl) }), signal ) @@ -1619,6 +1623,7 @@ export class AcpRemoteRuntime implements AgentRuntime { sessionId: binding.acpSessionId, context, modelBridge: prepared.modelBridge, + imageToolUrl: prepared.acceptance.imageToolUrl, sendHistoryWithNextPrompt: false } pendingModelBridge = undefined @@ -1668,7 +1673,7 @@ export class AcpRemoteRuntime implements AgentRuntime { '创建会话', state.agent.newSession({ cwd: this.options.workspacePath, - mcpServers: [] + mcpServers: this.imageMcpServers(binding.bindingId, prepared.acceptance.imageToolUrl) }), signal ) @@ -1697,6 +1702,7 @@ export class AcpRemoteRuntime implements AgentRuntime { sessionId: response.sessionId, context, modelBridge: prepared.modelBridge, + imageToolUrl: prepared.acceptance.imageToolUrl, sendHistoryWithNextPrompt: true } pendingModelBridge = undefined @@ -1725,6 +1731,10 @@ export class AcpRemoteRuntime implements AgentRuntime { } } + private imageMcpServers(bindingId: string, url?: string) { + return url ? [{ type: 'http' as const, name: remoteImageToolMcpName(bindingId), url, headers: [] }] : [] + } + private async getSession( conversationId: string, request: AgentExecutionRequest, @@ -1782,6 +1792,13 @@ export class AcpRemoteRuntime implements AgentRuntime { current.modelBridge = prepared.modelBridge if (this.options.modelProfile !== undefined) { current.ownedPromptAttached = false + } else if (prepared.acceptance.imageToolUrl || current.imageToolUrl) { + const state = await this.getState(current.context, signal) + await this.awaitOperation(current.context, 'Refresh image tools', state.agent.resumeSession({ + sessionId: current.sessionId, cwd: this.options.workspacePath, + mcpServers: this.imageMcpServers(current.binding.bindingId, prepared.acceptance.imageToolUrl) + }), signal) + current.imageToolUrl = prepared.acceptance.imageToolUrl } return current } catch (error) { diff --git a/src/main/agent/continue-runtime.test.ts b/src/main/agent/continue-runtime.test.ts index 097dd3fe..91738932 100644 --- a/src/main/agent/continue-runtime.test.ts +++ b/src/main/agent/continue-runtime.test.ts @@ -464,6 +464,34 @@ describe('ContinueAgentRuntime', () => { expect(gateway.grantCustomMcp).not.toHaveBeenCalled() }) + it.each([undefined, 'existing-capability'])('cleans up image capabilities after custom MCP preparation fails (existing=%s)', async existingToken => { + const gateway = { + getEndpoint: () => 'http://127.0.0.1:4567/mcp', + bindImageTool: vi.fn(() => existingToken ?? 'image-capability'), + grantCustomMcp: vi.fn(() => 'custom-capability'), + prepareCustomMcpTools: vi.fn(async () => { throw new Error('Custom MCP unavailable') }), + revoke: vi.fn() + } + const context = { requestId: randomUUID(), conversationId: 'conversation-1', messageId: 'message-1', workMode: 'execute' as const } + const runtime = new ContinueAgentRuntime({ + binaryPath: '', configPath: 'C:\\safe config\\continue.yaml', defaultWorkspace: process.cwd(), + hostCacheRoot: 'C:\\safe\\continue-host', + knowledgeGateway: gateway as unknown as KnowledgeMcpGateway, + mcpServers: [{ id: randomUUID(), name: 'Custom', description: '', enabled: true, allowDynamicTools: false, + assignments: ['continue'], secretConfigured: false, transport: 'http', url: 'https://mcp.test' }], + createHostAdapter: () => ({ getPreparedHost: mocks.prepareHost, run: mocks.runHost, dispose: mocks.disposeHost }) + }) + try { + await expect((async () => { + for await (const event of runtime.run({ ...context, prompt: 'Draw an image', knowledgeCapabilityToken: existingToken, + imageToolBinding: { context, describe: async () => 'Images', call: vi.fn() } + }, new AbortController().signal)) void event + })()).rejects.toThrow('Custom MCP unavailable') + expect(gateway.revoke.mock.calls).toEqual(existingToken ? [['custom-capability']] : [['custom-capability'], ['image-capability']]) + expect(mocks.runHost).not.toHaveBeenCalled() + } finally { await runtime.dispose() } + }) + it('adds assigned Skill instructions to the Continue prompt', async () => { let hostOptions: ContinueHostAdapterOptions | undefined const runtime = new ContinueAgentRuntime({ diff --git a/src/main/agent/continue-runtime.ts b/src/main/agent/continue-runtime.ts index 50f7d757..9f49eb55 100644 --- a/src/main/agent/continue-runtime.ts +++ b/src/main/agent/continue-runtime.ts @@ -455,48 +455,46 @@ export class ContinueAgentRuntime implements AgentRuntime { } const execute = request.workMode === 'execute' + const imageCapabilityToken = request.imageToolBinding && execute + ? this.options.knowledgeGateway?.bindImageTool(request.imageToolBinding, signal, request.knowledgeCapabilityToken) + : undefined const knowledgeEndpoint = this.options.knowledgeGateway?.getEndpoint() const knowledgeCapability = - request.knowledgeCapabilityToken && knowledgeEndpoint + (imageCapabilityToken ?? request.knowledgeCapabilityToken) && knowledgeEndpoint ? { endpoint: knowledgeEndpoint, - token: request.knowledgeCapabilityToken + token: (imageCapabilityToken ?? request.knowledgeCapabilityToken)! } : undefined let customMcpCapability: | { endpoint: string; token: string } | undefined - if ( - execute && - knowledgeEndpoint && - this.options.mcpServers?.length - ) { - const token = this.options.knowledgeGateway?.grantCustomMcp( - request.requestId, - this.options.mcpServers, - signal - ) - if (token) { - customMcpCapability = { - endpoint: knowledgeEndpoint, - token - } - try { - await this.options.knowledgeGateway?.prepareCustomMcpTools( - token, - signal - ) - } catch (error) { - this.options.knowledgeGateway?.revoke(token) - throw error - } - } - } let result: ContinueHostRunResult const emittedTools = new Map() let emittedText = '' const requestQuestionIds = new Set() try { + if ( + execute && + knowledgeEndpoint && + this.options.mcpServers?.length + ) { + const token = this.options.knowledgeGateway?.grantCustomMcp( + request.requestId, + this.options.mcpServers, + signal + ) + if (token) { + customMcpCapability = { + endpoint: knowledgeEndpoint, + token + } + await this.options.knowledgeGateway?.prepareCustomMcpTools( + token, + signal + ) + } + } const host = this.getHostAdapter( binaryPath, execute || knowledgeCapability ? 'agent' : 'chat' @@ -643,6 +641,7 @@ export class ContinueAgentRuntime implements AgentRuntime { customMcpCapability.token ) } + if (imageCapabilityToken && imageCapabilityToken !== request.knowledgeCapabilityToken) this.options.knowledgeGateway?.revoke(imageCapabilityToken) } if (!result.text) { throw new Error('Continue CLI 未返回内容') diff --git a/src/main/agent/deepseek-harness-acp-e2e.test.ts b/src/main/agent/deepseek-harness-acp-e2e.test.ts index 66eb8cc4..9fae535d 100644 --- a/src/main/agent/deepseek-harness-acp-e2e.test.ts +++ b/src/main/agent/deepseek-harness-acp-e2e.test.ts @@ -41,6 +41,64 @@ import { LocalRuntimeRegistry } from './local-runtime-registry' import { createAgentRuntime } from './create-runtime' import { defaultRuntimeSettings } from '../../shared/contracts' import type { ResolvedRuntimeSettings } from '../runtime-settings-store' +import { AssistantDatabase } from '../assistant/assistant-database' +import { ImageGenerationService } from './image-generation-service' + +vi.mock('electron', () => ({ nativeImage: {} })) + +it('generates and edits durable images through the controlled Harness Main proxy and refreshes it in Ask', async () => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'goodbuddy-dsh-images-'))) + const database = new AssistantDatabase(join(root, 'assistant.sqlite')) + database.initialize(root) + const conversationId = crypto.randomUUID() + const profileId = crypto.randomUUID() + const fetcher = vi.fn(async () => Response.json({ data: [{ b64_json: 'iVBORw0KGgo=' }] })) + const service = new ImageGenerationService({ database, fetcher, getSettings: async () => ({ defaultImageModelProfileId: profileId, modelProfiles: [{ + id: profileId, name: 'Image fixture', modelName: 'fixture', protocol: 'openai-images-generations', + allowConversationInvocation: true, baseUrl: 'https://image.test/v1', authentication: 'none' + }] }) }) + let input: Record = { intent: 'create', prompt: 'Blue circle' } + const inProcess = createInProcessLaunch(root, { + stream(options) { + const prompt = latestUserText(options) + const tool = options.tools?.find(tool => tool.name === 'generate_image') + if (prompt === 'ask-image') { + expect(tool).toBeUndefined() + return textResponse('Ask is read-only') + } + expect(tool).toBeDefined() + expect(tool?.description).toContain(profileId) + if (!toolResultText(options, prompt)) return toolCall(prompt, 'generate_image', input) + expect(toolResultText(options, prompt)).toContain('completed') + return textResponse('IMAGE_SAVED') + } + }) + const runtime = new DeepSeekHarnessRuntime({ defaultWorkspace: root, baseUrl: 'https://chat.test/v1', model: 'fixture', + launch: inProcess.launch, credentialRefs: { [CREDENTIAL_REF]: 'fixture' }, toolProvider: new ModelToolProvider(root), + initializationTimeoutMs: 20_000, promptTimeoutMs: 20_000, shutdownTimeoutMs: 5_000 }) + try { + for (const intent of ['create', 'edit'] as const) { + const messageId = crypto.randomUUID() + const requestId = crypto.randomUUID() + database.saveLocalConversations([{ header: { id: conversationId, title: 'Images', updatedAt: Date.now() }, + messages: [{ id: messageId, role: 'assistant', content: '', createdAt: Date.now(), state: 'complete' }] }]) + const events = await collect(runtime.run({ requestId, conversationId, prompt: `${intent}-${requestId}`, workMode: 'execute', + imageToolBinding: service.bind({ conversationId, messageId, requestId, workMode: 'execute' }) }, new AbortController().signal, async () => 'once')) + expect(events.at(-1)?.type).toBe('done') + const operation = database.getConversation(conversationId).messages.find(message => message.id === messageId)!.imageOperations![0]! + expect(operation.state).toBe('completed') + input = { intent: 'edit', prompt: 'Turn red', sourceArtifactIds: operation.artifactIds } + } + await collect(runtime.run({ requestId: crypto.randomUUID(), conversationId, prompt: 'ask-image', workMode: 'ask' }, new AbortController().signal)) + expect(fetcher).toHaveBeenCalledTimes(2) + expect(String(fetcher.mock.calls[1]![0])).toMatch(/images\/edits$/u) + } finally { + await runtime.dispose() + await service.dispose() + database.close() + await rm(root, { recursive: true, force: true }) + } +}, 60_000) const CREDENTIAL_REF = 'GOODBUDDY_HARNESS_MODEL_API_KEY' const SKILL_CALL_ID = 'e2e-skill-call' diff --git a/src/main/agent/deepseek-harness-runtime.ts b/src/main/agent/deepseek-harness-runtime.ts index 6ba14de6..0e342605 100644 --- a/src/main/agent/deepseek-harness-runtime.ts +++ b/src/main/agent/deepseek-harness-runtime.ts @@ -290,7 +290,7 @@ function isMainWebTool( ) } -function dshCompatibleWebInputSchema( +function dshCompatibleMainInputSchema( schema: Record ): Record { const compatible: Record = {} @@ -311,7 +311,7 @@ function dshCompatibleWebInputSchema( propertySchema && typeof propertySchema === 'object' && !Array.isArray(propertySchema) - ? dshCompatibleWebInputSchema( + ? dshCompatibleMainInputSchema( propertySchema as Record ) : propertySchema @@ -325,7 +325,7 @@ function dshCompatibleWebInputSchema( typeof value === 'object' && !Array.isArray(value) ) { - compatible.items = dshCompatibleWebInputSchema( + compatible.items = dshCompatibleMainInputSchema( value as Record ) continue @@ -335,7 +335,7 @@ function dshCompatibleWebInputSchema( candidate && typeof candidate === 'object' && !Array.isArray(candidate) - ? dshCompatibleWebInputSchema( + ? dshCompatibleMainInputSchema( candidate as Record ) : candidate @@ -350,8 +350,8 @@ function proxyToolInputSchema( ReturnType >[number] ): Record { - return isMainWebTool(tool) - ? dshCompatibleWebInputSchema(tool.inputSchema) + return isMainWebTool(tool) || tool.name === 'generate_image' + ? dshCompatibleMainInputSchema(tool.inputSchema) : tool.inputSchema } @@ -368,7 +368,7 @@ function boundedProxyToolCatalog( const catalog = tools.filter( (tool) => isMainWebTool(tool) || - (workMode === 'execute' && tool.source === 'mcp') + (workMode === 'execute' && (tool.source === 'mcp' || tool.name === 'generate_image')) ) if (catalog.length > MAX_MCP_PROXY_TOOLS) { throw new Error( @@ -386,7 +386,7 @@ function boundedProxyToolCatalog( names.add(tool.name) const description = tool.description.slice( 0, - MAX_MCP_TOOL_DESCRIPTION_CHARACTERS + tool.name === 'generate_image' ? imageToolDescriptionLimit : MAX_MCP_TOOL_DESCRIPTION_CHARACTERS ) const inputSchema = proxyToolInputSchema(tool) let serialized: string @@ -838,6 +838,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime { const run = this.activeRuns.get(params.sessionId) if (!run?.toolProvider || run.closed) return { tools: [] } const context = { + imageToolBinding: run.request.imageToolBinding, conversationId: run?.request.conversationId ?? 'deepseek-harness-tool-catalog', @@ -855,7 +856,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime { ) const catalog = boundedProxyToolCatalog(tools, context.workMode) this.proxyToolCatalogs.set(params.sessionId, tools.filter( - (tool) => tool.source === 'mcp' || isMainWebTool(tool) + (tool) => tool.source === 'mcp' || isMainWebTool(tool) || tool.name === 'generate_image' )) return { tools: catalog } } @@ -881,6 +882,8 @@ export class DeepSeekHarnessRuntime implements AgentRuntime { ) } const context = { + imageToolBinding: run.request.imageToolBinding, + ...(name === 'generate_image' ? { toolCallId: typeof params.callId === 'string' ? params.callId : crypto.randomUUID() } : {}), conversationId: run.request.conversationId, browserTabId: run.request.browserTabId, workMode: @@ -894,7 +897,7 @@ export class DeepSeekHarnessRuntime implements AgentRuntime { const tool = tools.find( (candidate) => candidate.name === name && - (candidate.source === 'mcp' || + (candidate.source === 'mcp' || candidate.name === 'generate_image' || isMainWebTool(candidate)) ) if (!tool) { @@ -1754,3 +1757,4 @@ export class DeepSeekHarnessRuntime implements AgentRuntime { ).catch(() => undefined) } } +import { imageToolDescriptionLimit } from '../../shared/image-generation-contracts' diff --git a/src/main/agent/goodbuddy-harness-control-plane.ts b/src/main/agent/goodbuddy-harness-control-plane.ts index 02c1fa3c..adb42e1f 100644 --- a/src/main/agent/goodbuddy-harness-control-plane.ts +++ b/src/main/agent/goodbuddy-harness-control-plane.ts @@ -221,7 +221,7 @@ function parseProxyToolCatalog( !/^[a-zA-Z0-9_-]{1,64}$/u.test(tool.name) || names.has(tool.name) || typeof tool.description !== 'string' || - tool.description.length > 1_000 || + tool.description.length > (tool.name === 'generate_image' ? imageToolDescriptionLimit : 1_000) || !tool.inputSchema || typeof tool.inputSchema !== 'object' || Array.isArray(tool.inputSchema) @@ -885,6 +885,7 @@ export class GoodBuddyHarnessControlPlane { { sessionId, name: tool.name, + callId: randomUUID(), arguments: argumentsValue } ) @@ -1450,3 +1451,4 @@ export class GoodBuddyHarnessControlPlane { await this.disposing } } +import { imageToolDescriptionLimit } from '../../shared/image-generation-contracts' diff --git a/src/main/agent/image-generation-service.test.ts b/src/main/agent/image-generation-service.test.ts new file mode 100644 index 00000000..5decce14 --- /dev/null +++ b/src/main/agent/image-generation-service.test.ts @@ -0,0 +1,231 @@ +import { randomUUID } from 'node:crypto' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' +import { AssistantDatabase } from '../assistant/assistant-database' +import { ImageGenerationService, type ImageServiceSettings, type ImageGenerationServiceOptions } from './image-generation-service' +import { imageOperationSchema, type ImageOperation } from '../../shared/image-generation-contracts' +import { KnowledgeMcpGateway } from './knowledge-mcp-gateway' +import { ModelToolProvider } from './model-tool-provider' + +vi.mock('electron', () => ({ nativeImage: { createFromBuffer: () => ({ isEmpty: () => false, toPNG: () => Buffer.from(png, 'base64') }) } })) +const png = 'iVBORw0KGgo=' +const cleanups: (() => void | Promise)[] = [] +afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup() }) + +function setup(fetcher = vi.fn(async () => Response.json({ data: [{ b64_json: png }] })), observers: Pick = {}) { + const database = new AssistantDatabase(':memory:') + database.initialize(process.cwd()) + cleanups.push(() => database.close()) + const context = { conversationId: randomUUID(), messageId: randomUUID(), requestId: randomUUID(), workMode: 'execute' as const } + const message = { id: context.messageId, role: 'assistant' as const, content: '', createdAt: Date.now(), state: 'complete' as const } + const header = { id: context.conversationId, title: 'Images', updatedAt: Date.now() } + database.saveLocalConversations([{ header, messages: [message] }]) + const profile = { id: randomUUID(), name: 'Image A', modelName: 'image-a', protocol: 'openai-images-generations', + allowConversationInvocation: true, baseUrl: 'https://provider.test/v1', authentication: 'api-key' as const, apiKey: 'test-secret' } + const settings: ImageServiceSettings = { modelProfiles: [profile], defaultImageModelProfileId: profile.id } + const events: ImageOperation[] = [] + const service = new ImageGenerationService({ database, getSettings: async () => settings, fetcher, + onOperation: operation => { imageOperationSchema.parse(operation); events.push(operation); observers.onOperation?.(operation) }, onUsage: observers.onUsage, onError: observers.onError }) + service.initialize() + cleanups.push(() => service.dispose()) + return { database, service, context, fetcher, settings, profile, events, header, message } +} + +describe('Main conversation image service', () => { + it('does not let a destroyed Renderer or usage observer prevent submission or overwrite completion', async () => { + const onError = vi.fn() + const h = setup(vi.fn(async () => Response.json({ data: [{ b64_json: png }], usage: { input_tokens: 2, output_tokens: 3 } })), { + onOperation: () => { throw new Error('Renderer is destroyed') }, + onUsage: () => { throw new Error('Usage observer failed') }, onError + }) + const operation = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'observer') + expect(operation.state).toBe('completed') + expect(h.service.getOperation(h.context.conversationId, operation.id).state).toBe('completed') + expect(h.fetcher).toHaveBeenCalledOnce() + expect(onError).toHaveBeenCalled() + }) + + it('returns the known persistence failure instead of a stale running state when the database stops accepting writes', async () => { + const onError = vi.fn() + const h = setup(undefined, { onError }) + h.fetcher.mockImplementation(async () => { + vi.spyOn(h.database, 'saveConversationImageOperation').mockImplementation(() => { throw new Error('Disk full') }) + return Response.json({ data: [{ b64_json: png }] }) + }) + const operation = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'disk-full') + expect(operation).toMatchObject({ state: 'failed', error: 'Disk full', artifactIds: [] }) + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'Disk full' })) + expect(h.fetcher).toHaveBeenCalledOnce() + }) + it('saves before publishing, preserves metadata against stale renderer saves, and edits durable history', async () => { + const h = setup() + const first = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue circle' }, 'first') + expect(first.state).toBe('completed') + expect(h.database.getArtifact(first.artifactIds[0]!).content).toContain(png) + h.database.saveLocalConversations([{ header: h.header, messages: [h.message] }]) + expect(h.service.getOperation(h.context.conversationId, first.id)).toEqual(first) + expect(h.database.getConversation(h.context.conversationId).messages[0]!.artifactIds).toEqual(first.artifactIds) + h.settings.defaultImageModelProfileId = randomUUID() + const second = await h.service.bind({ ...h.context, requestId: randomUUID() }).call({ intent: 'edit', prompt: 'Turn red', sourceArtifactIds: first.artifactIds }, 'edit') + expect(second.modelProfileId).toBe(first.modelProfileId) + expect(String(h.fetcher.mock.calls[1]![0])).toMatch(/images\/edits$/u) + expect((h.fetcher.mock.calls[1]![1]!.body as FormData).get('image')).toBeInstanceOf(Blob) + expect(h.events.map(event => event.state)).toEqual(['running', 'saving', 'completed', 'running', 'saving', 'completed']) + }) + + it('persists upload bytes and rejects cross-conversation or missing source references', async () => { + const h = setup() + const ids = h.service.persistUploads(h.context, [{ name: 'upload.png', mediaType: 'image/png', data: png }]) + h.database.saveLocalConversations([{ header: h.header, messages: [h.message] }]) + expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageSourceArtifactIds).toEqual(ids) + const binding = h.service.bind(h.context) + expect((await binding.call({ intent: 'edit', prompt: 'Blue', sourceArtifactIds: ids }, 'upload')).state).toBe('completed') + await expect(binding.call({ intent: 'edit', prompt: 'Blue', sourceArtifactIds: [randomUUID()] }, 'foreign')).rejects.toThrow('not available') + await expect(binding.call({ intent: 'edit', prompt: 'Blue' }, 'missing')).rejects.toThrow('requires source') + expect(h.fetcher).toHaveBeenCalledTimes(1) + }) + + it('rejects absent, disabled, deleted defaults without silent fallback and rechecks current mode', async () => { + const h = setup() + let mode: 'ask' | 'execute' = 'execute' + const binding = h.service.bind(h.context, () => mode) + expect(await binding.describe()).toContain(h.profile.id) + h.profile.allowConversationInvocation = false + await expect(binding.call({ intent: 'create', prompt: 'Blue' }, 'disabled')).rejects.toThrow('unavailable') + h.profile.allowConversationInvocation = true + h.settings.defaultImageModelProfileId = randomUUID() + await expect(binding.call({ intent: 'create', prompt: 'Blue' }, 'deleted')).rejects.toThrow('unavailable') + h.settings.defaultImageModelProfileId = null + await expect(binding.call({ intent: 'create', prompt: 'Blue' }, 'absent')).rejects.toThrow('default') + mode = 'ask' + await expect(binding.call({ intent: 'create', prompt: 'Blue', modelProfileId: h.profile.id }, 'ask')).rejects.toThrow('Execute') + expect(h.fetcher).not.toHaveBeenCalled() + }) + + it('continues after chat abort and pins the accepted profile while a replacement binding starts', async () => { + let resolve!: (response: Response) => void + const h = setup(vi.fn(() => new Promise(done => { resolve = done }))) + const controller = new AbortController() + const pending = h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'call', controller.signal) + await vi.waitFor(() => expect(h.fetcher).toHaveBeenCalledOnce()) + const providerSignal = h.fetcher.mock.calls[0]![1]!.signal! + controller.abort() + expect((await pending).state).toBe('running') + expect(providerSignal.aborted).toBe(false) + h.profile.modelName = 'replacement' + h.profile.allowConversationInvocation = false + resolve(Response.json({ data: [{ b64_json: png }] })) + await vi.waitFor(() => expect(h.events.at(-1)?.state).toBe('completed')) + expect(h.events.at(-1)).toMatchObject({ modelName: 'image-a', modelProfileName: 'Image A' }) + expect(JSON.parse(h.fetcher.mock.calls[0]![1]!.body as string).model).toBe('image-a') + await expect(h.service.bind({ ...h.context, requestId: randomUUID() }).call({ intent: 'create', prompt: 'Other' }, 'other')).rejects.toThrow('unavailable') + }) + + it('deduplicates one invocation, creates a new operation on manual regenerate, and retains late cancellation output', async () => { + let resolve!: (response: Response) => void + const h = setup(vi.fn(() => new Promise(done => { resolve = done }))) + const binding = h.service.bind(h.context) + const input = { intent: 'create', prompt: 'Blue' } + const one = binding.call(input, 'same') + const two = binding.call(input, 'same') + await vi.waitFor(() => expect(h.events).toHaveLength(1)) + h.service.cancel(h.context.conversationId, h.events[0]!.id) + resolve(Response.json({ data: [{ b64_json: png }] })) + const completed = await one + expect(await two).toEqual(completed) + expect(completed).toMatchObject({ state: 'completed', cancellationRequested: true }) + expect(h.fetcher).toHaveBeenCalledOnce() + h.fetcher.mockImplementation(async () => Response.json({ data: [{ b64_json: png }] })) + const regenerated = await h.service.regenerate(h.context, completed.id) + expect(regenerated.id).not.toBe(completed.id) + expect(h.fetcher).toHaveBeenCalledTimes(2) + }) + + it('never falls back to a paid generation after an edit rejection and redacts provider errors', async () => { + const h = setup(vi.fn(async () => Response.json({ error: { message: 'Image editing not supported Authorization: Bearer private-key' } }, { status: 405 }))) + const ids = h.service.persistUploads(h.context, [{ name: 'source', mediaType: 'image/png', data: png }]) + const result = await h.service.bind(h.context).call({ intent: 'edit', prompt: 'Blue', sourceArtifactIds: ids }, 'edit') + expect(result.state).toBe('failed') + expect(result.error).toContain('405') + expect(result.error).not.toContain('private-key') + expect(h.fetcher).toHaveBeenCalledOnce() + }) + + it('does not resurrect a deleted conversation or retain its late artifact', async () => { + let resolve!: (response: Response) => void + const h = setup(vi.fn(() => new Promise(done => { resolve = done }))) + const pending = h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'deleted') + await vi.waitFor(() => expect(h.fetcher).toHaveBeenCalledOnce()) + h.database.deleteLocalConversation(h.context.conversationId) + resolve(Response.json({ data: [{ b64_json: png }] })) + await pending + expect(() => h.database.getConversation(h.context.conversationId)).toThrow() + expect(h.database.listArtifacts()).toHaveLength(0) + }) + + it('marks unfinished operations unconfirmed on initialization without issuing requests', () => { + const h = setup() + h.database.saveConversationImageOperation({ id: randomUUID(), conversationId: h.context.conversationId, messageId: h.context.messageId, + requestId: h.context.requestId, callId: 'old', modelProfileId: h.profile.id, modelName: h.profile.name, + input: { intent: 'create', prompt: 'Blue', sourceArtifactIds: [] }, state: 'running', artifactIds: [], createdAt: 1, updatedAt: 1 }) + h.service.initialize() + expect(h.database.getConversation(h.context.conversationId).messages[0]!.imageOperations![0]!.state).toBe('unconfirmed') + expect(h.fetcher).not.toHaveBeenCalled() + }) + + it('reports provider-side uncertainty after network loss and local cancellation without retrying', async () => { + const h = setup(vi.fn(async () => { throw new TypeError('fetch failed') })) + const uncertain = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'network') + expect(uncertain.state).toBe('unconfirmed') + h.fetcher.mockImplementation((_url, init) => new Promise((_resolve, reject) => { + init!.signal!.addEventListener('abort', () => reject(init!.signal!.reason), { once: true }) + })) + const pending = h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'cancel') + await vi.waitFor(() => expect(h.fetcher).toHaveBeenCalledTimes(2)) + h.service.cancel(h.context.conversationId, h.events.at(-1)!.id) + expect(await pending).toMatchObject({ state: 'stopped', cancellationRequested: true }) + expect(h.fetcher).toHaveBeenCalledTimes(2) + }) + + it('reuses an accepted invocation across bindings and does not skip a disabled historical model', async () => { + const h = setup() + const created = await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'same') + expect((await h.service.bind(h.context).call({ intent: 'create', prompt: 'Blue' }, 'same')).id).toBe(created.id) + const other = { ...h.profile, id: randomUUID(), name: 'Other', modelName: 'other' } + h.settings.modelProfiles = [h.profile, other] + h.settings.defaultImageModelProfileId = other.id + h.profile.allowConversationInvocation = false + await expect(h.service.bind(h.context).call({ intent: 'edit', prompt: 'Red', sourceArtifactIds: created.artifactIds }, 'blocked-edit')).rejects.toThrow('unavailable') + const edited = await h.service.bind(h.context).call({ intent: 'edit', prompt: 'Red', sourceArtifactIds: created.artifactIds, modelProfileId: other.id }, 'explicit-edit') + expect(edited.modelProfileId).toBe(other.id) + expect(h.fetcher).toHaveBeenCalledTimes(2) + }) + + it('dispatches the native Model tool through its scoped binding and denies Ask', async () => { + const h = setup() + const provider = new ModelToolProvider(process.cwd()) + cleanups.push(() => provider.dispose()) + const context = { conversationId: h.context.conversationId, workMode: 'execute' as const, imageToolBinding: h.service.bind(h.context), toolCallId: 'native' } + const result = await provider.callTool('generate_image', { intent: 'create', prompt: 'Blue' }, new AbortController().signal, context) + expect(JSON.stringify(result)).toContain('completed') + expect(JSON.stringify(result)).not.toContain(png) + await expect(provider.callTool('generate_image', { intent: 'create', prompt: 'Blue' }, new AbortController().signal, { ...context, workMode: 'ask' })).rejects.toThrow() + }) + + it('round trips image-only MCP discovery/call and does not abort accepted work on capability revocation', async () => { + const h = setup() + const gateway = new KnowledgeMcpGateway({} as never) + await gateway.start() + cleanups.push(() => gateway.dispose()) + const token = gateway.bindImageTool(h.service.bind(h.context), new AbortController().signal) + const client = new Client({ name: 'image-test', version: '1' }) + cleanups.push(() => client.close()) + await client.connect(new StreamableHTTPClientTransport(new URL(gateway.getEndpoint()!), { requestInit: { headers: { Authorization: `Bearer ${token}` } } })) + expect((await client.listTools()).tools.map(tool => tool.name)).toEqual(['generate_image']) + const result = await client.callTool({ name: 'generate_image', arguments: { intent: 'create', prompt: 'Blue' } }) + expect(JSON.stringify(result)).toContain('completed') + expect(h.fetcher).toHaveBeenCalledOnce() + gateway.revoke(token) + }) +}) diff --git a/src/main/agent/image-generation-service.ts b/src/main/agent/image-generation-service.ts new file mode 100644 index 00000000..bbbecb17 --- /dev/null +++ b/src/main/agent/image-generation-service.ts @@ -0,0 +1,209 @@ +import { randomUUID } from 'node:crypto' +import { + imageToolDescription, imageToolInputSchema, + type ImageOperation, type ImageRequestContext, type ImageToolInput +} from '../../shared/image-generation-contracts' +import type { AssistantDatabase } from '../assistant/assistant-database' +import { ModelAgentRuntime, type ModelRuntimeOptions } from './model-runtime' +import { withImageConversationContext } from './image-conversation-context' +import type { AgentImage, RuntimeEvent } from './runtime' +import { redactSensitiveText, safeToolErrorDetail } from './approval-summary' +import type { ImageToolBinding } from './image-tool-binding' +export type { ImageToolBinding } from './image-tool-binding' + +export type ConversationImageProfile = { + id: string + name: string + protocol: string + modelName: string + allowConversationInvocation?: boolean +} & Pick + +export type ImageServiceSettings = { + modelProfiles: readonly ConversationImageProfile[] + defaultImageModelProfileId?: string | null +} +export type ImageGenerationServiceOptions = { + database: Pick + getSettings(): Promise + onOperation?(operation: ImageOperation): void + onUsage?(event: Extract): void + /** Diagnostic sink for observer or terminal-persistence failures; must not throw. */ + onError?(error: Error): void + fetcher?: typeof fetch + requestTimeoutMs?: number + toolWaitMs?: number +} + +/** Main owns requests; request-scoped bindings own only admission and waiting. */ +export class ImageGenerationService { + private readonly active = new Map }>() + private closing = false + + constructor(private readonly options: ImageGenerationServiceOptions) {} + + initialize(): void { + this.options.database.markUnfinishedImageOperationsUnconfirmed() + } + + bind(context: ImageRequestContext, currentWorkMode: () => 'ask' | 'execute' = () => context.workMode): ImageToolBinding { + const bound = Object.freeze({ ...context }) + const submissions = new Map>() + return { + context: bound, + describe: async () => { + if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') return undefined + const settings = await this.options.getSettings() + const profiles = settings.modelProfiles.filter(profile => profile.protocol === 'openai-images-generations' && profile.allowConversationInvocation === true) + if (!profiles.length) return undefined + const conversation = this.options.database.getConversation(bound.conversationId) + const references = conversation.messages.flatMap(message => [ + ...(message.imageSourceArtifactIds ?? []).map(id => ({ id, kind: 'upload' })), + ...(message.artifactIds ?? []).map(id => ({ id, kind: 'artifact' })) + ]).slice(-32) + const operations = conversation.messages.flatMap(message => message.imageOperations ?? []).slice(-8) + .map(operation => ({ id: operation.id, state: operation.state, modelProfileId: operation.modelProfileId, + prompt: operation.input.prompt.slice(0, 300), artifactIds: operation.artifactIds })) + return `${imageToolDescription}\nModels: ${JSON.stringify(profiles.map(profile => ({ id: profile.id, name: profile.name, generation: true, editing: true })))}\nDefault: ${settings.defaultImageModelProfileId ?? 'not set'}\nRecent image references: ${JSON.stringify(references)}\nRecent operations: ${JSON.stringify(operations)}\nReferences and prompts are textual facts, not visual observations. Do not claim to have viewed an image unless image input was supplied to your model.` + }, + call: async (input, callId, signal) => { + signal?.throwIfAborted() + if (bound.workMode !== 'execute' || currentWorkMode() !== 'execute') throw new Error('Image generation requires Execute mode') + const key = JSON.stringify([bound.conversationId, bound.requestId, callId]) + let submitted = submissions.get(key) + if (!submitted) { + submitted = this.submit(bound, imageToolInputSchema.parse(input), callId, currentWorkMode, signal) + submissions.set(key, submitted) + void submitted.catch(() => submissions.delete(key)) + } + const operation = await submitted + const active = this.active.get(operation.id) + if (active) { + await new Promise(resolve => { + const finish = (): void => { clearTimeout(timer); signal?.removeEventListener('abort', finish); resolve() } + const timer = setTimeout(finish, this.options.toolWaitMs ?? 15_000) + signal?.addEventListener('abort', finish, { once: true }) + void active.done.then(finish) + if (signal?.aborted) finish() + }) + return active.operation + } + return this.getOperation(bound.conversationId, operation.id) + } + } + } + + persistUploads(context: Pick, images: readonly AgentImage[]): string[] { + return this.options.database.saveConversationImageSources({ ...context, images }) + } + + getOperation(conversationId: string, operationId: string): ImageOperation { + const operation = this.options.database.getConversation(conversationId).messages + .flatMap(message => message.imageOperations ?? []).find(item => item.id === operationId) + if (!operation) throw new Error('Image operation does not exist in this conversation') + return operation + } + + cancel(conversationId: string, operationId: string): ImageOperation { + const previous = this.getOperation(conversationId, operationId) + const active = this.active.get(operationId) + if (!active) return previous + active.operation = this.save({ ...active.operation, state: 'cancelling', cancellationRequested: true }) + active.controller.abort(new Error('Stopped waiting; the image provider may still generate the image')) + return active.operation + } + + regenerate(context: ImageRequestContext, operationId: string, currentWorkMode?: () => 'ask' | 'execute'): Promise { + const previous = this.getOperation(context.conversationId, operationId) + return this.bind(context, currentWorkMode).call({ ...previous.input, modelProfileId: previous.modelProfileId }, randomUUID()) + } + + cancelConversation(conversationId: string): void { + for (const active of this.active.values()) { + if (active.operation.conversationId === conversationId) this.cancel(conversationId, active.operation.id) + } + } + + async dispose(): Promise { + this.closing = true + for (const active of this.active.values()) active.controller.abort(new Error('Application is closing; image result is unconfirmed')) + await Promise.all([...this.active.values()].map(active => active.done)) + } + + private save(operation: ImageOperation, image?: { mimeType: 'image/png' | 'image/jpeg' | 'image/webp'; data: string }): ImageOperation { + const saved = this.options.database.saveConversationImageOperation({ ...operation, updatedAt: Date.now() }, image) + try { + this.options.onOperation?.(saved) + } catch (error) { + // A closed Renderer must not stop an accepted provider request or change its outcome. + this.report(error) + } + return saved + } + + private report(error: unknown): void { + this.options.onError?.(new Error(redactSensitiveText(safeToolErrorDetail(error) ?? 'Image operation notification or persistence failed'))) + } + + private async submit(context: ImageRequestContext, input: ImageToolInput, callId: string, currentWorkMode: () => 'ask' | 'execute', signal?: AbortSignal): Promise { + if (this.closing) throw new Error('Image service is closing') + const settings = await this.options.getSettings() + const conversation = this.options.database.getConversation(context.conversationId) + const existing = conversation.messages.flatMap(message => message.imageOperations ?? []) + .find(operation => operation.requestId === context.requestId && operation.callId === callId) + if (existing) return existing + if (!conversation.messages.some(message => message.id === context.messageId)) throw new Error('Originating image message must be persisted before submission') + const allowed = new Set(conversation.messages.flatMap(message => [ + ...(message.artifactIds ?? []), ...(message.imageSourceArtifactIds ?? []), + ...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? []) + ])) + if (input.sourceArtifactIds.some(id => !allowed.has(id))) throw new Error('Source image is not available in this conversation') + const origins = new Set(conversation.messages.flatMap(message => message.imageOperations ?? []) + .filter(operation => operation.artifactIds.some(id => input.sourceArtifactIds.includes(id))) + .map(operation => operation.modelProfileId)) + if (!input.modelProfileId && origins.size > 1) throw new Error('Source images use different models; specify the image model') + const profileId = input.modelProfileId ?? (input.intent === 'edit' ? [...origins][0] : undefined) ?? settings.defaultImageModelProfileId + if (!profileId) throw new Error('Specify an image model or set the default image model') + const profile = settings.modelProfiles.find(item => item.id === profileId) + if (!profile || !profile.allowConversationInvocation || profile.protocol !== 'openai-images-generations') throw new Error('Selected image model is unavailable; select another model') + if (profile.authentication === 'api-key' && !profile.apiKey) throw new Error('Selected image model has no API key configured') + const connection = structuredClone(profile) + const request = withImageConversationContext({ requestId: context.requestId, conversationId: context.conversationId, prompt: input.prompt, imageContextArtifactIds: input.sourceArtifactIds }, id => this.options.database.getArtifact(id)) + if (input.intent === 'edit' && (request.imageContextNotice || request.images?.length !== input.sourceArtifactIds.length)) throw new Error('Source image is missing or unsupported; select the image again') + signal?.throwIfAborted() + if (this.closing) throw new Error('Image service is closing') + if (currentWorkMode() !== 'execute') throw new Error('Image generation requires Execute mode') + const operation = this.save({ id: randomUUID(), conversationId: context.conversationId, messageId: context.messageId, requestId: context.requestId, + callId, modelProfileId: connection.id, modelName: connection.modelName, modelProfileName: connection.name, + input: { ...input, quality: input.quality ?? connection.imageGenerationQuality ?? 'auto' }, + state: 'running', artifactIds: [], createdAt: Date.now(), updatedAt: Date.now() }) + const runtime = new ModelAgentRuntime({ ...connection, model: connection.modelName, protocol: 'openai-images-generations', + imageGenerationQuality: input.quality ?? connection.imageGenerationQuality, fetcher: this.options.fetcher, requestTimeoutMs: this.options.requestTimeoutMs }) + const active = { operation, controller: new AbortController(), done: Promise.resolve() } + this.active.set(operation.id, active) + active.done = (async () => { + try { + for await (const event of runtime.generateImage(request, active.controller.signal)) { + if (event.type === 'model-usage') { + try { this.options.onUsage?.(event) } catch (error) { this.report(error) } + } + if (event.type === 'generated-image') { + active.operation = this.save({ ...active.operation, state: 'saving' }) + active.operation = this.save(active.operation, event) + } + } + } catch (error) { + const state = this.closing ? 'unconfirmed' : active.controller.signal.aborted ? 'stopped' : /timeout|timed out|fetch failed|network|超时/iu.test(String(error)) ? 'unconfirmed' : 'failed' + const detail = safeToolErrorDetail(error) ?? 'Image request failed' + const suffix = detail.match(/（HTTP [^）]+）$/u)?.[0] ?? '' + const publicDetail = redactSensitiveText(suffix ? detail.slice(0, -suffix.length) : detail) + suffix + active.operation = { ...active.operation, state, error: publicDetail.slice(0, 2_000), updatedAt: Date.now() } + try { active.operation = this.save(active.operation) } catch (storageError) { this.report(storageError) } + } finally { + await runtime.dispose() + this.active.delete(operation.id) + } + })() + return operation + } +} diff --git a/src/main/agent/image-runtime-integration.test.ts b/src/main/agent/image-runtime-integration.test.ts new file mode 100644 index 00000000..925e4625 --- /dev/null +++ b/src/main/agent/image-runtime-integration.test.ts @@ -0,0 +1,95 @@ +// @vitest-environment node +import { randomUUID } from 'node:crypto' +import { existsSync } from 'node:fs' +import { mkdtemp, rm } from 'node:fs/promises' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, it, vi } from 'vitest' +import { AssistantDatabase } from '../assistant/assistant-database' +import { ImageGenerationService } from './image-generation-service' +import { KnowledgeMcpGateway } from './knowledge-mcp-gateway' +import { ModelAgentRuntime } from './model-runtime' +import { OpenCodeRuntime } from './opencode-runtime' +import { ContinueAgentRuntime } from './continue-runtime' + +vi.mock('electron', () => ({ nativeImage: {} })) +const openCodeBinary = join(process.cwd(), '.runtime-resources', process.arch, process.platform === 'win32' ? 'opencode.exe' : 'opencode') +const continueBinary = join(process.cwd(), 'node_modules', '@continuedev', 'cli', 'dist', 'cn.js') + +it.for(['model', 'opencode', 'continue'] as const)('generates and edits through the real local %s tool loop', { timeout: 120_000 }, async (name, { skip }) => { + if ((name === 'opencode' && !existsSync(openCodeBinary)) || (name === 'continue' && !existsSync(continueBinary))) return skip() + const root = await mkdtemp(join(tmpdir(), 'goodbuddy-image-runtime-')) + const database = new AssistantDatabase(join(root, 'assistant.sqlite')) + database.initialize(root) + const conversationId = randomUUID() + const imageProfileId = randomUUID() + let toolInput: Record = { intent: 'create', prompt: 'Blue circle' } + const textRequests: unknown[] = [] + const model = createServer((request, response) => { + void (async () => { + let content = '' + for await (const chunk of request) content += String(chunk) + const body = JSON.parse(content) as { messages?: { role: string; content: unknown }[]; tools?: { function: { name: string } }[] } + textRequests.push(body) + let lastUser = -1 + body.messages?.forEach((message, index) => { if (message.role === 'user') lastUser = index }) + const results = body.messages?.slice(lastUser + 1).filter(message => message.role === 'tool') ?? [] + const tool = body.tools?.find(tool => tool.function.name.endsWith('generate_image')) + if (!tool) throw new Error('Image tool was not exposed') + const call = results.length === 0 + const delta = call ? { role: 'assistant', tool_calls: [{ index: 0, id: randomUUID(), type: 'function', + function: { name: tool.function.name, arguments: JSON.stringify(toolInput) } }] } : { role: 'assistant', content: 'IMAGE_SAVED' } + response.writeHead(200, { 'content-type': 'text/event-stream' }) + for (const [value, finish] of [[delta, null], [{}, call ? 'tool_calls' : 'stop']]) { + response.write(`data: ${JSON.stringify({ id: randomUUID(), object: 'chat.completion.chunk', created: 1, model: 'fixture', + choices: [{ index: 0, delta: value, finish_reason: finish }] })}\n\n`) + } + response.end('data: [DONE]\n\n') + })().catch(error => { response.writeHead(500).end(String(error)) }) + }) + await new Promise(resolve => model.listen(0, '127.0.0.1', resolve)) + const port = (model.address() as { port: number }).port + const fetcher = vi.fn(async () => Response.json({ data: [{ b64_json: 'iVBORw0KGgo=' }] })) + const service = new ImageGenerationService({ database, fetcher, getSettings: async () => ({ defaultImageModelProfileId: imageProfileId, modelProfiles: [{ + id: imageProfileId, name: 'Image fixture', modelName: 'fixture', protocol: 'openai-images-generations', + allowConversationInvocation: true, authentication: 'none', baseUrl: 'https://image.test/v1' + }] }) }) + const gateway = new KnowledgeMcpGateway({} as never) + await gateway.start() + const profile = { id: randomUUID(), name: 'Chat fixture', modelName: 'fixture', protocol: 'openai-chat-completions' as const, + authentication: 'none' as const, baseUrl: `http://127.0.0.1:${port}/v1` } + const common = { defaultWorkspace: root, knowledgeGateway: gateway, modelProfile: profile } + const runtime = name === 'model' + ? new ModelAgentRuntime({ ...profile, model: profile.modelName, defaultWorkspace: root }) + : name === 'opencode' + ? new OpenCodeRuntime({ ...common, embedded: true, binaryPath: openCodeBinary, configPath: '' }) + : new ContinueAgentRuntime({ ...common, binaryPath: '', bundledBinaryPath: continueBinary, configPath: '', hostCacheRoot: join(root, 'continue') }) + try { + for (const intent of ['create', 'edit'] as const) { + const messageId = randomUUID() + const requestId = randomUUID() + database.saveLocalConversations([{ header: { id: conversationId, title: 'Image integration', updatedAt: Date.now() }, + messages: [{ id: messageId, role: 'assistant', content: '', createdAt: Date.now(), state: 'complete' }] }]) + const binding = service.bind({ conversationId, messageId, requestId, workMode: 'execute' }) + const events = [] + for await (const event of runtime.run({ requestId, conversationId, prompt: `${intent} image ${requestId}`, workMode: 'execute', imageToolBinding: binding }, new AbortController().signal, async () => 'once')) events.push(event) + expect(events.some(event => event.type === 'done')).toBe(true) + const operation = database.getConversation(conversationId).messages.find(message => message.id === messageId)!.imageOperations?.[0] + expect(operation?.state).toBe('completed') + expect(database.getArtifact(operation!.artifactIds[0]!).content).toContain('iVBORw0KGgo=') + toolInput = { intent: 'edit', prompt: 'Turn red', sourceArtifactIds: operation!.artifactIds } + } + expect(fetcher).toHaveBeenCalledTimes(2) + expect(String(fetcher.mock.calls[1]![0])).toMatch(/images\/edits$/u) + expect(textRequests.length).toBeGreaterThanOrEqual(4) + } finally { + await runtime.dispose() + await gateway.dispose() + await service.dispose() + model.closeAllConnections() + await new Promise(resolve => model.close(() => resolve())) + database.close() + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/src/main/agent/image-tool-binding.ts b/src/main/agent/image-tool-binding.ts new file mode 100644 index 00000000..d494d2c1 --- /dev/null +++ b/src/main/agent/image-tool-binding.ts @@ -0,0 +1,16 @@ +import { z } from 'zod' +import { imageToolInputSchema, imageToolName, type ImageOperation, type ImageRequestContext } from '../../shared/image-generation-contracts' + +export type ImageToolBinding = { + context: Readonly + describe(): Promise + call(input: unknown, callId: string, waitSignal?: AbortSignal): Promise +} + +export async function imageToolDefinition(binding: ImageToolBinding | undefined) { + const description = await binding?.describe() + if (!description) return undefined + const inputSchema = z.toJSONSchema(imageToolInputSchema, { target: 'draft-7', io: 'input' }) + Reflect.deleteProperty(inputSchema, '$schema') + return { name: imageToolName, displayName: 'Generate or edit image', description, inputSchema, source: 'builtin' as const } +} diff --git a/src/main/agent/knowledge-mcp-gateway.ts b/src/main/agent/knowledge-mcp-gateway.ts index 572e8adb..506830cf 100644 --- a/src/main/agent/knowledge-mcp-gateway.ts +++ b/src/main/agent/knowledge-mcp-gateway.ts @@ -102,6 +102,7 @@ export { } export type GoodBuddyBuiltinToolName = + | 'generate_image' | ScopedDataToolName | BrowserToolName @@ -184,6 +185,7 @@ export type KnowledgeLibraryListItem = { } type Capability = { + imageToolBinding?: ImageToolBinding requestId: string libraryIds: readonly string[] magicNotesAccess: MagicNotesCapabilityAccess @@ -530,6 +532,20 @@ export class KnowledgeMcpGateway { } } + bindImageTool(binding: ImageToolBinding, signal: AbortSignal, existingToken?: string): string { + signal.throwIfAborted() + if (existingToken) { + const capability = this.getCapability(existingToken) + if (capability.requestId !== binding.context.requestId) throw new Error('Image capability request does not match') + capability.imageToolBinding = binding + return existingToken + } + return this.storeCapability({ + requestId: binding.context.requestId, libraryIds: [], magicNotesAccess: 'none', configAccess: 'none', + customMcpServers: [], signal, imageToolBinding: binding + }) + } + grantCustomMcp( requestId: string, servers: readonly ResolvedMcpServer[], @@ -760,6 +776,7 @@ export class KnowledgeMcpGateway { getAvailableToolNames(token: string): GoodBuddyBuiltinToolName[] { const capability = this.getCapability(token) return [ + ...(capability.imageToolBinding ? ['generate_image' as const] : []), ...(capability.libraryIds.length > 0 ? knowledgeToolNames : []), @@ -1570,6 +1587,7 @@ export class KnowledgeMcpGateway { } ) const capability = this.getCapability(token) + const imageTool = await imageToolDefinition(capability.imageToolBinding) const browserTools = capability.browserConversationId && capability.browserTabId ? new BrowserModelTools({ service: this.browserService!, @@ -1596,6 +1614,7 @@ export class KnowledgeMcpGateway { session.listedTools = true return { tools: [ + ...(imageTool ? [{ name: imageTool.name, description: imageTool.description, inputSchema: imageTool.inputSchema as Tool['inputSchema'], annotations: { readOnlyHint: false, destructiveHint: false } }] : []), ...scopedTools, ...browserDefinitions, ...[...customBindings.values()].map( @@ -1610,6 +1629,12 @@ export class KnowledgeMcpGateway { async (call, extra) => { const name = call.params.name const input = call.params.arguments ?? {} + if (name === 'generate_image') { + const capability = this.getCapability(token) + if (!capability.imageToolBinding) throw new Error('Image capability is unavailable') + const operation = await capability.imageToolBinding.call(input, `${session.registryKey}:${extra.requestId}`, AbortSignal.any([extra.signal, capability.signal])) + return { content: [{ type: 'text' as const, text: JSON.stringify(operation) }] } + } if (availableTools.has(name as ScopedDataToolName)) { const definition = scopedDataToolByName.get( name as ScopedDataToolName @@ -1868,3 +1893,4 @@ export class KnowledgeMcpGateway { } } } +import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding' diff --git a/src/main/agent/model-runtime.ts b/src/main/agent/model-runtime.ts index 63240458..3c277864 100644 --- a/src/main/agent/model-runtime.ts +++ b/src/main/agent/model-runtime.ts @@ -2681,9 +2681,15 @@ export class ModelAgentRuntime implements AgentRuntime { return history } + /** Strict image service entry point; direct-image conversations retain their fallback policy. */ + generateImage(request: AgentExecutionRequest, signal: AbortSignal): AsyncGenerator { + return this.runImageGeneration(request, signal, false) + } + private async *runImageGeneration( request: AgentExecutionRequest, - signal: AbortSignal + signal: AbortSignal, + allowEditingFallback = true ): AsyncGenerator { const images = request.images ?? [] const prompt = imageConversationPrompt(request) @@ -2744,7 +2750,7 @@ export class ModelAgentRuntime implements AgentRuntime { ) const providerMessage = getErrorMessage(errorPayload) if ( - images.length > 0 && + allowEditingFallback && images.length > 0 && isImageEditingUnavailable(response.status, providerMessage) ) { signal.throwIfAborted() @@ -3417,6 +3423,7 @@ export class ModelAgentRuntime implements AgentRuntime { wakeNestedEvents = undefined } const toolContext: ModelToolCallContext = { + imageToolBinding: request.imageToolBinding, conversationId: request.conversationId, browserTabId: request.browserTabId, browserConversationId: request.browserConversationId, @@ -3743,6 +3750,7 @@ export class ModelAgentRuntime implements AgentRuntime { let roundContextBytes = 0 const roundCallIds = new Set() for (const call of response.toolCalls) { + toolContext.toolCallId = call.id signal.throwIfAborted() if (roundCallIds.has(call.id)) { throw new Error('模型重复使用了工具调用 ID') diff --git a/src/main/agent/model-tool-provider.ts b/src/main/agent/model-tool-provider.ts index 31986b67..2018c659 100644 --- a/src/main/agent/model-tool-provider.ts +++ b/src/main/agent/model-tool-provider.ts @@ -290,6 +290,8 @@ export type ModelToolResult = { } export type ModelToolCallContext = { + imageToolBinding?: ImageToolBinding + toolCallId?: string conversationId: string browserTabId?: BrowserTabId browserConversationId?: string @@ -1290,6 +1292,7 @@ export class ModelToolProvider implements ModelToolProviderLike { ): Promise { signal.throwIfAborted() const scopedTools = this.getScopedTools(context) + const imageTool = context.workMode === 'execute' ? await imageToolDefinition(context.imageToolBinding) : undefined const webTools = this.webSearchEnabled ? this.getWebSearchDefinitions() : [] @@ -1312,6 +1315,7 @@ export class ModelToolProvider implements ModelToolProviderLike { const browserTools = this.getBrowserTools(context) return [ ...workspaceTools, + ...(imageTool ? [imageTool] : []), ...processTools, ...subagentTools, ...outputTools, @@ -1439,6 +1443,10 @@ export class ModelToolProvider implements ModelToolProviderLike { ): Promise { signal.throwIfAborted() assertToolAuthorizedForWorkMode(name, context) + if (name === 'generate_image') { + if (!context.imageToolBinding || !context.toolCallId) throw new Error('Image tool request binding is unavailable') + return createTextToolResult(JSON.stringify(await context.imageToolBinding.call(argumentsValue, context.toolCallId, signal))) + } if (name === 'knowledge_list') { if ( !this.knowledgeGateway || @@ -2020,3 +2028,4 @@ function emitDirectModelSubagentUsage( ) }) } +import { imageToolDefinition, type ImageToolBinding } from './image-tool-binding' diff --git a/src/main/agent/opencode-runtime.ts b/src/main/agent/opencode-runtime.ts index 3cd78267..4f1756d9 100644 --- a/src/main/agent/opencode-runtime.ts +++ b/src/main/agent/opencode-runtime.ts @@ -2161,9 +2161,13 @@ export class OpenCodeRuntime implements AgentRuntime { let customMcpToken: string | undefined; const attemptedMcpNames: string[] = []; const subscriptionController = new AbortController(); + const imageCapabilityToken = request.imageToolBinding && request.workMode === 'execute' + ? this.options.knowledgeGateway?.bindImageTool(request.imageToolBinding, signal, request.knowledgeCapabilityToken) + : undefined; + const scopedCapabilityToken = imageCapabilityToken ?? request.knowledgeCapabilityToken; try { if ( - request.knowledgeCapabilityToken && + scopedCapabilityToken && this.usesEmbeddedPermissionMediation() && this.options.knowledgeGateway?.getEndpoint() ) { @@ -2186,7 +2190,7 @@ export class OpenCodeRuntime implements AgentRuntime { url: knowledgeEndpoint, enabled: true, headers: { - Authorization: `Bearer ${request.knowledgeCapabilityToken}`, + Authorization: `Bearer ${scopedCapabilityToken}`, }, oauth: false, }, @@ -2212,7 +2216,7 @@ export class OpenCodeRuntime implements AgentRuntime { // experimental/tool/ids. Its model tool namespace is deterministic: // "_". knowledgeToolIds = this.options.knowledgeGateway - .getAvailableToolNames(request.knowledgeCapabilityToken) + .getAvailableToolNames(scopedCapabilityToken) .map((toolName) => `${knowledgeMcpName}_${toolName}`); } if ( @@ -2996,6 +3000,7 @@ export class OpenCodeRuntime implements AgentRuntime { } } finally { subscriptionController.abort(); + if (imageCapabilityToken && imageCapabilityToken !== request.knowledgeCapabilityToken) this.options.knowledgeGateway?.revoke(imageCapabilityToken); for (const name of attemptedMcpNames) { await this.mutateMcp(new AbortController().signal, () => { // Queue wait must not consume the disconnect timeout. diff --git a/src/main/agent/remote-runtime-channel.ts b/src/main/agent/remote-runtime-channel.ts index ecfc43c8..15c5a42d 100644 --- a/src/main/agent/remote-runtime-channel.ts +++ b/src/main/agent/remote-runtime-channel.ts @@ -18,6 +18,7 @@ import type { remoteSemanticTranscriptPageResultSchema, } from '../../shared/remote-agent-contracts' import type { z } from 'zod' +import type { ImageToolBinding } from './image-tool-binding' import type { RemoteQuestionResponse } from '../../shared/remote-question-contracts' import type { ModelBridgePolicy @@ -111,7 +112,9 @@ export interface RemoteRuntimeChannel { ): Promise /** Accepts the prompt and starts its Ask or Execute Runtime process. */ preparePrompt( - preparation: RemotePromptOperationPreparation + preparation: RemotePromptOperationPreparation, + imageToolBinding?: ImageToolBinding, + waitSignal?: AbortSignal ): Promise /** Starts an Agent-owned ACP prompt exactly once for its stable operation. */ startOwnedPrompt?( diff --git a/src/main/agent/runtime.ts b/src/main/agent/runtime.ts index f98ca889..26e51032 100644 --- a/src/main/agent/runtime.ts +++ b/src/main/agent/runtime.ts @@ -146,6 +146,7 @@ export type AgentExecutionRequest = Omit & { trustedInstructions?: string /** Main-process-only request-scoped authorization for built-in data tools. */ knowledgeCapabilityToken?: string + imageToolBinding?: ImageToolBinding /** Main-process-only browser tab fixed for this request. */ browserTabId?: BrowserTabId /** @@ -160,3 +161,4 @@ export type AgentExecutionRequest = Omit & { /** Main-only durable native subagent state for partial recovery updates. */ remoteRecoveredSubagents?: readonly RemoteRecoveredSubagent[] } +import type { ImageToolBinding } from './image-tool-binding' diff --git a/src/main/assistant/assistant-database.test.ts b/src/main/assistant/assistant-database.test.ts index 7124d376..3aa714e0 100644 --- a/src/main/assistant/assistant-database.test.ts +++ b/src/main/assistant/assistant-database.test.ts @@ -15,8 +15,43 @@ import { ASSISTANT_DATABASE_SCHEMA_VERSION } from './assistant-database' import { agentRuntimeSelectionKey } from '../../shared/runtime-selection-contracts' +import type { ImageOperation } from '../../shared/image-generation-contracts' const temporaryDirectories: string[] = [] + +it('atomically completes conversation images and retains operations and upload references across saves and reopening', async () => { + const database = await createDatabase() + const conversationId = randomUUID() + const messageId = randomUUID() + const header = { id: conversationId, title: 'Image operation', updatedAt: Date.now() } + const message = { id: messageId, role: 'assistant' as const, content: '', state: 'complete' as const, createdAt: Date.now() } + database.saveLocalConversations([{ header, messages: [message] }]) + const operation: ImageOperation = { id: randomUUID(), conversationId, messageId, requestId: randomUUID(), callId: 'image-call', + modelProfileId: randomUUID(), modelName: 'Image model', input: { intent: 'create', prompt: 'Blue', sourceArtifactIds: [] }, + state: 'running', createdAt: Date.now(), updatedAt: Date.now(), artifactIds: [] } + database.saveConversationImageOperation(operation) + const sources = database.saveConversationImageSources({ conversationId, messageId, + images: [{ name: 'source.png', mediaType: 'image/png', data: 'iVBORw0KGgo=' }] }) + const completed = database.saveConversationImageOperation(operation, { mimeType: 'image/png', data: 'iVBORw0KGgo=' }) + expect(database.saveConversationImageOperation(operation, { mimeType: 'image/png', data: 'iVBORw0KGgo=' })).toEqual(completed) + expect(database.listArtifacts()).toHaveLength(2) + database.saveLocalConversations([{ header, messages: [message] }]) + database.close() + database.initialize(process.cwd()) + const restored = database.getConversation(conversationId).messages[0]! + expect(restored.imageOperations).toEqual([completed]) + expect(restored.artifactIds).toEqual(completed.artifactIds) + expect(restored.imageSourceArtifactIds).toEqual(sources) + expect(database.getArtifact(sources[0]!).content).toContain('iVBORw0KGgo=') + const failed = { ...operation, id: randomUUID() } + database.saveConversationImageOperation(failed) + expect(() => database.saveConversationImageOperation(failed, { mimeType: 'image/png', data: 'a'.repeat(5 * 1024 * 1024) })).toThrow('5MB') + expect(database.listArtifacts()).toHaveLength(2) + expect(database.getConversation(conversationId).messages[0]!.imageOperations?.find(item => item.id === failed.id)?.state).toBe('running') + database.deleteLocalConversation(conversationId) + expect(() => database.saveConversationImageOperation(failed, { mimeType: 'image/png', data: 'iVBORw0KGgo=' })).toThrow('no longer exists') + database.close() +}) const channelDefaultProfileId = '00000000-0000-4000-8000-000000000001' diff --git a/src/main/assistant/assistant-database.ts b/src/main/assistant/assistant-database.ts index 5d35342c..548fce58 100644 --- a/src/main/assistant/assistant-database.ts +++ b/src/main/assistant/assistant-database.ts @@ -279,6 +279,8 @@ type MessageMetadata = { sourceReferences?: ConversationSnapshot['messages'][number]['sourceReferences'] knowledgeRetrieval?: ConversationSnapshot['messages'][number]['knowledgeRetrieval'] artifactIds?: string[] + imageOperations?: ImageOperation[] + imageSourceArtifactIds?: string[] imageContextNotice?: ConversationSnapshot['messages'][number]['imageContextNotice'] task?: ConversationSnapshot['messages'][number]['task'] attachments?: ConversationSnapshot['messages'][number]['attachments'] @@ -1177,6 +1179,8 @@ function toConversationSnapshot( sourceReferences: metadata.sourceReferences, knowledgeRetrieval: metadata.knowledgeRetrieval, artifactIds: metadata.artifactIds, + imageOperations: metadata.imageOperations, + imageSourceArtifactIds: metadata.imageSourceArtifactIds, imageContextNotice: metadata.imageContextNotice, task: metadata.task, attachments: metadata.attachments, @@ -1204,6 +1208,8 @@ function serializeConversationMessageMetadata( sourceReferences: message.sourceReferences, knowledgeRetrieval: message.knowledgeRetrieval, artifactIds: message.artifactIds, + imageOperations: message.imageOperations, + imageSourceArtifactIds: message.imageSourceArtifactIds, imageContextNotice: message.imageContextNotice, task: message.task, attachments: message.attachments, @@ -2862,7 +2868,7 @@ export class AssistantDatabase { WHERE id = ? AND channel IS NULL` ) const findMessage = database.prepare( - `SELECT conversation_id, role, + `SELECT conversation_id, role, metadata_json, EXISTS( SELECT 1 FROM tasks WHERE tasks.id = messages.request_id @@ -2941,6 +2947,7 @@ export class AssistantDatabase { | { conversation_id: string role: MessageRow['role'] + metadata_json: string remote_recoverable: number } | undefined @@ -2957,7 +2964,20 @@ export class AssistantDatabase { updateMessage.run( message.content, message.state, - serializeConversationMessageMetadata(message), + serializeConversationMessageMetadata({ + ...message, + ...(() => { + const stored = JSON.parse(existingMessage.metadata_json) as MessageMetadata + return { + imageOperations: stored.imageOperations, + imageSourceArtifactIds: stored.imageSourceArtifactIds, + artifactIds: stored.imageOperations?.some(operation => operation.artifactIds.length) ? [...new Set([ + ...(message.artifactIds ?? []), + ...(stored.imageOperations?.flatMap(operation => operation.artifactIds) ?? []) + ])].slice(-8) : message.artifactIds + } + })() + }), message.id ) continue @@ -5705,6 +5725,101 @@ export class AssistantDatabase { }) } + saveConversationImageOperation( + operation: ImageOperation, + image?: { mimeType: 'image/png' | 'image/jpeg' | 'image/webp'; data: string } + ): ImageOperation { + const database = this.requireDatabase() + database.exec('BEGIN IMMEDIATE') + try { + const row = database.prepare(`SELECT m.metadata_json, c.project_id FROM messages m + JOIN conversations c ON c.id = m.conversation_id + WHERE m.id = ? AND m.conversation_id = ?`).get(operation.messageId, operation.conversationId) as + { metadata_json: string; project_id: string | null } | undefined + if (!row) throw new Error('Image operation conversation or message no longer exists') + const metadata = JSON.parse(row.metadata_json) as MessageMetadata + const previous = metadata.imageOperations?.find(item => item.id === operation.id) + if (previous?.state === 'completed') { + database.exec('COMMIT') + return previous + } + let saved = operation + if (image) { + const artifact = this.createImageArtifact({ + projectId: row.project_id ?? undefined, + title: operation.input.prompt.slice(0, 120), + mimeType: image.mimeType, + base64: image.data + }) + saved = { ...operation, state: 'completed', artifactIds: [artifact.id] } + metadata.artifactIds = [...new Set([...(metadata.artifactIds ?? []), artifact.id])].slice(-8) + } + metadata.imageOperations = [...(metadata.imageOperations ?? []).filter(item => item.id !== saved.id), saved] + database.prepare('UPDATE messages SET metadata_json = ? WHERE id = ? AND conversation_id = ?') + .run(JSON.stringify(metadata), saved.messageId, saved.conversationId) + database.prepare('UPDATE conversations SET updated_at = MAX(updated_at, ?) WHERE id = ?') + .run(new Date(saved.updatedAt).toISOString(), saved.conversationId) + database.exec('COMMIT') + return saved + } catch (error) { + database.exec('ROLLBACK') + throw error + } + } + + saveConversationImageSources(input: { + conversationId: string + messageId: string + images: readonly { name: string; mediaType: 'image/png' | 'image/jpeg'; data: string }[] + }): string[] { + if (input.images.length > 8) throw new Error('At most eight source images are supported') + const database = this.requireDatabase() + database.exec('BEGIN IMMEDIATE') + try { + const row = database.prepare(`SELECT m.metadata_json, c.project_id FROM messages m + JOIN conversations c ON c.id = m.conversation_id + WHERE m.id = ? AND m.conversation_id = ?`).get(input.messageId, input.conversationId) as + { metadata_json: string; project_id: string | null } | undefined + if (!row) throw new Error('Image source conversation or message no longer exists') + const metadata = JSON.parse(row.metadata_json) as MessageMetadata + if (!metadata.imageSourceArtifactIds?.length) { + metadata.imageSourceArtifactIds = input.images.map(image => this.createImageArtifact({ + projectId: row.project_id ?? undefined, title: image.name, + mimeType: image.mediaType, base64: image.data + }).id) + database.prepare('UPDATE messages SET metadata_json = ? WHERE id = ?').run(JSON.stringify(metadata), input.messageId) + } + database.exec('COMMIT') + return metadata.imageSourceArtifactIds + } catch (error) { + database.exec('ROLLBACK') + throw error + } + } + + markUnfinishedImageOperationsUnconfirmed(): void { + const database = this.requireDatabase() + database.exec('BEGIN IMMEDIATE') + try { + const rows = database.prepare("SELECT id, metadata_json FROM messages WHERE json_type(metadata_json, '$.imageOperations') = 'array'") + .all() as { id: string; metadata_json: string }[] + for (const row of rows) { + const metadata = JSON.parse(row.metadata_json) as MessageMetadata + let changed = false + metadata.imageOperations = metadata.imageOperations?.map(operation => { + if (!['running', 'saving', 'cancelling'].includes(operation.state)) return operation + changed = true + return { ...operation, state: 'unconfirmed', updatedAt: Date.now() } + }) + if (changed) database.prepare('UPDATE messages SET metadata_json = ? WHERE id = ?').run(JSON.stringify(metadata), row.id) + } + database.exec('COMMIT') + } catch (error) { + database.exec('ROLLBACK') + throw error + } + } + createImageArtifact(input: { projectId?: string taskId?: string @@ -9764,3 +9879,4 @@ export class AssistantDatabase { return this.database } } +import type { ImageOperation } from '../../shared/image-generation-contracts' diff --git a/src/main/capabilities/capability-service.test.ts b/src/main/capabilities/capability-service.test.ts index b983fc2e..6646653d 100644 --- a/src/main/capabilities/capability-service.test.ts +++ b/src/main/capabilities/capability-service.test.ts @@ -137,6 +137,30 @@ afterEach(async () => { }) describe('CapabilityService', () => { + it('derives image assignments from saved profiles without persisting or exposing connection data', async () => { + const image = { id: '00000000-0000-4000-8000-000000000001', name: 'Image', protocol: 'openai-images-generations', allowConversationInvocation: true, apiKey: 'must-not-leak' } + let profiles = [image] + const getSavedModelProfiles = vi.fn(async () => profiles) + const { service, filePath, builtinRoot, importedRoot } = await createService({ getSavedModelProfiles }) + const all = ['model', 'opencode', 'continue', 'deepseek-harness'] + expect((await service.getSnapshot()).imageGeneration).toEqual({ modelProfiles: [{ id: image.id, name: image.name }], assignments: all }) + profiles = [image, { ...image, id: '00000000-0000-4000-8000-000000000002' }] + expect((await service.getSnapshot()).imageGeneration?.modelProfiles).toHaveLength(2) + profiles = [{ ...image, allowConversationInvocation: false }, profiles[1]!] + expect((await service.getSnapshot()).imageGeneration).toMatchObject({ assignments: all, modelProfiles: [{ id: profiles[1]!.id }] }) + profiles = [{ ...image, allowConversationInvocation: false }, { ...image, protocol: 'openai-chat-completions' }] + expect((await service.getSnapshot()).imageGeneration).toEqual({ modelProfiles: [], assignments: [] }) + profiles = [] + expect((await service.getSnapshot()).imageGeneration?.assignments).toEqual([]) + profiles = [image] + const reloaded = new CapabilityService(filePath, builtinRoot, importedRoot, cipher, { getSavedModelProfiles }) + expect((await reloaded.getSnapshot()).imageGeneration?.assignments).toEqual(all) + await service.setBuiltinMcpServerEnabled('knowledge-base', false) + expect(await readFile(filePath, 'utf8')).not.toContain('imageGeneration') + expect((await service.getSnapshot()).imageGeneration?.assignments).toEqual(all) + expect((await new CapabilityService(filePath, builtinRoot, importedRoot, cipher).getSnapshot()).imageGeneration).toEqual({ modelProfiles: [], assignments: [] }) + }) + it('memoizes concurrent loads and retries after a failed load', async () => { const initialStore = { load: vi.fn(async () => undefined), diff --git a/src/main/capabilities/capability-service.ts b/src/main/capabilities/capability-service.ts index c453ce8f..8a51a025 100644 --- a/src/main/capabilities/capability-service.ts +++ b/src/main/capabilities/capability-service.ts @@ -273,6 +273,12 @@ export type SkillImportInspection = { } export type CapabilityServiceOptions = Readonly<{ + getSavedModelProfiles?: () => Promise> platform?: NodeJS.Platform architecture?: string electronTarget?: boolean @@ -702,6 +708,7 @@ async function parseSkillZip(archivePath: string): Promise { } export class CapabilityService { + private readonly getSavedModelProfiles: NonNullable private state?: StoredCapabilities private loadPromise?: Promise private warnings: SettingsWarning[] = [] @@ -721,6 +728,8 @@ export class CapabilityService { private readonly cipher: CapabilityCipher, options: CapabilityServiceOptions = {} ) { + this.getSavedModelProfiles = + options.getSavedModelProfiles ?? (async () => []) this.platform = options.platform ?? process.platform this.architecture = options.architecture ?? process.arch this.electronTarget = @@ -971,12 +980,25 @@ export class CapabilityService { } async getSnapshot(): Promise { - const [state, catalog, browserProfileState] = await Promise.all([ + const [state, catalog, browserProfileState, savedModelProfiles] = await Promise.all([ this.load(), this.getSkillCatalog(), - this.browserProfiles.getSnapshot() + this.browserProfiles.getSnapshot(), + this.getSavedModelProfiles() ]) + const imageProfiles = savedModelProfiles + .filter((profile) => + profile.protocol === 'openai-images-generations' && + profile.allowConversationInvocation === true + ) + .map(({ id, name }) => ({ id, name })) return { + imageGeneration: { + modelProfiles: imageProfiles, + assignments: imageProfiles.length > 0 + ? ['model', 'opencode', 'continue', 'deepseek-harness'] + : [] + }, skills: catalog .map((skill) => ({ ...skill, diff --git a/src/main/index.ts b/src/main/index.ts index 64f8945e..b27380ff 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -31,6 +31,7 @@ import { type SelectedRuntimeTarget } from './agent/runtime-selection' import { CapabilityService } from './capabilities/capability-service' +import { ImageGenerationService } from './agent/image-generation-service' import { ContextManager } from './context-manager' import { registerIpcHandlers, @@ -211,6 +212,7 @@ const localRuntimeRegistry = new LocalRuntimeRegistry() let knowledgeService: KnowledgeService | undefined let knowledgeGateway: KnowledgeMcpGateway | undefined let assistantDatabase: AssistantDatabase | undefined +let imageGenerationService: ImageGenerationService | undefined let browserService: BrowserService | undefined let globalTlsPolicy: GlobalTlsPolicy | undefined let feedbackService: FeedbackService | undefined @@ -750,7 +752,8 @@ if (hasSingleInstanceLock) { ? join(process.resourcesPath, 'skills') : join(app.getAppPath(), 'resources', 'skills'), join(app.getPath('userData'), 'skills', 'imported'), - secureCipher + secureCipher, + { getSavedModelProfiles: () => settingsStore.getSavedModelProfiles() } ) const channelSettingsStore = new ChannelSettingsStore( join(app.getPath('userData'), 'channel-settings.json'), @@ -910,6 +913,23 @@ if (hasSingleInstanceLock) { } ) assistantDatabase = startupAssistantDatabase + imageGenerationService = new ImageGenerationService({ + database: startupAssistantDatabase, + getSettings: () => settingsStore.getResolvedSettings(), + onError: (error) => { + void desktopDiagnostics.recordFailure({ component: 'desktop', stage: 'image-generation', code: 'image-generation.observer.failed', error }).catch(() => undefined) + }, + onOperation: (operation) => { + if (mainWindow && !mainWindow.isDestroyed() && !mainWindow.webContents.isDestroyed()) { + mainWindow.webContents.send(ipcChannels.imageOperationChanged, operation) + } + }, + onUsage: (event) => startupAssistantDatabase.upsertModelUsageCall({ + requestId: event.requestId, callId: event.callId, runtime: event.runtime, + provider: event.provider, model: event.model, input: event.inputTokens, + output: event.outputTokens, cacheRead: event.cacheReadTokens, cacheWrite: event.cacheWriteTokens + }) + }) directModelSubagentScheduler = new SubagentScheduler({ concurrency: 3, queueLimit: 20, @@ -1130,6 +1150,7 @@ if (hasSingleInstanceLock) { createConfiguredRuntime(initialResolvedSettings), initializeAssistant: () => { startupAssistantDatabase.initialize(defaultWorkspace) + imageGenerationService!.initialize() startupAssistantDatabase.ensureChannelProjects( defaultWorkspace, initialRuntimeSettings.defaultModelProfileId @@ -1426,7 +1447,8 @@ if (hasSingleInstanceLock) { agentPackageManager, startupRemoteAgentServices.connectionManager, terminalSessionManager, - startupLocalToolEnvironmentService + startupLocalToolEnvironmentService, + imageGenerationService ) removeFeedbackIpcHandler = registerFeedbackIpcHandler( mainWindow, @@ -1522,6 +1544,7 @@ app.on('before-quit', (event) => { () => feedbackService?.dispose(), () => removeFeedbackIpcHandler?.(), () => dshExtensionInstaller?.dispose(), + () => imageGenerationService?.dispose(), () => removeIpcHandlers?.() ], [() => stopRuntimeReconfiguration?.()], diff --git a/src/main/ipc.test.ts b/src/main/ipc.test.ts index 0ebd8892..ee7b4b8e 100644 --- a/src/main/ipc.test.ts +++ b/src/main/ipc.test.ts @@ -28,6 +28,8 @@ import type { } from '../shared/ssh-host-contracts' import { defaultKnowledgeOntologySettings } from '../shared/knowledge-ontology' import { AssistantDatabase } from './assistant/assistant-database' +import { ImageGenerationService } from './agent/image-generation-service' +import type { AgentExecutionRequest } from './agent/runtime' import { KnowledgeService } from './knowledge/knowledge-service' import { BrowserNavigationStoppedError } from './browser/browser-service' import { RemotePromptCancelledError } from './agent/acp-remote-runtime' @@ -1310,6 +1312,7 @@ describe('registerIpcHandlers model download source routing', () => { }) vi.mock('electron', () => ({ + nativeImage: { createFromBuffer: (buffer: Buffer) => ({ isEmpty: () => false, toPNG: () => buffer }) }, app: { getName: vi.fn(() => 'GoodBuddy'), getVersion: vi.fn(() => '0.1.0') @@ -5058,7 +5061,9 @@ describe('registerIpcHandlers agent terminal state', () => { magicNotesEnabled = false, goodbuddyConfigService?: Record, capabilityServiceOverride?: Record, - browserControl?: Record + browserControl?: Record, + imageService?: ImageGenerationService, + imageDatabase?: AssistantDatabase ) { const assistantDatabase = { createTask: vi.fn(), @@ -5229,7 +5234,7 @@ describe('registerIpcHandlers agent terminal state', () => { }) ) const onRuntimeSettingsChanged = vi.fn(async () => {}) - const dispose = registerIpcHandlers( + const args: Parameters = [ window as never, runtime as never, 'CommandOrControl+Shift+Space', @@ -5256,7 +5261,7 @@ describe('registerIpcHandlers agent terminal state', () => { (knowledgeServiceOverride ?? { database: { listKnowledgeBases: vi.fn(() => []) } }) as never, - assistantDatabase as never, + (imageDatabase ?? assistantDatabase) as never, approvalBroker as never, {} as never, onRuntimeSettingsChanged, @@ -5279,7 +5284,9 @@ describe('registerIpcHandlers agent terminal state', () => { undefined, undefined, goodbuddyConfigService as never - ) + ] + args[43] = imageService + const dispose = registerIpcHandlers(...args) return { approvalBroker, assistantDatabase, @@ -5370,6 +5377,84 @@ describe('registerIpcHandlers agent terminal state', () => { knowledgeLibraryIds: [] }) + it('binds the production image service before text dispatch and routes regenerate, Ask rejection, and deletion through IPC', async () => { + const database = new AssistantDatabase(':memory:') + database.initialize(process.cwd()) + const conversationId = crypto.randomUUID() + const userId = crypto.randomUUID() + const messageId = crypto.randomUUID() + const header = { id: conversationId, title: 'Image IPC', updatedAt: Date.now(), workMode: 'execute' as const } + database.saveLocalConversations([{ header, messages: [] }]) + const profile = { id: crypto.randomUUID(), name: 'Image model', modelName: 'image-test', protocol: 'openai-images-generations', + baseUrl: 'https://image.test/v1', authentication: 'none' as const, allowConversationInvocation: true } + const fetcher = vi.fn(async () => Response.json({ data: [{ b64_json: 'iVBORw0KGgo=' }] })) + const service = new ImageGenerationService({ database, getSettings: async () => ({ modelProfiles: [profile], defaultImageModelProfileId: profile.id }), fetcher }) + service.initialize() + const run = vi.fn(async function* (request: AgentExecutionRequest) { + expect(database.getConversation(conversationId).messages.map(message => message.id)).toEqual([userId, messageId]) + expect(request.imageToolBinding?.context.messageId).toBe(messageId) + expect(await request.imageToolBinding?.describe()).toContain(profile.id) + expect((await request.imageToolBinding!.call({ intent: 'create', prompt: 'A blue circle' }, 'image-call')).state).toBe('completed') + yield { type: 'done' as const, requestId: request.requestId } + }) + const harness = createHarness({ runtimeId: 'model', capability: 'text', supportsToolExecution: true, run }, + undefined, 'always', undefined, false, undefined, undefined, undefined, false, undefined, undefined, undefined, service, database) + const event = trustedEvent(harness.webContents) + try { + const requestId = crypto.randomUUID() + await harness.handler!(event, { requestId, conversationId, prompt: 'Draw a circle', workMode: 'execute', + currentUserMessageId: userId, currentAssistantMessageId: messageId }) + await vi.waitFor(() => expect(database.getTask(requestId).status).toBe('completed')) + const original = database.getConversation(conversationId).messages[1]!.imageOperations![0]! + expect(database.getArtifact(original.artifactIds[0]!).content).toContain('iVBORw0KGgo=') + const regenerate = electronMocks.handlers.get(ipcChannels.imageOperationRegenerate)! + const cancel = electronMocks.handlers.get(ipcChannels.imageOperationCancel)! + expect(() => cancel(event, { conversationId, operationId: original.id, prompt: 'injected' })).toThrow() + const regenerated = await regenerate(event, { conversationId, operationId: original.id }) as typeof original + expect(regenerated.id).not.toBe(original.id) + expect(regenerated.messageId).toBe(messageId) + expect(fetcher).toHaveBeenCalledTimes(2) + database.saveLocalConversations([{ header: { ...header, workMode: 'ask' }, messages: [] }]) + await expect(regenerate(event, { conversationId, operationId: original.id })).rejects.toThrow('Execute') + expect(fetcher).toHaveBeenCalledTimes(2) + const cancelConversation = vi.spyOn(service, 'cancelConversation') + await electronMocks.handlers.get(ipcChannels.conversationsDeleteLocal)!(event, conversationId) + expect(cancelConversation).toHaveBeenCalledWith(conversationId) + expect(() => database.getConversation(conversationId)).toThrow() + } finally { + await harness.dispose() + await service.dispose() + database.close() + } + }) + + it('persists uploads before dispatch while withholding image bytes from a nonvisual text model', async () => { + const database = new AssistantDatabase(':memory:') + database.initialize(process.cwd()) + const conversationId = crypto.randomUUID() + const userId = crypto.randomUUID() + database.saveLocalConversations([{ header: { id: conversationId, title: 'Upload', updatedAt: Date.now() }, messages: [] }]) + const service = new ImageGenerationService({ database, getSettings: async () => ({ modelProfiles: [] }) }) + const run = vi.fn(async function* (request: AgentExecutionRequest) { + expect(request.images).toBeUndefined() + expect(request.trustedInstructions).toContain('cannot view') + const sources = database.getConversation(conversationId).messages.find(message => message.id === userId)!.imageSourceArtifactIds! + expect(sources).toHaveLength(1) + expect(database.getArtifact(sources[0]!).content).toContain('iVBORw0KGgo=') + yield { type: 'done' as const, requestId: request.requestId } + }) + const harness = createHarness({ runtimeId: 'model', capability: 'text', supportsToolExecution: true, run }, + undefined, 'always', undefined, false, undefined, undefined, undefined, false, undefined, undefined, undefined, service, database) + harness.contextManager.enrichRequest.mockImplementation(request => ({ ...request, images: [{ name: 'Upload', mediaType: 'image/png', data: 'iVBORw0KGgo=' }] })) + try { + const requestId = crypto.randomUUID() + await harness.handler!(trustedEvent(harness.webContents), { requestId, conversationId, prompt: 'Turn it red', workMode: 'execute', + currentUserMessageId: userId, currentAssistantMessageId: crypto.randomUUID() }) + await vi.waitFor(() => expect(database.getTask(requestId).status).toBe('completed')) + expect(run).toHaveBeenCalledOnce() + } finally { await harness.dispose(); await service.dispose(); database.close() } + }) + it('restores image context at the IPC boundary and persists the quiet notice on its artifact event', async () => { const png = 'iVBORw0KGgo=' const artifactId = '00000000-0000-4000-8000-000000000301' diff --git a/src/main/ipc.ts b/src/main/ipc.ts index 3c1aa6cd..f6b6da97 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -382,6 +382,8 @@ import { } from './magic-notes/magic-note-analyzer' import { AgentEventBuffer } from './agent-event-buffer' import { withImageConversationContext } from './agent/image-conversation-context' +import type { ImageGenerationService } from './agent/image-generation-service' +import { imageOperationTargetSchema } from '../shared/image-operation-ipc' import { ExecutionSpaceResolver, REMOTE_EXECUTION_SPACE_UNAVAILABLE @@ -1334,7 +1336,8 @@ export function registerIpcHandlers( 'getHostConnectionState' | 'onHostConnectionStateChange' >, terminalSessionManager?: TerminalSessionManager, - localToolEnvironmentService?: LocalToolEnvironmentService + localToolEnvironmentService?: LocalToolEnvironmentService, + imageGenerationService?: ImageGenerationService ): () => Promise { type ActiveRequestLease = { controller: AbortController @@ -4018,6 +4021,34 @@ export function registerIpcHandlers( const attachedRequest = contextManager.enrichRequest( parsedRequest ) + let uploadedImageSourceIds: string[] = [] + if (imageGenerationService && parsedRequest.currentUserMessageId && parsedRequest.currentAssistantMessageId) { + const conversation = assistantDatabase.getConversation(parsedRequest.conversationId) + const now = Date.now() + assistantDatabase.saveLocalConversations([{ + header: { + id: conversation.id, projectId: conversation.projectId, title: conversation.title, + updatedAt: now, workMode: normalizedWorkMode, runtimeSelection: parsedRequest.runtimeSelection, + knowledgeLibraryIds: conversation.knowledgeLibraryIds, + knowledgeRetrievalMode: conversation.knowledgeRetrievalMode, + contextMetrics: conversation.contextMetrics, contextCompressionState: conversation.contextCompressionState, + branch: conversation.branch + }, + messages: [ + ...(conversation.messages.some(message => message.id === parsedRequest.currentUserMessageId) ? [] : [{ + id: parsedRequest.currentUserMessageId, role: 'user' as const, + content: parsedRequest.prompt, createdAt: now, state: 'complete' as const + }]), + ...(conversation.messages.some(message => message.id === parsedRequest.currentAssistantMessageId) ? [] : [{ + id: parsedRequest.currentAssistantMessageId, role: 'assistant' as const, + content: '', createdAt: now, state: 'streaming' as const + }]) + ] + }]) + if (attachedRequest.images?.length) { + uploadedImageSourceIds = imageGenerationService.persistUploads({ conversationId: parsedRequest.conversationId, messageId: parsedRequest.currentUserMessageId }, attachedRequest.images) + } + } const enrichedRequest = imageGeneration ? withImageConversationContext( attachedRequest, @@ -4045,7 +4076,7 @@ export function registerIpcHandlers( !agentRuntimeSelected ? capabilityService.getWebSearchCapabilityStatus?.() : undefined, - configAccess !== 'none' && !enrichedRequest.projectId + (configAccess !== 'none' && !enrichedRequest.projectId) || (imageGenerationService && !imageGeneration && enrichedRequest.images?.length) ? settingsStore.getResolvedSettings() : undefined, selectedRuntimeTarget @@ -4077,6 +4108,15 @@ export function registerIpcHandlers( : undefined : resolvedRuntimeSettings?.workspacePath const controller = new AbortController() + const imageToolBinding = !imageGeneration && imageGenerationService && enrichedRequest.currentAssistantMessageId + ? imageGenerationService.bind({ + conversationId: enrichedRequest.conversationId, + messageId: enrichedRequest.currentAssistantMessageId, + requestId: enrichedRequest.requestId, + workMode: normalizedWorkMode + }, () => normalizeInteractiveWorkMode(assistantDatabase.getConversation(enrichedRequest.conversationId).workMode)) + : undefined + const imageToolAvailable = Boolean(await imageToolBinding?.describe()) const scopedCapability = await grantScopedDataCapability({ gateway: knowledgeGateway, browserControl: browserControl as BrowserCapabilityControl | undefined, @@ -4101,6 +4141,7 @@ export function registerIpcHandlers( }) const knowledgeCapabilityToken = scopedCapability.token const availableTools = [ + ...(imageToolAvailable ? ['generate_image'] : []), ...(webSearchEnabled ? ['web_search', 'web_fetch'] : []), ...(!agentRuntimeSelected && !imageGeneration ? ['workspace_rg', 'workspace_read_text', 'output_read', 'subagent_delegate'] @@ -4142,6 +4183,28 @@ export function registerIpcHandlers( if (scopedCapability.browserTabId) { request.browserTabId = scopedCapability.browserTabId } + if (imageToolBinding) { + request.imageToolBinding = imageToolBinding + const imageReferenceIds = [...uploadedImageSourceIds, ...(request.imageContextArtifactIds ?? [])] + if (imageReferenceIds.length) { + request.trustedInstructions = [request.trustedInstructions, + `Image references for this request: ${JSON.stringify(imageReferenceIds)}. These are references, not visual observations; use explicit sourceArtifactIds when editing.`].filter(Boolean).join('\n') + } + if (request.images?.length && resolvedRuntimeSettings) { + const effective = request.runtimeSelection + ? applyRuntimeSelection(resolvedRuntimeSettings, request.runtimeSelection).settings + : resolvedRuntimeSettings + const supportsImages = selectedRuntimeTarget === 'model' ? effective.supportsImageInput + : selectedRuntimeTarget === 'opencode' ? effective.opencodeModelProfile?.supportsImageInput + : selectedRuntimeTarget === 'continue' ? effective.continueModelProfile?.supportsImageInput + : effective.deepseekHarnessModelProfile?.supportsImageInput + if (supportsImages !== true) { + request.images = undefined + request.trustedInstructions = [request.trustedInstructions, + 'The uploaded images were saved as conversation image references for the image tool. Your chat model cannot view them. Do not claim visual observations; use the user description to edit them.'].filter(Boolean).join('\n') + } + } + } const managedSshExecution = agentRuntimeSelected && configExecutionSpace?.kind === 'ssh' @@ -6762,6 +6825,7 @@ export function registerIpcHandlers( (event, input: unknown) => { assertTrustedSender(event, window) const conversationId = assistantIdSchema.parse(input) + imageGenerationService?.cancelConversation(conversationId) const queuedItems = assistantDatabase.listConversationQueueItems(conversationId) for (const item of queuedItems) { @@ -6789,6 +6853,24 @@ export function registerIpcHandlers( } ) + registerHandler(ipcChannels.imageOperationCancel, (event, input: unknown) => { + assertTrustedSender(event, window) + const target = imageOperationTargetSchema.parse(input) + if (!imageGenerationService) throw new Error('Image service is unavailable') + return imageGenerationService.cancel(target.conversationId, target.operationId) + }) + registerHandler(ipcChannels.imageOperationRegenerate, (event, input: unknown) => { + assertTrustedSender(event, window) + const target = imageOperationTargetSchema.parse(input) + if (!imageGenerationService) throw new Error('Image service is unavailable') + const previous = imageGenerationService.getOperation(target.conversationId, target.operationId) + const currentWorkMode = (): 'ask' | 'execute' => normalizeInteractiveWorkMode(assistantDatabase.getConversation(target.conversationId).workMode) + return imageGenerationService.regenerate({ + conversationId: target.conversationId, messageId: previous.messageId, + requestId: randomUUID(), workMode: currentWorkMode() + }, target.operationId, currentWorkMode) + }) + registerHandler( ipcChannels.conversationQueueList, (event, input: unknown) => { diff --git a/src/main/remote-agent/main-image-tool-session.ts b/src/main/remote-agent/main-image-tool-session.ts new file mode 100644 index 00000000..cf7407a7 --- /dev/null +++ b/src/main/remote-agent/main-image-tool-session.ts @@ -0,0 +1,56 @@ +import type { ImageToolBinding } from '../agent/image-tool-binding' +import type { RuntimeProtocolBinaryChannel } from './protocol-remote-runtime-channel' +import { + decodeRemoteImageToolMessage, encodeRemoteImageToolMessage, + remoteImageToolCallSchema, remoteImageToolReplySchema +} from '../../shared/remote-image-tool-contracts' + +export class MainImageToolSession { + private readonly wait = new AbortController() + private readonly unsubscribe: () => void + private readonly removeAbort: () => void + + constructor( + private readonly channel: RuntimeProtocolBinaryChannel, + private readonly binding: ImageToolBinding, + signal?: AbortSignal + ) { + this.unsubscribe = channel.onClose(() => this.close()) + const abort = (): void => this.close() + signal?.addEventListener('abort', abort, { once: true }) + this.removeAbort = () => signal?.removeEventListener('abort', abort) + if (signal?.aborted) this.close() + void this.receive().catch(() => this.close()) + } + + close(): void { + if (this.wait.signal.aborted) return + // This aborts binding.call's wait, never the Main-owned image operation. + this.wait.abort() + this.removeAbort() + this.unsubscribe() + this.channel.close() + } + + private async receive(): Promise { + while (!this.wait.signal.aborted) { + const frame = await this.channel.receive(this.wait.signal) + const call = remoteImageToolCallSchema.parse(decodeRemoteImageToolMessage(frame.payload)) + await frame.consume() + void this.dispatch(call).catch(() => this.close()) + } + } + + private async dispatch(call: ReturnType): Promise { + let reply: ReturnType + try { + this.wait.signal.throwIfAborted() + if (this.binding.context.workMode !== 'execute') throw new Error('Image tools are unavailable in Ask mode') + const result = await this.binding.call(call.input, call.callId, this.wait.signal) + reply = remoteImageToolReplySchema.parse({ callId: call.callId, result }) + } catch (error) { + reply = { callId: call.callId, error: (error instanceof Error ? error.message : 'Image tool failed').slice(0, 4_000) } + } + if (!this.wait.signal.aborted) await this.channel.send(encodeRemoteImageToolMessage(reply), this.wait.signal) + } +} diff --git a/src/main/remote-agent/protocol-remote-runtime-channel.test.ts b/src/main/remote-agent/protocol-remote-runtime-channel.test.ts index e8a83d48..84cdff23 100644 --- a/src/main/remote-agent/protocol-remote-runtime-channel.test.ts +++ b/src/main/remote-agent/protocol-remote-runtime-channel.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import type { ImageToolBinding } from '../agent/image-tool-binding' import { AGENT_PROTOCOL_LIMITS, ChannelProtocolError, @@ -489,6 +490,46 @@ async function openChannel(fixture = connectionFixture()) { } describe('ProtocolRemoteRuntimeChannel', () => { + it('refreshes image binding descriptions per prompt, hides Ask, and closes failed preparations', async () => { + const fixture = await openChannel() + const describe = vi.fn(async () => 'First image catalog') + const binding: ImageToolBinding = { + context: { conversationId: 'conversation', messageId: 'message', requestId: 'operation-1', workMode: 'execute' }, + describe, call: vi.fn() + } + const prepare = (requestId: string, workMode: 'ask' | 'execute' = 'execute') => ({ + bindingId: 'binding-1', operationId: requestId, requestId, workMode, + controllerId: 'controller-1', controllerGeneration: 1, connectionGeneration: fixture.channel.generation, + channelEpoch: fixture.channel.channelEpoch, hostId: 'host-1', hostRevision: 1, hostKeyGeneration: 1, + workspaceIdentity: 'workspace-1', agentInstallationId: 'agent-1', runtimeId: 'opencode', + runtimeBundleDigest, runtimeAdapterDigest: digest('d'), deadlineAt: new Date(Date.now() + 10000).toISOString(), + budget: { maximumInputBytes: 1024, maximumOutputBytes: 1024 }, promptSequence: 0 + }) + fixture.client.responder = (method, params) => { + if (method !== 'runtime/preparePrompt') return defaultResponse(method, params) + const input = params as ReturnType & { imageTool?: { description: string } } + return { bindingId: input.bindingId, operationId: input.operationId, requestId: input.requestId, + workMode: input.workMode, deadlineAt: input.deadlineAt, acceptedAt: new Date().toISOString(), + ...(input.imageTool ? { imageToolUrl: 'http://127.0.0.1:1234/image' } : {}) } + } + try { + const wait = new AbortController() + const first = await fixture.channel.preparePrompt(prepare('operation-1'), binding, wait.signal) + fixture.channel.setRecoveryBoundary(first.deadlineAt, wait.signal) + expect(fixture.client.blobBinaries[0]!.closed).toBe(false) + describe.mockResolvedValue('Refreshed image catalog') + await fixture.channel.preparePrompt(prepare('operation-2'), binding, wait.signal) + expect(fixture.client.blobBinaries[0]!.closed).toBe(true) + expect(fixture.client.requests.at(-1)!.params).toMatchObject({ imageTool: { description: 'Refreshed image catalog' } }) + await fixture.channel.preparePrompt(prepare('operation-3', 'ask'), binding, wait.signal) + expect(describe).toHaveBeenCalledTimes(2) + expect(fixture.client.blobBinaries[1]!.closed).toBe(true) + expect(fixture.client.requests.at(-1)!.params).not.toHaveProperty('imageTool', expect.anything()) + fixture.client.responder = () => { throw new Error('preparation response lost') } + await expect(fixture.channel.preparePrompt(prepare('operation-4'), binding, wait.signal)).rejects.toThrow() + expect(fixture.client.blobBinaries[2]!.closed).toBe(true) + } finally { await fixture.channel.close() } + }) it('sends question replies on the authenticated control channel and rejects foreign bindings', async () => { const fixture = await openChannel() const response = { bindingId: 'binding-1', operationId: 'operation-1', questionId: 'question-1', answers: [] } diff --git a/src/main/remote-agent/protocol-remote-runtime-channel.ts b/src/main/remote-agent/protocol-remote-runtime-channel.ts index 58f58a2b..c18245ab 100644 --- a/src/main/remote-agent/protocol-remote-runtime-channel.ts +++ b/src/main/remote-agent/protocol-remote-runtime-channel.ts @@ -1,4 +1,6 @@ import { z } from 'zod' +import type { ImageToolBinding } from '../agent/image-tool-binding' +import { MainImageToolSession } from './main-image-tool-session' import { remoteQuestionResponseSchema, type RemoteQuestionResponse } from '../../shared/remote-question-contracts' import { AGENT_PROTOCOL_LIMITS, @@ -584,38 +586,51 @@ export class ProtocolRemoteRuntimeChannel } async preparePrompt( - preparation: z.input + preparation: z.input, + imageToolBinding?: ImageToolBinding, + waitSignal?: AbortSignal ): Promise< z.infer > { - const parsed = remotePromptOperationPreparationSchema.parse( - preparation - ) - this.#assertPreparationIdentity(parsed) - const raw = await this.#request( - 'runtime/preparePrompt', - parsed - ) - const acceptance = parseRemoteResult( - remotePromptOperationAcceptanceSchema, - raw, - 'Remote Runtime prompt acceptance is invalid' - ) + this.#imageTool?.close() + this.#imageTool = undefined + const description = preparation.workMode === 'execute' ? await imageToolBinding?.describe() : undefined + waitSignal?.throwIfAborted() + let imageTool: z.infer['imageTool'] + if (description && imageToolBinding) { + this.#assertCurrent() + const binary = this.#state.client.allocateBinaryChannel?.({ kind: 'blob' }) + if (!binary) throw new Error('Remote image tool transport is unavailable') + imageTool = { channelId: binary.channelId, channelEpoch: binary.channelEpoch, description } + this.#imageTool = new MainImageToolSession(binary, imageToolBinding, waitSignal) + } try { - assertRemotePromptAcceptanceMatchesPreparation( - parsed, - acceptance + const parsed = remotePromptOperationPreparationSchema.parse({ ...preparation, imageTool }) + this.#assertPreparationIdentity(parsed) + const raw = await this.#request('runtime/preparePrompt', parsed) + const acceptance = parseRemoteResult( + remotePromptOperationAcceptanceSchema, raw, + 'Remote Runtime prompt acceptance is invalid' ) - } catch { - throw new ProtocolRemoteRuntimeChannelError( - 'Remote Runtime prompt acceptance identity is invalid', - 'binding-mismatch' - ) + try { + assertRemotePromptAcceptanceMatchesPreparation(parsed, acceptance) + if (Boolean(imageTool) !== Boolean(acceptance.imageToolUrl)) throw new Error('Image tool registration mismatch') + } catch { + throw new ProtocolRemoteRuntimeChannelError( + 'Remote Runtime prompt acceptance identity is invalid', 'binding-mismatch' + ) + } + this.#assertCurrent() + return acceptance + } catch (error) { + this.#imageTool?.close() + this.#imageTool = undefined + throw error } - this.#assertCurrent() - return acceptance } + #imageTool?: MainImageToolSession + async startOwnedPrompt( request: z.input ): Promise> { @@ -754,7 +769,8 @@ export class ProtocolRemoteRuntimeChannel 'protocol' ) } - this.clearRecoveryBoundary() + this.#removeRecoveryAbort?.() + this.#removeRecoveryAbort = undefined this.#recoveryDeadlineAt = deadline this.#recoverySignal = signal const abort = (): void => { @@ -774,6 +790,8 @@ export class ProtocolRemoteRuntimeChannel } clearRecoveryBoundary(): void { + this.#imageTool?.close() + this.#imageTool = undefined this.#removeRecoveryAbort?.() this.#removeRecoveryAbort = undefined this.#recoverySignal = undefined @@ -1628,6 +1646,7 @@ export class ProtocolRemoteRuntimeChannel this.#state.capabilityGeneration ) this.#closing = true + this.#imageTool?.close() this.clearRecoveryBoundary() this.#lifetime.abort( new DOMException('Remote Runtime channel closed', 'AbortError') diff --git a/src/main/runtime-settings-store.test.ts b/src/main/runtime-settings-store.test.ts index 7d287d87..824cf14a 100644 --- a/src/main/runtime-settings-store.test.ts +++ b/src/main/runtime-settings-store.test.ts @@ -80,6 +80,57 @@ afterEach(async () => { }) describe('RuntimeSettingsStore', () => { + it('round-trips conversation image settings and retains unavailable defaults without resolving credentials', async () => { + const { store, filePath } = await createStore() + const initial = await store.getPublicSettings() + expect(initial.modelProfiles[0]?.allowConversationInvocation).toBe(false) + expect(initial.defaultImageModelProfileId).toBeUndefined() + const imageId = '00000000-0000-4000-8000-000000000099' + const text = { + id: initial.modelProfiles[0]!.id, name: 'Text', + baseUrl: 'https://example.com', modelName: 'text', + protocol: 'openai-chat-completions' as const, + authentication: 'api-key' as const, + imageGenerationQuality: 'auto' as const, + apiKey: { action: 'keep' as const } + } + const image = { + ...text, id: imageId, name: 'Image model', + protocol: 'openai-images-generations' as const, + allowConversationInvocation: true, + apiKey: { action: 'replace' as const, value: 'image-secret' } + } + const input = settings({ modelProfiles: [text, image], defaultModelProfileId: text.id, defaultImageModelProfileId: imageId }) + await store.update(runtimeSettingsInputSchema.parse(input)) + const reloaded = new RuntimeSettingsStore(filePath, cipher, {}) + expect((await reloaded.getPublicSettings()).modelProfiles[1]).toMatchObject({ allowConversationInvocation: true, apiKeyConfigured: true }) + expect((await reloaded.getResolvedSettings()).defaultImageModelProfileId).toBe(imageId) + expect((await reloaded.getResolvedSettings()).modelProfiles[1]).toMatchObject({ allowConversationInvocation: true, apiKey: 'image-secret' }) + expect((await reloaded.getPublicSettings()).configured?.modelProfiles[1]?.allowConversationInvocation).toBe(true) + const decrypt = vi.fn(() => { throw new Error('Credential unavailable') }) + const credentialFree = new RuntimeSettingsStore(filePath, { ...cipher, decrypt }, {}) + expect(await credentialFree.getSavedModelProfiles()).toEqual([ + { id: text.id, name: text.name, protocol: text.protocol, allowConversationInvocation: false }, + { id: imageId, name: image.name, protocol: image.protocol, allowConversationInvocation: true } + ]) + expect(decrypt).not.toHaveBeenCalled() + await reloaded.update(settings()) + expect((await reloaded.getSavedModelProfiles())[1]?.allowConversationInvocation).toBe(true) + expect((await reloaded.getPublicSettings()).defaultImageModelProfileId).toBe(imageId) + const disabled = await reloaded.update({ ...input, modelProfiles: [text, { ...image, allowConversationInvocation: false, apiKey: { action: 'keep' } }] }) + expect(disabled.defaultImageModelProfileId).toBe(imageId) + expect(disabled.modelProfiles[1]).toMatchObject({ allowConversationInvocation: false, apiKeyConfigured: true }) + const removed = await reloaded.update({ ...input, modelProfiles: [text] }) + expect(removed.defaultImageModelProfileId).toBe(imageId) + expect((await new RuntimeSettingsStore(filePath, cipher, {}).getPublicSettings()).defaultImageModelProfileId).toBe(imageId) + expect((await reloaded.update({ ...input, modelProfiles: [text], defaultImageModelProfileId: null })).defaultImageModelProfileId).toBeNull() + }) + + it('rejects malformed conversation image settings', () => { + expect(runtimeSettingsInputSchema.safeParse(settings({ defaultImageModelProfileId: 'not-a-uuid' })).success).toBe(false) + expect(runtimeSettingsInputSchema.safeParse({ ...settings(), modelProfiles: [{ id: '00000000-0000-4000-8000-000000000001', name: 'Image', baseUrl: 'https://example.com', modelName: 'image', protocol: 'openai-images-generations', authentication: 'none', imageGenerationQuality: 'auto', apiKey: { action: 'keep' }, allowConversationInvocation: 'true' }] }).success).toBe(false) + }) + it('defaults to automatic Execute tool authorization and preserves an explicitly saved deny policy', async () => { const { store, filePath } = await createStore() await expect(store.getPublicSettings()).resolves.toMatchObject({ toolApproval: 'always' }) diff --git a/src/main/runtime-settings-store.ts b/src/main/runtime-settings-store.ts index 48289bac..bf12166c 100644 --- a/src/main/runtime-settings-store.ts +++ b/src/main/runtime-settings-store.ts @@ -292,12 +292,14 @@ const storedSettingsSchema = version20StoredSettingsSchema modelProfiles: z .array( currentStoredModelProfileSchema.extend({ + allowConversationInvocation: z.boolean().optional(), requestHeaders: modelRequestHeadersSchema, requestBody: modelRequestBodySchema }) ) .min(1) - .max(20) + .max(20), + defaultImageModelProfileId: z.string().uuid().nullable().optional() }) type StoredSettings = z.infer @@ -424,6 +426,7 @@ export type ResolvedRuntimeSettings = { apiKey?: string modelProfiles: ResolvedModelProfile[] defaultModelProfileId: string + defaultImageModelProfileId?: string | null opencodeModelProfile?: ResolvedModelProfile continueModelProfile?: ResolvedModelProfile deepseekHarnessModelProfile?: ResolvedModelProfile @@ -458,6 +461,7 @@ export type RuntimePolicySettings = Pick< > export type ResolvedModelProfile = { + allowConversationInvocation?: boolean id: string name: string baseUrl: string @@ -1599,6 +1603,7 @@ export class RuntimeSettingsStore { protocol: effective.protocol, authentication: effective.authentication, supportsImageInput: effective.supportsImageInput, + allowConversationInvocation: profile.allowConversationInvocation ?? false, contextWindowTokens: effective.contextWindowTokens, maximumOutputTokens: effective.maximumOutputTokens, imageGenerationQuality: effective.imageGenerationQuality, @@ -1619,6 +1624,7 @@ export class RuntimeSettingsStore { protocol: profile.protocol, authentication: profile.authentication, supportsImageInput: profile.supportsImageInput, + allowConversationInvocation: profile.allowConversationInvocation ?? false, contextWindowTokens: profile.contextWindowTokens, maximumOutputTokens: profile.maximumOutputTokens, imageGenerationQuality: profile.imageGenerationQuality, @@ -1704,6 +1710,7 @@ export class RuntimeSettingsStore { protocol: resolved.protocol, authentication: resolved.authentication, supportsImageInput: resolved.supportsImageInput, + allowConversationInvocation: profile.allowConversationInvocation ?? false, contextWindowTokens: resolved.contextWindowTokens, maximumOutputTokens: resolved.maximumOutputTokens, imageGenerationQuality: @@ -1728,6 +1735,7 @@ export class RuntimeSettingsStore { protocol: profile.protocol, authentication: profile.authentication, supportsImageInput: profile.supportsImageInput, + allowConversationInvocation: profile.allowConversationInvocation ?? false, contextWindowTokens: profile.contextWindowTokens, maximumOutputTokens: profile.maximumOutputTokens, imageGenerationQuality: @@ -1860,6 +1868,7 @@ export class RuntimeSettingsStore { credentialSource: effective.credentialSource, modelProfiles, defaultModelProfileId: settings.defaultModelProfileId, + defaultImageModelProfileId: settings.defaultImageModelProfileId, opencodeModelSource: agent.opencodeBaseUrl ? { kind: 'platform' } : settings.opencodeModelSource, @@ -1891,6 +1900,18 @@ export class RuntimeSettingsStore { return this.toPublicSettings(await this.load()) } + async getSavedModelProfiles(): Promise>> { + return (await this.load()).modelProfiles.map((profile) => ({ + id: profile.id, + name: profile.name, + protocol: profile.protocol, + allowConversationInvocation: profile.allowConversationInvocation ?? false + })) + } + captureRollback(): Promise { let result: RuntimeSettingsRollback | undefined const operation = this.updateQueue.then(async () => { @@ -1990,6 +2011,7 @@ export class RuntimeSettingsStore { modelProfiles, defaultModelProfileId: settings.defaultModelProfileId, opencodeModelProfile, + defaultImageModelProfileId: settings.defaultImageModelProfileId, continueModelProfile, deepseekHarnessModelProfile, ...agent, @@ -2128,6 +2150,7 @@ export class RuntimeSettingsStore { protocol: input.modelProtocol, authentication: input.modelAuthentication, supportsImageInput: profile.supportsImageInput, + allowConversationInvocation: profile.allowConversationInvocation ?? false, contextWindowTokens: profile.contextWindowTokens, maximumOutputTokens: profile.maximumOutputTokens, imageGenerationQuality: input.imageGenerationQuality, @@ -2143,6 +2166,7 @@ export class RuntimeSettingsStore { protocol: profile.protocol, authentication: profile.authentication, supportsImageInput: profile.supportsImageInput, + allowConversationInvocation: profile.allowConversationInvocation ?? false, contextWindowTokens: profile.contextWindowTokens, maximumOutputTokens: profile.maximumOutputTokens, imageGenerationQuality: profile.imageGenerationQuality, @@ -2197,6 +2221,9 @@ export class RuntimeSettingsStore { protocol: profile.protocol, authentication: profile.authentication, supportsImageInput: profile.supportsImageInput ?? false, + allowConversationInvocation: + profile.allowConversationInvocation ?? + existing?.allowConversationInvocation ?? false, contextWindowTokens: profile.contextWindowTokens, maximumOutputTokens: profile.maximumOutputTokens, imageGenerationQuality: profile.imageGenerationQuality, @@ -2491,6 +2518,9 @@ export class RuntimeSettingsStore { modelProfiles, defaultModelProfileId, opencodeModelSource, + defaultImageModelProfileId: input.defaultImageModelProfileId === undefined + ? current.defaultImageModelProfileId + : input.defaultImageModelProfileId, continueModelSource, deepseekHarnessModelSource: requestedDeepSeekHarnessSource, diff --git a/src/preload/image-operations.test.ts b/src/preload/image-operations.test.ts new file mode 100644 index 00000000..64dcfadf --- /dev/null +++ b/src/preload/image-operations.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from 'vitest' +import type { DesktopApi } from '../shared/contracts' +import { ipcChannels } from '../shared/ipc-channels' + +const electron = vi.hoisted(() => ({ + contextBridge: { exposeInMainWorld: vi.fn() }, + ipcRenderer: { invoke: vi.fn(), on: vi.fn(), removeListener: vi.fn() }, + webUtils: {} +})) +vi.mock('electron', () => electron) + +describe('image operation preload', () => { + it('forwards explicit actions and delivers operation data without Electron events', async () => { + await import('./index') + const api = (electron.contextBridge.exposeInMainWorld.mock.calls[0]![1] as DesktopApi).conversations.imageOperations + const target = { conversationId: 'conversation', operationId: 'operation' } + await api.cancel(target) + await api.regenerate(target) + expect(electron.ipcRenderer.invoke.mock.calls).toEqual([ + [ipcChannels.imageOperationCancel, target], [ipcChannels.imageOperationRegenerate, target] + ]) + const listener = vi.fn() + const unsubscribe = api.onChanged(listener) + const [channel, handler] = electron.ipcRenderer.on.mock.calls[0]! + const operation = { ...target, state: 'completed' } + handler({ sender: 'private' }, operation) + expect(listener).toHaveBeenCalledExactlyOnceWith(operation) + unsubscribe() + expect(electron.ipcRenderer.removeListener).toHaveBeenCalledExactlyOnceWith(channel, handler) + }) +}) diff --git a/src/preload/index.ts b/src/preload/index.ts index caeb604b..441f08db 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1057,6 +1057,15 @@ const desktopApi: DesktopApi = { } }, conversations: { + imageOperations: { + cancel: (input) => ipcRenderer.invoke(ipcChannels.imageOperationCancel, input), + regenerate: (input) => ipcRenderer.invoke(ipcChannels.imageOperationRegenerate, input), + onChanged: (listener) => { + const handler = (_event: Electron.IpcRendererEvent, operation: import('../shared/image-generation-contracts').ImageOperation): void => listener(operation) + ipcRenderer.on(ipcChannels.imageOperationChanged, handler) + return () => ipcRenderer.removeListener(ipcChannels.imageOperationChanged, handler) + } + }, list: () => ipcRenderer.invoke( ipcChannels.conversationsList diff --git a/src/renderer/src/App.test.tsx b/src/renderer/src/App.test.tsx index 1ea70d49..9d8e95b6 100644 --- a/src/renderer/src/App.test.tsx +++ b/src/renderer/src/App.test.tsx @@ -483,6 +483,11 @@ const api: DesktopApi = { }, }, conversations: { + imageOperations: { + cancel: vi.fn(), + regenerate: vi.fn(), + onChanged: vi.fn(() => () => undefined), + }, list: vi.fn(async () => []), replace: vi.fn(async () => {}), saveLocal: vi.fn(async () => {}), @@ -8926,6 +8931,19 @@ describe("App", () => { expect(mode).toHaveAccessibleName("工作模式：Ask · 只读问答"); }); + it('opens model settings from no-tool image guidance without losing the request draft', async () => { + vi.mocked(api.agent.getStatus).mockResolvedValue({ id: 'model', label: 'Legacy chat', available: true, supportsToolExecution: false, capability: 'chat', detail: 'Ready' }); + render(); + expect(await screen.findByText('当前聊天模型无法自动调用图片工具，可使用已有直连图片工作流。')).toBeVisible(); + fireEvent.change(screen.getByLabelText('向 GoodBuddy 提问'), { target: { value: 'Keep my image request' } }); + fireEvent.click(screen.getByRole('button', { name: '前往图片模型设置' })); + expect(await screen.findByRole('tab', { name: '模型连接' })).toHaveAttribute('aria-selected', 'true'); + fireEvent.click(screen.getByRole('button', { name: '关闭设置' })); + expect(screen.getByLabelText('向 GoodBuddy 提问')).toHaveValue('Keep my image request'); + expect(run).not.toHaveBeenCalled(); + expect(api.conversations.imageOperations.regenerate).not.toHaveBeenCalled(); + }); + it("allows a direct model to submit Execute with GoodBuddy approvals", async () => { vi.mocked(api.agent.getStatus).mockResolvedValue({ id: "model", @@ -10290,6 +10308,93 @@ describe("App", () => { anchorClick.mockRestore(); }); + it("updates the original image message after leaving its conversation and queues history editing across Runtime switches", async () => { + const conversationId = crypto.randomUUID(); + const messageId = crypto.randomUUID(); + const otherId = crypto.randomUUID(); + const operation: import('../../shared/image-generation-contracts').ImageOperation = { + id: crypto.randomUUID(), conversationId, messageId, requestId: crypto.randomUUID(), callId: 'image-call', + modelProfileId: crypto.randomUUID(), modelName: 'image-model', modelProfileName: '图片连接', + input: { intent: 'create', prompt: 'Blue circle', sourceArtifactIds: [] }, state: 'running', artifactIds: [], createdAt: 1, updatedAt: 1, + }; + const secondOperation = { ...operation, id: crypto.randomUUID(), modelProfileName: 'Second image connection' }; + let imageListener: ((value: typeof operation) => void) | undefined; + vi.mocked(api.conversations.imageOperations.onChanged).mockImplementationOnce(listener => { imageListener = listener; return () => { imageListener = undefined; }; }); + vi.mocked(api.conversations.list).mockResolvedValueOnce([ + { id: conversationId, projectId, title: '图片原会话', updatedAt: 2, workMode: 'execute', messages: [ + { id: messageId, role: 'assistant', content: '图片稍后保存', createdAt: 1, state: 'complete', imageOperations: [operation, secondOperation] }, + ] }, + { id: otherId, projectId, title: '另一会话', updatedAt: 1, messages: [ + { id: crypto.randomUUID(), role: 'assistant', content: '另一会话正文', createdAt: 1, state: 'complete' }, + ] }, + ]); + const artifact = { id: crypto.randomUUID(), kind: 'image' as const, title: 'Blue circle', mimeType: 'image/png', + content: 'data:image/png;base64,iVBORw0KGgo=', byteSize: 8, createdAt: '2026-09-15T00:00:00Z', updatedAt: '2026-09-15T00:00:00Z' }; + vi.mocked(api.artifacts.get).mockResolvedValueOnce(artifact); + render(); + expect(await screen.findByText('图片稍后保存')).toBeVisible(); + fireEvent.click(screen.getByRole('button', { name: /^另一会话/u })); + expect(await screen.findByText('另一会话正文')).toBeVisible(); + act(() => imageListener?.({ ...operation, state: 'completed', artifactIds: [artifact.id], updatedAt: 3 })); + expect(screen.queryByRole('img', { name: 'Blue circle' })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: /^图片原会话/u })); + expect(await screen.findByRole('img', { name: 'Blue circle' })).toBeVisible(); + expect(screen.getByText(/图片已保存/u)).toBeVisible(); + expect(screen.getAllByRole('region', { name: '图片生成' }).map(card => within(card).getByRole('status').textContent)).toEqual([ + '图片已保存 · 图片连接', '正在生成图片 · Second image connection', + ]); + expect(run).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole('button', { name: '继续修改' })); + fireEvent.click(screen.getByRole('button', { name: /sonnet-5/u })); + fireEvent.click(screen.getByRole('menuitemradio', { name: /^OpenCode · 默认模型/u })); + await waitFor(() => expect(api.agent.getStatus).toHaveBeenLastCalledWith({ provider: 'opencode' })); + fireEvent.change(screen.getByLabelText('向 GoodBuddy 提问'), { target: { value: 'Turn the circle red' } }); + fireEvent.click(screen.getByLabelText('发送')); + await waitFor(() => expect(run).toHaveBeenCalledOnce()); + expect(run.mock.calls[0]![0]).toMatchObject({ runtimeSelection: { provider: 'opencode' }, imageContextArtifactIds: [artifact.id] }); + expect(api.conversationQueue.enqueueUser).toHaveBeenCalledWith(expect.objectContaining({ imageContextArtifactIds: [artifact.id] })); + expect(api.context.addPastedImage).not.toHaveBeenCalled(); + }); + + it('recovers failed image editing through settings and composer attachments without resubmitting the operation', async () => { + const conversationId = crypto.randomUUID(); + const operation: import('../../shared/image-generation-contracts').ImageOperation = { + id: crypto.randomUUID(), conversationId, messageId: crypto.randomUUID(), requestId: crypto.randomUUID(), callId: 'failed-edit', + modelProfileId: crypto.randomUUID(), modelName: 'original-image-model', modelProfileName: 'Original image connection', + input: { intent: 'edit', prompt: 'Change the background', sourceArtifactIds: [crypto.randomUUID()] }, + state: 'failed', error: 'Original source unavailable', artifactIds: [], createdAt: 1, updatedAt: 1, + }; + vi.mocked(api.conversations.list).mockResolvedValueOnce([{ id: conversationId, projectId, title: 'Failed image editing', updatedAt: 2, workMode: 'execute', messages: [ + { id: operation.messageId, role: 'assistant', content: '', createdAt: 1, state: 'complete', imageOperations: [operation] }, + ] }]); + const source = { id: crypto.randomUUID(), name: 'replacement.png', size: 64, preview: 'Replacement image', kind: 'image' as const, contentUrl: 'data:image/png;base64,iVBORw0KGgo=' }; + vi.mocked(api.context.selectFiles).mockResolvedValueOnce([source]); + render(); + fireEvent.click(await screen.findByRole('button', { name: '重新选择素材' })); + const composer = screen.getByLabelText('向 GoodBuddy 提问'); + expect(composer).toHaveValue('使用图片模型“Original image connection”：Change the background'); + expect(composer).toHaveFocus(); + expect(await within(composer.closest('.composer')!).findByText('replacement.png')).toBeVisible(); + expect(api.context.selectFiles).toHaveBeenCalledOnce(); + expect(run).not.toHaveBeenCalled(); + expect(api.conversations.imageOperations.regenerate).not.toHaveBeenCalled(); + fireEvent.change(composer, { target: { value: 'My edited image request' } }); + vi.mocked(api.context.selectFiles).mockResolvedValueOnce([]); + fireEvent.click(screen.getByRole('button', { name: '重新选择素材' })); + await waitFor(() => expect(api.context.selectFiles).toHaveBeenCalledTimes(2)); + expect(composer).toHaveValue('My edited image request'); + fireEvent.click(screen.getByRole('button', { name: '重新选择模型' })); + expect(await screen.findByRole('tab', { name: '模型连接' })).toHaveAttribute('aria-selected', 'true'); + fireEvent.click(screen.getByRole('button', { name: '关闭设置' })); + expect(screen.getByLabelText('向 GoodBuddy 提问')).toHaveValue('My edited image request'); + expect(screen.getByText('Original source unavailable')).toBeVisible(); + expect(api.conversations.imageOperations.regenerate).not.toHaveBeenCalled(); + fireEvent.click(screen.getByLabelText('发送')); + await waitFor(() => expect(run).toHaveBeenCalledOnce()); + expect(run.mock.calls[0]![0]).toMatchObject({ conversationId, prompt: 'My edited image request', contextIds: [source.id] }); + expect(run.mock.calls[0]![0].imageContextArtifactIds).toBeUndefined(); + }); + it("can dispatch a request to the parallel expert team", async () => { render(); diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 12db646d..1dc00777 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -1,3 +1,5 @@ +import { ImageCapabilityNotice } from "./ImageCapabilityNotice"; +import type { ImageOperation } from "../../shared/image-generation-contracts"; import { ArrowDown, Bot, @@ -803,6 +805,9 @@ type ChatScrollSnapshot = { }; function ChatHistoryPane({ + onOpenImageModelSettings, + onReselectImageSources, + onEditImage, active, artifactById, conversationHtmlRenderingEnabled, @@ -824,6 +829,9 @@ function ChatHistoryPane({ taskStrip, visibleMessageCount, }: { + onOpenImageModelSettings: () => void; + onReselectImageSources: (operation: ImageOperation) => void; + onEditImage: (artifact: AssistantArtifact) => void; active: boolean; artifactById: ReadonlyMap; conversationHtmlRenderingEnabled: boolean; @@ -1107,6 +1115,9 @@ function ChatHistoryPane({ )} message.id), ); const messages = [ - ...conversation.messages.map((message) => { - const localMessage = localMessageById.get(message.id); + ...conversation.messages.map((persistedMessage) => { + const localMessage = localMessageById.get(persistedMessage.id); + const operations = new Map(persistedMessage.imageOperations?.map(operation => [operation.id, operation])); + for (const operation of localMessage?.imageOperations ?? []) { + if (operation.updatedAt > (operations.get(operation.id)?.updatedAt ?? -1)) operations.set(operation.id, operation); + } + const imageMetadata = { + imageOperations: operations.size ? [...operations.values()] : undefined, + imageSourceArtifactIds: persistedMessage.imageSourceArtifactIds ?? localMessage?.imageSourceArtifactIds, + artifactIds: [...new Set([...(persistedMessage.artifactIds ?? []), + ...[...operations.values()].flatMap(operation => operation.artifactIds)])].slice(-8), + }; + const message = { ...persistedMessage, ...imageMetadata }; if (!conversation.activeRequest && local.activeRequest?.messageId === message.id) { return { ...message, pendingQuestions: undefined }; } if (!localIsNewer) { // Pending prompts are live-only and are omitted from persisted messages. if (localMessage?.state === "streaming" && message.state === "streaming") { - return { ...message, approval: localMessage.approval, pendingQuestions: localMessage.pendingQuestions }; + return { ...localMessage, ...imageMetadata }; } return message; } @@ -1495,7 +1519,8 @@ function mergePersistedConversations( ) { return message; } - return localMessage; + return { ...localMessage, ...imageMetadata, + artifactIds: [...new Set([...(localMessage.artifactIds ?? []), ...(message.artifactIds ?? [])])].slice(-8) }; }), ...local.messages.filter((message) => !serverMessageIds.has(message.id)), ]; @@ -2505,6 +2530,8 @@ function App(): React.JSX.Element { Record >({}); const attachments = attachmentsByConversation[activeId] ?? []; + const [imageReferencesByConversation, setImageReferencesByConversation] = useState>({}); + const imageReferences = imageReferencesByConversation[activeId] ?? []; const attachmentsRef = useRef(new Map()); const updateAttachments = useCallback( ( @@ -2618,6 +2645,7 @@ function App(): React.JSX.Element { ); const hydratingArtifactIds = useRef(new Set()); const inputRef = useRef(null); + const attachmentButtonRef = useRef(null); const [chatScrollSnapshots, setChatScrollSnapshots] = useState< Record >({}); @@ -6017,7 +6045,7 @@ function App(): React.JSX.Element { const missingIds = [ ...new Set( (activeConversation?.messages ?? []).flatMap( - (message) => message.artifactIds ?? [], + (message) => [...(message.artifactIds ?? []), ...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? [])], ), ), ] @@ -6123,11 +6151,25 @@ function App(): React.JSX.Element { ); const removeAgentListener = window.goodbuddy.agent.onEvent(handleAgentEvent); + const removeImageListener = window.goodbuddy.conversations.imageOperations.onChanged((operation) => { + setConversations(current => current.map(conversation => conversation.id !== operation.conversationId ? conversation : { + ...conversation, + updatedAt: Math.max(conversation.updatedAt, operation.updatedAt), + messages: conversation.messages.map(message => message.id !== operation.messageId ? message : { + ...message, + imageOperations: message.imageOperations?.some(item => item.id === operation.id) + ? message.imageOperations.map(item => item.id === operation.id ? operation : item) + : [...(message.imageOperations ?? []), operation], + artifactIds: [...new Set([...(message.artifactIds ?? []), ...operation.artifactIds])].slice(-8), + }), + })); + }); const removeOpenSettingsListener = window.goodbuddy.app.onOpenSettings(() => setView("settings"), ); return () => { removeAgentListener(); + removeImageListener(); removeOpenSettingsListener(); }; }, [handleAgentEvent, setView]); @@ -7103,6 +7145,7 @@ function App(): React.JSX.Element { includeMemoryContext: !command, prompt, attachments: attachmentSnapshot, + imageContextArtifactIds: imageReferences.map(artifact => artifact.id), knowledgeLibraryIds: knowledgeLibraryIdsSnapshot, knowledgeRetrievalMode: knowledgeRetrievalModeSnapshot, }; @@ -7119,6 +7162,7 @@ function App(): React.JSX.Element { setRuntimeMenuOpen(false); setInput(""); updateAttachments([]); + setImageReferencesByConversation(current => ({ ...current, [conversationId]: [] })); if (command) { setSelectedRuntimeCommand(""); } @@ -7236,6 +7280,22 @@ function App(): React.JSX.Element { ), ); try { + const saveOrigin = conversationPersistenceQueueRef.current.then(async () => { + const origin = { + ...conversationSnapshot, + title: conversationSnapshot.title === "新对话" ? prompt.slice(0, 24) : conversationSnapshot.title, + updatedAt: assistantMessage.createdAt, + workMode: workModeSnapshot, + messages: [...historySnapshot, userMessage, assistantMessage], + }; + await window.goodbuddy.conversations.saveLocal([{ + header: toLocalConversationHeader(origin), + messages: [toConversationMessage(userMessage), toConversationMessage(assistantMessage)], + }]); + persistedLocalConversationsRef.current.set(conversationId, origin); + }); + conversationPersistenceQueueRef.current = saveOrigin.catch(() => undefined); + await saveOrigin; await window.goodbuddy.agent.run({ requestId, conversationId, @@ -7254,9 +7314,9 @@ function App(): React.JSX.Element { knowledgeLibraryIds: knowledgeLibraryIdsSnapshot, knowledgeRetrievalMode: knowledgeRetrievalModeSnapshot, contextIds: attachmentSnapshot.map((attachment) => attachment.id), - ...(runtimeSelectionSnapshot.provider === "model" - ? { - imageContextArtifactIds: [...historySnapshot] + imageContextArtifactIds: queuedInput?.imageContextArtifactIds?.length + ? queuedInput.imageContextArtifactIds + : attachmentSnapshot.some(attachment => attachment.kind === 'image') ? undefined : [...historySnapshot] .reverse() .find( (message) => @@ -7264,8 +7324,6 @@ function App(): React.JSX.Element { message.state === "complete" && message.artifactIds?.length, )?.artifactIds, - } - : {}), contextCompressionState: conversationSnapshot.contextCompressionState, history: retainedHistorySnapshot.map((message) => ({ role: message.role, @@ -7588,6 +7646,30 @@ function App(): React.JSX.Element { } }; + const editImage = useCallback((artifact: AssistantArtifact): void => { + const conversationId = activeConversationIdRef.current; + setContextError(undefined); + setImageReferencesByConversation(current => ({ ...current, + [conversationId]: [...(current[conversationId] ?? []).filter(item => item.id !== artifact.id), artifact].slice(-8), + })); + inputRef.current?.focus(); + }, []); + + const openImageModelSettings = useCallback((): void => { + setSettingsInitialCategory("model"); + setView("settings"); + }, [setView]); + + const reselectImageSources = useCallback((operation: ImageOperation): void => { + if (operation.conversationId !== activeId) return; + setInput(current => current || t("chat.images.recoveryPrompt", { + model: operation.modelProfileName ?? operation.modelName, + prompt: operation.input.prompt, + })); + inputRef.current?.focus(); + attachmentButtonRef.current?.click(); + }, [activeId, setInput, t]); + const selectContextFiles = async (paths?: string[]): Promise => { if (selectingContextFilesRef.current) { return; @@ -9229,6 +9311,9 @@ function App(): React.JSX.Element { locale={locale} onCopyMessage={copyMessage} onDownloadImage={downloadImage} + onEditImage={editImage} + onOpenImageModelSettings={openImageModelSettings} + onReselectImageSources={reselectImageSources} onOpenCitationContext={openCitationContext} onOpenCitationSource={openCitationSource} onOpenImage={openImageViewer} @@ -9343,12 +9428,20 @@ function App(): React.JSX.Element { running={conversationExecutionRunning} />
+ + profile.protocol === "openai-images-generations" && profile.allowConversationInvocation === true + ) ?? false} + onOpenModelSettings={openImageModelSettings} + /> {composerOptionSummary && (
{composerOptionSummary}
)} - {(attachments.length > 0 || + {(attachments.length > 0 || imageReferences.length > 0 || selectingContextFiles) && (
+ {imageReferences.map(artifact =>
+ {artifact.content && } + {artifact.title}{t('chat.images.edit')} + +
)} {attachments.map((attachment) => (
void selectContextFiles()} title={t("composer.addAttachment")} type="button" diff --git a/src/renderer/src/CapabilitiesAndToolsSettingsSection.tsx b/src/renderer/src/CapabilitiesAndToolsSettingsSection.tsx index d2296ad6..c84b92c7 100644 --- a/src/renderer/src/CapabilitiesAndToolsSettingsSection.tsx +++ b/src/renderer/src/CapabilitiesAndToolsSettingsSection.tsx @@ -11,9 +11,11 @@ type CapabilitySettingsTab = 'skills' | 'mcp' | 'tools' export function CapabilitiesAndToolsSettingsSection({ magicNotesEnabled, + onOpenImageModelSettings, onNotify }: { magicNotesEnabled: boolean + onOpenImageModelSettings: () => void onNotify?: (notification: AppNotificationInput) => void }): React.JSX.Element { const { t } = useTranslation('settings') @@ -76,7 +78,10 @@ export function CapabilitiesAndToolsSettingsSection({ role="tabpanel" > {mcpVisited && ( - + )}
void + onReselectImageSources?: (operation: ImageOperation) => void + onEditImage?: (artifact: AssistantArtifact) => void artifactById: ReadonlyMap canRetry: boolean conversationId: string @@ -522,6 +529,9 @@ type ChatMessageRowProps = { } function ChatMessageRowView({ + onOpenImageModelSettings, + onReselectImageSources, + onEditImage, artifactById, canRetry, conversationId, @@ -541,6 +551,10 @@ function ChatMessageRowView({ retryContent }: ChatMessageRowProps): React.JSX.Element { const { t } = useTranslation('app') + const imageArtifactIds = useMemo(() => [...new Set([ + ...(message.artifactIds ?? []), + ...(message.imageOperations?.flatMap(operation => operation.artifactIds) ?? []) + ])], [message.artifactIds, message.imageOperations]) const compressionMarkers = message.contextCompressions ?? (message.contextCompression ? [message.contextCompression] : []) @@ -799,7 +813,8 @@ function ChatMessageRowView({ subagents={unorderedSubagents} /> )} - {message.artifactIds?.map((artifactId) => { + {message.imageOperations?.map(operation => )} + {imageArtifactIds.map((artifactId) => { const candidate = artifactById.get(artifactId) const artifact = candidate?.kind === 'image' && @@ -833,6 +848,7 @@ function ChatMessageRowView({
{artifact.title}
+ {onEditImage && } +
+ + )} +
+ ) +} diff --git a/src/renderer/src/ImageOperationStatus.test.tsx b/src/renderer/src/ImageOperationStatus.test.tsx new file mode 100644 index 00000000..5526338d --- /dev/null +++ b/src/renderer/src/ImageOperationStatus.test.tsx @@ -0,0 +1,74 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ImageOperationStatus } from './ImageOperationStatus' +import type { ImageOperation } from '../../shared/image-generation-contracts' + +afterEach(() => { cleanup(); vi.unstubAllGlobals() }) + +describe('image operation controls', () => { + it('offers explicit failed-edit recovery without changing regeneration inputs', async () => { + const regenerate = vi.fn(async () => undefined) + Object.defineProperty(window, 'goodbuddy', { configurable: true, value: { conversations: { imageOperations: { regenerate } } } }) + const operation: ImageOperation = { + id: crypto.randomUUID(), conversationId: crypto.randomUUID(), messageId: crypto.randomUUID(), + requestId: crypto.randomUUID(), callId: 'edit-call', modelProfileId: crypto.randomUUID(), modelName: 'original-model', + input: { intent: 'edit', prompt: 'Change the background', sourceArtifactIds: [crypto.randomUUID()] }, + state: 'failed', error: 'Source unavailable', artifactIds: [], createdAt: 1, updatedAt: 1 + } + const onOpenImageModelSettings = vi.fn() + const onReselectImageSources = vi.fn() + const view = render() + fireEvent.click(screen.getByRole('button', { name: '重新选择模型' })) + fireEvent.click(screen.getByRole('button', { name: '重新选择素材' })) + expect(onOpenImageModelSettings).toHaveBeenCalledOnce() + expect(onReselectImageSources).toHaveBeenCalledExactlyOnceWith(operation) + expect(regenerate).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: '重新生成' })) + await waitFor(() => expect(regenerate).toHaveBeenCalledExactlyOnceWith({ conversationId: operation.conversationId, operationId: operation.id })) + view.rerender() + expect(screen.queryByRole('button', { name: '重新选择模型' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: '重新选择素材' })).not.toBeInTheDocument() + view.rerender() + expect(screen.getByRole('button', { name: '重新选择模型' })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: '重新选择素材' })).not.toBeInTheDocument() + }) + + it('cancels independently and offers an explicit new request with local error recovery', async () => { + const cancel = vi.fn(async () => undefined) + const regenerate = vi.fn(async () => { throw new Error('Selected image model is unavailable') }) + Object.defineProperty(window, 'goodbuddy', { configurable: true, value: { conversations: { imageOperations: { cancel, regenerate } } } }) + const operation: ImageOperation = { id: crypto.randomUUID(), conversationId: crypto.randomUUID(), messageId: crypto.randomUUID(), + requestId: crypto.randomUUID(), callId: 'image-call', modelProfileId: crypto.randomUUID(), modelName: 'image-model', + input: { intent: 'create', prompt: 'Blue circle', sourceArtifactIds: [] }, state: 'running', artifactIds: [], createdAt: 1, updatedAt: 1 } + const view = render() + fireEvent.click(screen.getByRole('button', { name: '取消生成' })) + await waitFor(() => expect(cancel).toHaveBeenCalledExactlyOnceWith({ conversationId: operation.conversationId, operationId: operation.id })) + view.rerender() + expect(screen.getByText('将发起新的模型请求。')).toBeInTheDocument() + fireEvent.click(await screen.findByRole('button', { name: '重新生成' })) + expect(await screen.findByRole('alert')).toHaveTextContent('Selected image model is unavailable') + expect(screen.getByText('Blue circle')).toBeInTheDocument() + expect(screen.getByRole('button', { name: '重新生成' })).toBeEnabled() + expect(screen.getByRole('button', { name: '重新生成' })).toHaveAccessibleDescription('Selected image model is unavailable') + expect(regenerate).toHaveBeenCalledOnce() + }) + + it('distinguishes pending cancellation, stopped waiting, and a saved late result', () => { + const operation: ImageOperation = { + id: crypto.randomUUID(), conversationId: crypto.randomUUID(), messageId: crypto.randomUUID(), + requestId: crypto.randomUUID(), callId: 'image-call', modelProfileId: crypto.randomUUID(), modelName: 'image-model', + input: { intent: 'create', prompt: 'Blue circle', sourceArtifactIds: [] }, + state: 'cancelling', cancellationRequested: true, artifactIds: [], createdAt: 1, updatedAt: 1 + } + const view = render() + expect(screen.getByText('已请求取消。提供商可能继续生成并计费。')).toBeVisible() + expect(screen.queryByText(/已停止等待/)).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: '取消生成' })).toBeDisabled() + view.rerender() + expect(screen.getByText('已停止等待。提供商可能继续生成并计费，晚到的结果会保存在这里。')).toBeVisible() + view.rerender() + expect(screen.getByText('取消未生效，已生成并保存图片。')).toBeVisible() + expect(screen.queryByText(/已停止等待/)).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: '重新生成' })).toBeEnabled() + }) +}) diff --git a/src/renderer/src/ImageOperationStatus.tsx b/src/renderer/src/ImageOperationStatus.tsx new file mode 100644 index 00000000..b5bad78e --- /dev/null +++ b/src/renderer/src/ImageOperationStatus.tsx @@ -0,0 +1,62 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import type { ImageOperation } from '../../shared/image-generation-contracts' +import { displayErrorMessage } from './error-message' + +export function ImageOperationStatus({ operation, onOpenImageModelSettings, onReselectImageSources }: { + operation: ImageOperation + onOpenImageModelSettings?: () => void + onReselectImageSources?: (operation: ImageOperation) => void +}): React.JSX.Element { + const { t } = useTranslation('app') + const [pending, setPending] = useState(false) + const [error, setError] = useState() + const active = ['running', 'saving', 'cancelling'].includes(operation.state) + const needsRecovery = !active && (operation.state === 'failed' || Boolean(error)) + const act = async (): Promise => { + setPending(true) + setError(undefined) + try { + const api = window.goodbuddy.conversations.imageOperations + await api[active ? 'cancel' : 'regenerate']({ conversationId: operation.conversationId, operationId: operation.id }) + } catch (reason) { + setError(displayErrorMessage(reason, t('chat.images.states.failed'))) + } finally { + setPending(false) + } + } + return
+
+ {t(`chat.images.states.${operation.state}`)} · {operation.modelProfileName ?? operation.modelName} + {operation.cancellationRequested &&

{t( + operation.state === 'completed' + ? 'chat.images.cancelTooLateNotice' + : active ? 'chat.images.cancellingNotice' : 'chat.images.cancelNotice' + )}

} + {operation.state === 'unconfirmed' &&

{t('chat.images.unconfirmedNotice')}

} + {operation.error &&

{operation.error}

} +
{t('chat.images.details')}
{operation.input.prompt}
+ {operation.modelName} · {operation.input.quality ?? 'auto'} + {operation.input.sourceArtifactIds.length > 0 &&
{t('chat.images.sources', { count: operation.input.sourceArtifactIds.length })}
} +
+
+ + {!active && {t('chat.images.regenerateNotice')}} + {needsRecovery && onOpenImageModelSettings && ( + + )} + {needsRecovery && operation.input.intent === 'edit' && onReselectImageSources && ( + + )} +
+ {needsRecovery && {t('chat.images.recoveryNotice')}} + {error && } +
+
+} diff --git a/src/renderer/src/McpSettingsSection.tsx b/src/renderer/src/McpSettingsSection.tsx index d392a249..2f0559b5 100644 --- a/src/renderer/src/McpSettingsSection.tsx +++ b/src/renderer/src/McpSettingsSection.tsx @@ -96,8 +96,10 @@ function editorFromServer(server: McpServerSummary): McpEditor { } export function McpSettingsSection({ + onOpenImageModelSettings, magicNotesEnabled = false }: { + onOpenImageModelSettings: () => void magicNotesEnabled?: boolean }): React.JSX.Element { const { t } = useTranslation('integrations') @@ -574,11 +576,49 @@ export function McpSettingsSection({
{t('mcp.profiles.count', { - count: builtinMcpServers.length + count: builtinMcpServers.length + 1 })}
+
+
+
+ {t('conversationImages.title')} + {t('conversationImages.managed')} +
+
+

+ {t('conversationImages.description')} +

+

{t('conversationImages.runtimeScope')}

+
+ {t('mcp.builtin.assignedTo')} + {configurableMcpTargets.map((target) => ( + + ))} +
+ +
{builtinMcpServers.map((server) => { const expansionId = `builtin:${server.id}` const expanded = expandedItemIds.has(expansionId) diff --git a/src/renderer/src/SettingsPanel.test.tsx b/src/renderer/src/SettingsPanel.test.tsx index ddde3c79..d698ddc7 100644 --- a/src/renderer/src/SettingsPanel.test.tsx +++ b/src/renderer/src/SettingsPanel.test.tsx @@ -4790,6 +4790,11 @@ describe('SettingsPanel runtime files', () => { target: { value: 'openai-images-generations' } }) const qualitySelect = screen.getByLabelText('图片质量 默认模型') + const invocationSwitch = screen.getByRole('switch', { name: '允许 AI 在会话中调用' }) + expect(invocationSwitch).not.toBeChecked() + expect(screen.queryByRole('radio', { name: '设为默认生图模型' })).not.toBeInTheDocument() + fireEvent.click(invocationSwitch) + fireEvent.click(screen.getByRole('radio', { name: '设为默认生图模型' })) expect(qualitySelect).toHaveValue('auto') fireEvent.change(qualitySelect, { target: { value: 'high' } @@ -4807,10 +4812,12 @@ describe('SettingsPanel runtime files', () => { modelProfiles: [ expect.objectContaining({ protocol: 'openai-images-generations', + allowConversationInvocation: true, imageGenerationQuality: 'high' }) ], - imageGenerationQuality: 'high' + imageGenerationQuality: 'high', + defaultImageModelProfileId: modelProfileId }) ) ) @@ -5179,6 +5186,77 @@ describe('SettingsPanel runtime files', () => { expect(screen.queryByText('智能心跳')).not.toBeInTheDocument() }) + it.each([false, true])('shows immutable image assignments and source navigation when enabled=%s', async (enabled) => { + const snapshot = { + ...capabilitySnapshot, + imageGeneration: { + modelProfiles: enabled ? [{ id: modelProfileId, name: 'Image' }] : [], + assignments: enabled ? ['model', 'opencode', 'continue', 'deepseek-harness'] : [] + } + } + getCapabilitySnapshot.mockResolvedValueOnce(snapshot).mockResolvedValueOnce(snapshot) + render( {})} onClose={vi.fn()} onSaved={vi.fn()} />) + fireEvent.click(screen.getByRole('tab', { name: '能力与工具' })) + await screen.findByText('文档写作') + fireEvent.click(screen.getByRole('tab', { name: 'MCP' })) + const card = screen.getByRole('article', { name: '图片生成与编辑' }) + await waitFor(() => { + const checkboxes = within(card).getAllByRole('checkbox') + expect(checkboxes).toHaveLength(4) + for (const checkbox of checkboxes) { + expect(checkbox).toBeDisabled() + if (enabled) expect(checkbox).toBeChecked() + else expect(checkbox).not.toBeChecked() + } + }) + expect(within(card).getByText('内置 · 自动管理')).toBeInTheDocument() + expect(within(card).queryByRole('switch')).not.toBeInTheDocument() + expect(within(card).queryByText(/暂不支持/)).not.toBeInTheDocument() + for (const checkbox of within(card).getAllByRole('checkbox')) { + expect(checkbox).toHaveAccessibleDescription('由图片模型的「允许 AI 在会话中调用」设置自动管理。') + fireEvent.click(checkbox) + fireEvent.click(checkbox.closest('label')!) + fireEvent.keyDown(checkbox, { key: ' ' }) + } + expect(setBuiltinMcpServerAssignments).not.toHaveBeenCalled() + expect(setBuiltinMcpServerEnabled).not.toHaveBeenCalled() + expect(saveMcpServer).not.toHaveBeenCalled() + const navigation = within(card).getByRole('button', { name: '前往图片模型设置' }) + navigation.focus() + expect(navigation).toHaveFocus() + fireEvent.click(navigation) + expect(screen.getByRole('tab', { name: '模型连接' })).toHaveAttribute('aria-selected', 'true') + expect(await screen.findByRole('button', { name: '添加自定义' })).toBeEnabled() + }) + + it('retains a disabled or deleted image default and keeps drafts on save failure', async () => { + const imageId = '00000000-0000-4000-8000-000000000099' + getRuntime.mockResolvedValueOnce({ + ...runtimeSettings, + defaultImageModelProfileId: imageId, + modelProfiles: [runtimeSettings.modelProfiles[0]!, { + ...runtimeSettings.modelProfiles[0]!, id: imageId, name: 'Image', + protocol: 'openai-images-generations', allowConversationInvocation: true + }] + }) + render( {})} onClose={vi.fn()} onSaved={vi.fn()} />) + fireEvent.click(screen.getByRole('tab', { name: '模型连接' })) + fireEvent.click(await screen.findByRole('button', { name: '编辑模型连接 Image' })) + fireEvent.click(screen.getByRole('switch', { name: '允许 AI 在会话中调用' })) + expect(screen.getByText('默认生图模型不可用，请开启该模型或明确选择其他图片模型。')).toBeInTheDocument() + updateRuntime.mockRejectedValueOnce(new Error('Image settings save failed')) + fireEvent.click(screen.getByRole('button', { name: '保存设置' })) + await screen.findByText('Image settings save failed') + expect(screen.getByRole('switch', { name: '允许 AI 在会话中调用' })).not.toBeChecked() + expect(updateRuntime).toHaveBeenLastCalledWith(expect.objectContaining({ defaultImageModelProfileId: imageId })) + fireEvent.click(screen.getByRole('button', { name: '删除模型连接 Image' })) + fireEvent.click(screen.getByRole('button', { name: '保存设置' })) + await waitFor(() => expect(updateRuntime).toHaveBeenLastCalledWith(expect.objectContaining({ + defaultImageModelProfileId: imageId, + modelProfiles: [expect.objectContaining({ id: modelProfileId })] + }))) + }) + it('groups Skills and MCP under Capabilities and tools', async () => { render( ({ ...profile, supportsImageInput: profile.supportsImageInput ?? false, + allowConversationInvocation: profile.allowConversationInvocation ?? false, maximumOutputTokens: profile.maximumOutputTokens ?? defaultAnthropicMaximumOutputTokens, @@ -356,6 +357,7 @@ function hydrateRuntimeSettings( modelProfiles: (value: ModelProfileDraft[]) => void selectedModelProfileId: (value: RuntimeDraftSelection) => void defaultModelProfileId: (value: string) => void + defaultImageModelProfileId: (value: string | null) => void opencodeModelSource: (value: RuntimeModelSource) => void continueModelSource: (value: RuntimeModelSource) => void deepseekHarnessModelSource: (value: RuntimeModelSource) => void @@ -404,6 +406,7 @@ function hydrateRuntimeSettings( : fallbackProfileId ) setters.defaultModelProfileId(value.defaultModelProfileId) + setters.defaultImageModelProfileId(value.defaultImageModelProfileId ?? null) setters.opencodeModelSource(configured.opencodeModelSource) setters.continueModelSource(configured.continueModelSource) setters.deepseekHarnessModelSource( @@ -671,6 +674,8 @@ export function SettingsPanel({ const [selectedModelProfileId, setSelectedModelProfileId] = useState('') const [defaultModelProfileId, setDefaultModelProfileId] = useState('') + const [defaultImageModelProfileId, setDefaultImageModelProfileId] = + useState(null) const [opencodeModelSource, setOpencodeModelSource] = useState({ kind: 'platform' }) const [continueModelSource, setContinueModelSource] = @@ -844,6 +849,7 @@ export function SettingsPanel({ modelProfiles: setModelProfiles, selectedModelProfileId: setSelectedModelProfileId, defaultModelProfileId: setDefaultModelProfileId, + defaultImageModelProfileId: setDefaultImageModelProfileId, opencodeModelSource: setOpencodeModelSource, continueModelSource: setContinueModelSource, deepseekHarnessModelSource: setDeepseekHarnessModelSource, @@ -905,6 +911,7 @@ export function SettingsPanel({ provider, modelProfiles, defaultModelProfileId, + defaultImageModelProfileId, opencodeModelSource, continueModelSource, deepseekHarnessModelSource, @@ -936,6 +943,7 @@ export function SettingsPanel({ provider: settings.provider, modelProfiles: toModelProfileDrafts(settings), defaultModelProfileId: settings.defaultModelProfileId, + defaultImageModelProfileId: settings.defaultImageModelProfileId ?? null, opencodeModelSource: savedConfiguredSettings?.opencodeModelSource, continueModelSource: @@ -1420,6 +1428,7 @@ export function SettingsPanel({ protocol: profile.protocol, authentication: profile.authentication, supportsImageInput: profile.supportsImageInput, + allowConversationInvocation: profile.allowConversationInvocation ?? false, contextWindowTokens: profile.contextWindowTokens, maximumOutputTokens: profile.maximumOutputTokens, imageGenerationQuality: profile.imageGenerationQuality, @@ -1506,6 +1515,7 @@ export function SettingsPanel({ apiKey: defaultProfileInput.apiKey, modelProfiles: profileInputs, defaultModelProfileId: defaultProfile.id, + defaultImageModelProfileId, opencodeModelSource, continueModelSource, deepseekHarnessModelSource, @@ -1802,6 +1812,7 @@ export function SettingsPanel({ protocol: 'openai-chat-completions', authentication: defaultRuntimeSettings.modelAuthentication, supportsImageInput: defaultRuntimeSettings.supportsImageInput, + allowConversationInvocation: false, maximumOutputTokens: defaultAnthropicMaximumOutputTokens, imageGenerationQuality: defaultRuntimeSettings.imageGenerationQuality, @@ -2929,6 +2940,15 @@ export function SettingsPanel({ {t('actions.addCustom')}
+ {defaultImageModelProfileId && !modelProfiles.some((profile) => + profile.id === defaultImageModelProfileId && + profile.protocol === 'openai-images-generations' && + profile.allowConversationInvocation + ) && ( +

+ {t('conversationImages.unavailable')} +

+ )}
)} + {profile.protocol === 'openai-images-generations' && ( +
+ + + {t('conversationImages.description')} + + {profile.allowConversationInvocation && ( + + )} +
+ )} {isAgentRuntimeModelProtocol(profile.protocol) && ( <>
@@ -4081,6 +4137,17 @@ export function SettingsPanel({ )} {activeTab === 'capabilities' && ( { + if (requestTabChange('model')) { + setModelType('llm') + const imageProfile = modelProfiles.find( + (profile) => profile.protocol === 'openai-images-generations' + ) + if (imageProfile) { + setSelectedModelProfileId(imageProfile.id) + } + } + }} magicNotesEnabled={magicNotesEnabled} onNotify={onNotify} /> diff --git a/src/renderer/src/WorkspacePrimitives.test.tsx b/src/renderer/src/WorkspacePrimitives.test.tsx index 40dde5cf..47351008 100644 --- a/src/renderer/src/WorkspacePrimitives.test.tsx +++ b/src/renderer/src/WorkspacePrimitives.test.tsx @@ -92,6 +92,18 @@ describe('WorkspacePrimitives', () => { ) }) + it('wraps long model names, prompts, and errors inside shared message status cards', () => { + expect(stylesheet).toMatch( + /\.message-retrieval-status\s*\{[^}]*overflow-wrap:\s*anywhere;/u + ) + expect(stylesheet).toMatch( + /\.message-retrieval-status > div\s*\{[^}]*min-width:\s*0;/u + ) + expect(stylesheet).toMatch( + /\.message-image-actions\s*\{[^}]*flex-wrap:\s*wrap;/u + ) + }) + it('keeps shared switch rows aligned without standalone dividers', () => { const toggleRowStyles = stylesheet.match( /\.toggle-row\s*\{(?[^}]*)\}/u diff --git a/src/renderer/src/i18n/locales/en-US/app.ts b/src/renderer/src/i18n/locales/en-US/app.ts index 8b380a05..eb3439f2 100644 --- a/src/renderer/src/i18n/locales/en-US/app.ts +++ b/src/renderer/src/i18n/locales/en-US/app.ts @@ -267,6 +267,26 @@ export const app = { }, exportSpeaker: '{{speaker}}:\n{{content}}', images: { + operation: 'Image generation', + edit: 'Continue editing', + cancel: 'Cancel generation', + regenerate: 'Generate again', + regenerateNotice: 'Starts a new model request.', + reselectModel: 'Choose another model', + reselectSources: 'Choose source images', + recoveryNotice: 'Generate again keeps the original model and sources. To change the model, name it explicitly in a new composer request. To change sources, upload attachments or choose Continue editing on a previous image.', + recoveryPrompt: 'Use image model "{{model}}": {{prompt}}', + noToolsNotice: 'This chat model cannot call image tools automatically. You can use the existing direct image workflow.', + directWorkflowNotice: 'Choose an image model in the Runtime selector and send your request. If none is configured, add an image connection in model settings.', + askNotice: 'Generating or editing images requires Execute mode. Change it using the work mode selector.', + openModelSettings: 'Open image model settings', + cancelNotice: 'Waiting was stopped. The provider may still generate and charge for the image; a late result will be saved here.', + cancellingNotice: 'Cancellation was requested. The provider may still generate and charge for the image.', + cancelTooLateNotice: 'Cancellation did not take effect. The generated image has been saved.', + unconfirmedNotice: 'The result could not be confirmed. Check before starting another request.', + details: 'Request details', + sources: '{{count}} source images', + states: { running: 'Generating image', saving: 'Saving image', completed: 'Image saved', cancelling: 'Cancelling generation', stopped: 'Stopped waiting', failed: 'Image generation failed', unconfirmed: 'Result unconfirmed' }, contextNotice: { 'reference-unavailable': 'Some previous images could not be loaded. Only available images and text context were used.', 'editing-unavailable': 'The provider does not support image editing. This image was generated from text without the reference images.' diff --git a/src/renderer/src/i18n/locales/en-US/integrations.ts b/src/renderer/src/i18n/locales/en-US/integrations.ts index 1a12423e..dcb2eea8 100644 --- a/src/renderer/src/i18n/locales/en-US/integrations.ts +++ b/src/renderer/src/i18n/locales/en-US/integrations.ts @@ -2,6 +2,13 @@ import type { TranslationShape } from '../../resource-types' import type { integrations as chineseIntegrations } from '../zh-CN/integrations' export const integrations = { + conversationImages: { + title: 'Image generation and editing', + managed: 'Builtin · Automatically managed', + description: 'Automatically managed by the image models’ “Allow AI to invoke in conversations” setting.', + runtimeScope: 'Assignments include applicable remote Runtimes through GoodBuddy Agent. Generation requires Execute and a chat model with tool calling; assignments do not indicate runtime readiness.', + openSettings: 'Go to image model settings' + }, channels: { tabs: { weixin: 'WeChat ClawBot', diff --git a/src/renderer/src/i18n/locales/en-US/settings.ts b/src/renderer/src/i18n/locales/en-US/settings.ts index 7fb0cfe8..b9c1f4df 100644 --- a/src/renderer/src/i18n/locales/en-US/settings.ts +++ b/src/renderer/src/i18n/locales/en-US/settings.ts @@ -2,6 +2,14 @@ import type { TranslationShape } from '../../resource-types' import type { settings as chineseSettings } from '../zh-CN/settings' export const settings = { + conversationImages: { + allow: 'Allow AI to invoke in conversations', + description: 'When enabled, assistants with tool calling can use this model to generate and edit images. Editing support depends on the image protocol. Results are saved in the originating conversation.', + callable: 'Conversation enabled', + defaultBadge: 'Default image model', + setDefault: 'Set as default image model', + unavailable: 'The default image model is unavailable. Enable it or explicitly choose another image model.' + }, modelDownloadSources: { modelscope: 'ModelScope', 'hugging-face': 'Hugging Face' diff --git a/src/renderer/src/i18n/locales/zh-CN/app.ts b/src/renderer/src/i18n/locales/zh-CN/app.ts index fa44c07d..fc66edfb 100644 --- a/src/renderer/src/i18n/locales/zh-CN/app.ts +++ b/src/renderer/src/i18n/locales/zh-CN/app.ts @@ -252,6 +252,26 @@ export const app = { }, exportSpeaker: '{{speaker}}：\n{{content}}', images: { + operation: '图片生成', + edit: '继续修改', + cancel: '取消生成', + regenerate: '重新生成', + regenerateNotice: '将发起新的模型请求。', + reselectModel: '重新选择模型', + reselectSources: '重新选择素材', + recoveryNotice: '重新生成仍使用原模型与素材。更换模型后，请在输入区明确指定模型并重新发送；更换素材可上传附件，或在历史图片上点击“继续修改”。', + recoveryPrompt: '使用图片模型“{{model}}”：{{prompt}}', + noToolsNotice: '当前聊天模型无法自动调用图片工具，可使用已有直连图片工作流。', + directWorkflowNotice: '在 Runtime 选择器中选择图片模型后发送请求；尚未配置时，请前往模型设置添加图片连接。', + askNotice: '生成或编辑图片需要 Execute 模式，请通过工作模式选择器切换。', + openModelSettings: '前往图片模型设置', + cancelNotice: '已停止等待。提供商可能继续生成并计费，晚到的结果会保存在这里。', + cancellingNotice: '已请求取消。提供商可能继续生成并计费。', + cancelTooLateNotice: '取消未生效，已生成并保存图片。', + unconfirmedNotice: '暂时无法确认结果，请在重新发起请求前核对。', + details: '请求详情', + sources: '{{count}} 张来源图片', + states: { running: '正在生成图片', saving: '正在保存图片', completed: '图片已保存', cancelling: '正在取消生成', stopped: '已停止等待', failed: '图片生成失败', unconfirmed: '结果未确认' }, contextNotice: { 'reference-unavailable': '部分历史图片无法读取，本次仅使用可用的图片与文字上下文。', 'editing-unavailable': '上游不支持图片编辑，本次按文字要求生成，未使用参考图片。' diff --git a/src/renderer/src/i18n/locales/zh-CN/integrations.ts b/src/renderer/src/i18n/locales/zh-CN/integrations.ts index e2bb2149..cf8c2b95 100644 --- a/src/renderer/src/i18n/locales/zh-CN/integrations.ts +++ b/src/renderer/src/i18n/locales/zh-CN/integrations.ts @@ -1,4 +1,11 @@ export const integrations = { + conversationImages: { + title: '图片生成与编辑', + managed: '内置 · 自动管理', + description: '由图片模型的「允许 AI 在会话中调用」设置自动管理。', + runtimeScope: '分配包含 GoodBuddy Agent 远程路径上的适用 Runtime。生成需要 Execute 模式及支持工具调用的聊天模型；勾选不代表 Runtime 已就绪。', + openSettings: '前往图片模型设置' + }, channels: { tabs: { weixin: '微信 ClawBot', diff --git a/src/renderer/src/i18n/locales/zh-CN/settings.ts b/src/renderer/src/i18n/locales/zh-CN/settings.ts index d5fdc962..c5603318 100644 --- a/src/renderer/src/i18n/locales/zh-CN/settings.ts +++ b/src/renderer/src/i18n/locales/zh-CN/settings.ts @@ -1,4 +1,12 @@ export const settings = { + conversationImages: { + allow: '允许 AI 在会话中调用', + description: '开启后，支持工具调用的聊天助手可使用此模型生成和编辑图片，编辑能力取决于图片协议。结果保存在发起请求的会话中。', + callable: '可在会话中调用', + defaultBadge: '默认生图模型', + setDefault: '设为默认生图模型', + unavailable: '默认生图模型不可用，请开启该模型或明确选择其他图片模型。' + }, modelDownloadSources: { modelscope: 'ModelScope', 'hugging-face': 'Hugging Face' diff --git a/src/renderer/src/styles.css b/src/renderer/src/styles.css index d9af554b..f3332c67 100644 --- a/src/renderer/src/styles.css +++ b/src/renderer/src/styles.css @@ -1020,6 +1020,7 @@ .message-retrieval-status { display: flex; + overflow-wrap: anywhere; align-items: flex-start; gap: var(--space-2); margin-top: var(--space-3); @@ -5749,6 +5750,7 @@ button > svg { .message-image-actions { display: flex; + flex-wrap: wrap; align-items: center; gap: var(--space-2); } diff --git a/src/shared/assistant-contracts.ts b/src/shared/assistant-contracts.ts index e8598c4c..ed690972 100644 --- a/src/shared/assistant-contracts.ts +++ b/src/shared/assistant-contracts.ts @@ -419,6 +419,8 @@ export const conversationMessageSchema = z .strict() .optional(), artifactIds: z.array(assistantIdSchema).max(8).optional(), + imageOperations: z.array(imageOperationSchema).optional(), + imageSourceArtifactIds: z.array(assistantIdSchema).max(8).optional(), imageContextNotice: z .enum(['reference-unavailable', 'editing-unavailable']) .optional(), @@ -1032,3 +1034,4 @@ export type AssistantExpert = z.output & { createdAt: string updatedAt: string } +import { imageOperationSchema } from './image-generation-contracts' diff --git a/src/shared/builtin-model-tools.test.ts b/src/shared/builtin-model-tools.test.ts index 60d718f3..2da7edba 100644 --- a/src/shared/builtin-model-tools.test.ts +++ b/src/shared/builtin-model-tools.test.ts @@ -11,12 +11,12 @@ describe('built-in model tool catalog', () => { expect(new Set(names).size).toBe(names.length) }) - it('places every direct-model group outside the shared browser capability', () => { + it('places direct-model groups outside the shared browser and managed image capabilities', () => { const groupedNames = builtinModelToolGroups.flatMap((group) => group.tools.map((tool) => tool.name) ) const directModelTools = builtinModelTools.filter( - (tool) => tool.group !== 'browser' + (tool) => tool.group !== 'browser' && tool.group !== 'image' ) expect(groupedNames).toHaveLength(directModelTools.length) @@ -24,6 +24,10 @@ describe('built-in model tool catalog', () => { expect(groupedNames).toEqual( expect.arrayContaining(directModelTools.map((tool) => tool.name)) ) + expect(groupedNames).not.toContain('generate_image') + expect(builtinModelTools.find((tool) => tool.name === 'generate_image')).toMatchObject({ + group: 'image', access: 'write' + }) for (const group of builtinModelToolGroups) { expect(group.tools.every((tool) => tool.group === group.id)).toBe(true) diff --git a/src/shared/builtin-model-tools.ts b/src/shared/builtin-model-tools.ts index 661a4770..8c0f9a9c 100644 --- a/src/shared/builtin-model-tools.ts +++ b/src/shared/builtin-model-tools.ts @@ -3,10 +3,17 @@ export type BuiltinModelToolSummary = { displayName: string description: string access: 'read' | 'write' - group: 'filesystem' | 'browser' | 'web' | 'programming' + group: 'filesystem' | 'browser' | 'web' | 'programming' | 'image' } export const builtinModelTools = [ + { + name: 'generate_image', + displayName: 'Generate or edit image', + description: 'Generate or edit an image with an enabled image model and save it in this conversation.', + access: 'write', + group: 'image' + }, { name: 'workspace_rg', displayName: '搜索工作区', diff --git a/src/shared/capability-contracts.ts b/src/shared/capability-contracts.ts index 35f3a31d..b3f2e2a0 100644 --- a/src/shared/capability-contracts.ts +++ b/src/shared/capability-contracts.ts @@ -368,6 +368,16 @@ export type WebSearchCapability = z.infer< export const capabilitySnapshotSchema = z .object({ skills: z.array(skillSummarySchema).max(256), + imageGeneration: z + .object({ + modelProfiles: z.array(z.object({ + id: z.string().uuid(), + name: z.string() + }).strict()).max(20), + assignments: capabilityAssignmentsSchema + }) + .strict() + .optional(), builtinMcpServers: z .array(builtinMcpServerStateSummarySchema) .max(4) diff --git a/src/shared/contracts.ts b/src/shared/contracts.ts index 5880ea16..6d09abf8 100644 --- a/src/shared/contracts.ts +++ b/src/shared/contracts.ts @@ -385,6 +385,7 @@ export const conversationQueueUserInputSchema = z includeMemoryContext: z.boolean().default(true), prompt: z.string().trim().min(1).max(100_000), attachments: z.array(conversationAttachmentSchema).max(8).default([]), + imageContextArtifactIds: z.array(assistantIdSchema).max(8).optional(), knowledgeLibraryIds: z .array(z.string().uuid()) .max(20) @@ -625,6 +626,7 @@ const modelProfileInputSchema = z protocol: modelProtocolSchema, authentication: modelAuthenticationSchema, supportsImageInput: z.boolean().optional(), + allowConversationInvocation: z.boolean().optional(), contextWindowTokens: z .number() .int() @@ -740,6 +742,7 @@ export const runtimeSettingsInputSchema = z apiKey: modelApiKeyUpdateSchema, modelProfiles: z.array(modelProfileInputSchema).min(1).max(20).optional(), defaultModelProfileId: modelProfileIdSchema.optional(), + defaultImageModelProfileId: modelProfileIdSchema.nullable().optional(), opencodeModelSource: runtimeModelSourceSchema.optional(), continueModelSource: runtimeModelSourceSchema.optional(), deepseekHarnessModelSource: runtimeModelSourceSchema @@ -1019,6 +1022,7 @@ export type ModelConnectionSettings = { protocol: ModelProtocol authentication: ModelAuthentication supportsImageInput?: boolean + allowConversationInvocation?: boolean contextWindowTokens?: number maximumOutputTokens?: number imageGenerationQuality: ImageGenerationQuality @@ -1111,6 +1115,7 @@ export type RuntimeSettings = { credentialSource: 'none' | 'encrypted' | 'environment' | 'unreadable' modelProfiles: ModelConnectionSettings[] defaultModelProfileId: string + defaultImageModelProfileId?: string | null opencodeModelSource: RuntimeModelSource continueModelSource: RuntimeModelSource deepseekHarnessModelSource?: RuntimeModelSource @@ -2047,6 +2052,7 @@ export type DesktopApi = { } } conversations: { + imageOperations: import('./image-operation-ipc').ImageOperationsApi list: () => Promise replace: (conversations: ConversationSnapshot[]) => Promise saveLocal: (batch: LocalConversationSaveBatch) => Promise diff --git a/src/shared/image-generation-contracts.ts b/src/shared/image-generation-contracts.ts new file mode 100644 index 00000000..8bfface3 --- /dev/null +++ b/src/shared/image-generation-contracts.ts @@ -0,0 +1,45 @@ +import { z } from 'zod' + +export const imageToolInputSchema = z.object({ + prompt: z.string().trim().min(1).max(32_000), + intent: z.enum(['create', 'edit']), + modelProfileId: z.string().uuid().optional(), + sourceArtifactIds: z.array(z.string().uuid()).max(8).default([]), + quality: z.enum(['auto', 'low', 'medium', 'high']).optional() +}).strict().superRefine((input, context) => { + if ((input.intent === 'edit') !== (input.sourceArtifactIds.length > 0)) { + context.addIssue({ code: 'custom', message: 'Editing requires source images; creating must not include source images.' }) + } +}) +export type ImageToolInput = z.infer + +export const imageOperationSchema = z.object({ + id: z.string().uuid(), + conversationId: z.string(), + messageId: z.string(), + requestId: z.string(), + callId: z.string(), + modelProfileId: z.string().uuid(), + modelName: z.string(), + modelProfileName: z.string().optional(), + input: imageToolInputSchema, + state: z.enum(['running', 'saving', 'completed', 'cancelling', 'stopped', 'failed', 'unconfirmed']), + createdAt: z.number(), + updatedAt: z.number(), + artifactIds: z.array(z.string().uuid()).default([]), + error: z.string().optional(), + cancellationRequested: z.boolean().optional() +}).strict() +export type ImageOperation = z.infer + +/** Bound by Main/Agent transport, never accepted as model tool arguments. */ +export type ImageRequestContext = { + conversationId: string + messageId: string + requestId: string + workMode: 'ask' | 'execute' +} + +export const imageToolName = 'generate_image' as const +export const imageToolDescriptionLimit = 16_000 +export const imageToolDescription = 'Generate or edit an image using a configured image model. Use intent=create for a new unrelated image, or intent=edit with explicit sourceArtifactIds. Never invent profile or artifact IDs. Clarify ambiguous references. A failed edit must not be replaced by generation. Results contain operation and artifact references, not image bytes. Do not retry a paid request automatically. A running operation continues after this chat stops; inspect its conversation card for completion.' diff --git a/src/shared/image-operation-ipc.ts b/src/shared/image-operation-ipc.ts new file mode 100644 index 00000000..df2e5b57 --- /dev/null +++ b/src/shared/image-operation-ipc.ts @@ -0,0 +1,14 @@ +import { z } from 'zod' +import type { ImageOperation } from './image-generation-contracts' + +export const imageOperationTargetSchema = z.object({ + conversationId: z.string().min(1).max(128), + operationId: z.string().uuid() +}).strict() + +export type ImageOperationTarget = z.infer +export type ImageOperationsApi = { + cancel(input: ImageOperationTarget): Promise + regenerate(input: ImageOperationTarget): Promise + onChanged(listener: (operation: ImageOperation) => void): () => void +} diff --git a/src/shared/ipc-channels.ts b/src/shared/ipc-channels.ts index 1c347280..f445f688 100644 --- a/src/shared/ipc-channels.ts +++ b/src/shared/ipc-channels.ts @@ -190,6 +190,9 @@ export const ipcChannels = { conversationsBranchLocal: 'conversations:branch-local', conversationsDeleteLocal: 'conversations:delete-local', conversationsChanged: 'conversations:changed', + imageOperationCancel: 'image-operation:cancel', + imageOperationRegenerate: 'image-operation:regenerate', + imageOperationChanged: 'image-operation:changed', conversationQueueList: 'conversation-queue:list', conversationQueueEnqueueUser: 'conversation-queue:enqueue-user', conversationQueueRemove: 'conversation-queue:remove', diff --git a/src/shared/remote-agent-contracts.ts b/src/shared/remote-agent-contracts.ts index 65bd8433..4a847e44 100644 --- a/src/shared/remote-agent-contracts.ts +++ b/src/shared/remote-agent-contracts.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { remoteImageToolSchema } from './remote-image-tool-contracts' import { workspaceManagementActionSchema } from './workspace-management-contracts' import { acpFrameDirectionSchema, @@ -164,6 +165,7 @@ const remotePromptOperationIdentityFields = { runtimeAdapterDigest: sha256DigestSchema, modelBridge: remotePromptModelBridgeSchema.optional(), modelProfile: agentPromptModelProfileSchema.optional(), + imageTool: remoteImageToolSchema.optional(), promptSequence: z .number() .int() @@ -350,6 +352,7 @@ export type RemoteSemanticTranscriptEvent = z.infer< export const remotePromptOperationAcceptanceSchema = z .object({ + imageToolUrl: z.string().url().optional(), bindingId: agentIdentifierSchema, operationId: agentIdentifierSchema, requestId: agentIdentifierSchema, diff --git a/src/shared/remote-image-tool-contracts.ts b/src/shared/remote-image-tool-contracts.ts new file mode 100644 index 00000000..3b0b3ea0 --- /dev/null +++ b/src/shared/remote-image-tool-contracts.ts @@ -0,0 +1,34 @@ +import { z } from 'zod' +import { agentIdentifierSchema, positiveAgentSequenceSchema } from './agent-protocol/contracts' +import { imageOperationSchema, imageToolDescriptionLimit, imageToolInputSchema } from './image-generation-contracts' + +export const remoteImageToolSchema = z.object({ + channelId: agentIdentifierSchema, + channelEpoch: positiveAgentSequenceSchema, + description: z.string().min(1).max(imageToolDescriptionLimit) +}).strict() + +export const remoteImageToolCallSchema = z.object({ + callId: z.string().uuid(), + input: imageToolInputSchema +}).strict() + +export const remoteImageToolReplySchema = z.object({ + callId: z.string().uuid(), + result: imageOperationSchema.optional(), + error: z.string().max(4_000).optional() +}).strict() + +// References and operation metadata only. Image bytes never cross this channel. +export const remoteImageToolMaximumBytes = 256 * 1024 + +export function encodeRemoteImageToolMessage(value: unknown): Uint8Array { + const bytes = new TextEncoder().encode(JSON.stringify(value)) + if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large') + return bytes +} + +export function decodeRemoteImageToolMessage(bytes: Uint8Array): unknown { + if (bytes.byteLength > remoteImageToolMaximumBytes) throw new Error('Image tool message is too large') + return JSON.parse(new TextDecoder().decode(bytes)) as unknown +} diff --git a/src/shared/remote-image-tool-node.ts b/src/shared/remote-image-tool-node.ts new file mode 100644 index 00000000..cc44fee5 --- /dev/null +++ b/src/shared/remote-image-tool-node.ts @@ -0,0 +1,6 @@ +import { createHash } from 'node:crypto' + +// Used by Main and the Agent daemon, never by Renderer contracts. +export function remoteImageToolMcpName(bindingId: string): string { + return `goodbuddy_image_${createHash('sha256').update(bindingId).digest('hex').slice(0, 24)}` +}