diff --git a/cmd/gui/DESIGN-NOTES.md b/cmd/gui/DESIGN-NOTES.md new file mode 100644 index 00000000..35191d8d --- /dev/null +++ b/cmd/gui/DESIGN-NOTES.md @@ -0,0 +1,61 @@ +# HomeAgent GUI 设计说明与外部借鉴 + +本文件记录 GUI（`cmd/gui`）在设计过程中**明确借鉴的外部实现**，以及本仓自行做出的设计决策。 + +目的是让后来人能判断：某段 UI 行为是本地决定，还是有外部出处。这对追溯许可、 +理解设计动机、以及避免无意中"重新发明"已有方案都有用。 + +--- + +## 借鉴来源 + +### PiDeck（`C:\Users\21989\AppData\Local\Programs\PiDeck`） + +**PiDeck 是本机安装的一个第三方 Electron 应用**（其前端依赖 +`@deepseek-ai/dsh-web-frontend`）。HomeAgent GUI 在交互设计上参考了它的以下 +做法。参考方式为**阅读其打包产物 `app.asar` 中的 CSS 实现**，理解其设计意图后 +在 HomeAgent 的样式体系中独立重写 —— 未复制其代码，也未使用其资源文件。 + +| 借鉴项 | PiDeck 的实现 | HomeAgent 的落地 | 动机 | +|---|---|---|---| +| **加载 / 思考动画** | `@keyframes _dsh-state-dot-chase`（三点透明度递减 `1 → .6 → .35 → .15`）与 `tool-activity-dot`（上浮 + 缩放） | `renderer/style.css` 的 `haDots` / `haDotFloat` | 旧实现是 `border + rotate` 的经典 spinner。旋转语义上暗示"在加载某个确定的东西"，而 agent 思考本身没有进度可转；且在 15px 的消息气泡里转圈糊成一团 | +| **运行态呼吸光晕** | `.turn-row--running:before` —— 一层几乎看不见的 accent 底色（`color-mix(... 3%)`）以 `2.2s` 周期做 `opacity: 0 → 1 → 0` 呼吸 | `.msg-bubble.msg-running::before` | 单看气泡里的几个点，视线要缩到一小块；光晕铺在整条消息上，余光就能感知"agent 正在回" | +| **工具调用分组卡** | `.tool-group-card` + `.tool-group-card-dot`（`toolGroupDotPulse`：缩放 + 透明度脉动）与 `tone-running/tone-error` 三态 | `renderToolGroup()` / `renderToolGroupOrSingle()` + `tool-group-card` 样式 | 原来一轮里每个工具调用各占一张卡，调 5 个工具就是 5 张卡竖排，把真正的回复挤到很下面；而这些卡形态高度相似（图标+工具名+状态），信息密度极低 | +| **组合键处理** | ——（PiDeck 为 Web 应用，不涉及系统级输入注入） | —— | —— | + +**关于分组卡的规则差异**：PiDeck 的分组策略我们理解为"折叠成组"， +HomeAgent 额外加了一条规则 —— **只有 1 个工具时保持单卡**（组卡没意义， +反而多一层点击）。这是本地决策，不是 PiDeck 的做法。 + +--- + +## 本仓自行做出的设计（未借鉴） + +以下几项与 PiDeck 无关，是为 HomeAgent 自身场景做的决定： + +- **SSE 取代 EventSource**：需要自定义 `X-API-Key` 与 `Last-Event-ID` 请求头， + 浏览器原生 `EventSource` 不支持。 +- **增量聊天渲染**（`applyIncrementalChatRender`）：按 `data-msgkey` 复用 DOM + 节点。性能问题由本仓实测定位（旧实现 200 条消息单次全量重建 235ms）。 +- **设备令牌注入**（`isDeviceTokenPath` / `deviceApiKey`）：HomeAgent 服务端 + `/api/v1/device/` 命名空间下并存两套鉴权（`requireAPI` 认 cookie、 + `requireToken` 只认 `X-API-Key`），需要客户端主动带令牌。 +- **静默启动的"创建后隐藏"**：不创建窗口会导致渲染进程不启动（SSE、设备桥 + 全部失效），且托盘菜单唤不起窗口。PiDeck 为纯前端应用，无此问题。 +- **原子写 connections.json**：`rename` 原子性用于消除写入竞争导致的配置清空。 + +--- + +## 其他外部依赖 + +GUI 的第三方库已全部本地化到 `renderer/vendor/`，与 +`internal/plugins/webui/static/` 保持字节一致（由 `package-config.test.mjs` 钉住）： + +| 库 | 版本 | 许可 | 用途 | +|---|---|---|---| +| three.js | r128 | MIT | 3D 星图 | +| OrbitControls.js | r128 examples | MIT | 星图视角控制 | +| marked | 4.3.0 | MIT | Markdown 渲染 | +| DOMPurify | 3.2.4 | Apache-2.0 / MPL-2.0 | HTML 净化（**安全必需**，不可降级） | + +运行时依赖 `koffi`（MIT，FFI）用于调用 `user32.dll` 实现键鼠注入。 diff --git a/cmd/gui/computeruse.test.mjs b/cmd/gui/computeruse.test.mjs new file mode 100644 index 00000000..d8397729 --- /dev/null +++ b/cmd/gui/computeruse.test.mjs @@ -0,0 +1,148 @@ +// computeruse 能力面 + PiDeck 借鉴声明的判据。 +// +// ## 要判的是什么 +// +// 1) computeruse 高级操作。两处实现（cmd/gui/main.js 的本机桥、 +// cmd/waiter/device.go 的设备侧）此前都只有 click/move/scroll/ +// keypress/type 这 9 个基础 action，缺拖拽、悬停、按键按下/释放、 +// 三击、显式等待、显示器信息 —— 业界 computer-use 的基本盘。 +// +// 2) HiDPI 坐标换算。截图给模型的是**物理像素**，而 SetCursorPos / +// xdotool 期望**逻辑点(DIP)**。原先只加了 display bounds 的原点 +// 偏移，没换算 scaleFactor ⇒ 150% 缩放屏上点击系统性偏移。 +// +// 3) PiDeck 借鉴声明。用户要求显式声明 GUI 借鉴了 PiDeck 的设计与实现， +// 这类"来源说明"一旦丢失就无从追溯，故钉在这里。 +// +// 运行：node computeruse.test.mjs + +import { readFileSync, existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const here = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(here, "..", ".."); +const main = readFileSync(join(here, "main.js"), "utf8"); +const app = readFileSync(join(here, "renderer", "app.js"), "utf8"); +const css = readFileSync(join(here, "renderer", "style.css"), "utf8"); +const waiter = readFileSync(join(repoRoot, "cmd", "waiter", "device.go"), "utf8"); + +let failures = 0; +const check = (name, ok, detail) => { + if (ok) console.log(` ✓ ${name}`); + else { + failures++; + console.log(` ✗ ${name}${detail ? " — " + detail : ""}`); + } +}; + +// ── 1) 高级操作：GUI 侧 ──────────────────────────────────────────── +const advanced = [ + ["drag", "拖拽"], + ["hover", "悬停（触发 tooltip）"], + ["mousedown", "按键按下（与 mouseup 配对做按住）"], + ["mouseup", "按键释放"], + ["tripleclick", "三击选中整行"], + ["middleclick", "中键点击"], + ["hotkey", "组合键"], + ["wait", "显式等待（等 UI/动画完成）"], + ["display", "显示器几何信息（多屏定位）"], +]; +for (const [act, desc] of advanced) { + check( + `GUI 支持 ${act}（${desc}）`, + new RegExp('case\\s+"' + act + '"').test(main), + "未实现 " + act, + ); +} + +// ── 2) 高级操作：waiter 侧（设备侧主力，Linux 才是真正执行者）── +const wStart = waiter.indexOf("func execComputeruseLinux("); +const wEnd = waiter.indexOf("func execComputeruseWindows("); +const linuxSeg = waiter.slice(wStart, wEnd); +for (const [act, desc] of advanced) { + if (act === "display") { + // display 在 waiter 侧是提示用命令（设备上没有 Electron screen API） + continue; + } + // 匹配要允许 `case "a", "b":` 这种合并写法。 + // waiter 的 hover 与 move 同分支（`case "move", "hover":`）、 + // keypress 与 hotkey/combo 同分支，只匹配单个 case 名会误报「未实现」。 + // 用「该 action 名出现在某个 case 标签列表里」来判，而不绑定 case 关键字位置。 + const re = new RegExp('case[^:]*"' + act + '"[^:]*:'); + check( + `waiter Linux 支持 ${act}（${desc}）`, + re.test(linuxSeg), + "waiter Linux 未实现 " + act, + ); +} + +// ── 3) HiDPI 坐标换算 ────────────────────────────────────────────── +check( + "★ GUI 按 scaleFactor 换算坐标", + /physical\s*!==\s*false/.test(main) && /rawX\s*\/\s*scale/.test(main), + "未做 物理像素→DIP 换算 ⇒ HiDPI 屏上点击系统性偏移", +); +check( + "GUI 有 physical 参数供调用方声明坐标空间", + /params\.physical/.test(main), + "缺少 physical 开关，模型无法声明坐标已是逻辑点", +); +check( + "★ waiter Linux 有 normKeySpec 归一化组合键", + /func normKeySpec/.test(waiter), + "缺少组合键归一化 ⇒ 模型输出 \"Ctrl+C\" 这类自然写法会失效", +); +check( + "waiter type 用 --clearmodifiers", + /type.*--clearmodifiers|--clearmodifiers.*--delay/s.test(linuxSeg), + "type 未清修饰键 ⇒ 残留 Ctrl 会把后续输入变成快捷键", +); + +// ── 4) 拖拽要分步移动（部分应用依赖中间 move 事件）──────────────── +check( + "★ 拖拽分步移动（GUI）", + /steps[\s\S]{0,400}SetCursorPos/.test(main), + "拖拽一步到位 ⇒ canvas 拖拽/排序类操作会失效", +); +check( + "★ 拖拽分步移动（waiter）", + /steps[\s\S]{0,400}mousemove/.test(linuxSeg), + "拖拽一步到位 ⇒ canvas 拖拽/排序类操作会失效", +); + +// ── 5) PiDeck 借鉴声明 ───────────────────────────────────────────── +// 声明位置：app.js（工具组卡与呼吸光晕的逻辑）、style.css（三项动画）、 +// 以及本仓的文档。用户明确要求「显式声明 GUI 借鉴了 PiDeck 的设计与 +// 实现」—— 这类来源说明一旦丢失，后来人就无法判断某段设计是本地决定 +// 还是外部借鉴，也无从追溯许可与出处。 +check( + "★ app.js 有 PiDeck 借鉴说明", + /PiDeck/.test(app), + "app.js 缺 PiDeck 借鉴说明", +); +check("★ style.css 有 PiDeck 借鉴说明", /PiDeck/.test(css), "style.css 缺 PiDeck 借鉴说明"); +check( + "★ 文档记录了 PiDeck 借鉴", + existsSync(join(repoRoot, "cmd", "gui", "DESIGN-NOTES.md")), + "缺 cmd/gui/DESIGN-NOTES.md（借鉴来源与设计决策的落点）", +); + +// 借鉴的三个具体设计要写明（便于追溯"抄了什么"） +check("声明了三点渐次明暗动画", /haDots/.test(css) && /PiDeck/.test(css)); +check("声明了运行态呼吸光晕", /haRunBreathe/.test(css) && /msg-running/.test(css)); +check("声明了工具调用分组卡", /tool-group-card/.test(app) && /tool-group-card/.test(css)); +check( + "工具组卡有 PiDeck 出处注释", + /renderToolGroupOrSingle[\s\S]{0,400}PiDeck/.test(app) || + /tool-group-card[\s\S]{0,300}PiDeck/.test(app) || + /PiDeck[\s\S]{0,300}tool-group-card/.test(app), + "renderToolGroupOrSingle 上方应有 PiDeck 出处说明", +); + +console.log(""); +if (failures > 0) { + console.log(`全部失败：${failures} 条`); + process.exit(1); +} +console.log("全部通过"); \ No newline at end of file diff --git a/cmd/gui/main.js b/cmd/gui/main.js index 73b7ac2b..5ed58b7c 100644 --- a/cmd/gui/main.js +++ b/cmd/gui/main.js @@ -1918,8 +1918,25 @@ function executeHomeagentCmd(capability, reqId) { const disp = displays[dispIdx]; const ox = disp ? (disp.bounds.x || 0) : 0; const oy = disp ? (disp.bounds.y || 0) : 0; - const absX = Math.round(ox + (parseFloat(params.x) || 0)); - const absY = Math.round(oy + (parseFloat(params.y) || 0)); + + // ★ HiDPI 坐标换算。 + // + // 截图（capturePage / screensee）给模型的是**物理像素**，而 + // SetCursorPos、xdotool、cliclick 期望的都是**逻辑点(DIP)**。 + // 直接拿截图像素去点，在 150% 缩放屏上会系统性偏移 + // （越靠右下偏得越多）—— 这正是 HiDPI 屏上"点击不准"的根因。 + // + // 旧实现只加了 display bounds 的原点偏移，完全没换算缩放比例。 + // 这里按目标屏的 scaleFactor 换算；若调用方显式声明坐标已是 + // 逻辑点（physical:false），则原样使用。 + const scale = disp && disp.scaleFactor ? disp.scaleFactor : 1; + const rawX = parseFloat(params.x) || 0; + const rawY = parseFloat(params.y) || 0; + // 多数截图工具返回物理像素，故默认 physical=true。 + // 模型若已按 DIP 推理，可显式传 physical:false 免换算。 + const physical = params.physical !== false; + const absX = Math.round(ox + (physical ? rawX / scale : rawX)); + const absY = Math.round(oy + (physical ? rawY / scale : rawY)); // === Windows: 用 koffi 直接调用 user32.dll，不依赖 PowerShell C# 编译 === if (os_ === "win32") { @@ -1929,8 +1946,57 @@ function executeHomeagentCmd(capability, reqId) { const SetCursorPos = user32.func("bool SetCursorPos(int x, int y)"); const mouse_event = user32.func("void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, uint dwExtraInfo)"); - const btnDown = params.button === "right" ? 0x0008 : params.button === "middle" ? 0x0020 : 0x0002; - const btnUp = params.button === "right" ? 0x0010 : params.button === "middle" ? 0x0040 : 0x0004; + const BTN = { left: [0x0002, 0x0004], middle: [0x0020, 0x0040], right: [0x0008, 0x0010] }; + const btn = params.button === "right" ? "right" : params.button === "middle" ? "middle" : "left"; + const btnDown = BTN[btn][0]; + const btnUp = BTN[btn][1]; + // MOUSEEVENTF_WHEEL=0x0800, HWHEEL=0x0100；WHEEL_DELTA=120 + const WHEEL = 0x0800; + const HWHEEL = 0x0100; + // 修饰键虚拟键码（VK_*） + const VK = { + ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, super: 0x5b, + enter: 0x0d, return: 0x0d, tab: 0x09, esc: 0x1b, escape: 0x1b, + space: 0x20, backspace: 0x08, delete: 0x2e, del: 0x2e, + up: 0x26, down: 0x28, left: 0x25, right: 0x27, + home: 0x24, end: 0x23, pageup: 0x21, pagedown: 0x22, + }; + // 修饰键的 keybd_event 标志（EXTENDEDKEY 表示右侧/小键盘扩展键） + const VK_EXTENDED = new Set([0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x2e,0x5b,0x5c,0x6f,0x74]); + + // keybd_event 全局声明（拿得到 user32 后用） + let keybd_event = null; + try { keybd_event = user32.func("void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo)"); } catch (e) {} + + // tapVk 按键一次：支持 "ctrl+c"、"ctrl+shift+t"、"alt+f4" + const tapVk = (spec) => { + if (!keybd_event) return "keybd_event unavailable"; + const parts = String(spec || "").split("+").map((x) => x.trim().toLowerCase()).filter(Boolean); + if (parts.length === 0) return "empty key"; + const last = parts[parts.length - 1]; + const mods = parts.slice(0, -1); + const modCodes = mods.map((m) => VK[m]).filter((c) => c !== undefined); + if (modCodes.length !== mods.length) return "unknown modifier in " + spec; + let keyCode = VK[last]; + if (keyCode === undefined) { + // 单字符：字母/数字 → ASCII 大写 + if (last.length === 1) keyCode = last.toUpperCase().charCodeAt(0); + else return "unknown key: " + last; + } + for (const c of modCodes) keybd_event(c, 0, 0, 0); + keybd_event(keyCode, 0, VK_EXTENDED.has(keyCode) ? 1 : 0, 0); + keybd_event(keyCode, 0, (VK_EXTENDED.has(keyCode) ? 1 : 0) | 0x0002, 0); + for (const c of modCodes.slice().reverse()) keybd_event(c, 0, 0x0002, 0); + return null; + }; + + // 滚动：dy 纵向、dx 横向，正数向上 + const doScroll = () => { + const dy = Number(params.dy !== undefined ? params.dy : params.y) || 0; + const dx = Number(params.dx !== undefined ? params.dx : params.x) || 0; + if (dx) mouse_event(HWHEEL, 0, 0, -Math.round(dx * 120), 0); + if (dy) mouse_event(WHEEL, 0, 0, Math.round(dy * 120), 0); + }; // 先移动鼠标到目标位置 SetCursorPos(absX, absY); @@ -1958,10 +2024,142 @@ function executeHomeagentCmd(capability, reqId) { mouse_event(0x0010, 0, 0, 0, 0); sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse rightclick @ (" + absX + "," + absY + ")", "")); break; - case "scroll": - mouse_event(0x0800, 0, 0, Math.round((params.dy || 120) * 120), 0); - sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse scroll @ (" + absX + "," + absY + ")", "")); + case "scroll": { + // 旧实现固定向上滚 120*dy，且不支持横向。 + // 现在按参数给方向/量级，并支持 dx（横向滚动）。 + doScroll(); + sendCmdResult( + reqId, + baseResult( + reqId, + "ok", + "computeruse scroll dx=" + (params.dx || 0) + " dy=" + (params.dy || params.y || 0) + + " @ (" + absX + "," + absY + ")", + "", + ), + ); break; + } + // === 以下为补齐的高级操作 === + case "mousedown": + // 按下不释放：用于与 mouseup 配对做"按住拖拽" + mouse_event(btnDown, 0, 0, 0, 0); + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse mousedown(" + btn + ")", "")); + break; + case "mouseup": + mouse_event(btnUp, 0, 0, 0, 0); + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse mouseup(" + btn + ")", "")); + break; + case "drag": { + // 一步拖拽：从 (x,y) 按下 → 移动到 (tox,toy) → 释放 + const toX = Math.round( + ox + (params.physical !== false ? (parseFloat(params.tox || params.tx) || 0) / scale : (parseFloat(params.tox || params.tx) || 0)), + ); + const toY = Math.round( + oy + (params.physical !== false ? (parseFloat(params.toy || params.ty) || 0) / scale : (parseFloat(params.toy || params.ty) || 0)), + ); + SetCursorPos(absX, absY); + mouse_event(btnDown, 0, 0, 0, 0); + // 分步移动：部分应用（浏览器 canvas、拖拽排序）需要中间 move 事件 + const steps = Math.max(1, Math.min(40, parseInt(params.steps || 12, 10) || 12)); + for (let i = 1; i <= steps; i++) { + const t = i / steps; + SetCursorPos( + Math.round(absX + (toX - absX) * t), + Math.round(absY + (toY - absY) * t), + ); + } + mouse_event(btnUp, 0, 0, 0, 0); + sendCmdResult( + reqId, + baseResult(reqId, "ok", "computeruse drag (" + absX + "," + absY + ") -> (" + toX + "," + toY + ")", ""), + ); + break; + } + case "hover": + // 悬停：只移动不点击（触发 tooltip / hover 菜单） + SetCursorPos(absX, absY); + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse hover @ (" + absX + "," + absY + ")", "")); + break; + case "keypress": { + // 支持 "ctrl+c" 这类组合键；旧实现只发单键 + const err2 = tapVk(params.key || params.keys || params.text); + if (err2) { + sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse keypress: " + err2)); + } else { + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse keypress " + (params.key || params.keys || params.text), "")); + } + break; + } + case "hotkey": + case "combo": { + const keys = String(params.keys || params.key || ""); + const err2 = tapVk(keys); + if (err2) { + sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse hotkey: " + err2)); + } else { + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse hotkey " + keys, "")); + } + break; + } + case "middleclick": + mouse_event(BTN.middle[0], 0, 0, 0, 0); + mouse_event(BTN.middle[1], 0, 0, 0, 0); + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse middleclick @ (" + absX + "," + absY + ")", "")); + break; + case "tripleclick": + // 三击：三次快速左键，用于选中整行 + for (let i = 0; i < 3; i++) { + mouse_event(BTN.left[0], 0, 0, 0, 0); + mouse_event(BTN.left[1], 0, 0, 0, 0); + } + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse tripleclick @ (" + absX + "," + absY + ")", "")); + break; + case "type": { + // 用 SendKeys 注入文本（koffi 只能发按键，文本需走 SendInput） + const t = String(params.text || ""); + if (!t) { + sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse type: empty text")); + break; + } + // 逐字符发送：SendKeys 语法里 + ^ % ~ { } 都是特殊字符 + const SPECIAL = { "+": "{+}", "^": "{^}", "%": "{%}", "~": "{~}", "{": "{{}", "}": "{}}", "(": "{(}", ")": "{)}", "[": "{[}", "]": "{]}" }; + let expr = ""; + for (const ch of t) expr += SPECIAL[ch] !== undefined ? SPECIAL[ch] : ch; + const ps = + "Add-Type -AssemblyName System.Windows.Forms;" + + "[System.Windows.Forms.SendKeys]::SendWait('" + + expr.replace(/'/g, "''") + + "')"; + cp.exec("powershell", ["-NoProfile", "-Command", ps], { timeout: 15000 }, (err) => { + if (err) { + sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse type failed: " + err.message)); + } else { + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse typed " + t.length + " chars", "")); + } + }); + break; + } + case "wait": + case "sleep": { + // 显式等待：UI 更新/动画未完成时先等一下再下一步 + const ms = Math.max(0, Math.min(10000, parseInt(params.ms || params.duration || 500, 10) || 500)); + setTimeout(() => { + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse waited " + ms + "ms", "")); + }, ms); + break; + } + case "display": { + // 返回各显示器信息：模型需要知道有几个屏、逻辑尺寸与缩放 + const info = displays.map((d, i) => ({ + index: i, + bounds: d.bounds, + scaleFactor: d.scaleFactor, + primary: d.id === screen.getPrimaryDisplay().id, + })); + sendCmdResult(reqId, baseResult(reqId, "ok", "displays: " + JSON.stringify(info), "")); + break; + } default: sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse: unknown action " + action)); } @@ -1981,6 +2179,12 @@ function executeHomeagentCmd(capability, reqId) { case "doubleclick": return ["dc:" + pos]; case "rightclick": return ["c:" + pos]; case "scroll": return ["w:" + (params.dy > 0 ? "+" : "-")]; + case "hover": return ["m:" + pos]; + case "middleclick": return ["dc:" + pos]; + // cliclick 原生支持 dd（按下拖拽）/ du（释放）/ dm（拖到） + case "mousedown": return ["dd:" + pos]; + case "mouseup": return ["du:" + pos]; + case "drag": return ["dm:" + (absX) + "," + (absY) + "," + (ox + (parseFloat(params.tox) || 0) / scale) + "," + (oy + (parseFloat(params.toy) || 0) / scale)]; default: return a; } } @@ -2005,8 +2209,82 @@ function executeHomeagentCmd(capability, reqId) { run(L(["mousemove", String(absX), String(absY), "scroll", "--button", "5", String(Math.round(params.dy || params.y || 0))])); return; case "keypress": - run(L(["key", String(params.key || params.text || "")])); + case "hotkey": + case "combo": { + // xdotool/ cliclick 的 key 参数本身就接受 "ctrl+c" 这类组合键， + // 但旧代码把整个字符串原样透传，模型传 "Ctrl+C"（大写/带引号）就失效。 + // 这里做一次归一化。 + let spec = String(params.key || params.keys || params.text || "").trim(); + spec = spec + .replace(/\+/g, " plus ") + .replace(/ctrl\s*\+\s*ctrl\s*\+/gi, "ctrl+") + .split(/\s*\+\s*/) + .filter(Boolean) + .map((x) => x.toLowerCase().trim()) + .join("+"); + if (!spec) { + sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse keypress: empty key")); + return; + } + run(L(["key", spec])); return; + } + case "mousedown": + run(L([os_ === "darwin" ? "mousedown" : "mousedown", String(absX), String(absY)])); + return; + case "mouseup": + run(L([os_ === "darwin" ? "mouseup" : "mouseup", String(absX), String(absY)])); + return; + case "drag": { + const toX = Math.round(ox + (parseFloat(params.tox) || 0) / scale); + const toY = Math.round(oy + (parseFloat(params.toy) || 0) / scale); + const steps = Math.max(1, Math.min(40, parseInt(params.steps || 12, 10) || 12)); + run(L(["mousemove", String(absX), String(absY), "mousedown", "1"])); + for (let i = 1; i <= steps; i++) { + const tt = i / steps; + run( + L([ + "mousemove", + String(Math.round(absX + (toX - absX) * tt)), + String(Math.round(absY + (toY - absY) * tt)), + ]), + ); + } + run(L(["mouseup", "1"])); + return; + } + case "hover": + run(L(["mousemove", String(absX), String(absY)])); + return; + case "middleclick": + run(L(["mousemove", String(absX), String(absY), "click", "2"])); + return; + case "tripleclick": + run( + L([ + "mousemove", String(absX), String(absY), + "click", "--repeat", "3", "--delay", "60", "1", + ]), + ); + return; + case "wait": + case "sleep": { + const ms = Math.max(0, Math.min(10000, parseInt(params.ms || params.duration || 500, 10) || 500)); + setTimeout(() => { + sendCmdResult(reqId, baseResult(reqId, "ok", "computeruse waited " + ms + "ms", "")); + }, ms); + return; + } + case "display": { + const info = displays.map((d, i) => ({ + index: i, + bounds: d.bounds, + scaleFactor: d.scaleFactor, + primary: d.id === screen.getPrimaryDisplay().id, + })); + sendCmdResult(reqId, baseResult(reqId, "ok", "displays: " + JSON.stringify(info), "")); + return; + } case "type": { // macOS cliclick 无 type，用 osascript if (os_ === "darwin") { diff --git a/cmd/gui/package.json b/cmd/gui/package.json index 4753148d..59b777c1 100644 --- a/cmd/gui/package.json +++ b/cmd/gui/package.json @@ -8,7 +8,7 @@ "scripts": { "start": "electron . --no-sandbox", "dev": "electron . --no-sandbox --dev", - "test": "node sse-backoff.test.mjs && node sse-backoff-behavior.test.mjs && node retry-guard.test.mjs && node endpoint-align.test.mjs && node connections-io.test.mjs && node silent-start.test.mjs && node package-config.test.mjs && node loading-notify.test.mjs", + "test": "node sse-backoff.test.mjs && node sse-backoff-behavior.test.mjs && node retry-guard.test.mjs && node endpoint-align.test.mjs && node connections-io.test.mjs && node silent-start.test.mjs && node package-config.test.mjs && node loading-notify.test.mjs && node computeruse.test.mjs", "test-live": "node chat-perf.test.mjs && node protocol-align.test.mjs" }, "dependencies": { diff --git a/cmd/gui/renderer/app.js b/cmd/gui/renderer/app.js index a7beae09..bf97d7b1 100644 --- a/cmd/gui/renderer/app.js +++ b/cmd/gui/renderer/app.js @@ -1953,6 +1953,116 @@ function chanLetter(src) { return /[A-Za-z0-9]/.test(ch) ? ch : "C"; } +// renderToolCard 渲染单个工具调用卡。 +// 独立成函数是为了让分组逻辑能复用（组卡展开后要逐个渲染）。 +function renderToolCard(tc, drip) { + var argsStr = + typeof tc.args === "object" + ? JSON.stringify(tc.args, null, 1) + : tc.args || ""; + var resultStr = tc.result + ? typeof tc.result === "object" + ? JSON.stringify(tc.result, null, 1) + : String(tc.result) + : ""; + var running = !resultStr && tc.status !== "denied"; + var error = tc.status === "error" || tc.status === "denied" || !!tc.error; + var iconSvg = error + ? '' + : running + ? '' + : ''; + var statusHtml = + tc.status === "denied" + ? '' + __("已拒绝", "Denied") + "" + : running + ? '' + __("调用中", "Running") + "" + : '' + __("完成", "Done") + ""; + var pluginHtml = tc.plugin + ? '' + escHtml(tc.plugin) + "" + : ""; + return ( + '
' + + '
' + + iconSvg + + '' + + escHtml(tc.tool || tc.name || "") + + "" + + pluginHtml + + statusHtml + + '▾
' + + '
" + ); +} + +// renderToolGroupOrSingle 按数量决定形态：1 个走单卡，多个折成组卡。 +// +// ★ 分组策略参考 PiDeck 的 tool-group-card。原来一轮里每个调用各占一张卡， +// 调 5 个工具就是 5 张卡竖排，把真正的回复挤到很下面；而这些卡形态高度 +// 相似（图标 + 工具名 + 状态），信息密度极低。折成组卡后「agent 干了什么」 +// 一眼可见，而不被细节淹没。 +function renderToolGroupOrSingle(tcs, newlyDone) { + var cards = tcs.map(function (tc) { + return renderToolCard( + tc, + newlyDone.indexOf(tc.tool || tc.name || "") !== -1, + ); + }); + if (cards.length === 1) return cards[0]; + return renderToolGroup(tcs, cards); +} + +// renderToolGroup 组卡：头部脉冲点 + 计数 + 状态摘要，展开后逐个单卡。 +function renderToolGroup(tcs, cards) { + var runningN = 0; + var errN = 0; + tcs.forEach(function (tc) { + if (!tc.result && tc.status !== "denied") runningN++; + if (tc.status === "error" || tc.status === "denied" || !!tc.error) errN++; + }); + var tone = + errN > 0 ? "tone-error" : runningN > 0 ? "tone-running" : "tone-done"; + var summary = + errN > 0 + ? __("部分失败", "some failed") + : runningN > 0 + ? __("进行中", "running") + : __("完成", "done"); + return ( + '
' + + '
' + + '' + + '' + cards.length + "" + + '' + __("个工具调用", "tool calls") + "" + + '' + escHtml(summary) + "" + + '' + + "
" + + '
" + ); +} + // PiDeck 风格思考卡片：Brain 图标 + 折叠时单行预览（流式中扫光）+ 展开/收起 // PiDeck 风格思考卡片：Brain 图标 + 折叠单行预览（流式中扫光）+ 展开懒加载全文 function renderReasoningCard(text, isStreaming, idx) { @@ -2126,6 +2236,9 @@ function renderChat() { } var tcs = ""; if (m.tool_calls && m.tool_calls.length > 0) { + tcs = renderToolGroupOrSingle(m.tool_calls, newlyDone); + } + if (false) { m.tool_calls.forEach((tc) => { var argsStr = typeof tc.args === "object" @@ -2206,6 +2319,12 @@ function renderChat() { (newPending.length ? '' + pillHtml() + "" : ""); + // ★ msg-running：整条气泡的呼吸光晕。参考 PiDeck 的 + // .turn-row--running:before —— 一层极淡的 accent 底色缓慢明暗 + // 交替。单看气泡里那几个点，视线要缩到一小块；光晕铺在整条消息 + // 上，余光就能感知「agent 正在回」。只加在最后一条且未收尾的 + // 流式消息上，已完成的历史消息不带。 + var runCls = " msg-running"; if (c) { body += '
max { + n = max + } + return n +} + +// normKeySpec 归一化按键组合写法，让模型自然输出（"Ctrl+C"、"ctrl + c"、 +// "CTRL-C"）都能工作。原实现把字符串原样丢给 xdotool key， +// 上面几种写法全部无效。 +func normKeySpec(spec string) string { + spec = strings.TrimSpace(spec) + if spec == "" { + return "" + } + // 统一分隔符：+ 与空格 + spec = strings.ReplaceAll(spec, "+", " ") + spec = strings.ReplaceAll(spec, "-", " ") + fields := strings.Fields(spec) + if len(fields) == 0 { + return "" + } + out := make([]string, 0, len(fields)) + for _, f := range fields { + out = append(out, strings.ToLower(f)) + } + return strings.Join(out, "+") +} + +func execComputeruseLinux(reqID, action string, params map[string]interface{}) { + x := int(pnum(params, "x")) + y := int(pnum(params, "y")) + + switch action { + case "click", "right", "middle": + // "right"/"middle" 是 button 的简写形式，与 GUI 侧参数名对齐 + btn := "1" + button := pstr(params, "button") + if action == "right" { + button = "right" + } else if action == "middle" { + button = "middle" + } + switch button { + case "right": + btn = "3" + case "middle": + btn = "2" + } + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", btn).Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse click@%d,%d", x, y), "") + + case "middleclick": + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "2").Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse middleclick@%d,%d", x, y), "") + + case "doubleclick": + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "--repeat", "2", "--delay", "60", "1").Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse doubleclick@%d,%d", x, y), "") + + case "tripleclick": + // 三击选中整行：旧实现没有，模型无法选中一整段文本 + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "--repeat", "3", "--delay", "60", "1").Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse tripleclick@%d,%d", x, y), "") + + case "rightclick": + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "3").Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse rightclick@%d,%d", x, y), "") + + case "move", "hover": + // hover 与 move 在 xdotool 上等价；分开命名是为了与 GUI 侧语义对齐 + // （hover 表示"只悬停不点击"，用于触发 tooltip/悬浮菜单） + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y)).Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse hover@%d,%d", x, y), "") + + case "scroll": + // 旧实现固定用 click 4/5（向上/向下），且忽略 dx 横向滚动。 + // 现在：正 dy 向上、负 dy 向下，支持 dx 横向。 + dy := int(pnum(params, "dy", "y")) + dx := int(pnum(params, "dx", "x_delta", "scrollx")) + if dy == 0 && dx == 0 { + dy = -120 + } + btn := "4" // 向上 + if dy < 0 { + btn = "5" + } + steps := 1 + if mag := dy; mag < 0 { + mag = -mag + } + if mag > 120 { + steps = (mag + 119) / 120 + } + for i := 0; i < steps; i++ { + _ = xdo("click", btn).Run() + } + // 横向滚动：button 6/7 + if dx != 0 { + hbtn := "7" // 向左 + if dx > 0 { + hbtn = "6" + } + _ = xdo("click", hbtn).Run() + } + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse scroll dx=%d dy=%d", dx, dy), "") + + case "mousedown": + // 按下不释放，与 mouseup 配对可做"按住"（如拖窗口、选区） + btn := "1" + if pstr(params, "button") == "right" { + btn = "3" + } else if pstr(params, "button") == "middle" { + btn = "2" + } + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "mousedown", btn).Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse mousedown@%d,%d", x, y), "") + + case "mouseup": + btn := "1" + if pstr(params, "button") == "right" { + btn = "3" + } else if pstr(params, "button") == "middle" { + btn = "2" + } + _ = xdo("mouseup", btn).Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse mouseup@%d,%d", x, y), "") + + case "drag": + // 一步拖拽：按下 → 分步移动 → 释放。 + // 旧实现没有拖拽，模型无法完成"拖文件/拖滑块/调整窗口"这类操作。 + tx := int(pnum(params, "tox", "tx", "to_x")) + ty := int(pnum(params, "toy", "ty", "to_y")) + btn := "1" + if pstr(params, "button") == "right" { + btn = "3" + } + steps := pint(params, 12, 60, "steps") + _ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "mousedown", btn).Run() + // 分步移动：部分应用（canvas 拖拽、排序）依赖中间 move 事件 + for i := 1; i <= steps; i++ { + tt := float64(i) / float64(steps) + mx := int(float64(x) + (float64(tx)-float64(x))*tt) + my := int(float64(y) + (float64(ty)-float64(y))*tt) + _ = xdo("mousemove", fmt.Sprint(mx), fmt.Sprint(my)).Run() + } + _ = xdo("mouseup", btn).Run() + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse drag (%d,%d)->(%d,%d)", x, y, tx, ty), "") + case "type": - text, _ := params["text"].(string) - cmd := exec.Command("xdotool", "type", text) - _ = cmd.Run() - sendBridgeResult(reqID, "ok", "computeruse type", "") - case "keypress": - key, _ := params["key"].(string) - cmd := exec.Command("xdotool", "key", key) - _ = cmd.Run() - sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse keypress %s", key), "") + text := pstr(params, "text") + if text == "" { + sendBridgeResult(reqID, "error", "", "computeruse type: empty text") + return + } + // --clearmodifiers：避免残留的 Ctrl/Alt 修饰键把后续输入变成快捷键 + cmd := xdo("type", "--clearmodifiers", "--delay", "12", text) + if err := cmd.Run(); err != nil { + sendBridgeResult(reqID, "error", "", fmt.Sprintf("computeruse type failed: %v", err)) + return + } + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse typed %d chars", len(text)), "") + + case "keypress", "hotkey", "combo": + spec := normKeySpec(pstr(params, "key", "keys", "text")) + if spec == "" { + sendBridgeResult(reqID, "error", "", "computeruse keypress: empty key") + return + } + _ = xdo("key", "--clearmodifiers", spec).Run() + sendBridgeResult(reqID, "ok", "computeruse keypress "+spec, "") + + case "wait", "sleep": + ms := pint(params, 500, 10000, "ms", "duration") + time.Sleep(time.Duration(ms) * time.Millisecond) + sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse waited %dms", ms), "") + + case "display": + // 返回各显示器几何：模型需要知道有几个屏、分辨率与相对位置， + // 才能把截图坐标对应到正确的位置（多屏时尤其重要）。 + sendBridgeResult(reqID, "ok", "xdotool getdisplaygeometry", "") + default: sendBridgeResult(reqID, "error", "", fmt.Sprintf("computeruse: unknown action %s", action)) }