gui: 新增 computeruse 能力(agent 跨平台控制鼠标键盘)

- executeHomeagentCmd 加 case computeruse: 参数 JSON {x,y,action,button,text,key}
- action: click/move/doubleclick/rightclick/scroll/keypress/type
- 跨平台: Linux=xdotool, macOS=cliclick(及osascript输入), Windows=PowerShell user32
- 坐标基于 screensueDisplay 目标屏幕原点偏移
- caps 增补 computeruse

已装本机 xdotool 备测

cmd/gui/main.js

@@ -1726,6 +1726,224 @@ function executeHomeagentCmd(capability, reqId) {
       );
       return;
     }
+    case "computeruse": {
+      // computeruse：agent 控制鼠标/键盘在目标屏幕操作（点击/移动/滚动/按键/输入）
+      // 参数：homeagent-computeruse {"x":100,"y":200,"action":"click","button":"left"}
+      //   action: click/doubleclick/rightclick/move/scroll/type/keypress
+      //   x/y: 相对屏幕左上角(px)；scroll 用 dx/dy; type 用 text
+      const cp = require("child_process");
+      const platform = process.platform;
+      const raw = String(capability || "")
+        .replace(/^computeruse/, "")
+        .trim();
+      // 解析 JSON 参数
+      let params = {};
+      const jsonM = raw.match(/\{[\s\S]*\}/);
+      if (jsonM) {
+        try {
+          params = JSON.parse(jsonM[0]);
+        } catch (e) {
+          sendCmdResult(
+            reqId,
+            baseResult(
+              reqId,
+              "error",
+              "",
+              "computeruse: invalid JSON params: " + e.message,
+            ),
+          );
+          return;
+        }
+      } else {
+        // 支持 "x y action" 简写
+        const t = raw.split(/\s+/).filter(Boolean);
+        params.x = parseFloat(t[0]);
+        params.y = parseFloat(t[1]);
+        params.action = t[2] || "click";
+      }
+      const action = params.action || "click";
+      // 跨平台输入模拟：Linux=xdotool / macOS=cliclick(或用osascript) / Windows=PowerShell user32
+      const os_ = platform === "win32" ? "win32" : platform === "darwin" ? "darwin" : "linux";
+      let tool = null; // {cmd, args, shell}
+      if (os_ === "linux") {
+        if (cp.spawnSync("which", ["xdotool"]).status === 0) tool = { cmd: "xdotool", shell: false };
+      } else if (os_ === "darwin") {
+        if (cp.spawnSync("which", ["cliclick"]).status === 0) tool = { cmd: "cliclick", shell: false };
+      } // win32 用 PowerShell 内联，下面单独处理
+      if (os_ === "linux" && !tool) {
+        sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse: xdotool not installed"));
+        return;
+      }
+      if (os_ === "darwin" && !tool) {
+        sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse: cliclick not installed"));
+        return;
+      }
+      // 目标屏幕左上角原点偏移：沿用 screensueDisplay；坐标相对该屏幕
+      let dispIdx = 0;
+      try {
+        const prefs = loadGuiPrefs();
+        dispIdx = parseInt((prefs.deviceBridge || {}).screensueDisplay || "0", 10) || 0;
+      } catch (e) {}
+      const displays = screen.getAllDisplays() || [];
+      if (dispIdx >= displays.length) dispIdx = 0;
+      const disp = displays[dispIdx];
+      const ox = disp ? (disp.bounds.x || 0) : 0;
+      const oy = disp ? (disp.bounds.y || 0) : 0;
+      const absX = Math.round(ox + (parseFloat(params.x) || 0));
+      const absY = Math.round(oy + (parseFloat(params.y) || 0));
+      const run = (cmdStr, done) => {
+        if (os_ === "win32") {
+          // Windows: 用 PowerShell user32 SendInput
+          cp.execFile(
+            "powershell",
+            ["-NoProfile", "-Command", cmdStr],
+            { timeout: 15000, maxBuffer: 1024 * 1024 },
+            (err) => {
+              sendCmdResult(
+                reqId,
+                baseResult(
+                  reqId,
+                  err ? "error" : "ok",
+                  err ? "" : "computeruse " + action + " @ (" + absX + "," + absY + ")" + (done ? " " + done : ""),
+                  err ? err.message : "",
+                ),
+              );
+            },
+          );
+          return;
+        }
+        if (os_ === "darwin") {
+          // macOS cliclick
+          cp.execFile(
+            "cliclick",
+            cmdStr,
+            { timeout: 15000, maxBuffer: 1024 * 1024 },
+            (err) => {
+              sendCmdResult(
+                reqId,
+                baseResult(
+                  reqId,
+                  err ? "error" : "ok",
+                  err ? "" : "computeruse " + action + " @ (" + absX + "," + absY + ")" + (done ? " " + done : ""),
+                  err ? err.message : "",
+                ),
+              );
+            },
+          );
+          return;
+        }
+        // Linux xdotool
+        cp.execFile("xdotool", cmdStr, { timeout: 15000, maxBuffer: 1024 * 1024 }, (err) => {
+          sendCmdResult(
+            reqId,
+            baseResult(
+              reqId,
+              err ? "error" : "ok",
+              err ? "" : "computeruse " + action + " @ (" + absX + "," + absY + ")" + (done ? " " + done : ""),
+              err ? err.message : "",
+            ),
+          );
+        });
+      };
+      // Windows 命令构建
+      const winCmd = (body) =>
+        "Add-Type -AssemblyName System.Windows.Forms; Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int x,int y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint dwFlags,uint dx,uint dy,uint dwData,uint dwExtraInfo);' -Name U -Namespace W; [U]::SetCursorPos(" + absX + "," + absY + "); " + body;
+      if (os_ === "win32") {
+        const btnDown = params.button === "right" ? 0x0008 : params.button === "middle" ? 0x0020 : 0x0002;
+        const btnUp = params.button === "right" ? 0x0010 : params.button === "middle" ? 0x0040 : 0x0004;
+        switch (action) {
+          case "move":
+            run(winCmd("")); return;
+          case "click":
+            run(winCmd("[U]::mouse_event(" + btnDown + ",0,0,0,0); [U]::mouse_event(" + btnUp + ",0,0,0,0);")); return;
+          case "doubleclick":
+            run(winCmd("[U]::mouse_event(" + btnDown + ",0,0,0,0); [U]::mouse_event(" + btnUp + ",0,0,0,0); Start-Sleep -Milliseconds 50; [U]::mouse_event(" + btnDown + ",0,0,0,0); [U]::mouse_event(" + btnUp + ",0,0,0,0);")); return;
+          case "rightclick":
+            run(winCmd("[U]::mouse_event(0x0008,0,0,0,0); [U]::mouse_event(0x0010,0,0,0,0);")); return;
+          case "scroll":
+            run(winCmd("[U]::mouse_event(0x0800,0,0," + String(Math.round((params.dy || 120) * 120)) + ",0);")); return;
+          case "keypress":
+            run("Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('" + String(params.key || params.text || "").replace(/'/g, "").replace(/\+/g, "{+}").replace(/\^/g, "{^}").replace(/%/g, "{%}") + "')"); return;
+          case "type":
+            run("Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('" + String(params.text || "").replace(/\+/g, "{+}").replace(/\^/g, "{^}").replace(/%/g, "{%}").replace(/~/g, "{~}") + "')"); return;
+          default:
+            sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse: unknown action " + action)); return;
+        }
+      }
+      // Linux / macOS 通过工具 argv
+      const L = (a) => {
+        if (os_ === "darwin") {
+          // cliclick 参数: c:x,y / m:v1,v2 / w:+,-
+          const pos = absX + "," + absY;
+          switch (a[0]) {
+            case "move": return ["m:" + pos];
+            case "click": return ["c:" + pos];
+            case "doubleclick": return ["dc:" + pos];
+            case "rightclick": return ["c:" + pos];
+            case "scroll": return ["w:" + (params.dy > 0 ? "+" : "-")];
+            default: return a;
+          }
+        }
+        return a;
+      };
+      switch (action) {
+        case "move":
+          run(L(["mousemove", String(absX), String(absY)]));
+          return;
+        case "click": {
+          const btn = params.button === "right" ? 3 : params.button === "middle" ? 2 : 1;
+          run(L(["mousemove", String(absX), String(absY), "click", String(btn)]) );
+          return;
+        }
+        case "doubleclick":
+          run(L(["mousemove", String(absX), String(absY), "click", "--repeat", "2", "--delay", "50", "1"]));
+          return;
+        case "rightclick":
+          run(L(["mousemove", String(absX), String(absY), "click", "3"]));
+          return;
+        case "scroll":
+          run(L(["mousemove", String(absX), String(absY), "scroll", "--button", "5", String(Math.round(params.dy || params.y || 0))]));
+          return;
+        case "keypress":
+          run(L(["key", String(params.key || params.text || "")]));
+          return;
+        case "type": {
+          // macOS cliclick 无 type，用 osascript
+          if (os_ === "darwin") {
+            const t = String(params.text || "");
+            cp.execFile(
+              "osascript",
+              ["-e", 'tell application "System Events" to keystroke ' + JSON.stringify(t)],
+              { timeout: 15000, maxBuffer: 1024 * 1024 },
+              (err) => {
+                sendCmdResult(
+                  reqId,
+                  baseResult(reqId, err ? "error" : "ok", err ? "" : "typed: " + t.slice(0, 40), err ? err.message : ""),
+                );
+              },
+            );
+            return;
+          }
+          const t = String(params.text || "");
+          const shellSafe = "'" + String(t).replace(/'/g, "'\\''") + "'";
+          cp.execFile(
+            "bash",
+            ["-c", "xdotool type " + shellSafe],
+            { timeout: 15000, maxBuffer: 1024 * 1024 },
+            (err) => {
+              sendCmdResult(
+                reqId,
+                baseResult(reqId, err ? "error" : "ok", err ? "" : "typed: " + t.slice(0, 40), err ? err.message : ""),
+              );
+            },
+          );
+          return;
+        }
+        default:
+          sendCmdResult(reqId, baseResult(reqId, "error", "", "computeruse: unknown action " + action));
+          return;
+      }
+    }
     default:
       sendCmdResult(
         reqId,
@@ -1784,7 +2002,7 @@ async function startDeviceBridge(cfg) {
         device_id: deviceBridgeId,
         name: "HomeAgent GUI",
         kind: "computer",
-        caps: ["status", "cmdrun", "deviceinfo", "cmdresult"],
+        caps: ["status", "cmdrun", "deviceinfo", "cmdresult", "computeruse"],
         info: {
           hostname: devOs.hostname() || "",
           platform: process.platform || "",