mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-15 12:15:16 +00:00
feat(gui): 注入改 SendInput（默认）+ 用户操作时暂缓；拆解 Coopanion 桌宠动作系统
## 拆解 Coopanion（用户指出的参考项目）
新增 DESKTOP-PET-NOTES.md，记录 Pal-AI-Lab/Coopanion 的桌宠动作系统
（packages/cortico-world-desktop-pet）：
· 4 个工具：pet_say(气泡说话,可夹带动作标记) / pet_ask(提问,异步回答) /
pet_walk_to(走到 0-1 比例或 left/center/right/cursor,含走到鼠标位置) /
pet_act(只做动作; sit/sleep 是持续态)
· 动作词表 VOCAB：10 个表情 + 13 个动作，每个词多个中文别名，
英文 id 与中文别名进同一张 Map；未知词丢弃而非报错
· 脚本标记：【词,词】= 先做动作再换新气泡；<词> = 打字到那里时做、不换气泡。
让文字与动作在时间轴上对齐 —— 实现成本低但表现力强
· 设计要点：身体是 agent 唯一可见输出通道（用户看不见消息正文）；
互动事件不单独唤醒模型，跟着下批事件到
可借鉴（已写入文档待办）：姿态表达状态、动作与文字对齐、持续态、
动作解析容错、待机自主行为。不建议照搬：桌宠承载全部对话输出、ASR、戳摸甩。
## 注入层改 SendInput（参考它的实现）
原 PostMessage 路径只能操作原生 Win32 程序 —— Chromium/Electron 自绘界面
与游戏会忽略合成消息。这不是「平台硬限制」，是所选 API 的限制。
Coopanion 用 SendInput 注入系统输入队列，可靠性高得多。
新增三档模式 gui-prefs.deviceBridge.agentCursor：
sendinput（默认） SendInput，可靠；会动真光标，故先等用户停手
overlay PostMessage，不动用户鼠标；只能操作原生程序
real 保留旧代码路径
另有 deferMs：用户活动时最多等多久（默认 3000，上限 10000）
★ SendInput 绝对坐标是 0–65535 归一化值，不是像素。按虚拟桌面归一化
并带 MOUSEEVENTV_VIRTUALDESK，多屏下也正确（Coopanion 只按主屏归一化）。
★ 文本用 KEYEVENTF.UNICODE 逐字符发，不受键盘布局与输入法影响（比 WM_CHAR 可靠）。
## 修「用户活动感知」把自己也算进去的 bug
GetLastInputInfo 是**全局**的 —— agent 自己的 SendInput 也被算作「用户输入」。
实测：deferMs 设 0 仍被暂缓，回执写「开始时空闲仅 0ms」，即上一条命令的
注入刚把 idleMs 清零。若不排除自身注入，会永远处于「用户正在操作」状态。
按 Coopanion 的三重判定修正：
1. ownTick 自己最后一次注入时刻，之后 40ms 内的输入忽略
2. ownCursor 自己放下的光标位置，偏离 >2px 说明是用户动的
3. 无符号差值 GetTickCount 每 2^32ms 回绕，必须按无符号比较
## 修同一条命令被执行两遍
sendinput 分支把逻辑包进 runAction 箭头函数，里面的 return 只退出它自己，
于是执行完继续 fall through 到旧 real 路径。实测 8 条命令回了 16 个
cmd_result，且真光标被额外抢一次。补上外层 return。
## 判据
computeruse.test.mjs 47 条（npm test 共 131 条全绿），新增 4 条钉住本轮
踩过的坑，均做变异验证：
· 三档模式存在（overlay/real 只在条件里，不能要求赋值形式）
· 用户活动排除自身注入；且盯 send() 里的**调用点** —— 只查函数名时
删掉调用仍会误报绿（已实测并改正）
· SendInput 坐标归一化 + VIRTUALDESK
· sendinput 分支必须 return（防双执行）
## 待办（已写入 GUI-PLAN.md）
· loadGuiPrefs 不透传 agentCursor，导致 deferMs 配置读取不生效（实测）
· 显式调 SetProcessDpiAwarenessContext(-4)，不依赖 Electron 默认值
· mascot 姿态：思考/等待/出错
· 真实点击效果仍未验证（当前环境无可注入的可见窗口）
This commit is contained in:
JianFeeeee
@ -1,45 +1,78 @@
|
||||
// Windows 后台输入注入：不经过系统光标，直接把消息投递到目标窗口。
|
||||
// Windows 输入注入：两种模式。
|
||||
//
|
||||
// 为什么不调 SetCursorPos：
|
||||
// 那样会抢走用户的真实光标，用户在 agent 操作期间无法正常使用电脑。
|
||||
// 本模块用 PostMessage 把输入直接送到目标窗口的 hwnd，系统光标纹丝不动。
|
||||
// ┌─ SendInput（默认，可靠）────────────────────────────────────────┐
|
||||
// │ 注入到**系统输入队列**，等价真实硬件事件。Chromium/Electron/ │
|
||||
// │ 游戏等自绘界面都能接收。代价：会移动用户的真实光标。 │
|
||||
// │ 参考实现：Pal-AI-Lab/Coopanion 的 cortico-world-cua/win32.ts │
|
||||
// └────────────────────────────────────────────────────────────────┘
|
||||
// ┌─ PostMessage（后台，不干扰用户）───────────────────────────────┐
|
||||
// │ 直接投递给**某个窗口句柄**，用户鼠标纹丝不动。代价：自绘界面 │
|
||||
// │ （Chromium/Electron/游戏）常忽略合成消息，点不动。 │
|
||||
// └────────────────────────────────────────────────────────────────┘
|
||||
//
|
||||
// 已实测确认（本轮 probe-inject/electron-probe.js）：
|
||||
// PostMessage(EDIT, WM_CHAR, 'H') 后读回控件文本 = "Ha"
|
||||
// 且 GetCursorPos 前后一致 ⇒ 用户鼠标确实未被移动。
|
||||
// ★ 已实测确认（本轮）：
|
||||
// · SendInput 结构体尺寸 MOUSEINPUT=32 / KEYBDINPUT=24 / INPUT=40（x64），
|
||||
// layout 正确，SendInput 返回 1（成功送出）。
|
||||
// · GetLastInputInfo 可用，配合 GetTickCount 可算「用户空闲多久」。
|
||||
// · koffi 指针参数必须传 **Buffer**；传普通 JS 对象不写回
|
||||
// （GetCursorPos({x,y}) 读回仍是原值）。
|
||||
//
|
||||
// ★ koffi 3.x 的硬约定（踩过坑，务必遵守）：
|
||||
// 指针参数必须传 **Buffer**。传普通 JS 对象不会写回 ——
|
||||
// GetCursorPos({x,y}) 读回仍是原值（无效）
|
||||
// GetCursorPos(Buffer) 读到真实坐标（有效）
|
||||
// 写错会导致拿错目标窗口 hwnd ⇒ 点击落到别的窗口上。
|
||||
// ★ SendInput 绝对坐标是 **0–65535 归一化值**，不是像素 —— 最容易踩的坑。
|
||||
// 本次实现按**虚拟桌面**归一化并带 MOUSEEVENTF_VIRTUALDESK，
|
||||
// 因此多显示器下坐标也正确（Coopanion 只按主屏归一化）。
|
||||
|
||||
let koffi = null;
|
||||
let u32 = null;
|
||||
let k32 = null;
|
||||
let cached = null;
|
||||
|
||||
// koffi 类型名是**全局注册**的：同进程重复注册同名 struct 会抛
|
||||
// "Duplicate type name"。统一用 reg() 尝试原名、失败换 HA_ 前缀。
|
||||
function reg(name, def) {
|
||||
try {
|
||||
return koffi.struct(name, def);
|
||||
} catch (e) {
|
||||
return koffi.struct("HA_" + name, def);
|
||||
}
|
||||
}
|
||||
|
||||
function lib() {
|
||||
if (cached) return cached;
|
||||
koffi = require("koffi");
|
||||
u32 = koffi.load("user32.dll");
|
||||
// ★ koffi 的类型名是**全局注册**的：同一个进程里重复 koffi.struct("POINT", ...)
|
||||
// 会抛 "Duplicate type name 'POINT'"。main.js / 探针脚本可能已声明过同名结构体，
|
||||
// 所以这里先试着复用，失败就换个名字注册。
|
||||
let POINT;
|
||||
try {
|
||||
POINT = koffi.struct("POINT", { x: "long", y: "long" });
|
||||
} catch (e) {
|
||||
POINT = koffi.struct("HA_POINT", { x: "long", y: "long" });
|
||||
}
|
||||
k32 = koffi.load("kernel32.dll");
|
||||
|
||||
const MOUSEINPUT = reg("MOUSEINPUT", {
|
||||
dx: "long", dy: "long", mouseData: "uint32_t",
|
||||
dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t",
|
||||
});
|
||||
const KEYBDINPUT = reg("KEYBDINPUT", {
|
||||
wVk: "uint16_t", wScan: "uint16_t", dwFlags: "uint32_t",
|
||||
time: "uint32_t", dwExtraInfo: "uintptr_t",
|
||||
});
|
||||
const HARDWAREINPUT = reg("HARDWAREINPUT", {
|
||||
uMsg: "uint32_t", wParamL: "uint16_t", wParamH: "uint16_t",
|
||||
});
|
||||
const INPUT = koffi.struct(
|
||||
MOUSEINPUT.name.startsWith("HA_") ? "HA_INPUT" : "INPUT",
|
||||
{ type: "uint32_t", u: koffi.union({ mi: MOUSEINPUT, ki: KEYBDINPUT, hi: HARDWAREINPUT }) },
|
||||
);
|
||||
const LASTINPUTINFO = reg("LASTINPUTINFO", { cbSize: "uint32_t", dwTime: "uint32_t" });
|
||||
|
||||
const f = (sig) => u32.func(sig);
|
||||
cached = {
|
||||
POINT,
|
||||
POINT: reg("POINT", { x: "long", y: "long" }),
|
||||
INPUT,
|
||||
LASTINPUTINFO,
|
||||
INPUT_SIZE: koffi.sizeof(INPUT),
|
||||
SendInput: f("uint32_t __stdcall SendInput(uint32_t count, " + INPUT.name + " *inputs, int size)"),
|
||||
GetLastInputInfo: f("bool __stdcall GetLastInputInfo(_Inout_ " + LASTINPUTINFO.name + " *info)"),
|
||||
GetTickCount: k32.func("uint32_t __stdcall GetTickCount()"),
|
||||
GetSystemMetrics: f("int GetSystemMetrics(int i)"),
|
||||
SetProcessDpiAwarenessContext: f("bool __stdcall SetProcessDpiAwarenessContext(intptr_t value)"),
|
||||
// 指针参数一律用 void *，调用时传 Buffer
|
||||
WindowFromPoint: f("void *WindowFromPoint(void *pt)"),
|
||||
ChildWindowFromPointEx: f(
|
||||
"void *ChildWindowFromPointEx(void *hwndParent, void *pt, uint flags)",
|
||||
),
|
||||
ChildWindowFromPointEx: f("void *ChildWindowFromPointEx(void *hwndParent, void *pt, uint flags)"),
|
||||
GetAncestor: f("void *GetAncestor(void *hwnd, uint flags)"),
|
||||
IsWindow: f("int IsWindow(void *hwnd)"),
|
||||
IsWindowVisible: f("int IsWindowVisible(void *hwnd)"),
|
||||
@ -49,49 +82,41 @@ function lib() {
|
||||
GetCursorPos: f("bool GetCursorPos(void *pt)"),
|
||||
PostMessageW: f("intptr_t PostMessageW(void *hwnd, uint msg, uintptr_t w, intptr_t l)"),
|
||||
SendMessageW: f("intptr_t SendMessageW(void *hwnd, uint msg, uintptr_t w, intptr_t l)"),
|
||||
// 这些用于"这类窗口吃不吃合成消息"的能力探测
|
||||
GetWindowLongPtrW: f("intptr_t GetWindowLongPtrW(void *hwnd, int i)"),
|
||||
};
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Win32 常量
|
||||
// ── Win32 常量 ─────────────────────────────────────────────────────
|
||||
const GA_ROOT = 2;
|
||||
const CWP_SKIPINVISIBLE = 0x0002;
|
||||
const CWP_SKIPTRANSPARENT = 0x0004;
|
||||
const GWL_STYLE = -16;
|
||||
const GWL_EXSTYLE = -20;
|
||||
const WS_EX_LAYERED = 0x00080000;
|
||||
const WS_EX_TRANSPARENT = 0x00000020;
|
||||
const WS_EX_NOACTIVATE = 0x08000000;
|
||||
|
||||
// SendInput
|
||||
const INPUT_MOUSE = 0, INPUT_KEYBOARD = 1;
|
||||
const ME = {
|
||||
MOVE: 0x1, LEFTDOWN: 0x2, LEFTUP: 0x4, RIGHTDOWN: 0x8, RIGHTUP: 0x10,
|
||||
MIDDLEDOWN: 0x20, MIDDLEUP: 0x40, WHEEL: 0x800, HWHEEL: 0x1000,
|
||||
VIRTUALDESK: 0x4000, ABSOLUTE: 0x8000,
|
||||
};
|
||||
const KE = { EXTENDEDKEY: 0x1, KEYUP: 0x2, UNICODE: 0x4 };
|
||||
const WHEEL_DELTA = 120;
|
||||
|
||||
// PostMessage
|
||||
const WM_MOUSEMOVE = 0x0200;
|
||||
const WM_LBUTTONDOWN = 0x0201;
|
||||
const WM_LBUTTONUP = 0x0202;
|
||||
const WM_LBUTTONDBLCLK = 0x0203;
|
||||
const WM_RBUTTONDOWN = 0x0204;
|
||||
const WM_RBUTTONUP = 0x0205;
|
||||
const WM_MBUTTONDOWN = 0x0207;
|
||||
const WM_MBUTTONUP = 0x0208;
|
||||
const WM_MOUSEWHEEL = 0x020A;
|
||||
const WM_MOUSEHWHEEL = 0x020E;
|
||||
const WM_KEYDOWN = 0x0100;
|
||||
const WM_KEYUP = 0x0101;
|
||||
const WM_CHAR = 0x0102;
|
||||
const WM_SYSKEYDOWN = 0x0104;
|
||||
const WM_SYSKEYUP = 0x0105;
|
||||
const MK_LBUTTON = 0x0001;
|
||||
const MK_RBUTTON = 0x0002;
|
||||
const MK_MBUTTON = 0x0010;
|
||||
const MK_CONTROL = 0x0008;
|
||||
const MK_SHIFT = 0x0004;
|
||||
const WM_LBUTTONDOWN = 0x0201, WM_LBUTTONUP = 0x0202, WM_LBUTTONDBLCLK = 0x0203;
|
||||
const WM_RBUTTONDOWN = 0x0204, WM_RBUTTONUP = 0x0205;
|
||||
const WM_MBUTTONDOWN = 0x0207, WM_MBUTTONUP = 0x0208;
|
||||
const WM_MOUSEWHEEL = 0x020A, WM_MOUSEHWHEEL = 0x020E;
|
||||
const WM_KEYDOWN = 0x0100, WM_KEYUP = 0x0101, WM_CHAR = 0x0102;
|
||||
const MK_LBUTTON = 0x0001, MK_RBUTTON = 0x0002, MK_MBUTTON = 0x0010;
|
||||
|
||||
const SM_XVIRTUALSCREEN = 76;
|
||||
const SM_YVIRTUALSCREEN = 77;
|
||||
const SM_CXVIRTUALSCREEN = 78;
|
||||
const SM_CYVIRTUALSCREEN = 79;
|
||||
const SM_XVIRTUALSCREEN = 76, SM_YVIRTUALSCREEN = 77;
|
||||
const SM_CXVIRTUALSCREEN = 78, SM_CYVIRTUALSCREEN = 79;
|
||||
|
||||
// 虚拟键码（与 main.js 的 tapVk 共用同一张表）
|
||||
const VK = {
|
||||
ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, super: 0x5b,
|
||||
enter: 0x0d, return: 0x0d, tab: 0x09, esc: 0x1b, escape: 0x1b,
|
||||
@ -102,13 +127,14 @@ const VK = {
|
||||
const VK_EXTENDED = new Set([0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x2e, 0x5b, 0x5c, 0x6f, 0x74]);
|
||||
const isExtVK = (vk) => VK_EXTENDED.has(vk);
|
||||
|
||||
function getSystemMetrics() {
|
||||
const m = u32.func("int GetSystemMetrics(int i)");
|
||||
// ── 公共小工具 ─────────────────────────────────────────────────────
|
||||
function virtualScreen() {
|
||||
const k = lib();
|
||||
return {
|
||||
x: m(SM_XVIRTUALSCREEN),
|
||||
y: m(SM_YVIRTUALSCREEN),
|
||||
w: m(SM_CXVIRTUALSCREEN),
|
||||
h: m(SM_CYVIRTUALSCREEN),
|
||||
x: k.GetSystemMetrics(SM_XVIRTUALSCREEN),
|
||||
y: k.GetSystemMetrics(SM_YVIRTUALSCREEN),
|
||||
w: k.GetSystemMetrics(SM_CXVIRTUALSCREEN),
|
||||
h: k.GetSystemMetrics(SM_CYVIRTUALSCREEN),
|
||||
};
|
||||
}
|
||||
|
||||
@ -116,8 +142,257 @@ function hwndStr(h) {
|
||||
return h ? "0x" + h.toString(16) : "(null)";
|
||||
}
|
||||
|
||||
// 屏幕物理坐标 → 目标窗口 hwnd（含子窗口命中）
|
||||
// ★ pt 必须是 Buffer
|
||||
/** 屏幕坐标 → 0–65535 归一化（按虚拟桌面）。SendInput 绝对坐标用这个。 */
|
||||
function toAbsolute(x, y) {
|
||||
const v = virtualScreen();
|
||||
const nx = Math.round(((x - v.x) * 65535) / Math.max(1, v.w - 1));
|
||||
const ny = Math.round(((y - v.y) * 65535) / Math.max(1, v.h - 1));
|
||||
return { nx: Math.max(0, Math.min(65535, nx)), ny: Math.max(0, Math.min(65535, ny)) };
|
||||
}
|
||||
|
||||
// ── 用户活动感知（「用户在操作就暂缓」的核心）───────────────────────
|
||||
//
|
||||
// ★ 关键陷阱：GetLastInputInfo 是**全局**的 —— agent 自己的 SendInput
|
||||
// 也会被算作「输入」。若不排除自己的注入，每次注入后 idleMs() 都归零，
|
||||
// 于是默认永远处于「用户正在操作」状态，所有命令都被暂缓。
|
||||
// （实测踩到：deferMs 设为 0 仍被暂缓，因为上一条命令刚注入过。）
|
||||
//
|
||||
// 参照 Pal-AI-Lab/Coopanion 的做法，用三重判定区分「自己注入」vs「用户操作」：
|
||||
// 1. ownTick —— 自己最后一次注入的时刻，之后 40ms 内的输入忽略
|
||||
// 2. ownCursor —— 自己放的光标位置；偏离 >2px 说明是用户动的
|
||||
// 3. 无符号差值 —— GetTickCount 每 2^32 ms 回绕，必须按无符号比较
|
||||
let ownTick = 0; // 自己最后一次 SendInput 的 tick
|
||||
let ownCursor = null; // 自己放下的光标位置 {x,y}
|
||||
let ownMovedAt = 0; // 检测到光标被外力移动的时刻
|
||||
|
||||
/** 无符号「a - b」，正确处理 2^32 回绕。 */
|
||||
function since(a, b) {
|
||||
return (a - b) >>> 0;
|
||||
}
|
||||
|
||||
function cursorNow() {
|
||||
const k = lib();
|
||||
const pt = Buffer.alloc(8);
|
||||
if (!k.GetCursorPos(pt)) return null;
|
||||
return { x: pt.readInt32LE(0), y: pt.readInt32LE(4) };
|
||||
}
|
||||
|
||||
/** 标记「刚刚由我们自己注入了输入」。 */
|
||||
function markOwnInput() {
|
||||
ownTick = lib().GetTickCount();
|
||||
ownCursor = cursorNow();
|
||||
}
|
||||
|
||||
/** 距上次**用户**输入过了多少毫秒（排除自己的注入）。无记录时返回 Infinity。 */
|
||||
function idleMs() {
|
||||
const k = lib();
|
||||
const now = k.GetTickCount();
|
||||
|
||||
// 1) 若光标偏离我们自己放的位置 >2px，说明是用户移动的
|
||||
const cur = cursorNow();
|
||||
if (ownCursor && cur) {
|
||||
if (Math.abs(cur.x - ownCursor.x) > 2 || Math.abs(cur.y - ownCursor.y) > 2) {
|
||||
ownCursor = cur;
|
||||
ownMovedAt = now;
|
||||
}
|
||||
}
|
||||
|
||||
// 2) 取全局最后输入时刻，并判断它是否发生在「我们自己注入」之后
|
||||
const info = { cbSize: koffi.sizeof(k.LASTINPUTINFO), dwTime: 0 };
|
||||
if (!k.GetLastInputInfo(info)) return Number.MAX_SAFE_INTEGER;
|
||||
const last = info.dwTime;
|
||||
// ownTick 之后 40ms 内的输入视为我们自己造成的（SendInput 生效有延迟）
|
||||
const afterOwn =
|
||||
ownTick === 0 || (since(last, ownTick) > 40 && since(last, ownTick) < 0x80000000);
|
||||
let userAt = afterOwn ? last : 0;
|
||||
|
||||
// 3) 光标移动也算用户活动，取两者中更晚的
|
||||
if (ownMovedAt && (!userAt || since(ownMovedAt, userAt) < 0x80000000)) userAt = ownMovedAt;
|
||||
|
||||
return userAt ? since(now, userAt) : Number.MAX_SAFE_INTEGER;
|
||||
}
|
||||
|
||||
/** 用户是否在最近 thresholdMs 内操作过（已排除 agent 自身注入）。 */
|
||||
function userIsActive(thresholdMs = 1200) {
|
||||
return idleMs() < thresholdMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* 等到用户停手再执行。
|
||||
*
|
||||
* 为什么需要：SendInput 会移动真实光标。若用户正在打字/拖拽，agent
|
||||
* 插进来会抢走光标、打断输入。做法是轮询等待用户空闲，超过 maxWait
|
||||
* 则放弃等待（返回 false）—— 宁可跳过本次操作，也不硬抢。
|
||||
*/
|
||||
async function waitForUserIdle(maxWaitMs = 3000, thresholdMs = 800) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < maxWaitMs) {
|
||||
if (!userIsActive(thresholdMs)) return true;
|
||||
await new Promise((r) => setTimeout(r, 120));
|
||||
}
|
||||
return !userIsActive(thresholdMs);
|
||||
}
|
||||
|
||||
// ── SendInput 实现 ─────────────────────────────────────────────────
|
||||
const mouseEvent = (dwFlags, dx = 0, dy = 0, mouseData = 0) => ({
|
||||
type: INPUT_MOUSE,
|
||||
u: { mi: { dx, dy, mouseData, dwFlags, time: 0, dwExtraInfo: 0 } },
|
||||
});
|
||||
const keyEvent = (vk, up, extended, unicode) => ({
|
||||
type: INPUT_KEYBOARD,
|
||||
u: {
|
||||
ki: {
|
||||
wVk: unicode ? 0 : vk,
|
||||
wScan: unicode ? vk : 0,
|
||||
dwFlags: (up ? KE.KEYUP : 0) | (extended ? KE.EXTENDEDKEY : 0) | (unicode ? KE.UNICODE : 0),
|
||||
time: 0,
|
||||
dwExtraInfo: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function send(events) {
|
||||
const k = lib();
|
||||
const sent = k.SendInput(events.length, events, k.INPUT_SIZE);
|
||||
if (sent !== events.length) {
|
||||
// 常见原因：目标窗口权限更高（UIPI 阻挡）。这必须报出来，
|
||||
// 不能静默 —— 否则会表现为「命令成功但什么都没发生」。
|
||||
throw new Error(
|
||||
"SendInput 只送出 " + sent + "/" + events.length + " 个事件（可能被更高权限的窗口挡住）",
|
||||
);
|
||||
}
|
||||
// 记录「这次是我们自己注入的」，避免被 idleMs() 误判为用户活动
|
||||
markOwnInput();
|
||||
return true;
|
||||
}
|
||||
|
||||
const BUTTON_FLAGS = {
|
||||
left: [ME.LEFTDOWN, ME.LEFTUP],
|
||||
right: [ME.RIGHTDOWN, ME.RIGHTUP],
|
||||
middle: [ME.MIDDLEDOWN, ME.MIDDLEUP],
|
||||
};
|
||||
|
||||
/** 移动真实光标（SendInput）。坐标：屏幕物理像素。 */
|
||||
function siMove(x, y) {
|
||||
const { nx, ny } = toAbsolute(x, y);
|
||||
send([mouseEvent(ME.MOVE | ME.ABSOLUTE | ME.VIRTUALDESK, nx, ny)]);
|
||||
return { ok: true, x, y };
|
||||
}
|
||||
|
||||
function siButton(x, y, action, button) {
|
||||
const b = BUTTON_FLAGS[button] || BUTTON_FLAGS.left;
|
||||
if (action === "move") return siMove(x, y);
|
||||
siMove(x, y); // 先到位，再按键：否则会点到上一个位置
|
||||
if (action === "click") {
|
||||
send([mouseEvent(b[0]), mouseEvent(b[1])]);
|
||||
} else if (action === "doubleclick") {
|
||||
send([mouseEvent(b[0]), mouseEvent(b[1]), mouseEvent(b[0]), mouseEvent(b[1])]);
|
||||
} else if (action === "tripleclick") {
|
||||
send([
|
||||
mouseEvent(b[0]), mouseEvent(b[1]),
|
||||
mouseEvent(b[0]), mouseEvent(b[1]),
|
||||
mouseEvent(b[0]), mouseEvent(b[1]),
|
||||
]);
|
||||
} else if (action === "mousedown") {
|
||||
send([mouseEvent(b[0])]);
|
||||
} else if (action === "mouseup") {
|
||||
send([mouseEvent(b[1])]);
|
||||
} else {
|
||||
return { ok: false, error: "unsupported action " + action };
|
||||
}
|
||||
return { ok: true, x, y, action, button: button || "left" };
|
||||
}
|
||||
|
||||
/**
|
||||
* 拖拽：按下 → 分步移动 → 释放。
|
||||
* 分步是必需的：canvas 拖拽、列表排序、滑动条依赖中间 mousemove 事件，
|
||||
* 一步到位会失效。
|
||||
*/
|
||||
function siDrag(sx, sy, tx, ty, button, steps) {
|
||||
const b = BUTTON_FLAGS[button] || BUTTON_FLAGS.left;
|
||||
const n = Math.max(1, Math.min(60, steps || 12));
|
||||
siMove(sx, sy);
|
||||
send([mouseEvent(b[0])]);
|
||||
for (let i = 1; i <= n; i++) {
|
||||
const t = i / n;
|
||||
siMove(sx + (tx - sx) * t, sy + (ty - sy) * t);
|
||||
}
|
||||
siMove(tx, ty);
|
||||
send([mouseEvent(b[1])]);
|
||||
return { ok: true, from: { x: sx, y: sy }, to: { x: tx, y: ty }, steps: n };
|
||||
}
|
||||
|
||||
/**
|
||||
* 滚轮。dy/dx 单位为「格」（1 格 = WHEEL_DELTA=120）。
|
||||
* 约定：dy > 0 = 向上滚（与 Windows 惯例一致，也与本仓既有实现一致）。
|
||||
*/
|
||||
function siScroll(dy, dx) {
|
||||
const events = [];
|
||||
if (dy) events.push(mouseEvent(ME.WHEEL, 0, 0, (Math.round(dy) * WHEEL_DELTA) >>> 0));
|
||||
if (dx) events.push(mouseEvent(ME.HWHEEL, 0, 0, (Math.round(dx) * WHEEL_DELTA) >>> 0));
|
||||
if (events.length) send(events);
|
||||
return { ok: true, dy: dy || 0, dx: dx || 0 };
|
||||
}
|
||||
|
||||
/** 组合键。spec 形如 "ctrl+shift+t"。 */
|
||||
function siKeyTap(spec) {
|
||||
const parts = String(spec || "")
|
||||
.split("+")
|
||||
.map((x) => x.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
if (!parts.length) return { ok: false, error: "empty key" };
|
||||
const last = parts[parts.length - 1];
|
||||
const mods = parts.slice(0, -1);
|
||||
const codes = [];
|
||||
for (const m of mods) {
|
||||
const c = VK[m];
|
||||
if (c === undefined) return { ok: false, error: "unknown modifier: " + m };
|
||||
codes.push(c);
|
||||
}
|
||||
let keyCode = VK[last];
|
||||
if (keyCode === undefined) {
|
||||
if (last.length === 1) keyCode = last.toUpperCase().charCodeAt(0);
|
||||
else return { ok: false, error: "unknown key: " + last };
|
||||
}
|
||||
// 按下的按正序、释放按反序 —— 与真实手指动作一致
|
||||
const events = [
|
||||
...codes.map((c) => keyEvent(c, false, false, false)),
|
||||
keyEvent(keyCode, false, isExtVK(keyCode), false),
|
||||
keyEvent(keyCode, true, isExtVK(keyCode), false),
|
||||
...[...codes].reverse().map((c) => keyEvent(c, true, false, false)),
|
||||
];
|
||||
send(events);
|
||||
return { ok: true, key: spec };
|
||||
}
|
||||
|
||||
/**
|
||||
* 文本输入：用 KEYEVENTF.UNICODE 逐字符发送。
|
||||
* 比 WM_CHAR / SendKeys 可靠：不受键盘布局和输入法状态影响，
|
||||
* 中文、emoji 也能进（emoji 需按 UTF-16 代理对逐个发）。
|
||||
*/
|
||||
function siType(text) {
|
||||
const t = String(text || "");
|
||||
if (!t) return { ok: false, error: "empty text" };
|
||||
const events = [];
|
||||
for (let i = 0; i < t.length; i++) {
|
||||
const ch = t[i];
|
||||
if (ch === "\n") {
|
||||
events.push(keyEvent(VK.enter, false, false, false), keyEvent(VK.enter, true, false, false));
|
||||
continue;
|
||||
}
|
||||
if (ch === "\r") continue;
|
||||
if (ch === "\t") {
|
||||
events.push(keyEvent(VK.tab, false, false, false), keyEvent(VK.tab, true, false, false));
|
||||
continue;
|
||||
}
|
||||
const unit = t.charCodeAt(i); // UTF-16 code unit（代理对会被拆成两个，正好）
|
||||
events.push(keyEvent(unit, false, false, true), keyEvent(unit, true, false, true));
|
||||
}
|
||||
if (events.length) send(events);
|
||||
return { ok: true, chars: t.length };
|
||||
}
|
||||
|
||||
// ── PostMessage 实现（后台，不干扰用户）────────────────────────────
|
||||
function windowAtPoint(sx, sy) {
|
||||
const k = lib();
|
||||
const pt = Buffer.alloc(8);
|
||||
@ -125,7 +400,6 @@ function windowAtPoint(sx, sy) {
|
||||
pt.writeInt32LE(Math.round(sy), 4);
|
||||
let h = k.WindowFromPoint(pt);
|
||||
if (!h) return null;
|
||||
// WindowFromPoint 只返回顶层窗口；对 UI 框架要下钻到真实接收消息的子窗口
|
||||
const flags = CWP_SKIPINVISIBLE | CWP_SKIPTRANSPARENT;
|
||||
let guard = 0;
|
||||
for (;;) {
|
||||
@ -136,7 +410,6 @@ function windowAtPoint(sx, sy) {
|
||||
return h;
|
||||
}
|
||||
|
||||
// 屏幕坐标 → 目标窗口客户区坐标（lParam 用的就是客户区坐标）
|
||||
function toClient(hwnd, sx, sy) {
|
||||
const k = lib();
|
||||
const pt = Buffer.alloc(8);
|
||||
@ -146,68 +419,53 @@ function toClient(hwnd, sx, sy) {
|
||||
return { x: pt.readInt32LE(0), y: pt.readInt32LE(4) };
|
||||
}
|
||||
|
||||
// 窗口是否"看起来"能接收合成消息（启发式判断，不保证）
|
||||
function acceptsSynthetic(hwnd) {
|
||||
const k = lib();
|
||||
if (!hwnd || !k.IsWindow(hwnd)) return { ok: false, why: "invalid hwnd" };
|
||||
if (!k.IsWindowVisible(hwnd)) return { ok: false, why: "window hidden" };
|
||||
const ex = k.GetWindowLongPtrW(hwnd, GWL_EXSTYLE);
|
||||
// WS_EX_TRANSPARENT 的窗口通常不接收命中测试类消息
|
||||
if (ex & WS_EX_TRANSPARENT) return { ok: false, why: "WS_EX_TRANSPARENT" };
|
||||
return { ok: true, ex };
|
||||
}
|
||||
|
||||
function post(hwnd, msg, w, l) {
|
||||
const k = lib();
|
||||
const r = k.PostMessageW(hwnd, msg, w, l);
|
||||
return r !== 0;
|
||||
return lib().PostMessageW(hwnd, msg, w, l) !== 0;
|
||||
}
|
||||
const pack = (x, y) => ((y & 0xffff) << 16) | (x & 0xffff);
|
||||
|
||||
function pack(x, y) {
|
||||
// lParam: 低 16 位 x，高 16 位 y（signed 16-bit）
|
||||
return ((y & 0xffff) << 16) | (x & 0xffff);
|
||||
}
|
||||
|
||||
// ── 鼠标操作 ───────────────────────────────────────────────────────
|
||||
const BTN_MSG = {
|
||||
const PM_BTN = {
|
||||
left: { down: WM_LBUTTONDOWN, up: WM_LBUTTONUP, dbl: WM_LBUTTONDBLCLK, mk: MK_LBUTTON },
|
||||
right: { down: WM_RBUTTONDOWN, up: WM_RBUTTONUP, mk: MK_RBUTTON },
|
||||
middle: { down: WM_MBUTTONDOWN, up: WM_MBUTTONUP, mk: MK_MBUTTON },
|
||||
};
|
||||
|
||||
function mouseOp(sx, sy, action, opts) {
|
||||
function pmMouse(sx, sy, action, opts) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空（该坐标无窗口）" };
|
||||
const c = toClient(hwnd, sx, sy);
|
||||
const lp = pack(c.x, c.y);
|
||||
const btn = BTN_MSG[(opts && opts.button) || "left"] || BTN_MSG.left;
|
||||
const post_ = (m, w) => post(hwnd, m, w, lp);
|
||||
|
||||
const btn = PM_BTN[(opts && opts.button) || "left"] || PM_BTN.left;
|
||||
const P = (m, w) => post(hwnd, m, w, lp);
|
||||
switch (action) {
|
||||
case "move":
|
||||
return { ok: post_(WM_MOUSEMOVE, 0), hwnd, client: c };
|
||||
case "click": {
|
||||
post_(btn.down, btn.mk);
|
||||
const up = post_(btn.up, 0);
|
||||
return { ok: up, hwnd, client: c };
|
||||
}
|
||||
case "doubleclick": {
|
||||
post_(btn.down, btn.mk);
|
||||
post_(btn.up, 0);
|
||||
post_(btn.dbl, MK_LBUTTON);
|
||||
const up = post_(btn.up, 0);
|
||||
return { ok: up, hwnd, client: c };
|
||||
}
|
||||
return { ok: P(WM_MOUSEMOVE, 0), hwnd, client: c };
|
||||
case "click":
|
||||
P(btn.down, btn.mk);
|
||||
return { ok: P(btn.up, 0), hwnd, client: c };
|
||||
case "doubleclick":
|
||||
P(btn.down, btn.mk); P(btn.up, 0);
|
||||
P(btn.dbl, MK_LBUTTON);
|
||||
return { ok: P(btn.up, 0), hwnd, client: c };
|
||||
case "mousedown":
|
||||
return { ok: post_(btn.down, btn.mk), hwnd, client: c };
|
||||
return { ok: P(btn.down, btn.mk), hwnd, client: c };
|
||||
case "mouseup":
|
||||
return { ok: post_(btn.up, 0), hwnd, client: c };
|
||||
return { ok: P(btn.up, 0), hwnd, client: c };
|
||||
default:
|
||||
return { ok: false, error: "unsupported mouse action " + action };
|
||||
}
|
||||
}
|
||||
|
||||
function tripleClick(sx, sy) {
|
||||
function pmTripleClick(sx, sy) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空" };
|
||||
const c = toClient(hwnd, sx, sy);
|
||||
@ -216,59 +474,46 @@ function tripleClick(sx, sy) {
|
||||
post(hwnd, WM_LBUTTONUP, 0, lp);
|
||||
post(hwnd, WM_LBUTTONDBLCLK, MK_LBUTTON, lp);
|
||||
post(hwnd, WM_LBUTTONUP, 0, lp);
|
||||
return { ok: true, hwnd, client: c };
|
||||
return { ok: true, hwnd };
|
||||
}
|
||||
|
||||
function drag(sx, sy, tx, ty, opts) {
|
||||
function pmDrag(sx, sy, tx, ty, opts) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空" };
|
||||
const btn = BTN_MSG[(opts && opts.button) || "left"] || BTN_MSG.left;
|
||||
const btn = PM_BTN[(opts && opts.button) || "left"] || PM_BTN.left;
|
||||
const steps = Math.max(1, Math.min(60, (opts && opts.steps) || 12));
|
||||
const c0 = toClient(hwnd, sx, sy);
|
||||
post(hwnd, btn.down, btn.mk, pack(c0.x, c0.y));
|
||||
// 分步移动：部分应用（canvas 拖拽、排序、滑块）依赖中间 move 事件
|
||||
for (let i = 1; i <= steps; i++) {
|
||||
const t = i / steps;
|
||||
const c = toClient(hwnd, sx + (tx - sx) * t, sy + (ty - sy) * t);
|
||||
post(hwnd, WM_MOUSEMOVE, btn.mk, pack(c.x, c.y));
|
||||
}
|
||||
const cEnd = toClient(hwnd, tx, ty);
|
||||
const up = post(hwnd, btn.up, 0, pack(cEnd.x, cEnd.y));
|
||||
return { ok: up, hwnd, steps };
|
||||
const ce = toClient(hwnd, tx, ty);
|
||||
const ok = post(hwnd, btn.up, 0, pack(ce.x, ce.y));
|
||||
return { ok, hwnd, steps };
|
||||
}
|
||||
|
||||
// 滚轮：lParam 用**屏幕**坐标，不是客户区坐标
|
||||
function scroll(sx, sy, dy, dx) {
|
||||
function pmScroll(sx, sy, dy, dx) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空" };
|
||||
const lp = pack(Math.round(sx), Math.round(sy));
|
||||
const lp = pack(Math.round(sx), Math.round(sy)); // 滚轮用屏幕坐标
|
||||
let ok = true;
|
||||
if (dy) ok = post(hwnd, WM_MOUSEWHEEL, deltaWord(Math.round(dy)), lp) && ok;
|
||||
if (dx) ok = post(hwnd, WM_MOUSEHWHEEL, deltaWord(Math.round(dx)), lp) && ok;
|
||||
if (dy) ok = post(hwnd, WM_MOUSEWHEEL, (Math.round(dy) * WHEEL_DELTA) >>> 0, lp) && ok;
|
||||
if (dx) ok = post(hwnd, WM_MOUSEHWHEEL, (Math.round(dx) * WHEEL_DELTA) >>> 0, lp) && ok;
|
||||
return { ok, hwnd };
|
||||
}
|
||||
|
||||
// 把像素量换算成滚轮"格"（WHEEL_DELTA=120）
|
||||
function deltaWord(v) {
|
||||
const clicks = Math.round(v / 120) || (v > 0 ? 1 : -1);
|
||||
// high word 存 delta，low word 存按键状态
|
||||
return (clicks << 16) >>> 0;
|
||||
}
|
||||
|
||||
// ── 键盘操作 ───────────────────────────────────────────────────────
|
||||
function keyTap(hwnd, spec) {
|
||||
const parts = String(spec || "")
|
||||
.split("+")
|
||||
.map((x) => x.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
function pmKeyTap(hwnd, spec) {
|
||||
const parts = String(spec || "").split("+").map((x) => x.trim().toLowerCase()).filter(Boolean);
|
||||
if (!parts.length) return { ok: false, error: "empty key" };
|
||||
const last = parts[parts.length - 1];
|
||||
const mods = parts.slice(0, -1);
|
||||
const modCodes = [];
|
||||
const codes = [];
|
||||
for (const m of mods) {
|
||||
const c = VK[m];
|
||||
if (c === undefined) return { ok: false, error: "unknown modifier: " + m };
|
||||
modCodes.push(c);
|
||||
codes.push(c);
|
||||
}
|
||||
let keyCode = VK[last];
|
||||
if (keyCode === undefined) {
|
||||
@ -276,20 +521,14 @@ function keyTap(hwnd, spec) {
|
||||
else return { ok: false, error: "unknown key: " + last };
|
||||
}
|
||||
let ok = true;
|
||||
for (const c of modCodes) {
|
||||
ok = post(hwnd, WM_KEYDOWN, c, 0) && ok;
|
||||
}
|
||||
for (const c of codes) ok = post(hwnd, WM_KEYDOWN, c, 0) && ok;
|
||||
ok = post(hwnd, WM_KEYDOWN, keyCode, isExtVK(keyCode) ? 1 : 0) && ok;
|
||||
ok = post(hwnd, WM_KEYUP, keyCode, isExtVK(keyCode) ? 1 : 0) && ok;
|
||||
for (const c of modCodes.slice().reverse()) {
|
||||
ok = post(hwnd, WM_KEYUP, c, 0) && ok;
|
||||
}
|
||||
return { ok, mods: modCodes, key: keyCode };
|
||||
for (const c of codes.slice().reverse()) ok = post(hwnd, WM_KEYUP, c, 0) && ok;
|
||||
return { ok, key: spec };
|
||||
}
|
||||
|
||||
// 文本注入：走 WM_CHAR，不做输入法合成，因此对原生控件最可靠。
|
||||
// 换行/回车单独发 VK_RETURN，否则 WM_CHAR 的 \n 多数控件不认。
|
||||
function typeText(hwnd, text) {
|
||||
function pmTypeText(hwnd, text) {
|
||||
const t = String(text || "");
|
||||
if (!t) return { ok: false, error: "empty text" };
|
||||
let ok = true;
|
||||
@ -318,18 +557,43 @@ module.exports = {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
// 坐标/桌面
|
||||
virtualScreen,
|
||||
toAbsolute,
|
||||
hwndStr,
|
||||
// 用户活动感知
|
||||
idleMs,
|
||||
userIsActive,
|
||||
waitForUserIdle,
|
||||
markOwnInput,
|
||||
cursorNow,
|
||||
// SendInput（默认路径）
|
||||
si: {
|
||||
move: siMove,
|
||||
button: siButton,
|
||||
drag: siDrag,
|
||||
scroll: siScroll,
|
||||
key: siKeyTap,
|
||||
type: siType,
|
||||
},
|
||||
// PostMessage（后台路径）
|
||||
pm: {
|
||||
mouse: pmMouse,
|
||||
tripleClick: pmTripleClick,
|
||||
drag: pmDrag,
|
||||
scroll: pmScroll,
|
||||
key: pmKeyTap,
|
||||
type: pmTypeText,
|
||||
},
|
||||
// 兼容旧名（PostMessage）
|
||||
windowAtPoint,
|
||||
toClient,
|
||||
acceptsSynthetic,
|
||||
getSystemMetrics,
|
||||
hwndStr,
|
||||
// 鼠标
|
||||
mouse: mouseOp,
|
||||
tripleClick,
|
||||
drag,
|
||||
scroll,
|
||||
// 键盘
|
||||
keyTap,
|
||||
typeText,
|
||||
mouse: pmMouse,
|
||||
tripleClick: pmTripleClick,
|
||||
drag: pmDrag,
|
||||
scroll: pmScroll,
|
||||
keyTap: pmKeyTap,
|
||||
typeText: pmTypeText,
|
||||
VK,
|
||||
};
|
||||
|
||||
@ -194,9 +194,41 @@ check(
|
||||
"指针参数未用 Buffer ⇒ WindowFromPoint 拿不到真实 hwnd，点击会落错窗口",
|
||||
);
|
||||
check(
|
||||
"★ overlay 默认开启且可切回 real",
|
||||
/cursorMode = "overlay"/.test(main) && /cursorMode = "real"/.test(main),
|
||||
"未提供 overlay/real 切换",
|
||||
"★ 注入模式默认 sendinput，且可切 overlay/real",
|
||||
// overlay/real 只出现在条件判断里（不是赋值），不能要求 "cursorMode = \"overlay\""
|
||||
/let cursorMode = "sendinput"/.test(main) &&
|
||||
/_ac\.mode === "overlay" \|\| _ac\.mode === "real"\) cursorMode = _ac\.mode/.test(main),
|
||||
"未提供 sendinput/overlay/real 三档切换",
|
||||
);
|
||||
check(
|
||||
"★ 用户活动感知排除自身注入（ownTick/ownCursor）",
|
||||
// GetLastInputInfo 是全局的，agent 自己的 SendInput 也会被算作「用户输入」。
|
||||
// 不排除则永远判定「用户正在操作」，所有命令被暂缓（实测踩到）。
|
||||
/ownTick/.test(injectSrc) && /ownCursor/.test(injectSrc),
|
||||
"未排除自身注入 ⇒ 永远处于「用户正在操作」状态",
|
||||
);
|
||||
check(
|
||||
"★ send() 成功后必须记录自身注入（否则 idleMs 归零）",
|
||||
// 这个断言必须盯**调用点**而不是函数名：只检查 /markOwnInput/ 时，
|
||||
// 把 send() 里的调用删掉仍会误报绿（已用品变异验证）。
|
||||
// function send(...) { ...; markOwnInput(); return true; }
|
||||
/function send\(events\)[\s\S]{0,600}?markOwnInput\(\);[\s\S]{0,60}?return true;/.test(
|
||||
injectSrc,
|
||||
),
|
||||
"send() 内未调用 markOwnInput ⇒ 自己的注入会被误判为用户操作",
|
||||
);
|
||||
check(
|
||||
"★ SendInput 绝对坐标按虚拟桌面归一化到 0–65535",
|
||||
// 像素直接传给 SendInput 会落在错误位置（它要的是归一化值）。
|
||||
/65535/.test(injectSrc) && /VIRTUALDESK/.test(injectSrc),
|
||||
"绝对坐标未归一化，或未带 VIRTUALDESK（多屏会错位）",
|
||||
);
|
||||
check(
|
||||
"★ sendinput 分支结束后必须 return（防双执行）",
|
||||
// runAction 里的 return 只退出它自己；少了外层 return 会 fall through
|
||||
// 到旧 real 路径 —— 实测 8 条命令回了 16 个结果。
|
||||
/startDefer\(\(proceed\)[\s\S]{0,400}?\n\s+return;\n\s*\} catch \(e\) \{/.test(main),
|
||||
"缺少外层 return，同一条命令会被执行两遍",
|
||||
);
|
||||
check(
|
||||
"自绘光标层不吃点击（setIgnoreMouseEvents）",
|
||||
|
||||
178
cmd/gui/main.js
178
cmd/gui/main.js
@ -1423,6 +1423,15 @@ function sendCmdResult(a, b) {
|
||||
}
|
||||
}
|
||||
|
||||
// 安全设置自绘光标的「忙碌」呼吸态（agentCursor 可能未就绪）
|
||||
function agentCursorBusySafe(dispIdx, busy) {
|
||||
try {
|
||||
if (agentCursor && typeof agentCursor.agentCursorBusy === "function") {
|
||||
agentCursor.agentCursorBusy(busy, dispIdx);
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
function baseResult(reqId, status, output, error) {
|
||||
return {
|
||||
op: "cmd_result",
|
||||
@ -1948,17 +1957,33 @@ function executeHomeagentCmd(capability, reqId) {
|
||||
}
|
||||
const action = params.action || "click";
|
||||
|
||||
// ★ 模式 A：agent 独立光标 + 后台注入。
|
||||
// ★ 注入模式：sendinput（默认）/ overlay / real
|
||||
//
|
||||
// 取值 "overlay"（不动用户鼠标）/ "real"（旧行为，操作真实鼠标）。
|
||||
// 未配置时默认 "overlay" —— 用户明确要求不要抢鼠标。
|
||||
let cursorMode = "overlay";
|
||||
// sendinput SendInput 注入到系统输入队列。等价真实硬件事件，
|
||||
// Chromium/Electron/游戏等自绘界面都能接收 —— 可靠。
|
||||
// 代价：会移动用户的真实光标，故操作前先等用户停手。
|
||||
// 参考 Pal-AI-Lab/Coopanion 的 cortico-world-cua。
|
||||
// overlay PostMessage 直投窗口句柄。用户鼠标纹丝不动，但自绘
|
||||
// 界面常忽略合成消息 ⇒ 只能操作原生 Win32 程序。
|
||||
// real 同上但用旧代码路径（保留以兼容）。
|
||||
//
|
||||
// 默认 sendinput：用户要求「可靠 + 用户正在操作则暂缓」。
|
||||
let cursorMode = "sendinput";
|
||||
try {
|
||||
const _p = loadGuiPrefs();
|
||||
const _ac = (_p && _p.deviceBridge && _p.deviceBridge.agentCursor) || {};
|
||||
if (_ac.mode === "real") cursorMode = "real";
|
||||
if (_ac.mode === "overlay" || _ac.mode === "real") cursorMode = _ac.mode;
|
||||
} catch (e) {}
|
||||
const injOk = agentInject.available();
|
||||
const useOverlay = cursorMode === "overlay" && injOk;
|
||||
const useSendInput = cursorMode === "sendinput" && injOk;
|
||||
// 用户活动暂缓：等用户停手再注入（毫秒）。0 = 不等待。
|
||||
let deferMs = 3000;
|
||||
try {
|
||||
const _p2 = loadGuiPrefs();
|
||||
const _ac2 = (_p2 && _p2.deviceBridge && _p2.deviceBridge.agentCursor) || {};
|
||||
if (typeof _ac2.deferMs === "number") deferMs = Math.max(0, Math.min(10000, _ac2.deferMs));
|
||||
} catch (e) {}
|
||||
const useOverlay = cursorMode === "overlay" && agentInject.available();
|
||||
|
||||
// 把内部参数提到 action 分支之前共用
|
||||
const cuNumber = (v, dflt) => {
|
||||
@ -2019,6 +2044,147 @@ function executeHomeagentCmd(capability, reqId) {
|
||||
const absX = Math.round(ox + (physical ? rawX : rawX * scale));
|
||||
const absY = Math.round(oy + (physical ? rawY : rawY * scale));
|
||||
|
||||
// === 模式 sendinput：SendInput 注入 + 用户在操作则暂缓 ===
|
||||
//
|
||||
// SendInput 会动真实光标，因此注入前先等用户停手（deferMs 上限）。
|
||||
// 等不到就跳过本次操作并明确报错 —— 宁可失败，也不打断用户。
|
||||
if (os_ === "win32" && useSendInput) {
|
||||
try {
|
||||
const busy = () => agentCursorBusySafe(dispIdx, true);
|
||||
const idle = () => agentCursorBusySafe(dispIdx, false);
|
||||
const fin = (ok, msg) => {
|
||||
idle();
|
||||
sendCmdResult(reqId, baseResult(reqId, ok ? "ok" : "error", msg, ok ? "" : msg));
|
||||
};
|
||||
const fail = (m) => fin(false, m);
|
||||
|
||||
// 等待用户停手
|
||||
//
|
||||
// 注意：executeHomeagentCmd 是**同步函数**，不能 await。
|
||||
// 故改为「同步查询 + 轮询回调」：先看用户是否在操作；若是，
|
||||
// 定时轮询到空闲或超时再继续；全程不阻塞主进程。
|
||||
const startDefer = (cb) => {
|
||||
if (deferMs <= 0 || !agentInject.userIsActive(1200)) return cb(true);
|
||||
const before = agentInject.idleMs();
|
||||
const t0 = Date.now();
|
||||
const tick = () => {
|
||||
if (!agentInject.userIsActive(800)) return cb(true);
|
||||
if (Date.now() - t0 >= deferMs) {
|
||||
idle();
|
||||
sendCmdResult(
|
||||
reqId,
|
||||
baseResult(
|
||||
reqId,
|
||||
"error",
|
||||
"",
|
||||
"用户正在操作（已等待 " + deferMs + "ms，开始时空闲仅 " + before +
|
||||
"ms）。为避免抢走光标已跳过本次操作。可在设置里调整 agentCursor.deferMs。",
|
||||
),
|
||||
);
|
||||
return cb(false);
|
||||
}
|
||||
setTimeout(tick, 120);
|
||||
};
|
||||
tick();
|
||||
};
|
||||
|
||||
const runAction = () => {
|
||||
const btnName = params.button === "right" ? "right" : params.button === "middle" ? "middle" : "left";
|
||||
const toX = () => Math.round(ox + (physical ? (parseFloat(params.tox || params.tx) || 0) : (parseFloat(params.tox || params.tx) || 0) * scale));
|
||||
const toY = () => Math.round(oy + (physical ? (parseFloat(params.toy || params.ty) || 0) : (parseFloat(params.toy || params.ty) || 0) * scale));
|
||||
|
||||
// 自绘光标同步到目标位置（视觉指示 agent 在做什么）
|
||||
const needsXY = ["click", "doubleclick", "rightclick", "middleclick", "tripleclick",
|
||||
"mousedown", "mouseup", "move", "hover", "drag", "scroll"].includes(action);
|
||||
if (needsXY) agentCursor.agentCursorMove(absX, absY, dispIdx);
|
||||
busy();
|
||||
|
||||
switch (action) {
|
||||
case "move":
|
||||
case "hover": {
|
||||
const r = agentInject.si.move(absX, absY);
|
||||
return fin(r.ok, "sendinput " + action + " @ (" + absX + "," + absY + ")");
|
||||
}
|
||||
case "click":
|
||||
case "rightclick":
|
||||
case "middleclick":
|
||||
case "doubleclick":
|
||||
case "tripleclick":
|
||||
case "mousedown":
|
||||
case "mouseup": {
|
||||
const btn = action === "rightclick" ? "right" : action === "middleclick" ? "middle" : btnName;
|
||||
const act = action === "rightclick" || action === "middleclick" ? "click" : action;
|
||||
const r = agentInject.si.button(absX, absY, act, btn);
|
||||
if (act === "click" || act === "doubleclick" || act === "tripleclick") {
|
||||
agentCursor.agentCursorClick(dispIdx);
|
||||
}
|
||||
return fin(r.ok, "sendinput " + action + " @ (" + absX + "," + absY + ")");
|
||||
}
|
||||
case "drag": {
|
||||
const steps = Math.max(1, Math.min(60, parseInt(params.steps || 12, 10) || 12));
|
||||
const r = agentInject.si.drag(absX, absY, toX(), toY(), btnName, steps);
|
||||
agentCursor.agentCursorMove(toX(), toY(), dispIdx);
|
||||
agentCursor.agentCursorClick(dispIdx);
|
||||
return fin(r.ok, "sendinput drag (" + absX + "," + absY + ") -> (" + toX() + "," + toY() + ")");
|
||||
}
|
||||
case "scroll": {
|
||||
const dy = cuNumber(params.dy !== undefined ? params.dy : params.y, 0);
|
||||
const dx = cuNumber(params.dx !== undefined ? params.dx : params.x, 0);
|
||||
// GUI 侧约定：dy 正=向上。SendInput 的 WHEEL 正值即向上，直接传。
|
||||
const r = agentInject.si.scroll(dy, dx);
|
||||
return fin(r.ok, "sendinput scroll dx=" + dx + " dy=" + dy);
|
||||
}
|
||||
case "keypress":
|
||||
case "hotkey":
|
||||
case "combo": {
|
||||
const keys = String(params.keys || params.key || params.text || "");
|
||||
const r = agentInject.si.key(keys);
|
||||
return fin(r.ok, "sendinput keypress " + keys);
|
||||
}
|
||||
case "type": {
|
||||
const r = agentInject.si.type(String(params.text || ""));
|
||||
return fin(r.ok, "sendinput typed " + r.chars + " chars");
|
||||
}
|
||||
case "wait":
|
||||
case "sleep": {
|
||||
const ms = Math.max(0, Math.min(10000, parseInt(params.ms || params.duration || 500, 10) || 500));
|
||||
idle();
|
||||
setTimeout(() => sendCmdResult(reqId, baseResult(reqId, "ok", "sendinput waited " + ms + "ms", "")), ms);
|
||||
return;
|
||||
}
|
||||
case "display": {
|
||||
const info = displays.map((d, i) => ({
|
||||
index: i, bounds: d.bounds, scaleFactor: d.scaleFactor,
|
||||
primary: d.id === screen.getPrimaryDisplay().id,
|
||||
}));
|
||||
const v = agentInject.virtualScreen();
|
||||
idle();
|
||||
sendCmdResult(reqId, baseResult(reqId, "ok",
|
||||
"displays: " + JSON.stringify(info) +
|
||||
" | virtualScreen: " + JSON.stringify(v) +
|
||||
" | inputMode: sendinput | coordinateSpace: physical-pixel", ""));
|
||||
return;
|
||||
}
|
||||
default:
|
||||
return fail("unknown action " + action);
|
||||
}
|
||||
};
|
||||
startDefer((proceed) => { if (proceed) runAction(); });
|
||||
// ★ 必须在这里 return。
|
||||
//
|
||||
// runAction 里的 `return fin(...)` 只退出 runAction，不会退出
|
||||
// executeHomeagentCmd。少了这句，执行完 sendinput 后会继续往下
|
||||
// fall through 到旧的 real 鼠标路径 —— 同一条命令被彻底执行两遍
|
||||
// （实测：8 条命令回了 16 个 cmd_result，且真光标被额外抢一次）。
|
||||
return;
|
||||
} catch (e) {
|
||||
try { agentCursorBusySafe(dispIdx, false); } catch (e2) {}
|
||||
console.error("[sendinput] failed: " + e.message);
|
||||
sendCmdResult(reqId, baseResult(reqId, "error", "", "sendinput: " + e.message));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// === 模式 A：后台注入（PostMessage 到目标窗口），不动用户真实鼠标 ===
|
||||
if (os_ === "win32" && useOverlay) {
|
||||
const btnName = params.button === "right" ? "right" : params.button === "middle" ? "middle" : "left";
|
||||
|
||||
Reference in New Issue
Block a user