mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-15 12:15:16 +00:00
feat(gui): 模式 A agent 独立光标 + 后台注入；修坐标换算；固化计划与交接文档
## 模式 A：agent 独立光标（不抢用户鼠标）
现有 computeruse 走 SetCursorPos + mouse_event，agent 一干活就抢走用户
真实鼠标，用户无法同时使用电脑。现改为：GUI 自绘 agent 专属光标，实际
输入经 PostMessage 直接投递到目标窗口句柄，用户鼠标全程不动。
新增：
cmd/gui/agent-cursor.js 透明置顶层窗口 + mascot 指针 + 移动/点击脉冲
动画；记录上次命中的 hwnd 供键盘动作使用
cmd/gui/agent-inject.js Windows 后台注入：WindowFromPoint 定位、
客户区坐标换算、PostMessage 投递鼠标/键盘/
滚轮/文本；拖拽分步移动
配置 gui-prefs.deviceBridge.agentCursor.mode = "overlay"(默认) | "real"。
明确的设计边界（不是实现缺陷，是平台限制）：
· 只对「接受窗口消息的程序」可靠。原生 Win32 程序可靠；Chromium/
Electron 自绘界面、游戏、DirectX 会忽略合成消息。
· 不做真实鼠标降级。用户既选了不抢鼠标，就不该偷偷改成操作真鼠标，
失败即 status=error 并说明原因。
· 键盘类动作依赖先前的鼠标操作（后台注入不动真光标，
GetCursorPos 拿到的是用户自己的光标位置）。
端到端 8/8 正确响应；「真实点击是否落到目标窗口」因当前环境无可注入的
可见窗口而未验证，已在 GUI-PLAN.md 列为待办。
## 修坐标换算（我自己上一轮引入的 bug）
上一轮我加了 rawX / scaleFactor 的换算，理由是「SetCursorPos 期望 DIP」。
**那个前提是错的** —— Electron 主进程是 per-monitor DPI-aware，Win32 坐标
不做虚拟化。实测：
Electron screen API (DIP) : 1260 x 840 scaleFactor=2
GetSystemMetrics : 2520 x 1680
SetCursorPos(200,200) -> GetCursorPos 读回 (200,200) 恒等，不缩放
加 /scale 会把原本正确的点击改坏：200% 屏上点 (600,400) 被送到
(400,267)，越靠右下偏得越远。已改回 physical ? rawX : rawX*scale，
drag 终点同步修正（原先两处换算不一致）。
易误判之处：在普通 node 进程里测同一函数读到的是 1260x840（虚拟化值），
会以为 Win32 用 DIP —— 必须在 Electron 内测。
## 判据
computeruse.test.mjs 43 条（npm test 共 127 条全绿），新增 5 条钉住
模式 A 的核心承诺：
· agent-inject / agent-cursor 均无 SetCursorPos 调用（查前先剥注释，
因为注释里会引用该名字解释「为什么不用它」）
· koffi 指针参数用 Buffer（普通对象不写回，会拿错 hwnd）
· overlay 默认开启且可切回 real
· 自绘光标层 setIgnoreMouseEvents 穿透
核心不变量已做变异验证：注入 SetCursorPos 后判据立即转红。
另修正坐标判据方向（原判据断言 /scale 是对的，与实测相反）。
## 文档
GUI-PLAN.md 改造计划与进度的单一事实来源：已完成/待办、坐标
实测结论与踩坑、koffi 3.x 使用约定、端到端测试方法、
环境注意事项（含本机无 Go 工具链、waiter 改动未编译验证）
SERVER-HANDOFF.md 服务端需同步的部分：放开 action 白名单（schema enum
+ switch default 双重限制，不改则 11 个新动作下发不到）、
透传 tox/toy/steps/dx/ms、统一 scroll 方向语义、
写入坐标约定防重蹈
This commit is contained in:
JianFeeeee
335
cmd/gui/agent-inject.js
Normal file
335
cmd/gui/agent-inject.js
Normal file
@ -0,0 +1,335 @@
|
||||
// Windows 后台输入注入：不经过系统光标，直接把消息投递到目标窗口。
|
||||
//
|
||||
// 为什么不调 SetCursorPos：
|
||||
// 那样会抢走用户的真实光标，用户在 agent 操作期间无法正常使用电脑。
|
||||
// 本模块用 PostMessage 把输入直接送到目标窗口的 hwnd，系统光标纹丝不动。
|
||||
//
|
||||
// 已实测确认（本轮 probe-inject/electron-probe.js）：
|
||||
// PostMessage(EDIT, WM_CHAR, 'H') 后读回控件文本 = "Ha"
|
||||
// 且 GetCursorPos 前后一致 ⇒ 用户鼠标确实未被移动。
|
||||
//
|
||||
// ★ koffi 3.x 的硬约定（踩过坑，务必遵守）：
|
||||
// 指针参数必须传 **Buffer**。传普通 JS 对象不会写回 ——
|
||||
// GetCursorPos({x,y}) 读回仍是原值（无效）
|
||||
// GetCursorPos(Buffer) 读到真实坐标（有效）
|
||||
// 写错会导致拿错目标窗口 hwnd ⇒ 点击落到别的窗口上。
|
||||
|
||||
let koffi = null;
|
||||
let u32 = null;
|
||||
let cached = null;
|
||||
|
||||
function lib() {
|
||||
if (cached) return cached;
|
||||
koffi = require("koffi");
|
||||
u32 = koffi.load("user32.dll");
|
||||
// ★ koffi 的类型名是**全局注册**的：同一个进程里重复 koffi.struct("POINT", ...)
|
||||
// 会抛 "Duplicate type name 'POINT'"。main.js / 探针脚本可能已声明过同名结构体，
|
||||
// 所以这里先试着复用，失败就换个名字注册。
|
||||
let POINT;
|
||||
try {
|
||||
POINT = koffi.struct("POINT", { x: "long", y: "long" });
|
||||
} catch (e) {
|
||||
POINT = koffi.struct("HA_POINT", { x: "long", y: "long" });
|
||||
}
|
||||
|
||||
const f = (sig) => u32.func(sig);
|
||||
cached = {
|
||||
POINT,
|
||||
// 指针参数一律用 void *，调用时传 Buffer
|
||||
WindowFromPoint: f("void *WindowFromPoint(void *pt)"),
|
||||
ChildWindowFromPointEx: f(
|
||||
"void *ChildWindowFromPointEx(void *hwndParent, void *pt, uint flags)",
|
||||
),
|
||||
GetAncestor: f("void *GetAncestor(void *hwnd, uint flags)"),
|
||||
IsWindow: f("int IsWindow(void *hwnd)"),
|
||||
IsWindowVisible: f("int IsWindowVisible(void *hwnd)"),
|
||||
GetWindowThreadProcessId: f("uint GetWindowThreadProcessId(void *hwnd, uint *pid)"),
|
||||
GetClientRect: f("int GetClientRect(void *hwnd, void *rect)"),
|
||||
ScreenToClient: f("int ScreenToClient(void *hwnd, void *pt)"),
|
||||
GetCursorPos: f("bool GetCursorPos(void *pt)"),
|
||||
PostMessageW: f("intptr_t PostMessageW(void *hwnd, uint msg, uintptr_t w, intptr_t l)"),
|
||||
SendMessageW: f("intptr_t SendMessageW(void *hwnd, uint msg, uintptr_t w, intptr_t l)"),
|
||||
// 这些用于"这类窗口吃不吃合成消息"的能力探测
|
||||
GetWindowLongPtrW: f("intptr_t GetWindowLongPtrW(void *hwnd, int i)"),
|
||||
};
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Win32 常量
|
||||
const GA_ROOT = 2;
|
||||
const CWP_SKIPINVISIBLE = 0x0002;
|
||||
const CWP_SKIPTRANSPARENT = 0x0004;
|
||||
const GWL_STYLE = -16;
|
||||
const GWL_EXSTYLE = -20;
|
||||
const WS_EX_LAYERED = 0x00080000;
|
||||
const WS_EX_TRANSPARENT = 0x00000020;
|
||||
const WS_EX_NOACTIVATE = 0x08000000;
|
||||
|
||||
const WM_MOUSEMOVE = 0x0200;
|
||||
const WM_LBUTTONDOWN = 0x0201;
|
||||
const WM_LBUTTONUP = 0x0202;
|
||||
const WM_LBUTTONDBLCLK = 0x0203;
|
||||
const WM_RBUTTONDOWN = 0x0204;
|
||||
const WM_RBUTTONUP = 0x0205;
|
||||
const WM_MBUTTONDOWN = 0x0207;
|
||||
const WM_MBUTTONUP = 0x0208;
|
||||
const WM_MOUSEWHEEL = 0x020A;
|
||||
const WM_MOUSEHWHEEL = 0x020E;
|
||||
const WM_KEYDOWN = 0x0100;
|
||||
const WM_KEYUP = 0x0101;
|
||||
const WM_CHAR = 0x0102;
|
||||
const WM_SYSKEYDOWN = 0x0104;
|
||||
const WM_SYSKEYUP = 0x0105;
|
||||
const MK_LBUTTON = 0x0001;
|
||||
const MK_RBUTTON = 0x0002;
|
||||
const MK_MBUTTON = 0x0010;
|
||||
const MK_CONTROL = 0x0008;
|
||||
const MK_SHIFT = 0x0004;
|
||||
|
||||
const SM_XVIRTUALSCREEN = 76;
|
||||
const SM_YVIRTUALSCREEN = 77;
|
||||
const SM_CXVIRTUALSCREEN = 78;
|
||||
const SM_CYVIRTUALSCREEN = 79;
|
||||
|
||||
// 虚拟键码（与 main.js 的 tapVk 共用同一张表）
|
||||
const VK = {
|
||||
ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, super: 0x5b,
|
||||
enter: 0x0d, return: 0x0d, tab: 0x09, esc: 0x1b, escape: 0x1b,
|
||||
space: 0x20, backspace: 0x08, delete: 0x2e, del: 0x2e,
|
||||
up: 0x26, down: 0x28, left: 0x25, right: 0x27,
|
||||
home: 0x24, end: 0x23, pageup: 0x21, pagedown: 0x22,
|
||||
};
|
||||
const VK_EXTENDED = new Set([0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x2e, 0x5b, 0x5c, 0x6f, 0x74]);
|
||||
const isExtVK = (vk) => VK_EXTENDED.has(vk);
|
||||
|
||||
function getSystemMetrics() {
|
||||
const m = u32.func("int GetSystemMetrics(int i)");
|
||||
return {
|
||||
x: m(SM_XVIRTUALSCREEN),
|
||||
y: m(SM_YVIRTUALSCREEN),
|
||||
w: m(SM_CXVIRTUALSCREEN),
|
||||
h: m(SM_CYVIRTUALSCREEN),
|
||||
};
|
||||
}
|
||||
|
||||
function hwndStr(h) {
|
||||
return h ? "0x" + h.toString(16) : "(null)";
|
||||
}
|
||||
|
||||
// 屏幕物理坐标 → 目标窗口 hwnd（含子窗口命中）
|
||||
// ★ pt 必须是 Buffer
|
||||
function windowAtPoint(sx, sy) {
|
||||
const k = lib();
|
||||
const pt = Buffer.alloc(8);
|
||||
pt.writeInt32LE(Math.round(sx), 0);
|
||||
pt.writeInt32LE(Math.round(sy), 4);
|
||||
let h = k.WindowFromPoint(pt);
|
||||
if (!h) return null;
|
||||
// WindowFromPoint 只返回顶层窗口；对 UI 框架要下钻到真实接收消息的子窗口
|
||||
const flags = CWP_SKIPINVISIBLE | CWP_SKIPTRANSPARENT;
|
||||
let guard = 0;
|
||||
for (;;) {
|
||||
const child = k.ChildWindowFromPointEx(h, pt, flags);
|
||||
if (!child || child === h || guard++ > 8) break;
|
||||
h = child;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
// 屏幕坐标 → 目标窗口客户区坐标（lParam 用的就是客户区坐标）
|
||||
function toClient(hwnd, sx, sy) {
|
||||
const k = lib();
|
||||
const pt = Buffer.alloc(8);
|
||||
pt.writeInt32LE(Math.round(sx), 0);
|
||||
pt.writeInt32LE(Math.round(sy), 4);
|
||||
k.ScreenToClient(hwnd, pt);
|
||||
return { x: pt.readInt32LE(0), y: pt.readInt32LE(4) };
|
||||
}
|
||||
|
||||
// 窗口是否"看起来"能接收合成消息（启发式判断，不保证）
|
||||
function acceptsSynthetic(hwnd) {
|
||||
const k = lib();
|
||||
if (!hwnd || !k.IsWindow(hwnd)) return { ok: false, why: "invalid hwnd" };
|
||||
if (!k.IsWindowVisible(hwnd)) return { ok: false, why: "window hidden" };
|
||||
const ex = k.GetWindowLongPtrW(hwnd, GWL_EXSTYLE);
|
||||
// WS_EX_TRANSPARENT 的窗口通常不接收命中测试类消息
|
||||
if (ex & WS_EX_TRANSPARENT) return { ok: false, why: "WS_EX_TRANSPARENT" };
|
||||
return { ok: true, ex };
|
||||
}
|
||||
|
||||
function post(hwnd, msg, w, l) {
|
||||
const k = lib();
|
||||
const r = k.PostMessageW(hwnd, msg, w, l);
|
||||
return r !== 0;
|
||||
}
|
||||
|
||||
function pack(x, y) {
|
||||
// lParam: 低 16 位 x，高 16 位 y（signed 16-bit）
|
||||
return ((y & 0xffff) << 16) | (x & 0xffff);
|
||||
}
|
||||
|
||||
// ── 鼠标操作 ───────────────────────────────────────────────────────
|
||||
const BTN_MSG = {
|
||||
left: { down: WM_LBUTTONDOWN, up: WM_LBUTTONUP, dbl: WM_LBUTTONDBLCLK, mk: MK_LBUTTON },
|
||||
right: { down: WM_RBUTTONDOWN, up: WM_RBUTTONUP, mk: MK_RBUTTON },
|
||||
middle: { down: WM_MBUTTONDOWN, up: WM_MBUTTONUP, mk: MK_MBUTTON },
|
||||
};
|
||||
|
||||
function mouseOp(sx, sy, action, opts) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空（该坐标无窗口）" };
|
||||
const c = toClient(hwnd, sx, sy);
|
||||
const lp = pack(c.x, c.y);
|
||||
const btn = BTN_MSG[(opts && opts.button) || "left"] || BTN_MSG.left;
|
||||
const post_ = (m, w) => post(hwnd, m, w, lp);
|
||||
|
||||
switch (action) {
|
||||
case "move":
|
||||
return { ok: post_(WM_MOUSEMOVE, 0), hwnd, client: c };
|
||||
case "click": {
|
||||
post_(btn.down, btn.mk);
|
||||
const up = post_(btn.up, 0);
|
||||
return { ok: up, hwnd, client: c };
|
||||
}
|
||||
case "doubleclick": {
|
||||
post_(btn.down, btn.mk);
|
||||
post_(btn.up, 0);
|
||||
post_(btn.dbl, MK_LBUTTON);
|
||||
const up = post_(btn.up, 0);
|
||||
return { ok: up, hwnd, client: c };
|
||||
}
|
||||
case "mousedown":
|
||||
return { ok: post_(btn.down, btn.mk), hwnd, client: c };
|
||||
case "mouseup":
|
||||
return { ok: post_(btn.up, 0), hwnd, client: c };
|
||||
default:
|
||||
return { ok: false, error: "unsupported mouse action " + action };
|
||||
}
|
||||
}
|
||||
|
||||
function tripleClick(sx, sy) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空" };
|
||||
const c = toClient(hwnd, sx, sy);
|
||||
const lp = pack(c.x, c.y);
|
||||
post(hwnd, WM_LBUTTONDOWN, MK_LBUTTON, lp);
|
||||
post(hwnd, WM_LBUTTONUP, 0, lp);
|
||||
post(hwnd, WM_LBUTTONDBLCLK, MK_LBUTTON, lp);
|
||||
post(hwnd, WM_LBUTTONUP, 0, lp);
|
||||
return { ok: true, hwnd, client: c };
|
||||
}
|
||||
|
||||
function drag(sx, sy, tx, ty, opts) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空" };
|
||||
const btn = BTN_MSG[(opts && opts.button) || "left"] || BTN_MSG.left;
|
||||
const steps = Math.max(1, Math.min(60, (opts && opts.steps) || 12));
|
||||
const c0 = toClient(hwnd, sx, sy);
|
||||
post(hwnd, btn.down, btn.mk, pack(c0.x, c0.y));
|
||||
// 分步移动：部分应用（canvas 拖拽、排序、滑块）依赖中间 move 事件
|
||||
for (let i = 1; i <= steps; i++) {
|
||||
const t = i / steps;
|
||||
const c = toClient(hwnd, sx + (tx - sx) * t, sy + (ty - sy) * t);
|
||||
post(hwnd, WM_MOUSEMOVE, btn.mk, pack(c.x, c.y));
|
||||
}
|
||||
const cEnd = toClient(hwnd, tx, ty);
|
||||
const up = post(hwnd, btn.up, 0, pack(cEnd.x, cEnd.y));
|
||||
return { ok: up, hwnd, steps };
|
||||
}
|
||||
|
||||
// 滚轮：lParam 用**屏幕**坐标，不是客户区坐标
|
||||
function scroll(sx, sy, dy, dx) {
|
||||
const hwnd = windowAtPoint(sx, sy);
|
||||
if (!hwnd) return { ok: false, error: "WindowFromPoint 返回空" };
|
||||
const lp = pack(Math.round(sx), Math.round(sy));
|
||||
let ok = true;
|
||||
if (dy) ok = post(hwnd, WM_MOUSEWHEEL, deltaWord(Math.round(dy)), lp) && ok;
|
||||
if (dx) ok = post(hwnd, WM_MOUSEHWHEEL, deltaWord(Math.round(dx)), lp) && ok;
|
||||
return { ok, hwnd };
|
||||
}
|
||||
|
||||
// 把像素量换算成滚轮"格"（WHEEL_DELTA=120）
|
||||
function deltaWord(v) {
|
||||
const clicks = Math.round(v / 120) || (v > 0 ? 1 : -1);
|
||||
// high word 存 delta，low word 存按键状态
|
||||
return (clicks << 16) >>> 0;
|
||||
}
|
||||
|
||||
// ── 键盘操作 ───────────────────────────────────────────────────────
|
||||
function keyTap(hwnd, spec) {
|
||||
const parts = String(spec || "")
|
||||
.split("+")
|
||||
.map((x) => x.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
if (!parts.length) return { ok: false, error: "empty key" };
|
||||
const last = parts[parts.length - 1];
|
||||
const mods = parts.slice(0, -1);
|
||||
const modCodes = [];
|
||||
for (const m of mods) {
|
||||
const c = VK[m];
|
||||
if (c === undefined) return { ok: false, error: "unknown modifier: " + m };
|
||||
modCodes.push(c);
|
||||
}
|
||||
let keyCode = VK[last];
|
||||
if (keyCode === undefined) {
|
||||
if (last.length === 1) keyCode = last.toUpperCase().charCodeAt(0);
|
||||
else return { ok: false, error: "unknown key: " + last };
|
||||
}
|
||||
let ok = true;
|
||||
for (const c of modCodes) {
|
||||
ok = post(hwnd, WM_KEYDOWN, c, 0) && ok;
|
||||
}
|
||||
ok = post(hwnd, WM_KEYDOWN, keyCode, isExtVK(keyCode) ? 1 : 0) && ok;
|
||||
ok = post(hwnd, WM_KEYUP, keyCode, isExtVK(keyCode) ? 1 : 0) && ok;
|
||||
for (const c of modCodes.slice().reverse()) {
|
||||
ok = post(hwnd, WM_KEYUP, c, 0) && ok;
|
||||
}
|
||||
return { ok, mods: modCodes, key: keyCode };
|
||||
}
|
||||
|
||||
// 文本注入：走 WM_CHAR，不做输入法合成，因此对原生控件最可靠。
|
||||
// 换行/回车单独发 VK_RETURN，否则 WM_CHAR 的 \n 多数控件不认。
|
||||
function typeText(hwnd, text) {
|
||||
const t = String(text || "");
|
||||
if (!t) return { ok: false, error: "empty text" };
|
||||
let ok = true;
|
||||
for (const ch of t) {
|
||||
if (ch === "\n" || ch === "\r") {
|
||||
ok = post(hwnd, WM_KEYDOWN, VK.enter, 0) && ok;
|
||||
ok = post(hwnd, WM_KEYUP, VK.enter, 0) && ok;
|
||||
continue;
|
||||
}
|
||||
if (ch === "\t") {
|
||||
ok = post(hwnd, WM_KEYDOWN, VK.tab, 0) && ok;
|
||||
ok = post(hwnd, WM_KEYUP, VK.tab, 0) && ok;
|
||||
continue;
|
||||
}
|
||||
ok = post(hwnd, WM_CHAR, ch.codePointAt(0), 0) && ok;
|
||||
}
|
||||
return { ok, chars: t.length };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
available() {
|
||||
try {
|
||||
lib();
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
windowAtPoint,
|
||||
toClient,
|
||||
acceptsSynthetic,
|
||||
getSystemMetrics,
|
||||
hwndStr,
|
||||
// 鼠标
|
||||
mouse: mouseOp,
|
||||
tripleClick,
|
||||
drag,
|
||||
scroll,
|
||||
// 键盘
|
||||
keyTap,
|
||||
typeText,
|
||||
VK,
|
||||
};
|
||||
Reference in New Issue
Block a user