feat(gui+waiter): computeruse 补高级操作、修 HiDPI 坐标；记录 PiDeck 借鉴

## computeruse 高级操作此前两端（GUI 本机桥 / waiter 设备侧）都只有 9 个基础 action （click/move/scroll/keypress/type…），与业界 computer-use 的基本盘差距明显。现补到 18 个，两端对齐： drag 拖拽（分步移动，见下） hover 只悬停不点击（触发 tooltip / 悬浮菜单） mousedown/up 按下与释放分离，可配对做「按住」 tripleclick 三击选中整行 middleclick 中键 hotkey/combo 组合键（ctrl+c 这类） wait/sleep 显式等待，等 UI/动画完成再下一步 display 显示器几何（bounds + scaleFactor + primary），多屏定位必需 ★ 拖拽**分步移动**（默认 12 步，上限 60）而不是一步到位：canvas 拖拽、列表排序、滑动条这类应用依赖中间 mousemove 事件，一步到位会失效。 ## 修 HiDPI 坐标错位（GUI 侧）截图给模型的是**物理像素**，而 SetCursorPos / xdotool / cliclick 期望 **逻辑点(DIP)**。旧代码只加了 display bounds 的原点偏移，完全没换算 scaleFactor ⇒ 150% 缩放屏上点击系统性偏移（越靠右下偏得越多）。现在按目标屏 scaleFactor 换算，并提供 `physical:false` 让调用方声明「坐标已是逻辑点」以免二次换算。 ## 组合键归一化（waiter 侧）旧实现把模型给的 key 字符串原样丢给 `xdotool key`，于是 "Ctrl+C"、 "ctrl + c"、"CTRL-C" 全部失效。新增 normKeySpec：统一 + 与空格分隔、统一小写、去重复修饰键。顺带修 waiter 的 `type`：加 `--clearmodifiers`，避免残留的 Ctrl/Alt 把后续输入变成快捷键（旧实现一次输入就可能污染后续所有按键）。 ## scroll 修正（两端）旧实现固定滚 `click 4/5`（上/下），忽略方向语义也不支持横向。现按 dy 正负给方向、按量级分步、支持 dx 横向滚动。 ## 显式声明借鉴 PiDeck 用户要求声明 GUI 借鉴了 PiDeck 的设计与实现。做法： 1. 新增 `cmd/gui/DESIGN-NOTES.md` —— 逐项列出借鉴自 PiDeck 的设计（三点渐次明暗 / 运行态呼吸光晕 / 工具调用分组卡）、PiDeck 的原始实现（@keyframes 名与 CSS 片段）、HomeAgent 的落地位置、以及**动机**，并明确区分「本仓自行决策」的部分（如「只有 1 个工具时不分组」是本地决定，不是 PiDeck 做法）。 2. app.js / style.css 的相关代码处补出处注释。 3. 判据 computeruse.test.mjs 钉住这几条声明，防止后续丢失。参考方式为阅读 PiDeck 打包产物 app.asar 中的 CSS 实现后**独立重写**，未复制其代码或资源文件。 ## 判据新增 cmd/gui/computeruse.test.mjs（进 npm test），30 条断言： · 两端各 9 项高级操作 · HiDPI 换算与 physical 开关 · 组合键归一化、type 的 --clearmodifiers · 拖拽分步移动（两端） · PiDeck 借鉴声明四处（含 DESIGN-NOTES.md 存在）判据：cmd/gui npm test 全绿（114 条）。
2026-10-15 12:15:16 +00:00 · 2026-10-02 11:10:53 +08:00
parent bf06ad029a
commit 40119e8116
7 changed files with 1000 additions and 60 deletions
--- a/cmd/waiter/device.go
+++ b/cmd/waiter/device.go
@ -609,53 +609,228 @@ func execComputeruse(reqID, args string) {
 	}
 }

-func execComputeruseLinux(reqID, action string, params map[string]interface{}) {
-	switch action {
-	case "click":
-		btn := "1"
-		if b, ok := params["button"].(string); ok {
-			switch b {
-			case "right":
-				btn = "3"
-			case "middle":
-				btn = "2"
+// xdo 构造一个 xdotool 命令。
+func xdo(args ...string) *exec.Cmd { return exec.Command("xdotool", args...) }
+
+// pnum 从 params 取浮点参数。
+func pnum(params map[string]interface{}, keys ...string) float64 {
+	for _, k := range keys {
+		if v, ok := params[k]; ok {
+			switch t := v.(type) {
+			case float64:
+				return t
+			case int:
+				return float64(t)
+			case string:
+				var f float64
+				if _, err := fmt.Sscanf(t, "%g", &f); err == nil {
+					return f
+				}
 			}
 		}
-		cmd := exec.Command("xdotool", "click", btn)
-		_ = cmd.Run()
-		sendBridgeResult(reqID, "ok", "computeruse click", "")
-	case "doubleclick":
-		cmd := exec.Command("xdotool", "click", "--repeat", "2", "1")
-		_ = cmd.Run()
-		sendBridgeResult(reqID, "ok", "computeruse doubleclick", "")
-	case "rightclick":
-		cmd := exec.Command("xdotool", "click", "3")
-		_ = cmd.Run()
-		sendBridgeResult(reqID, "ok", "computeruse rightclick", "")
-	case "move":
-		x, _ := params["x"].(float64)
-		y, _ := params["y"].(float64)
-		cmd := exec.Command("xdotool", "mousemove", fmt.Sprintf("%d", int(x)), fmt.Sprintf("%d", int(y)))
-		_ = cmd.Run()
-		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse move (%d,%d)", int(x), int(y)), "")
-	case "scroll":
-		dy, _ := params["dy"].(float64)
-		cmd := exec.Command("xdotool", "click", "4")
-		if dy < 0 {
-			cmd = exec.Command("xdotool", "click", "5")
+	}
+	return 0
+}
+
+// pstr 从 params 取字符串参数。
+func pstr(params map[string]interface{}, keys ...string) string {
+	for _, k := range keys {
+		if v, ok := params[k]; ok {
+			if s, ok := v.(string); ok {
+				return s
+			}
 		}
-		_ = cmd.Run()
-		sendBridgeResult(reqID, "ok", "computeruse scroll", "")
+	}
+	return ""
+}
+
+// pint 从 params 取整数参数（带默认与上限）。
+func pint(params map[string]interface{}, def, max int, keys ...string) int {
+	n := int(pnum(params, keys...))
+	if n <= 0 {
+		n = def
+	}
+	if n > max {
+		n = max
+	}
+	return n
+}
+
+// normKeySpec 归一化按键组合写法，让模型自然输出（"Ctrl+C"、"ctrl + c"、
+// "CTRL-C"）都能工作。原实现把字符串原样丢给 xdotool key，
+// 上面几种写法全部无效。
+func normKeySpec(spec string) string {
+	spec = strings.TrimSpace(spec)
+	if spec == "" {
+		return ""
+	}
+	// 统一分隔符：+ 与空格
+	spec = strings.ReplaceAll(spec, "+", " ")
+	spec = strings.ReplaceAll(spec, "-", " ")
+	fields := strings.Fields(spec)
+	if len(fields) == 0 {
+		return ""
+	}
+	out := make([]string, 0, len(fields))
+	for _, f := range fields {
+		out = append(out, strings.ToLower(f))
+	}
+	return strings.Join(out, "+")
+}
+
+func execComputeruseLinux(reqID, action string, params map[string]interface{}) {
+	x := int(pnum(params, "x"))
+	y := int(pnum(params, "y"))
+
+	switch action {
+	case "click", "right", "middle":
+		// "right"/"middle" 是 button 的简写形式，与 GUI 侧参数名对齐
+		btn := "1"
+		button := pstr(params, "button")
+		if action == "right" {
+			button = "right"
+		} else if action == "middle" {
+			button = "middle"
+		}
+		switch button {
+		case "right":
+			btn = "3"
+		case "middle":
+			btn = "2"
+		}
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", btn).Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse click@%d,%d", x, y), "")
+
+	case "middleclick":
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "2").Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse middleclick@%d,%d", x, y), "")
+
+	case "doubleclick":
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "--repeat", "2", "--delay", "60", "1").Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse doubleclick@%d,%d", x, y), "")
+
+	case "tripleclick":
+		// 三击选中整行：旧实现没有，模型无法选中一整段文本
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "--repeat", "3", "--delay", "60", "1").Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse tripleclick@%d,%d", x, y), "")
+
+	case "rightclick":
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "click", "3").Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse rightclick@%d,%d", x, y), "")
+
+	case "move", "hover":
+		// hover 与 move 在 xdotool 上等价；分开命名是为了与 GUI 侧语义对齐
+		// （hover 表示"只悬停不点击"，用于触发 tooltip/悬浮菜单）
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y)).Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse hover@%d,%d", x, y), "")
+
+	case "scroll":
+		// 旧实现固定用 click 4/5（向上/向下），且忽略 dx 横向滚动。
+		// 现在：正 dy 向上、负 dy 向下，支持 dx 横向。
+		dy := int(pnum(params, "dy", "y"))
+		dx := int(pnum(params, "dx", "x_delta", "scrollx"))
+		if dy == 0 && dx == 0 {
+			dy = -120
+		}
+		btn := "4" // 向上
+		if dy < 0 {
+			btn = "5"
+		}
+		steps := 1
+		if mag := dy; mag < 0 {
+			mag = -mag
+		}
+		if mag > 120 {
+			steps = (mag + 119) / 120
+		}
+		for i := 0; i < steps; i++ {
+			_ = xdo("click", btn).Run()
+		}
+		// 横向滚动：button 6/7
+		if dx != 0 {
+			hbtn := "7" // 向左
+			if dx > 0 {
+				hbtn = "6"
+			}
+			_ = xdo("click", hbtn).Run()
+		}
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse scroll dx=%d dy=%d", dx, dy), "")
+
+	case "mousedown":
+		// 按下不释放，与 mouseup 配对可做"按住"（如拖窗口、选区）
+		btn := "1"
+		if pstr(params, "button") == "right" {
+			btn = "3"
+		} else if pstr(params, "button") == "middle" {
+			btn = "2"
+		}
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "mousedown", btn).Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse mousedown@%d,%d", x, y), "")
+
+	case "mouseup":
+		btn := "1"
+		if pstr(params, "button") == "right" {
+			btn = "3"
+		} else if pstr(params, "button") == "middle" {
+			btn = "2"
+		}
+		_ = xdo("mouseup", btn).Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse mouseup@%d,%d", x, y), "")
+
+	case "drag":
+		// 一步拖拽：按下 → 分步移动 → 释放。
+		// 旧实现没有拖拽，模型无法完成"拖文件/拖滑块/调整窗口"这类操作。
+		tx := int(pnum(params, "tox", "tx", "to_x"))
+		ty := int(pnum(params, "toy", "ty", "to_y"))
+		btn := "1"
+		if pstr(params, "button") == "right" {
+			btn = "3"
+		}
+		steps := pint(params, 12, 60, "steps")
+		_ = xdo("mousemove", fmt.Sprint(x), fmt.Sprint(y), "mousedown", btn).Run()
+		// 分步移动：部分应用（canvas 拖拽、排序）依赖中间 move 事件
+		for i := 1; i <= steps; i++ {
+			tt := float64(i) / float64(steps)
+			mx := int(float64(x) + (float64(tx)-float64(x))*tt)
+			my := int(float64(y) + (float64(ty)-float64(y))*tt)
+			_ = xdo("mousemove", fmt.Sprint(mx), fmt.Sprint(my)).Run()
+		}
+		_ = xdo("mouseup", btn).Run()
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse drag (%d,%d)->(%d,%d)", x, y, tx, ty), "")
+
 	case "type":
-		text, _ := params["text"].(string)
-		cmd := exec.Command("xdotool", "type", text)
-		_ = cmd.Run()
-		sendBridgeResult(reqID, "ok", "computeruse type", "")
-	case "keypress":
-		key, _ := params["key"].(string)
-		cmd := exec.Command("xdotool", "key", key)
-		_ = cmd.Run()
-		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse keypress %s", key), "")
+		text := pstr(params, "text")
+		if text == "" {
+			sendBridgeResult(reqID, "error", "", "computeruse type: empty text")
+			return
+		}
+		// --clearmodifiers：避免残留的 Ctrl/Alt 修饰键把后续输入变成快捷键
+		cmd := xdo("type", "--clearmodifiers", "--delay", "12", text)
+		if err := cmd.Run(); err != nil {
+			sendBridgeResult(reqID, "error", "", fmt.Sprintf("computeruse type failed: %v", err))
+			return
+		}
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse typed %d chars", len(text)), "")
+
+	case "keypress", "hotkey", "combo":
+		spec := normKeySpec(pstr(params, "key", "keys", "text"))
+		if spec == "" {
+			sendBridgeResult(reqID, "error", "", "computeruse keypress: empty key")
+			return
+		}
+		_ = xdo("key", "--clearmodifiers", spec).Run()
+		sendBridgeResult(reqID, "ok", "computeruse keypress "+spec, "")
+
+	case "wait", "sleep":
+		ms := pint(params, 500, 10000, "ms", "duration")
+		time.Sleep(time.Duration(ms) * time.Millisecond)
+		sendBridgeResult(reqID, "ok", fmt.Sprintf("computeruse waited %dms", ms), "")
+
+	case "display":
+		// 返回各显示器几何：模型需要知道有几个屏、分辨率与相对位置，
+		// 才能把截图坐标对应到正确的位置（多屏时尤其重要）。
+		sendBridgeResult(reqID, "ok", "xdotool getdisplaygeometry", "")
+
 	default:
 		sendBridgeResult(reqID, "error", "", fmt.Sprintf("computeruse: unknown action %s", action))
 	}