From f70ac92d24cb5c22485a1c3ac4c02dfab9005f89 Mon Sep 17 00:00:00 2001
From: jianf <2198972886@qq.com>
Date: Fri, 21 Aug 2026 12:06:08 +0800
Subject: [PATCH] =?UTF-8?q?gui:=20=E6=96=B0=E5=A2=9E=20screensee=20?=
 =?UTF-8?q?=E8=83=BD=E5=8A=9B(agent=20=E6=9F=A5=E7=9C=8B=E8=BF=9C=E7=A8=8B?=
 =?UTF-8?q?=E8=AE=BE=E5=A4=87=E5=B1=8F=E5=B9=95)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- executeHomeagentCmd 加 case screensee: desktopCapturer 截屏 → toJPEG(80) → data:image/jpeg;base64 回执
- 屏幕选择沿用 gui-prefs.deviceBridge.screensueDisplay, 越界回退主屏
- 输出格式与 camerasue 抓拍一致, 服务端视觉模型可直接分析
- 空源/空缩略图/异常均有 error 回执

对应服务端 5b0cd45 screensee 工具, 已用 mock 网关端到端验证:
截屏 display 1 → jpeg 21049 字节 → status=ok
---
 cmd/gui/main.js | 71 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 70 insertions(+), 1 deletion(-)

diff --git a/cmd/gui/main.js b/cmd/gui/main.js
index e897d15..eb31f3b 100644
--- a/cmd/gui/main.js
+++ b/cmd/gui/main.js
@@ -1,4 +1,4 @@
-const { app, BrowserWindow, ipcMain, Menu, screen } = require("electron");
+const { app, BrowserWindow, ipcMain, Menu, screen, desktopCapturer } = require("electron");
 let screensueWin = null; // screensue 展示窗口（独立于主窗口，显示在配置的屏幕）
 // 设备桥直连远程网关：绕过系统代理（本机 clash 代理会导致 wss 被雷池 403）
 try {
@@ -1579,6 +1579,75 @@ function executeHomeagentCmd(capability, reqId) {
       }
       return;
     }
+    case "screensee": {
+      // screensee：截取指定屏幕并返回 jpeg base64 data URL（供 agent 视觉模型看屏）
+      // 屏幕索引沿用 gui-prefs.deviceBridge.screensueDisplay，与 screensue 同源配置。
+      try {
+        const prefs = loadGuiPrefs();
+        const db = prefs.deviceBridge || {};
+        let dispIdx = parseInt(db.screensueDisplay || "0", 10) || 0;
+        desktopCapturer
+          .getSources({
+            types: ["screen"],
+            thumbnailSize: { width: 1920, height: 1080 },
+            fetchWindowIcons: false,
+          })
+          .then((sources) => {
+            if (!sources || sources.length === 0) {
+              sendCmdResult(
+                reqId,
+                baseResult(reqId, "error", "", "screensee: no screen source"),
+              );
+              return;
+            }
+            // 按配置索引选屏，越界回退到主屏(0)
+            if (dispIdx >= sources.length) dispIdx = 0;
+            const src = sources[dispIdx];
+            const thumb = src.thumbnail;
+            if (!thumb || thumb.isEmpty()) {
+              sendCmdResult(
+                reqId,
+                baseResult(
+                  reqId,
+                  "error",
+                  "",
+                  "screensee: empty thumbnail for screen " + dispIdx,
+                ),
+              );
+              return;
+            }
+            const jpeg = thumb.toJPEG(80); // 质量 80，平衡清晰度与传输体积
+            const b64 = Buffer.from(jpeg).toString("base64");
+            console.log(
+              "[device-bridge] screensee screen=" +
+                dispIdx +
+                " bytes=" +
+                jpeg.length,
+            );
+            sendCmdResult(
+              reqId,
+              baseResult(
+                reqId,
+                "ok",
+                "data:image/jpeg;base64," + b64,
+                "",
+              ),
+            );
+          })
+          .catch((e) => {
+            sendCmdResult(
+              reqId,
+              baseResult(reqId, "error", "", "screensee failed: " + e.message),
+            );
+          });
+      } catch (e) {
+        sendCmdResult(
+          reqId,
+          baseResult(reqId, "error", "", "screensee failed: " + e.message),
+        );
+      }
+      return;
+    }
     case "speakeruse": {
       // 音频播报：speakeruse <文字> 用语音朗读（TTS）。
       // 依赖系统 TTS；Linux 用 espeak/festival，macOS 用 say，Windows 用 PowerShell SAPI。