gui: 新增 screensee 能力(agent 查看远程设备屏幕)

- executeHomeagentCmd 加 case screensee: desktopCapturer 截屏 → toJPEG(80) → data:image/jpeg;base64 回执
- 屏幕选择沿用 gui-prefs.deviceBridge.screensueDisplay, 越界回退主屏
- 输出格式与 camerasue 抓拍一致, 服务端视觉模型可直接分析
- 空源/空缩略图/异常均有 error 回执

对应服务端 5b0cd45 screensee 工具, 已用 mock 网关端到端验证:
截屏 display 1 → jpeg 21049 字节 → status=ok
This commit is contained in:
2026-08-21 12:06:08 +08:00
parent 0393daa644
commit f70ac92d24

View File

@ -1,4 +1,4 @@
const { app, BrowserWindow, ipcMain, Menu, screen } = require("electron");
const { app, BrowserWindow, ipcMain, Menu, screen, desktopCapturer } = require("electron");
let screensueWin = null; // screensue 展示窗口(独立于主窗口,显示在配置的屏幕)
// 设备桥直连远程网关:绕过系统代理(本机 clash 代理会导致 wss 被雷池 403
try {
@ -1579,6 +1579,75 @@ function executeHomeagentCmd(capability, reqId) {
}
return;
}
case "screensee": {
// screensee截取指定屏幕并返回 jpeg base64 data URL供 agent 视觉模型看屏)
// 屏幕索引沿用 gui-prefs.deviceBridge.screensueDisplay与 screensue 同源配置。
try {
const prefs = loadGuiPrefs();
const db = prefs.deviceBridge || {};
let dispIdx = parseInt(db.screensueDisplay || "0", 10) || 0;
desktopCapturer
.getSources({
types: ["screen"],
thumbnailSize: { width: 1920, height: 1080 },
fetchWindowIcons: false,
})
.then((sources) => {
if (!sources || sources.length === 0) {
sendCmdResult(
reqId,
baseResult(reqId, "error", "", "screensee: no screen source"),
);
return;
}
// 按配置索引选屏,越界回退到主屏(0)
if (dispIdx >= sources.length) dispIdx = 0;
const src = sources[dispIdx];
const thumb = src.thumbnail;
if (!thumb || thumb.isEmpty()) {
sendCmdResult(
reqId,
baseResult(
reqId,
"error",
"",
"screensee: empty thumbnail for screen " + dispIdx,
),
);
return;
}
const jpeg = thumb.toJPEG(80); // 质量 80平衡清晰度与传输体积
const b64 = Buffer.from(jpeg).toString("base64");
console.log(
"[device-bridge] screensee screen=" +
dispIdx +
" bytes=" +
jpeg.length,
);
sendCmdResult(
reqId,
baseResult(
reqId,
"ok",
"data:image/jpeg;base64," + b64,
"",
),
);
})
.catch((e) => {
sendCmdResult(
reqId,
baseResult(reqId, "error", "", "screensee failed: " + e.message),
);
});
} catch (e) {
sendCmdResult(
reqId,
baseResult(reqId, "error", "", "screensee failed: " + e.message),
);
}
return;
}
case "speakeruse": {
// 音频播报speakeruse <文字> 用语音朗读TTS
// 依赖系统 TTSLinux 用 espeak/festivalmacOS 用 sayWindows 用 PowerShell SAPI。