mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-15 12:15:16 +00:00
gui: 新增 screensee 能力(agent 查看远程设备屏幕)
- executeHomeagentCmd 加 case screensee: desktopCapturer 截屏 → toJPEG(80) → data:image/jpeg;base64 回执
- 屏幕选择沿用 gui-prefs.deviceBridge.screensueDisplay, 越界回退主屏
- 输出格式与 camerasue 抓拍一致, 服务端视觉模型可直接分析
- 空源/空缩略图/异常均有 error 回执
对应服务端 28019a6 screensee 工具, 已用 mock 网关端到端验证:
截屏 display 1 → jpeg 21049 字节 → status=ok
This commit is contained in:
@ -1,4 +1,4 @@
|
|||||||
const { app, BrowserWindow, ipcMain, Menu, screen } = require("electron");
|
const { app, BrowserWindow, ipcMain, Menu, screen, desktopCapturer } = require("electron");
|
||||||
let screensueWin = null; // screensue 展示窗口(独立于主窗口,显示在配置的屏幕)
|
let screensueWin = null; // screensue 展示窗口(独立于主窗口,显示在配置的屏幕)
|
||||||
// 设备桥直连远程网关:绕过系统代理(本机 clash 代理会导致 wss 被雷池 403)
|
// 设备桥直连远程网关:绕过系统代理(本机 clash 代理会导致 wss 被雷池 403)
|
||||||
try {
|
try {
|
||||||
@ -1579,6 +1579,75 @@ function executeHomeagentCmd(capability, reqId) {
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
case "screensee": {
|
||||||
|
// screensee:截取指定屏幕并返回 jpeg base64 data URL(供 agent 视觉模型看屏)
|
||||||
|
// 屏幕索引沿用 gui-prefs.deviceBridge.screensueDisplay,与 screensue 同源配置。
|
||||||
|
try {
|
||||||
|
const prefs = loadGuiPrefs();
|
||||||
|
const db = prefs.deviceBridge || {};
|
||||||
|
let dispIdx = parseInt(db.screensueDisplay || "0", 10) || 0;
|
||||||
|
desktopCapturer
|
||||||
|
.getSources({
|
||||||
|
types: ["screen"],
|
||||||
|
thumbnailSize: { width: 1920, height: 1080 },
|
||||||
|
fetchWindowIcons: false,
|
||||||
|
})
|
||||||
|
.then((sources) => {
|
||||||
|
if (!sources || sources.length === 0) {
|
||||||
|
sendCmdResult(
|
||||||
|
reqId,
|
||||||
|
baseResult(reqId, "error", "", "screensee: no screen source"),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 按配置索引选屏,越界回退到主屏(0)
|
||||||
|
if (dispIdx >= sources.length) dispIdx = 0;
|
||||||
|
const src = sources[dispIdx];
|
||||||
|
const thumb = src.thumbnail;
|
||||||
|
if (!thumb || thumb.isEmpty()) {
|
||||||
|
sendCmdResult(
|
||||||
|
reqId,
|
||||||
|
baseResult(
|
||||||
|
reqId,
|
||||||
|
"error",
|
||||||
|
"",
|
||||||
|
"screensee: empty thumbnail for screen " + dispIdx,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const jpeg = thumb.toJPEG(80); // 质量 80,平衡清晰度与传输体积
|
||||||
|
const b64 = Buffer.from(jpeg).toString("base64");
|
||||||
|
console.log(
|
||||||
|
"[device-bridge] screensee screen=" +
|
||||||
|
dispIdx +
|
||||||
|
" bytes=" +
|
||||||
|
jpeg.length,
|
||||||
|
);
|
||||||
|
sendCmdResult(
|
||||||
|
reqId,
|
||||||
|
baseResult(
|
||||||
|
reqId,
|
||||||
|
"ok",
|
||||||
|
"data:image/jpeg;base64," + b64,
|
||||||
|
"",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
sendCmdResult(
|
||||||
|
reqId,
|
||||||
|
baseResult(reqId, "error", "", "screensee failed: " + e.message),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
sendCmdResult(
|
||||||
|
reqId,
|
||||||
|
baseResult(reqId, "error", "", "screensee failed: " + e.message),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
case "speakeruse": {
|
case "speakeruse": {
|
||||||
// 音频播报:speakeruse <文字> 用语音朗读(TTS)。
|
// 音频播报:speakeruse <文字> 用语音朗读(TTS)。
|
||||||
// 依赖系统 TTS;Linux 用 espeak/festival,macOS 用 say,Windows 用 PowerShell SAPI。
|
// 依赖系统 TTS;Linux 用 espeak/festival,macOS 用 say,Windows 用 PowerShell SAPI。
|
||||||
|
|||||||
Reference in New Issue
Block a user