Files
HomeAgent/internal/plugins/remotedevice/device.go
JianFeeeee 28019a6bdb feat: screensee 工具 — agent 查看远程设备屏幕内容
与 screensue(向用户屏幕显示)配对: screensue 是给用户看, screensee 是 agent 看。

服务端实现:
1. remotedevice 新增 screensee 工具:
   - 下发 homeagent-screensee 命令 → 设备截屏回传 jpeg base64
   - seeHandler 回调(agent 核心注入)用视觉模型自动描述屏幕内容
   - 未授权/离线/超时完整错误路径; 结果留档 cmdresult
2. SDK LLMMessage 扩展多模态 Blocks(text/image_url):
   - llm_impl 转换为 agentAPI.ContentBlock, 视觉模型可看图
3. describeScreen: 默认提示词描述窗口/文字/界面状态;
   provider 参数可指定视觉源(临时切换后恢复)

GUI 端需配套(已发群): onDeviceMsg 加 case "screensee",
desktopCapturer 截屏 → jpeg base64 data URL 回执(同 camerasue 抓拍模式)。

测试: 端到端模拟设备截屏回传+视觉回调验证; 全项目 go test 通过
2026-08-21 11:31:45 +08:00

359 lines
14 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package remotedevice
import (
"fmt"
"sort"
"strings"
"time"
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
)
// devicectlDevice 把设备网关暴露为 IOManager 的一个 Device:
// Tools() 提供 devicedetect / device_ctl_status / device_ctl_cmdrun / device_ctl_cmdresult / screensee,
// Execute() 检查授权并路由到 WS 在线设备。
type devicectlDevice struct {
reg *Registry
persist func() // 授权变更后持久化
// screensee 回调:设备截屏回传后由 agent 核心消费(视觉描述)。
// 由插件 Start 注入;nil 时退化为仅返回 base64 数据。
seeHandler func(dataURL string, provider string) string
}
func (d *devicectlDevice) Name() string { return "devicectl" }
func (d *devicectlDevice) Type() agentIO.DeviceType { return agentIO.DeviceIO }
func (d *devicectlDevice) OutputCapabilities() agentIO.OutputCapability { return agentIO.CapStructured }
func (d *devicectlDevice) Description() string {
return "远程设备控制网关:查看已接入/已授权的设备并发送控制指令(经用户授权的设备)"
}
func (d *devicectlDevice) ChannelDef() agentIO.ChannelDef { return agentIO.ChannelDef{} }
func (d *devicectlDevice) Start() error { return nil }
func (d *devicectlDevice) Stop() error { return nil }
func (d *devicectlDevice) Tools() []agentIO.ToolDef {
return []agentIO.ToolDef{
{
Name: "devicedetect",
Description: "扫描并列出已接入设备网关的设备(含在线/离线状态与授权状态)。" +
"用途:查看当前有哪些设备连接了 HomeAgent、是否在线、是否已授权。" +
"参数 kind 可选,只返回该种类的设备。返回值 device_id 用于后续 device_ctl_* 工具。",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"kind": map[string]interface{}{"type": "string", "description": "按设备种类过滤(light/camera/computer/phone/...),可省略"},
},
},
},
{
Name: "device_ctl_status",
Description: "查询一台已在线设备的实时状态。" +
"仅返回设备上报的状态信息(如电量/温度/运行状态)。" +
"需要 device_id(来自 devicedetect)。设备必须已授权且在线。",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"device_id": map[string]interface{}{"type": "string", "description": "目标设备 ID"},
},
"required": []interface{}{"device_id"},
},
},
{
Name: "device_ctl_cmdrun",
Description: "向设备下发命令/操作(异步,accepted=true 后用 device_ctl_cmdresult 轮询结果)。" +
"command 支持两类(前缀区分):\n" +
"- shell-cmd: 在设备上执行原生 shell 命令,如 shell-cmd ls -la /tmp\n" +
"- homeagent-cmd: 调用设备端 HomeAgent 内置能力:\n" +
" · homeagent-screensue <显示内容/HTML> — 用户侧屏幕弹窗显示自定义内容(带参显示,如 homeagent-screensue 会议提醒:三点开会)\n" +
" · homeagent-screensue <秒> <内容> — 显示指定时长后自动关闭;不带参数则常驻\n" +
" · homeagent-camerasue — 抓拍单张 jpeg(结果为 base64 data URL)\n" +
" · homeagent-camerasue <N秒> — 录像 N 秒 mp4(二进制分块回传,cmdresult 含 data_base64 字段)\n" +
" · homeagent-speakeruse <文字> — 设备端 TTS 语音朗读文字\n" +
"⚡ 高危:设备必须已授权,且该操作会改变设备行为。" +
"返回 accepted=true 表示已下发并等待设备执行,之后可用 device_ctl_cmdresult 查询结果。" +
"若设备未授权或离线,返回错误信息。",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"device_id": map[string]interface{}{"type": "string", "description": "目标设备 ID"},
"command": map[string]interface{}{"type": "string", "description": "以 shell-cmd 或 homeagent-cmd 前缀开头。如 shell-cmd pwd、homeagent-screensue 三点开会、homeagent-camerasue 5(录5秒)、homeagent-speakeruse 你好"},
},
"required": []interface{}{"device_id", "command"},
},
},
{
Name: "device_ctl_cmdresult",
Description: "查询之前 device_ctl_cmdrun 下发命令的执行结果(按 req_id 或 device_id 最近一次)。" +
"若非阻塞或已超时,用此工具取回设备执行输出。",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"req_id": map[string]interface{}{"type": "string", "description": "命令请求 ID(device_ctl_cmdrun 返回)"},
"device_id": map[string]interface{}{"type": "string", "description": "设备 ID(查询最近一次结果)"},
},
},
},
{
Name: "screensee",
Description: "查看一台已授权设备的屏幕当前画面(截屏回传)。" +
"与 screensue(向用户屏幕显示内容)配对:screensue 是给用户看,screensee 是你看。" +
"返回屏幕截图的自动视觉描述;如需读取屏上文字可接着用 ocr_image。" +
"需要 device_id(来自 devicedetect)。设备必须已授权且在线。",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"device_id": map[string]interface{}{"type": "string", "description": "目标设备 ID"},
"provider": map[string]interface{}{"type": "string", "description": "可选:用于视觉描述的 LLM 源名称,不填则使用默认模型"},
},
"required": []interface{}{"device_id"},
},
},
{
Name: "deviceinfo",
Description: "探查一台设备接入网关时声明的详细信息与支持能力。" +
"返回设备的 OS/架构/CPU/内存/能力 caps 等(设备接入时上报,非实时)。" +
"需要 device_id(来自 devicedetect)。设备必须已授权。",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"device_id": map[string]interface{}{"type": "string", "description": "目标设备 ID"},
},
"required": []interface{}{"device_id"},
},
},
}
}
func (d *devicectlDevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
switch tool {
case "devicedetect":
return d.detect(args)
case "device_ctl_status":
return d.status(args)
case "device_ctl_cmdrun":
return d.cmdrun(args)
case "device_ctl_cmdresult":
return d.cmdresult(args)
case "screensee":
return d.screensee(args)
case "deviceinfo":
return d.info(args)
default:
return nil, fmt.Errorf("unknown device tool %s", tool)
}
}
// publicDevices 把设备列表转为简洁 JSON(无内部字段)。
func publicDevices(in []DeviceMeta) []map[string]interface{} {
out := make([]map[string]interface{}, 0, len(in))
for _, m := range in {
entry := map[string]interface{}{
"device_id": m.DeviceID,
"name": m.Name,
"kind": m.Kind,
"caps": m.Caps,
"authorized": m.Authorized,
"online": m.Online,
"last_seen": m.LastSeen,
}
if len(m.Info) > 0 {
entry["info"] = m.Info
}
out = append(out, entry)
}
return out
}
// detect 实现 devicedetect。
func (d *devicectlDevice) detect(args map[string]interface{}) (interface{}, error) {
kind, _ := args["kind"].(string)
devs := d.reg.List()
if kind != "" {
var filtered []DeviceMeta
for _, m := range devs {
if strings.EqualFold(m.Kind, kind) {
filtered = append(filtered, m)
}
}
devs = filtered
}
sort.Slice(devs, func(i, j int) bool { return devs[i].DeviceID < devs[j].DeviceID })
if len(devs) == 0 {
return map[string]interface{}{"devices": []interface{}{}, "message": "暂无设备接入"}, nil
}
return map[string]interface{}{"devices": publicDevices(devs)}, nil
}
// status 实现 device_ctl_status(只读,查询设备最后一次上报状态)。
func (d *devicectlDevice) status(args map[string]interface{}) (interface{}, error) {
id, _ := args["device_id"].(string)
if id == "" {
return nil, fmt.Errorf("device_id required")
}
m, ok := d.reg.Get(id)
if !ok {
return nil, fmt.Errorf("device %s 不存在", id)
}
if !m.Authorized {
return nil, fmt.Errorf("device %s 未授权,无法查询状态(需先在设备管理页或经 bind 授权)", id)
}
if !m.Online {
return publicDevices([]DeviceMeta{m}), nil // 带 offline=true
}
return publicDevices([]DeviceMeta{m}), nil
}
// cmdrun 实现 device_ctl_cmdrun:检查授权+在线,push 命令,异步等待结果。
func (d *devicectlDevice) cmdrun(args map[string]interface{}) (interface{}, error) {
id, _ := args["device_id"].(string)
cmd, _ := args["command"].(string)
if id == "" || cmd == "" {
return nil, fmt.Errorf("device_id and command required")
}
m, ok := d.reg.Get(id)
if !ok {
return nil, fmt.Errorf("device %s 不存在", id)
}
if !m.Authorized {
return nil, fmt.Errorf("device %s 未授权,无法执行命令(请先在设备管理页授权)", id)
}
if !m.Online {
return nil, fmt.Errorf("device %s 不在线,无法执行命令", id)
}
// 命令类型:shell-cmd / homeagent-* 前缀区分;无前缀按 shell 处理(兼容旧格式)
cmdType := "shell"
switch {
case strings.HasPrefix(cmd, "shell-cmd"):
cmdType = "shell"
cmd = strings.TrimSpace(strings.TrimPrefix(cmd, "shell-cmd"))
case strings.HasPrefix(cmd, "homeagent-cmd"):
cmdType = "homeagent"
cmd = strings.TrimSpace(strings.TrimPrefix(cmd, "homeagent-cmd"))
case strings.HasPrefix(cmd, "homeagent-"):
cmdType = "homeagent"
cmd = strings.TrimSpace(strings.TrimPrefix(cmd, "homeagent-"))
default:
cmdType = "shell"
}
reqID := newReqID()
if err := d.reg.PushCmd(id, reqID, cmd, cmdType); err != nil {
return nil, fmt.Errorf("下发命令失败: %w", err)
}
// 阻塞等待设备结果(带超时);结果同时由 registry 留档。
res, err := d.reg.AwaitResult(reqID, 30*time.Second)
if err != nil {
d.reg.SaveResult(reqID, map[string]interface{}{"accepted": true, "error": err.Error(), "pending": true})
return map[string]interface{}{"accepted": true, "req_id": reqID, "pending": true, "note": "命令已下发但设备未在超时内回执,可用 device_ctl_cmdresult 再查"}, nil
}
out := res
out["req_id"] = reqID
d.reg.SaveResult(reqID, out)
return out, nil
}
// cmdresult 实现 device_ctl_cmdresult。
func (d *devicectlDevice) cmdresult(args map[string]interface{}) (interface{}, error) {
reqID, _ := args["req_id"].(string)
deviceID, _ := args["device_id"].(string)
if reqID != "" {
res, ok := d.reg.GetResult(reqID)
if !ok {
return nil, fmt.Errorf("no result for req %s(设备可能尚未回执)", reqID)
}
return res, nil
}
if deviceID != "" {
// 返回该设备最近一次结果(简化:遍历 results 找 device_id 匹配的最近一条)
// 说明:当前只按 req_id 查询;device_id 查询留给后续迭代。
m, ok := d.reg.Get(deviceID)
if !ok {
return nil, fmt.Errorf("device %s 不存在", deviceID)
}
if !m.Authorized {
return nil, fmt.Errorf("device %s 未授权", deviceID)
}
return map[string]interface{}{"device_id": deviceID, "note": "请用 device_ctl_cmdrun 返回的 req_id 查询命令结果"}, nil
}
return nil, fmt.Errorf("req_id 或 device_id 至少提供一个")
}
// info 实现 deviceinfo:返回设备详情 + 支持能力。
func (d *devicectlDevice) info(args map[string]interface{}) (interface{}, error) {
id, _ := args["device_id"].(string)
if id == "" {
return nil, fmt.Errorf("device_id required")
}
m, ok := d.reg.Get(id)
if !ok {
return nil, fmt.Errorf("device %s 不存在", id)
}
if !m.Authorized {
return nil, fmt.Errorf("device %s 未授权,无法探查信息(请先在设备管理页授权)", id)
}
out := map[string]interface{}{
"device_id": m.DeviceID,
"name": m.Name,
"kind": m.Kind,
"caps": m.Caps,
"authorized": m.Authorized,
"online": m.Online,
"last_seen": m.LastSeen,
}
if len(m.Info) > 0 {
out["info"] = m.Info
}
return out, nil
}
// SetSeeHandler 注入 screensee 的视觉描述回调(agent 核心提供)。
func (d *devicectlDevice) SetSeeHandler(fn func(dataURL string, provider string) string) {
d.seeHandler = fn
}
// screensee 实现 screensee:向设备下发 homeagent-screensee 截屏命令,
// 等待回传 jpeg base64,交给 seeHandler(agent 核心)做视觉描述。
func (d *devicectlDevice) screensee(args map[string]interface{}) (interface{}, error) {
id, _ := args["device_id"].(string)
provider, _ := args["provider"].(string)
if id == "" {
return nil, fmt.Errorf("device_id required")
}
m, ok := d.reg.Get(id)
if !ok {
return nil, fmt.Errorf("device %s 不存在", id)
}
if !m.Authorized {
return nil, fmt.Errorf("device %s 未授权,无法查看屏幕(请先在设备管理页授权)", id)
}
if !m.Online {
return nil, fmt.Errorf("device %s 不在线", id)
}
reqID := newReqID()
if err := d.reg.PushCmd(id, reqID, "screensee", "homeagent"); err != nil {
return nil, fmt.Errorf("下发截屏命令失败: %w", err)
}
res, err := d.reg.AwaitResult(reqID, 30*time.Second)
if err != nil {
d.reg.SaveResult(reqID, map[string]interface{}{"accepted": true, "error": err.Error(), "pending": true})
return nil, fmt.Errorf("设备未在超时内回传屏幕画面: %w", err)
}
if res["status"] != "ok" {
errMsg, _ := res["error"].(string)
if errMsg == "" {
errMsg = fmt.Sprintf("status=%v", res["status"])
}
return nil, fmt.Errorf("设备截屏失败: %s", errMsg)
}
output, _ := res["output"].(string)
// 设备端回传 data URL(data:image/jpeg;base64,...)或裸 base64
if !strings.HasPrefix(output, "data:") {
output = "data:image/jpeg;base64," + output
}
d.reg.SaveResult(reqID, res)
if d.seeHandler == nil {
return map[string]interface{}{"image_data_url": output, "note": "无视觉描述处理器,仅返回原始图像数据"}, nil
}
desc := d.seeHandler(output, provider)
return map[string]interface{}{"description": desc}, nil
}