feat: screensee 工具 — agent 查看远程设备屏幕内容

与 screensue(向用户屏幕显示)配对: screensue 是给用户看, screensee 是 agent 看。

服务端实现:
1. remotedevice 新增 screensee 工具:
   - 下发 homeagent-screensee 命令 → 设备截屏回传 jpeg base64
   - seeHandler 回调(agent 核心注入)用视觉模型自动描述屏幕内容
   - 未授权/离线/超时完整错误路径; 结果留档 cmdresult
2. SDK LLMMessage 扩展多模态 Blocks(text/image_url):
   - llm_impl 转换为 agentAPI.ContentBlock, 视觉模型可看图
3. describeScreen: 默认提示词描述窗口/文字/界面状态;
   provider 参数可指定视觉源(临时切换后恢复)

GUI 端需配套(已发群): onDeviceMsg 加 case "screensee",
desktopCapturer 截屏 → jpeg base64 data URL 回执(同 camerasue 抓拍模式)。

测试: 端到端模拟设备截屏回传+视觉回调验证; 全项目 go test 通过
This commit is contained in:
JianFeeeee
2026-08-21 11:31:45 +08:00
parent f2e3215c77
commit 5b0cd45093
5 changed files with 206 additions and 1 deletions

View File

@ -295,3 +295,76 @@ func TestPushDataOfflineDevice(t *testing.T) {
t.Fatalf("expected not online error, got %v", err)
}
}
// ===== screensee:截屏回传 + 视觉描述回调 =====
func TestScreenseeEndToEnd(t *testing.T) {
reg := NewRegistry()
token := "test-token-see"
reg.SetAcceptToken(func(provided string) bool { return provided == token })
dev := &devicectlDevice{reg: reg}
var gotDataURL string
dev.SetSeeHandler(func(dataURL string, provider string) string {
gotDataURL = dataURL
return "屏幕上显示的是测试画面"
})
srv := httptest.NewServer(http.HandlerFunc(reg.ServeWS))
defer srv.Close()
cli := dialTestWS(t, srv.URL, token)
defer cli.close()
// 设备 hello + bind(bind 需 token 才能被授权流程识别,这里直接手动授权)
cli.sendText([]byte(`{"op":"hello","device":{"device_id":"see-dev","name":"屏幕机","kind":"computer","caps":["cmd"]}}`))
if _, _, err := cli.readMsg(); err != nil {
t.Fatalf("read hello_ack: %v", err)
}
reg.SetAuthorized("see-dev", true)
// 设备侧循环收命令并回执(模拟 GUI screensee 实现)
go func() {
for {
op, payload, err := cli.readMsg()
if err != nil {
return
}
if op != 0x1 {
continue
}
var msg map[string]interface{}
if json.Unmarshal(payload, &msg) != nil {
continue
}
if msg["op"] == "cmd" && msg["command"] == "screensee" {
reqID, _ := msg["req_id"].(string)
fakeJPEG := []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10} // JPEG magic
b64 := base64.StdEncoding.EncodeToString(fakeJPEG)
cli.sendText(mustJSON(map[string]interface{}{
"op": "cmd_result", "req_id": reqID, "device_id": "see-dev",
"status": "ok", "output": "data:image/jpeg;base64," + b64,
}))
}
}
}()
// agent 调用 screensee
res, err := dev.Execute("screensee", map[string]interface{}{"device_id": "see-dev"})
if err != nil {
t.Fatalf("screensee: %v", err)
}
m := res.(map[string]interface{})
if m["description"] != "屏幕上显示的是测试画面" {
t.Fatalf("unexpected description: %v", m["description"])
}
if !strings.HasPrefix(gotDataURL, "data:image/jpeg;base64,") {
t.Fatalf("handler received bad dataURL: %s", gotDataURL)
}
// 未授权设备应拒绝
reg.SetAuthorized("see-dev", false)
if _, err := dev.Execute("screensee", map[string]interface{}{"device_id": "see-dev"}); err == nil {
t.Fatal("expected unauthorized error")
}
}