diff --git a/internal/plugins/remotedevice/binary_test.go b/internal/plugins/remotedevice/binary_test.go index e70c927..db18ca2 100644 --- a/internal/plugins/remotedevice/binary_test.go +++ b/internal/plugins/remotedevice/binary_test.go @@ -295,3 +295,76 @@ func TestPushDataOfflineDevice(t *testing.T) { t.Fatalf("expected not online error, got %v", err) } } + +// ===== screensee:截屏回传 + 视觉描述回调 ===== + +func TestScreenseeEndToEnd(t *testing.T) { + reg := NewRegistry() + token := "test-token-see" + reg.SetAcceptToken(func(provided string) bool { return provided == token }) + + dev := &devicectlDevice{reg: reg} + var gotDataURL string + dev.SetSeeHandler(func(dataURL string, provider string) string { + gotDataURL = dataURL + return "屏幕上显示的是测试画面" + }) + + srv := httptest.NewServer(http.HandlerFunc(reg.ServeWS)) + defer srv.Close() + + cli := dialTestWS(t, srv.URL, token) + defer cli.close() + + // 设备 hello + bind(bind 需 token 才能被授权流程识别,这里直接手动授权) + cli.sendText([]byte(`{"op":"hello","device":{"device_id":"see-dev","name":"屏幕机","kind":"computer","caps":["cmd"]}}`)) + if _, _, err := cli.readMsg(); err != nil { + t.Fatalf("read hello_ack: %v", err) + } + reg.SetAuthorized("see-dev", true) + + // 设备侧循环收命令并回执(模拟 GUI screensee 实现) + go func() { + for { + op, payload, err := cli.readMsg() + if err != nil { + return + } + if op != 0x1 { + continue + } + var msg map[string]interface{} + if json.Unmarshal(payload, &msg) != nil { + continue + } + if msg["op"] == "cmd" && msg["command"] == "screensee" { + reqID, _ := msg["req_id"].(string) + fakeJPEG := []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10} // JPEG magic + b64 := base64.StdEncoding.EncodeToString(fakeJPEG) + cli.sendText(mustJSON(map[string]interface{}{ + "op": "cmd_result", "req_id": reqID, "device_id": "see-dev", + "status": "ok", "output": "data:image/jpeg;base64," + b64, + })) + } + } + }() + + // agent 调用 screensee + res, err := dev.Execute("screensee", map[string]interface{}{"device_id": "see-dev"}) + if err != nil { + t.Fatalf("screensee: %v", err) + } + m := res.(map[string]interface{}) + if m["description"] != "屏幕上显示的是测试画面" { + t.Fatalf("unexpected description: %v", m["description"]) + } + if !strings.HasPrefix(gotDataURL, "data:image/jpeg;base64,") { + t.Fatalf("handler received bad dataURL: %s", gotDataURL) + } + + // 未授权设备应拒绝 + reg.SetAuthorized("see-dev", false) + if _, err := dev.Execute("screensee", map[string]interface{}{"device_id": "see-dev"}); err == nil { + t.Fatal("expected unauthorized error") + } +} diff --git a/internal/plugins/remotedevice/device.go b/internal/plugins/remotedevice/device.go index 4547925..3a4f0a4 100644 --- a/internal/plugins/remotedevice/device.go +++ b/internal/plugins/remotedevice/device.go @@ -10,11 +10,15 @@ import ( ) // devicectlDevice 把设备网关暴露为 IOManager 的一个 Device: -// Tools() 提供 devicedetect / device_ctl_status / device_ctl_cmdrun / device_ctl_cmdresult, +// Tools() 提供 devicedetect / device_ctl_status / device_ctl_cmdrun / device_ctl_cmdresult / screensee, // Execute() 检查授权并路由到 WS 在线设备。 type devicectlDevice struct { reg *Registry persist func() // 授权变更后持久化 + + // screensee 回调:设备截屏回传后由 agent 核心消费(视觉描述)。 + // 由插件 Start 注入;nil 时退化为仅返回 base64 数据。 + seeHandler func(dataURL string, provider string) string } func (d *devicectlDevice) Name() string { return "devicectl" } @@ -89,6 +93,21 @@ func (d *devicectlDevice) Tools() []agentIO.ToolDef { }, }, }, + { + Name: "screensee", + Description: "查看一台已授权设备的屏幕当前画面(截屏回传)。" + + "与 screensue(向用户屏幕显示内容)配对:screensue 是给用户看,screensee 是你看。" + + "返回屏幕截图的自动视觉描述;如需读取屏上文字可接着用 ocr_image。" + + "需要 device_id(来自 devicedetect)。设备必须已授权且在线。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "device_id": map[string]interface{}{"type": "string", "description": "目标设备 ID"}, + "provider": map[string]interface{}{"type": "string", "description": "可选:用于视觉描述的 LLM 源名称,不填则使用默认模型"}, + }, + "required": []interface{}{"device_id"}, + }, + }, { Name: "deviceinfo", Description: "探查一台设备接入网关时声明的详细信息与支持能力。" + @@ -115,6 +134,8 @@ func (d *devicectlDevice) Execute(tool string, args map[string]interface{}) (int return d.cmdrun(args) case "device_ctl_cmdresult": return d.cmdresult(args) + case "screensee": + return d.screensee(args) case "deviceinfo": return d.info(args) default: @@ -283,3 +304,55 @@ func (d *devicectlDevice) info(args map[string]interface{}) (interface{}, error) } return out, nil } + +// SetSeeHandler 注入 screensee 的视觉描述回调(agent 核心提供)。 +func (d *devicectlDevice) SetSeeHandler(fn func(dataURL string, provider string) string) { + d.seeHandler = fn +} + +// screensee 实现 screensee:向设备下发 homeagent-screensee 截屏命令, +// 等待回传 jpeg base64,交给 seeHandler(agent 核心)做视觉描述。 +func (d *devicectlDevice) screensee(args map[string]interface{}) (interface{}, error) { + id, _ := args["device_id"].(string) + provider, _ := args["provider"].(string) + if id == "" { + return nil, fmt.Errorf("device_id required") + } + m, ok := d.reg.Get(id) + if !ok { + return nil, fmt.Errorf("device %s 不存在", id) + } + if !m.Authorized { + return nil, fmt.Errorf("device %s 未授权,无法查看屏幕(请先在设备管理页授权)", id) + } + if !m.Online { + return nil, fmt.Errorf("device %s 不在线", id) + } + reqID := newReqID() + if err := d.reg.PushCmd(id, reqID, "screensee", "homeagent"); err != nil { + return nil, fmt.Errorf("下发截屏命令失败: %w", err) + } + res, err := d.reg.AwaitResult(reqID, 30*time.Second) + if err != nil { + d.reg.SaveResult(reqID, map[string]interface{}{"accepted": true, "error": err.Error(), "pending": true}) + return nil, fmt.Errorf("设备未在超时内回传屏幕画面: %w", err) + } + if res["status"] != "ok" { + errMsg, _ := res["error"].(string) + if errMsg == "" { + errMsg = fmt.Sprintf("status=%v", res["status"]) + } + return nil, fmt.Errorf("设备截屏失败: %s", errMsg) + } + output, _ := res["output"].(string) + // 设备端回传 data URL(data:image/jpeg;base64,...)或裸 base64 + if !strings.HasPrefix(output, "data:") { + output = "data:image/jpeg;base64," + output + } + d.reg.SaveResult(reqID, res) + if d.seeHandler == nil { + return map[string]interface{}{"image_data_url": output, "note": "无视觉描述处理器,仅返回原始图像数据"}, nil + } + desc := d.seeHandler(output, provider) + return map[string]interface{}{"description": desc}, nil +} diff --git a/internal/plugins/remotedevice/plugin.go b/internal/plugins/remotedevice/plugin.go index 1295bdd..b23e74c 100644 --- a/internal/plugins/remotedevice/plugin.go +++ b/internal/plugins/remotedevice/plugin.go @@ -109,6 +109,8 @@ func (p *Plugin) Start(s *sdk.PluginSDK) error { // ---- devicectl Device(agent 工具) ---------------- p.dev = &devicectlDevice{reg: p.registry, persist: p.persistAuthorized} + // screensee 视觉描述回调:截屏回传后用视觉模型描述屏幕内容 + p.dev.SetSeeHandler(p.describeScreen) if err := s.RegisterChannel("devicectl", p.dev); err != nil { log.Printf("[remotedevice] register devicectl channel: %v", err) } @@ -250,6 +252,41 @@ func (p *Plugin) handleDeviceAuth(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]interface{}{"device_id": req.DeviceID, "authorized": req.Authorize}) } +// describeScreen 用视觉模型描述设备屏幕截图(screensee 回调)。 +// provider 为空时使用默认 LLM 源;模型不支持视觉时返回友好错误。 +func (p *Plugin) describeScreen(dataURL string, provider string) string { + if p.sdk == nil || p.sdk.LLM() == nil { + return "LLM 不可用,无法描述屏幕内容" + } + llm := p.sdk.LLM() + req := &sdk.LLMCompletionRequest{ + MaxTokens: 2048, + Messages: []sdk.LLMMessage{{ + Role: "user", + Blocks: []sdk.LLMContentBlock{ + {Type: "text", Text: "这是用户设备的屏幕截图。请详细描述屏幕上显示的内容:正在运行的窗口/应用、可见的文字内容、界面状态等。如果是代码编辑器或终端,尽量转述关键文字信息。"}, + {Type: "image_url", ImageURL: dataURL}, + }, + }}, + } + // 指定源:临时切换(低频操作,用完恢复原源) + if provider != "" { + prev := llm.CurrentSource() + if err := llm.SetSource(provider); err != nil { + log.Printf("[remotedevice] screensee set source %s: %v", provider, err) + } else if prev != "" { + defer func() { _ = llm.SetSource(prev) }() + } + } + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + resp, err := llm.Chat(ctx, req) + if err != nil { + return fmt.Sprintf("屏幕截图视觉描述失败: %v(当前模型可能不支持图像输入)", err) + } + return resp.Content +} + func (p *Plugin) Stop() error { if p.server != nil { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) diff --git a/internal/sdk/llm.go b/internal/sdk/llm.go index 6e1baf5..e585248 100644 --- a/internal/sdk/llm.go +++ b/internal/sdk/llm.go @@ -23,6 +23,16 @@ type LLMMessage struct { ReasoningContent string `json:"reasoning_content,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` ToolCalls []LLMToolCall `json:"tool_calls,omitempty"` + // Blocks 多模态内容块(与 Content 二选一;非空时优先)。 + // 支持 text 与 image_url 两类,用于视觉模型看图(如 screensee 截屏描述)。 + Blocks []LLMContentBlock `json:"blocks,omitempty"` +} + +// LLMContentBlock 是多模态消息中的单个内容块。 +type LLMContentBlock struct { + Type string `json:"type"` // "text" | "image_url" + Text string `json:"text,omitempty"` + ImageURL string `json:"image_url,omitempty"` // data URL 或 http(s) URL } // LLMToolCall 是中立的工具调用请求。 diff --git a/internal/sdk/llm_impl.go b/internal/sdk/llm_impl.go index a923898..dafbb4f 100644 --- a/internal/sdk/llm_impl.go +++ b/internal/sdk/llm_impl.go @@ -71,6 +71,18 @@ func (l *llmImpl) Chat(ctx context.Context, req *LLMCompletionRequest) (*LLMComp ReasoningContent: m.ReasoningContent, ToolCallID: m.ToolCallID, } + // 多模态 Blocks:text/image_url → agentAPI.ContentBlock + for _, b := range m.Blocks { + switch b.Type { + case "text": + msg.Blocks = append(msg.Blocks, agentAPI.ContentBlock{Type: "text", Text: b.Text}) + case "image_url": + msg.Blocks = append(msg.Blocks, agentAPI.ContentBlock{ + Type: "image_url", + ImageURL: &agentAPI.ImageURL{URL: b.ImageURL, Detail: "high"}, + }) + } + } if len(m.ToolCalls) > 0 { msg.ToolCalls = make([]agentAPI.ToolCall, len(m.ToolCalls)) for j, tc := range m.ToolCalls {