mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 17:38:10 +00:00
用户指出 remotedevice 的能力应当 outputch 化。查证后发现比"应当"更严重:
设备方向**根本没有出站实现**。
## 查到的三个缺口
1. `devicectlDevice` 一直声明 `OutputCapabilities() = CapStructured`(对外宣称可作输出目标),
但 `Execute` 的 switch 里**没有 "output" 分支** ⇒ `output_send__devicectl` 必然拿到
`unknown device tool output`,回模型"通过 [devicectl] 通道发送失败"。
2. 全插件没有 `RegisterOutputChannel`,也没有任何 EmitOutput/output_send 路径:
设备方向只有「工具(请求-响应)」与「设备→agent 注入」,**agent → 设备是断的**
(唯一的下行通道是个 HTTP 端点 `/api/v1/device/push`,不在 agent 的工具/通道模型里)。
3. 寻址是聚合的:所有设备共用一个名字 `devicectl`,没有 `device/<id>`;且设备 caps 只在
插件内部软检查(`SupportsTool`),**绕过**了内核的 `AllowedOutputs` 授权闸 ——
驻留子只要拿到 `device_ctl_cmdrun` 就能指挥**任意**设备。
## 按方向切分(不是一刀切)
**出站/消息类 → 每设备一个输出通道 `device/<id>`**(与入站同名):
- 上线注册、掉线注销(caps 由设备声明的 caps 映射:文本恒有;有屏→图/文件;
speaker→音频;可跑命令(cmd/cmdrun/cmdresult)或未声明已知能力→全能力,与
`deviceSupportsTool` 的旧设备兼容规则一致)。断连不注销会留下死通道骗模型。
- 于是自动获得:内核按 caps 在**发送前**拦(送图给纯文本音箱直接拒);
`output_list_channels` 能列出设备;`AllowedOutputs` 可按设备收窄给驻留子。
- 上下线钩子用 `Registry.SetPresenceHandler`(**同步回调**)而不是既有的 `ChangeChan`
(那是 select+default,缓冲满会丢事件;丢一次就留下死通道或漏注册)。
- `devicectl` 保留为聚合通道,并把它"声明了却不实现"的 output 补实:按
`meta.device_id`(或 meta 就是设备 id / args.device_id)路由;缺省时返回**可执行**的
报错(列出在线设备),而不是含糊失败。
**RPC 类保留为工具**(`devicedetect`/`screensee`/`computeruse`/`clipboard*`/
`device_ctl_status|cmdrun|cmdresult`):它们的返回值(图像/命令输出/状态)必须进模型
上下文,做成通道会丢掉这个语义。
**并给设备指令类工具补上同一道授权闸**(core/toolcall.go):`device_id` 指向的设备
必须是本 agent 被授权的 `device/<id>`。根 agent 默认完整授权 ⇒ 无行为变化;
驻留子收窄后,"拿到工具就能指挥任意设备"的缺口被堵上(新增 3 条 core 测试钉住)。
**设备端参考实现**(`internal/devicebridge/client/bridge.go` + waiter):新增 `op=push`
分发与 `OnPush` 回调(文本/结构化;二进制走既有 `cmd_speech_*` → `DataHandler`),
waiter 把它打到终端。
## 设计口径(用户当场纠偏,已写进代码注释与 harness README)
**主动转发只有 webui 与 cli 两个交互界面**(webui 订阅 EventAgentOutput 渲染气泡、
cli 用同步回程写回终端)。其它通道一律要求 agent **显式** `output_send__<通道>`。
我第一版给 remotedevice 加了 `EventAgentOutput` 订阅来自动回投设备——那是凭空造了
第三个转发者,违背"输出是 agent 的主动调用",已撤回(该测试一并删除)。
## 验收
- 单测:caps 映射词表;设备上线→注册通道(含同名 inputch)/掉线→注销;push 真落到
WS 设备;聚合通道 `devicectl` 的寻址(无 device_id 报可执行错误、按 meta 投递、
指定不存在设备报错);core 授权闸 3 例。
- 全量 `go test ./...` = 37 包 ok / 0 FAIL;`-race`(agent/plugins/plugin/devicebridge/sdk)干净。
- **真二进制端到端**(私有 netns + mock LLM + 真 WS 设备客户端 scripts/kernel-stress/devclient.py):
设备上线 → 通道表出现 `device/pydev-1`(caps=[text file image audio structured],由
`caps:["cmd"]` 映射)→ agent 经 `output_send__device/pydev-1` 主动发送 → 设备收到
`{"op":"push","payload":"内核推给你的消息","type":"text"}` → 设备掉线 → 通道从表中消失。
569 lines
13 KiB
Go
569 lines
13 KiB
Go
package client
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"runtime"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
// CmdHandler 是本地命令路由回调类型。
|
||
type CmdHandler func(reqID, command string)
|
||
|
||
// BridgeCmdHandler 接收服务端明确下发的路由信号(shell 或 homeagent)。
|
||
type BridgeCmdHandler func(reqID, command, cmdType string)
|
||
|
||
// CmdResult 是命令执行结果回调(用于异步通知 GUI 层)。
|
||
type CmdResultHandler func(reqID, status, output, errMsg string)
|
||
|
||
// DataHandler 是二进制数据接收回调(如 TTS 音频)。
|
||
type DataHandler func(reqID, kind, mime string, data []byte)
|
||
|
||
// PushHandler 接收 agent **主动投递**给本设备的内容。
|
||
//
|
||
// 与 cmd 的区别:cmd 是"让设备做一件事"(请求-响应,结果要回传),
|
||
// push 是"把这段内容交给设备"(agent 经 output_send__device/<id> 发起,
|
||
// 一种单向投递)。宿主按自己的形态落地:终端打出来、音箱念出来、屏幕显示。
|
||
//
|
||
// typ: text / structured / image / file / audio(二进制走 DataHandler,不走这里)
|
||
type PushHandler func(reqID, typ, payload, meta string)
|
||
|
||
// Bridge 是设备桥客户端核心结构体。
|
||
// 管理 WebSocket 连接、消息路由、心跳保活和命令分发。
|
||
// 授权状态由设备端本地存储(客户端鉴权),服务端不存储;
|
||
// 未授权时收到 cmd 直接拒绝执行并回执 error。
|
||
type Bridge struct {
|
||
mu sync.RWMutex
|
||
gateway string
|
||
token string
|
||
deviceID string
|
||
name string
|
||
kind string
|
||
caps []string
|
||
info map[string]interface{}
|
||
authorized bool // 客户端本地授权状态(用户在设备上手动开启)
|
||
|
||
ws *wsConn
|
||
stopCh chan struct{}
|
||
doneCh chan struct{}
|
||
started bool
|
||
|
||
// 回调
|
||
cmdHandler BridgeCmdHandler
|
||
resultHandler CmdResultHandler
|
||
dataHandler DataHandler
|
||
pushHandler PushHandler
|
||
|
||
// 二进制数据聚合(服务端→设备,如 TTS 音频)
|
||
speechAccum *speechBuffer
|
||
|
||
// 心跳间隔
|
||
pingInterval time.Duration
|
||
}
|
||
|
||
// speechBuffer 聚合服务端分块推送的二进制数据。
|
||
type speechBuffer struct {
|
||
reqID string
|
||
kind string
|
||
mime string
|
||
total int
|
||
data []byte
|
||
}
|
||
|
||
// New 创建设备桥客户端。
|
||
// gateway: ws://host:port(可选 /api/v1/device/ws 路径)
|
||
// token: 接入令牌
|
||
// deviceID: 设备唯一标识
|
||
// name: 设备显示名称
|
||
// caps: 能力列表(如 ["status","cmdrun","screensee","computeruse"])
|
||
// info: 额外设备信息(hostname, platform, arch 等),可为 nil
|
||
func New(gateway, token, deviceID, name string, caps []string, info map[string]interface{}) *Bridge {
|
||
if info == nil {
|
||
info = make(map[string]interface{})
|
||
}
|
||
// 填充默认信息
|
||
if _, ok := info["hostname"]; !ok {
|
||
hostname, _ := os.Hostname()
|
||
info["hostname"] = hostname
|
||
}
|
||
if _, ok := info["platform"]; !ok {
|
||
info["platform"] = runtime.GOOS
|
||
}
|
||
if _, ok := info["arch"]; !ok {
|
||
info["arch"] = runtime.GOARCH
|
||
}
|
||
if _, ok := info["cpus"]; !ok {
|
||
info["cpus"] = runtime.NumCPU()
|
||
}
|
||
|
||
return &Bridge{
|
||
gateway: gateway,
|
||
token: token,
|
||
deviceID: deviceID,
|
||
name: name,
|
||
kind: "computer",
|
||
caps: caps,
|
||
info: info,
|
||
stopCh: make(chan struct{}),
|
||
doneCh: make(chan struct{}),
|
||
pingInterval: 30 * time.Second,
|
||
}
|
||
}
|
||
|
||
// SetAuthorized 设置客户端本地授权状态(用户在设备上手动开启)。
|
||
// 授权后立即重新发送 hello 同步到服务端展示。
|
||
func (b *Bridge) SetAuthorized(auth bool) {
|
||
b.mu.Lock()
|
||
b.authorized = auth
|
||
b.mu.Unlock()
|
||
// 重新 hello 同步状态
|
||
b.mu.RLock()
|
||
ws := b.ws
|
||
connected := ws != nil && !ws.closed
|
||
b.mu.RUnlock()
|
||
if connected {
|
||
b.sendJSON(map[string]interface{}{
|
||
"op": "hello",
|
||
"device": map[string]interface{}{
|
||
"device_id": b.deviceID,
|
||
"name": b.name,
|
||
"kind": b.kind,
|
||
"caps": b.caps,
|
||
"info": b.info,
|
||
"authorized": auth,
|
||
},
|
||
})
|
||
}
|
||
}
|
||
|
||
// Authorized 返回当前客户端本地授权状态。
|
||
func (b *Bridge) Authorized() bool {
|
||
b.mu.RLock()
|
||
defer b.mu.RUnlock()
|
||
return b.authorized
|
||
}
|
||
|
||
// OnCmd 注册命令处理器。当收到 remotedevice 下发的 cmd 时调用。
|
||
func (b *Bridge) OnCmd(handler BridgeCmdHandler) {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
b.cmdHandler = handler
|
||
}
|
||
|
||
// OnResult 注册命令结果回调(用于异步通知)。
|
||
func (b *Bridge) OnResult(handler CmdResultHandler) {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
b.resultHandler = handler
|
||
}
|
||
|
||
// OnData 注册二进制数据接收回调(如 TTS 音频)。
|
||
func (b *Bridge) OnData(handler DataHandler) {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
b.dataHandler = handler
|
||
}
|
||
|
||
// OnPush 注册 agent 主动投递内容的回调(服务端 op=push)。
|
||
func (b *Bridge) OnPush(handler PushHandler) {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
b.pushHandler = handler
|
||
}
|
||
|
||
// SetPingInterval 设置心跳间隔(默认 30 秒)。
|
||
func (b *Bridge) SetPingInterval(d time.Duration) {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
b.pingInterval = d
|
||
}
|
||
|
||
// Start 启动设备桥连接。
|
||
// 会阻塞直到连接建立或超时失败。
|
||
func (b *Bridge) Start() error {
|
||
b.mu.Lock()
|
||
if b.started {
|
||
b.mu.Unlock()
|
||
return fmt.Errorf("devicebridge: already started")
|
||
}
|
||
b.started = true
|
||
b.mu.Unlock()
|
||
|
||
ws, err := dialWS(b.gateway, b.token, 10*time.Second)
|
||
if err != nil {
|
||
b.mu.Lock()
|
||
b.started = false
|
||
b.mu.Unlock()
|
||
return fmt.Errorf("devicebridge: dial: %w", err)
|
||
}
|
||
|
||
b.mu.Lock()
|
||
b.ws = ws
|
||
b.mu.Unlock()
|
||
|
||
// 发送 hello(含设备自报的授权状态,服务端仅展示不决策)
|
||
b.mu.RLock()
|
||
auth := b.authorized
|
||
b.mu.RUnlock()
|
||
b.sendJSON(map[string]interface{}{
|
||
"op": "hello",
|
||
"device": map[string]interface{}{
|
||
"device_id": b.deviceID,
|
||
"name": b.name,
|
||
"kind": b.kind,
|
||
"caps": b.caps,
|
||
"info": b.info,
|
||
"authorized": auth,
|
||
},
|
||
})
|
||
|
||
// 发送 bind
|
||
b.sendJSON(map[string]interface{}{
|
||
"op": "bind",
|
||
"device_id": b.deviceID,
|
||
"token": b.token,
|
||
})
|
||
|
||
go b.readLoop()
|
||
go b.pingLoop()
|
||
|
||
return nil
|
||
}
|
||
|
||
// Stop 停止设备桥连接。
|
||
func (b *Bridge) Stop() {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
if !b.started {
|
||
return
|
||
}
|
||
select {
|
||
case <-b.stopCh:
|
||
return
|
||
default:
|
||
close(b.stopCh)
|
||
}
|
||
if b.ws != nil {
|
||
_ = b.ws.close()
|
||
b.ws = nil
|
||
}
|
||
}
|
||
|
||
// Wait 等待设备桥连接关闭。
|
||
func (b *Bridge) Wait() {
|
||
<-b.doneCh
|
||
}
|
||
|
||
// DeviceID 返回设备 ID。
|
||
func (b *Bridge) DeviceID() string {
|
||
b.mu.RLock()
|
||
defer b.mu.RUnlock()
|
||
return b.deviceID
|
||
}
|
||
|
||
// Connected 返回是否已连接。
|
||
func (b *Bridge) Connected() bool {
|
||
b.mu.RLock()
|
||
defer b.mu.RUnlock()
|
||
return b.ws != nil && !b.ws.closed
|
||
}
|
||
|
||
// ===== 发送消息 =====
|
||
|
||
// SendResult 发送命令执行结果。
|
||
func (b *Bridge) SendResult(reqID, status, output, errMsg string) {
|
||
msg := map[string]interface{}{
|
||
"op": "cmd_result",
|
||
"req_id": reqID,
|
||
"status": status,
|
||
"device_id": b.deviceID,
|
||
}
|
||
if output != "" {
|
||
msg["output"] = output
|
||
}
|
||
if errMsg != "" {
|
||
msg["error"] = errMsg
|
||
}
|
||
b.sendJSON(msg)
|
||
}
|
||
|
||
// SendDataStart 开始二进制数据传输(设备→网关,如录像回传)。
|
||
func (b *Bridge) SendDataStart(reqID, kind, mime string, total int) {
|
||
b.sendJSON(map[string]interface{}{
|
||
"op": "cmd_data_start",
|
||
"req_id": reqID,
|
||
"kind": kind,
|
||
"mime": mime,
|
||
"total": total,
|
||
"chunk_size": 8192,
|
||
})
|
||
}
|
||
|
||
// SendDataChunk 发送一块二进制数据。
|
||
func (b *Bridge) SendDataChunk(data []byte) error {
|
||
b.mu.RLock()
|
||
ws := b.ws
|
||
b.mu.RUnlock()
|
||
if ws == nil || ws.closed {
|
||
return fmt.Errorf("devicebridge: not connected")
|
||
}
|
||
return ws.writeBinary(data)
|
||
}
|
||
|
||
// SendDataEnd 结束二进制数据传输。
|
||
func (b *Bridge) SendDataEnd(reqID, status, errMsg string) {
|
||
msg := map[string]interface{}{
|
||
"op": "cmd_data_end",
|
||
"req_id": reqID,
|
||
"status": status,
|
||
}
|
||
if errMsg != "" {
|
||
msg["error"] = errMsg
|
||
}
|
||
b.sendJSON(msg)
|
||
}
|
||
|
||
// SendDataChunked 便捷方法:自动分块发送完整二进制数据。
|
||
func (b *Bridge) SendDataChunked(reqID, kind, mime string, data []byte) {
|
||
total := len(data)
|
||
b.SendDataStart(reqID, kind, mime, total)
|
||
const chunkSize = 8192
|
||
for off := 0; off < total; off += chunkSize {
|
||
end := off + chunkSize
|
||
if end > total {
|
||
end = total
|
||
}
|
||
if err := b.SendDataChunk(data[off:end]); err != nil {
|
||
b.SendDataEnd(reqID, "error", err.Error())
|
||
return
|
||
}
|
||
}
|
||
b.SendDataEnd(reqID, "ok", "")
|
||
}
|
||
|
||
// SendEvent 发送设备主动上报事件。
|
||
func (b *Bridge) SendEvent(eventType string, payload interface{}) {
|
||
b.sendJSON(map[string]interface{}{
|
||
"op": "event",
|
||
"device_id": b.deviceID,
|
||
"type": eventType,
|
||
"payload": payload,
|
||
})
|
||
}
|
||
|
||
// SendStatus 发送设备状态更新。
|
||
func (b *Bridge) SendStatus(status string) {
|
||
b.sendJSON(map[string]interface{}{
|
||
"op": "status",
|
||
"device_id": b.deviceID,
|
||
"status": status,
|
||
})
|
||
}
|
||
|
||
// ===== 内部方法 =====
|
||
|
||
func (b *Bridge) sendJSON(v interface{}) {
|
||
b.mu.RLock()
|
||
ws := b.ws
|
||
b.mu.RUnlock()
|
||
if ws == nil || ws.closed {
|
||
return
|
||
}
|
||
payload := mustJSON(v)
|
||
_ = ws.writeText(payload)
|
||
}
|
||
|
||
func (b *Bridge) readLoop() {
|
||
defer func() {
|
||
b.mu.Lock()
|
||
b.started = false
|
||
if b.ws != nil {
|
||
_ = b.ws.close()
|
||
b.ws = nil
|
||
}
|
||
b.mu.Unlock()
|
||
close(b.doneCh)
|
||
}()
|
||
|
||
for {
|
||
select {
|
||
case <-b.stopCh:
|
||
return
|
||
default:
|
||
}
|
||
|
||
// 设置读超时(2 倍 ping 间隔)
|
||
b.mu.RLock()
|
||
ws := b.ws
|
||
interval := b.pingInterval
|
||
b.mu.RUnlock()
|
||
if ws == nil {
|
||
return
|
||
}
|
||
|
||
ws.setDeadline(time.Now().Add(interval * 2))
|
||
payload, isClose, opcode, err := ws.readFrame()
|
||
if err != nil {
|
||
if err == errPing {
|
||
_ = ws.writePong()
|
||
continue
|
||
}
|
||
// 超时或其他错误,退出
|
||
return
|
||
}
|
||
if isClose {
|
||
return
|
||
}
|
||
if opcode == 0x2 {
|
||
// 二进制帧:处于聚合状态时追加
|
||
b.handleBinaryFrame(payload)
|
||
continue
|
||
}
|
||
|
||
var msg map[string]interface{}
|
||
if err := json.Unmarshal(payload, &msg); err != nil {
|
||
continue
|
||
}
|
||
b.handleMessage(msg)
|
||
}
|
||
}
|
||
|
||
func (b *Bridge) handleMessage(msg map[string]interface{}) {
|
||
op, _ := msg["op"].(string)
|
||
switch op {
|
||
case "cmd":
|
||
reqID, _ := msg["req_id"].(string)
|
||
command, _ := msg["command"].(string)
|
||
cmdType, _ := msg["cmd_type"].(string)
|
||
if reqID == "" || command == "" {
|
||
return
|
||
}
|
||
// 客户端鉴权:未授权时拒绝执行(服务端不存储授权状态,无法被 agent 篡改)
|
||
b.mu.RLock()
|
||
auth := b.authorized
|
||
handler := b.cmdHandler
|
||
b.mu.RUnlock()
|
||
if !auth {
|
||
log.Printf("[devicebridge] cmd rejected (unauthorized) req=%s cmd=%s", reqID, truncateString(command, 60))
|
||
b.SendResult(reqID, "error", "", "设备未授权:请在设备本机开启远程控制授权")
|
||
return
|
||
}
|
||
// 记录日志
|
||
log.Printf("[devicebridge] cmd req=%s type=%s cmd=%s", reqID, cmdType, truncateString(command, 60))
|
||
|
||
if handler != nil {
|
||
handler(reqID, command, cmdType)
|
||
}
|
||
|
||
case "hello_ack", "bind_ack":
|
||
log.Printf("[devicebridge] %s device=%v", op, msg["device"])
|
||
|
||
case "cmd_speech_start":
|
||
reqID, _ := msg["req_id"].(string)
|
||
kind, _ := msg["kind"].(string)
|
||
mime, _ := msg["mime"].(string)
|
||
total := 0
|
||
if v, ok := msg["total"].(float64); ok {
|
||
total = int(v)
|
||
}
|
||
b.mu.Lock()
|
||
b.speechAccum = &speechBuffer{
|
||
reqID: reqID,
|
||
kind: kind,
|
||
mime: mime,
|
||
total: total,
|
||
}
|
||
b.mu.Unlock()
|
||
|
||
case "cmd_speech_end":
|
||
reqID, _ := msg["req_id"].(string)
|
||
b.mu.Lock()
|
||
acc := b.speechAccum
|
||
b.speechAccum = nil
|
||
b.mu.Unlock()
|
||
if acc == nil || acc.reqID != reqID {
|
||
return
|
||
}
|
||
data := acc.data
|
||
b.mu.RLock()
|
||
dh := b.dataHandler
|
||
b.mu.RUnlock()
|
||
if dh != nil {
|
||
dh(reqID, acc.kind, acc.mime, data)
|
||
}
|
||
|
||
case "push":
|
||
// agent 主动投递(output_send__device/<id>)。二进制负载走
|
||
// cmd_speech_* → DataHandler,这里只处理文本/结构化。
|
||
reqID, _ := msg["req_id"].(string)
|
||
typ, _ := msg["type"].(string)
|
||
payload, _ := msg["payload"].(string)
|
||
meta, _ := msg["meta"].(string)
|
||
if typ == "" {
|
||
typ = "text"
|
||
}
|
||
b.mu.RLock()
|
||
ph := b.pushHandler
|
||
b.mu.RUnlock()
|
||
if ph != nil {
|
||
ph(reqID, typ, payload, meta)
|
||
} else {
|
||
log.Printf("[devicebridge] push req=%s type=%s payload=%s", reqID, typ, truncateString(payload, 120))
|
||
}
|
||
|
||
default:
|
||
log.Printf("[devicebridge] unhandled op=%s", op)
|
||
}
|
||
}
|
||
|
||
func (b *Bridge) handleBinaryFrame(payload []byte) {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
if b.speechAccum == nil {
|
||
return
|
||
}
|
||
b.speechAccum.data = append(b.speechAccum.data, payload...)
|
||
// 防滥用:超出声明 total 的 2 倍或硬上限 64MB 时放弃
|
||
limit := b.speechAccum.total*2 + 1024
|
||
if limit < 64<<20 {
|
||
limit = 64 << 20
|
||
}
|
||
if len(b.speechAccum.data) > limit {
|
||
log.Printf("[devicebridge] speech data exceeded limit, dropped")
|
||
b.speechAccum = nil
|
||
}
|
||
}
|
||
|
||
func (b *Bridge) pingLoop() {
|
||
b.mu.RLock()
|
||
interval := b.pingInterval
|
||
b.mu.RUnlock()
|
||
|
||
ticker := time.NewTicker(interval)
|
||
defer ticker.Stop()
|
||
|
||
for {
|
||
select {
|
||
case <-b.stopCh:
|
||
return
|
||
case <-ticker.C:
|
||
b.mu.RLock()
|
||
ws := b.ws
|
||
b.mu.RUnlock()
|
||
if ws != nil && !ws.closed {
|
||
_ = ws.writeFrame(0x9, nil) // ping
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func truncateString(s string, n int) string {
|
||
if len(s) <= n {
|
||
return s
|
||
}
|
||
return s[:n] + "..."
|
||
}
|