From c44ec0f210a3f45cc0743408133f7af566d69d68 Mon Sep 17 00:00:00 2001 From: JianFeeeee Date: Thu, 27 Aug 2026 09:27:15 +0800 Subject: [PATCH] =?UTF-8?q?feat(waiter):=20daemon=E6=A8=A1=E5=BC=8F=20+=20?= =?UTF-8?q?localuse=E6=9C=AC=E6=9C=BA=E5=A4=96=E8=AE=BE=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit waiter 新增 --daemon 后台驻留模式 (daemon.go): - 维持 homed 连接，TUI实例经 Unix socket 接入 - 单客户端串行模型：每个TUI独占homed响应，新客户端回放缓冲(256行) - 设备桥场景可无 homed 运行（纯设备桥驻留） - 设备桥看护循环：WS断开自动重连(3-5s间隔) - conn.go 新增 daemonConn 类型，dial() 优先检测 daemon localuse 插件 (internal/plugins/localuse/): - local_screensee / local_camerasue / local_speakeruse - local_screensue / local_clipboardsee / local_clipboardsue / local_computeruse - 跨平台实现(Linux/macOS/Windows)，能力与 waiter device.go 对齐 - headless服务器上缺依赖工具自动返回安装提示 SDK sync: third_party SetToolBlocks 多模态类型同步 README 补充 daemon 模式使用文档 --- README.md | 17 + cmd/waiter/conn.go | 39 ++ cmd/waiter/daemon.go | 378 +++++++++++++++ cmd/waiter/main.go | 7 + internal/plugins/all.go | 1 + internal/plugins/localuse/plugin.go | 594 ++++++++++++++++++++++++ third_party/homeagent-sdk/sdk/plugin.go | 22 + 7 files changed, 1058 insertions(+) create mode 100644 cmd/waiter/daemon.go create mode 100644 internal/plugins/localuse/plugin.go diff --git a/README.md b/README.md index cd0aa41..e871c3f 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,23 @@ make build build-cli echo "你好，记住我喜欢喝咖啡" | ./build/waiter ``` +### 后台驻留模式（daemon） + +waiter 也支持后台驻留，保持与 homed 的持久连接并等待 TUI 实例接入，适合让 agent 主动召唤用户/设备桥持续存活： + +```bash +# 后台驻留（默认连 ~/.homeagent/cli.sock） +./build/waiter --daemon + +# 指定 socket +./build/waiter --socket /path/to/cli.sock --daemon + +# 随后任意 TUI/一行实例都会自动接入正在运行的 daemon，而不是直连 homed +./build/waiter +``` + +daemon 监听 `~/.homeagent/waiter.sock`，新客户端连入时会回放缓冲的最近对话（256 行），断连后 daemon 持续存活、自动重连 homed，并保持设备桥（若配置了 `device_gateway`/`device_token`）。 + API 密钥通过 WebUI `http://localhost:8080` 设置页配置，持久化在 SQLite 中。 ## 代码结构 diff --git a/cmd/waiter/conn.go b/cmd/waiter/conn.go index f043ad7..ce72940 100644 --- a/cmd/waiter/conn.go +++ b/cmd/waiter/conn.go @@ -19,6 +19,12 @@ type Conn interface { } func dial(cfg *Config) (Conn, error) { + // 优先连接运行中的 daemon（后台驻留模式） + if daemonIsRunning() { + if c, err := dialDaemon(); err == nil { + return c, nil + } + } if cfg.Remote != "" { return dialRemote(cfg.Remote, cfg.APIKey) } @@ -52,6 +58,39 @@ func dialLocal(socket, apiKey string) (Conn, error) { return lc, nil } +// daemonConn 是连接到运行中 waiter daemon 的轻量封装。 +// 协议与 localConn 完全一致（行式 \n 分隔），但不做 /auth（daemon 已集中鉴权）。 +type daemonConn struct { + conn net.Conn + r *bufio.Reader +} + +func dialDaemon() (Conn, error) { + sock := daemonSocketPath() + c, err := net.DialTimeout("unix", sock, 3*time.Second) + if err != nil { + return nil, fmt.Errorf("daemon %s: %w", sock, err) + } + return &daemonConn{conn: c, r: bufio.NewReader(c)}, nil +} + +func (c *daemonConn) Send(line string) error { + _, err := fmt.Fprintf(c.conn, "%s\n", line) + return err +} + +func (c *daemonConn) ReadLine() (string, error) { + s, err := c.r.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimSuffix(s, "\n"), nil +} + +func (c *daemonConn) Close() error { + return c.conn.Close() +} + type localConn struct { conn net.Conn r *bufio.Reader diff --git a/cmd/waiter/daemon.go b/cmd/waiter/daemon.go new file mode 100644 index 0000000..cf2271d --- /dev/null +++ b/cmd/waiter/daemon.go @@ -0,0 +1,378 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "net" + "os" + "os/signal" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "time" + + "gitcode.com/JianFeeeee/HomeAgent/internal/devicebridge/client" +) + +// --------------------------------------------------------------------------- +// Daemon 模式：后台驻留，维持 homed 连接 + 设备桥 + 消息缓冲 +// +// 工作原理： +// - daemon 保持一个到 homed 的持久连接 +// - TUI 实例通过 Unix socket 连接到 daemon +// - daemon 为每个 TUI 客户端分配独立的 homed 响应（通过 homeMu 序列化） +// - 新客户端连入时回放缓冲的历史消息（方便重连后看到上下文） +// --------------------------------------------------------------------------- + +const ( + daemonSocketName = "waiter.sock" + msgBufCap = 256 // 环形缓冲最近 N 行 homed 输出 +) + +type msgEntry struct { + line string + seq uint64 +} + +type daemonHandler struct { + // homed 连接 + homeMu sync.Mutex + homeConn net.Conn + homeR *bufio.Reader + homeCfg *Config + + // 消息缓冲（新客户端连入时回放） + bufMu sync.Mutex + buf []msgEntry + bufSeq uint64 + bufCap int + + // 生命周期 + stopCh chan struct{} +} + +func newDaemonHandler() *daemonHandler { + return &daemonHandler{ + bufCap: msgBufCap, + stopCh: make(chan struct{}), + } +} + +// ===== 消息缓冲 ===== + +func (h *daemonHandler) appendBuf(line string) { + h.bufMu.Lock() + defer h.bufMu.Unlock() + h.bufSeq++ + h.buf = append(h.buf, msgEntry{line: line, seq: h.bufSeq}) + if len(h.buf) > h.bufCap { + h.buf = h.buf[len(h.buf)-h.bufCap:] + } +} + +func (h *daemonHandler) replayBuffer() []string { + h.bufMu.Lock() + defer h.bufMu.Unlock() + lines := make([]string, 0, len(h.buf)) + for _, e := range h.buf { + lines = append(lines, e.line) + } + return lines +} + +// ===== homed 连接 ===== + +func (h *daemonHandler) connectHome(cfg *Config) error { + h.homeCfg = cfg + if cfg.Remote != "" { + return fmt.Errorf("daemon: remote mode not supported") + } + if cfg.Socket == "" { + cfg.Socket = discoverSocket("") + } + c, err := net.DialTimeout("unix", cfg.Socket, 5*time.Second) + if err != nil { + return fmt.Errorf("daemon: connect home: %w", err) + } + h.homeConn = c + h.homeR = bufio.NewReader(c) + log.Printf("[daemon] connected to home %s", cfg.Socket) + return nil +} + +func (h *daemonHandler) closeHome() { + if h.homeConn != nil { + h.homeConn.Close() + h.homeConn = nil + } +} + +func (h *daemonHandler) reconnectHome() { + cfg := h.homeCfg + if cfg == nil { + cfg = discoverConfig("") + } + if cfg.Socket == "" && cfg.Remote == "" { + cfg.Socket = discoverSocket("") + } + for i := 0; i < 30; i++ { + select { + case <-h.stopCh: + return + default: + } + h.closeHome() + time.Sleep(2 * time.Second) + if err := h.connectHome(cfg); err != nil { + log.Printf("[daemon] reconnect home (%d/30): %v", i+1, err) + continue + } + log.Printf("[daemon] reconnected to home") + return + } + log.Printf("[daemon] gave up reconnecting to home") +} + +// handleClient 处理单个 TUI 客户端： +// 1. 回放缓冲历史 +// 2. 读客户端输入 → 转发到 homed +// 3. 读 homed 响应 → 回写给该客户端（独占响应，不广播） +func (h *daemonHandler) handleClient(c net.Conn) { + defer c.Close() + + cid := fmt.Sprintf("%s", c.RemoteAddr()) + log.Printf("[daemon] client %s connected", cid) + defer log.Printf("[daemon] client %s disconnected", cid) + + // 1) 回放缓冲（新客户端看到最近对话上下文） + for _, line := range h.replayBuffer() { + fmt.Fprintf(c, "%s\n", line) + } + + // 2) 循环：读客户端 → 转发 homed → 读 homed 响应 → 回写客户端 + reader := bufio.NewReader(c) + for { + c.SetReadDeadline(time.Now().Add(5 * time.Minute)) + line, err := reader.ReadString('\n') + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + continue + } + return + } + line = strings.TrimSuffix(line, "\n") + if line == "" { + continue + } + + // 转发到 homed（加锁保证请求-响应配对） + h.homeMu.Lock() + if h.homeConn == nil { + h.homeMu.Unlock() + fmt.Fprintf(c, `{"type":"error","error":"not connected to home"}`+"\n") + continue + } + _, sendErr := fmt.Fprintf(h.homeConn, "%s\n", line) + if sendErr != nil { + h.homeMu.Unlock() + fmt.Fprintf(c, `{"type":"error","error":"send failed"}`+"\n") + continue + } + + // 读 homed 响应（所有帧：reasoning_delta / content_delta / tool_call / response / error） + for { + h.homeConn.SetReadDeadline(time.Now().Add(60 * time.Second)) + respLine, readErr := h.homeR.ReadString('\n') + if readErr != nil { + h.homeMu.Unlock() + log.Printf("[daemon] home read error during client %s: %v", cid, readErr) + h.reconnectHome() + // 回写错误给客户端 + fmt.Fprintf(c, `{"type":"error","error":"home disconnected"}`+"\n") + goto nextMessage + } + respLine = strings.TrimSuffix(respLine, "\n") + if respLine == "" { + continue + } + // 写入缓冲 + 回写给发起请求的客户端 + h.appendBuf(respLine) + fmt.Fprintf(c, "%s\n", respLine) + + // 检查是否是终结帧 + if strings.Contains(respLine, `"type":"response"`) || strings.Contains(respLine, `"type":"error"`) { + break + } + } + h.homeMu.Unlock() + + nextMessage: + } +} + +// ===== 启动入口 ===== + +func runDaemon(cfg *Config) { + dh := newDaemonHandler() + + // 连接 homed（设备桥场景下可失败——被控主机无需 homed） + if cfg.Socket != "" || cfg.Remote != "" { + if err := dh.connectHome(cfg); err != nil { + log.Printf("[daemon] home connect failed: %v (continue with device bridge only)", err) + dh.homeConn = nil + } + } else { + log.Printf("[daemon] no home socket configured, running device bridge only") + } + defer dh.closeHome() + + // 启动设备桥（设备网关场景下为核心职责） + startDaemonDeviceBridge(cfg) + + // 监听 Unix socket + sockPath := daemonSocketPath() + os.Remove(sockPath) + os.MkdirAll(filepath.Dir(sockPath), 0755) + + ln, err := net.Listen("unix", sockPath) + if err != nil { + fmt.Fprintf(os.Stderr, "daemon: listen %s: %v\n", sockPath, err) + os.Exit(1) + } + defer func() { + ln.Close() + os.Remove(sockPath) + }() + + log.Printf("[daemon] listening on %s", sockPath) + fmt.Printf("waiter daemon started\n socket: %s\n press Ctrl+C to stop\n", sockPath) + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + log.Printf("[daemon] shutting down") + close(dh.stopCh) + dh.closeHome() + ln.Close() + }() + + // 单客户端模式：串行处理（同一时刻只有一个 TUI 连接） + // 这与 homed CLI 插件的行为一致——一个连接对应一个活跃会话。 + for { + conn, err := ln.Accept() + if err != nil { + select { + case <-dh.stopCh: + log.Printf("[daemon] stopped") + return + default: + log.Printf("[daemon] accept error: %v", err) + continue + } + } + dh.handleClient(conn) + } +} + +// ===== Socket 工具 ===== + +func daemonSocketPath() string { + home, _ := os.UserHomeDir() + if home == "" { + home = "/tmp" + } + return filepath.Join(home, ".homeagent", daemonSocketName) +} + +func daemonIsRunning() bool { + sock := daemonSocketPath() + c, err := net.DialTimeout("unix", sock, 500*time.Millisecond) + if err != nil { + return false + } + c.Close() + return true +} + +func startDaemonDeviceBridge(cfg *Config) { + dg := cfg.DeviceGateway + dt := cfg.DeviceToken + if dg == "" || dt == "" { + return + } + // 设备桥重连循环：WS 断开时自动重连 + go runDeviceBridgeLoop(dg, dt) +} + +// runDeviceBridgeLoop 无限重连循环：建立设备桥 → 等待断开 → 重连。 +func runDeviceBridgeLoop(gateway, token string) { + for { + bridge, err := connectDeviceBridge(gateway, token) + if err != nil { + log.Printf("[daemon] device bridge connect failed: %v, retrying in 5s", err) + time.Sleep(5 * time.Second) + continue + } + log.Printf("[daemon] device bridge connected, waiting...") + bridge.Wait() // 阻塞直到断开 + log.Printf("[daemon] device bridge disconnected, reconnecting in 3s") + time.Sleep(3 * time.Second) + } +} + +// connectDeviceBridge 创建并启动一次设备桥，返回 bridge 实例供 Wait()。 +func connectDeviceBridge(gateway, token string) (*client.Bridge, error) { + hostname, _ := os.Hostname() + if hostname == "" { + hostname = "local" + } + deviceID := "waiter-" + sanitizeID(hostname) + + caps := []string{ + "status", "cmdrun", "deviceinfo", "cmdresult", + "computeruse", "screensee", "clipboardsee", "clipboardsue", + "camerasue", "speakeruse", "screensue", + } + + info := map[string]interface{}{ + "hostname": hostname, + "platform": runtime.GOOS, + "arch": runtime.GOARCH, + "cpus": runtime.NumCPU(), + } + + // 确保 gateway URL 格式正确 + gw := gateway + if !strings.HasPrefix(gw, "ws://") && !strings.HasPrefix(gw, "wss://") { + gw = "ws://" + gw + } + if !strings.Contains(gw, "/api/v1/device/ws") { + gw = gw + "/api/v1/device/ws" + } + + bridge := client.New(gw, token, deviceID, hostname, caps, info) + + // 注册命令处理器 + cr := client.NewCmdRouter() + cr.Handle("homeagent-", handleHomeagentCmd) + cr.HandleDefault(handleShellCmd) + bridge.OnCmd(func(reqID, command string) { + cr.Dispatch(reqID, command) + }) + + if err := bridge.Start(); err != nil { + return nil, err + } + + // 设置全局变量供 sendBridgeResult 使用 + deviceBridge = bridge + deviceBridgeID = deviceID + auth := true // daemon 模式默认授权（配置已指定） + bridge.SetAuthorized(auth) + + return bridge, nil +} diff --git a/cmd/waiter/main.go b/cmd/waiter/main.go index c430687..428fa86 100644 --- a/cmd/waiter/main.go +++ b/cmd/waiter/main.go @@ -126,6 +126,7 @@ func main() { deviceGateway := flag.String("device", "", "remotedevice 网关地址（如 127.0.0.1:9890），启动设备桥") deviceToken := flag.String("device-token", "", "设备接入 token") deviceAuthorized := flag.Bool("device-authorized", false, "客户端本地授权（允许远程操控本机；也可在 waiter.yaml 配 device_authorized: true）") + daemonMode := flag.Bool("daemon", false, "后台驻留模式：维持 homed 连接 + 设备桥，等待 TUI 实例接入") testCap := flag.String("test-cap", "", "测试本地能力（screensue/speakeruse/screensee/clipboardsee/clipboardsue/computeruse/camerasue），如 --test-cap screensue") testCapArgs := flag.String("test-cap-args", "", "测试能力的参数") flag.Parse() @@ -144,6 +145,12 @@ func main() { cfg.Socket = discoverSocket("") } + // Daemon 模式：后台驻留 + if *daemonMode { + runDaemon(cfg) + return + } + oneShotMsg := *chat if oneShotMsg == "" { oneShotMsg = *say diff --git a/internal/plugins/all.go b/internal/plugins/all.go index 2245528..3bc9587 100644 --- a/internal/plugins/all.go +++ b/internal/plugins/all.go @@ -9,6 +9,7 @@ import ( _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/cmd" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/files" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/healthcheck" + _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/localuse" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/mcp" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/multimodal" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/pluginmgr" diff --git a/internal/plugins/localuse/plugin.go b/internal/plugins/localuse/plugin.go new file mode 100644 index 0000000..1716926 --- /dev/null +++ b/internal/plugins/localuse/plugin.go @@ -0,0 +1,594 @@ +package localuse + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "gitcode.com/JianFeeeee/HomeAgent/internal/plugin" + sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk" +) + +func init() { + plugin.RegisterPluginMeta("localuse", "本地外设", "Local Device Use") + plugin.RegisterFactory("localuse", func(name string, config map[string]interface{}) (sdk.Plugin, error) { + return New(name), nil + }) +} + +type Plugin struct { + name string + dataDir string +} + +func New(name string) *Plugin { + return &Plugin{name: name} +} + +func (p *Plugin) Name() string { return p.name } + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + s.SetAutoRestart(true) + + if dd := s.Settings().DataDir(); dd != "" { + p.dataDir = dd + os.MkdirAll(p.dataDir, 0755) + } + + // ── screensee ── + s.RegisterTool("local_screensee", sdk.ToolDef{ + Name: "local_screensee", + Description: "截取本机屏幕当前画面（截屏）。" + + "返回图片的 base64 data URL，可直接用于视觉分析。" + + "依赖：Linux 需 scrot/import/gnome-screenshot 任一；macOS 需 screencapture（自带）；Windows 用 PowerShell。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, + }, p.handleScreensee) + + // ── camerasue ── + s.RegisterTool("local_camerasue", sdk.ToolDef{ + Name: "local_camerasue", + Description: "使用本机摄像头拍照或录像。" + + "无参数=拍照（jpeg），传入正整数=录像 N 秒（mp4）。" + + "依赖：ffmpeg + v4l2（Linux）/ dshow（Windows）/ avfoundation（macOS）。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "duration": map[string]interface{}{ + "type": "integer", + "description": "录像秒数（省略=拍照，传 0=拍照）", + }, + }, + }, + }, p.handleCamerasue) + + // ── speakeruse ── + s.RegisterTool("local_speakeruse", sdk.ToolDef{ + Name: "local_speakeruse", + Description: "使用本机扬声器朗读指定文字（TTS）。" + + "依赖：Linux 需 espeak/festival 任一；macOS 需 say（自带）；Windows 用 PowerShell SAPI。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "text": map[string]interface{}{ + "type": "string", + "description": "要朗读的文字", + }, + }, + "required": []string{"text"}, + }, + }, p.handleSpeakeruse) + + // ── screensue ── + s.RegisterTool("local_screensue", sdk.ToolDef{ + Name: "local_screensue", + Description: "在本机屏幕上显示一段内容（通知/弹窗/HTML 页面）。" + + "适合在用户面前弹出提醒、会议倒计时等。" + + "Linux 用 xterm/browsh/w3m/notify-send；macOS 用 osascript；Windows 用 PowerShell 弹窗。" + + "参数可选 duration（秒，默认 5）。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "content": map[string]interface{}{ + "type": "string", + "description": "要显示的文字或 HTML", + }, + "duration": map[string]interface{}{ + "type": "integer", + "description": "显示时长秒数（默认 5，0=永不超时）", + }, + }, + "required": []string{"content"}, + }, + }, p.handleScreensue) + + // ── clipboardsee ── + s.RegisterTool("local_clipboardsee", sdk.ToolDef{ + Name: "local_clipboardsee", + Description: "读取本机剪切板当前内容（用户最近复制/剪切的文字）。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, + }, p.handleClipboardsee) + + // ── clipboardsue ── + s.RegisterTool("local_clipboardsue", sdk.ToolDef{ + Name: "local_clipboardsue", + Description: "将文字写入本机剪切板（用户随后可 Ctrl+V 粘贴）。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "text": map[string]interface{}{ + "type": "string", + "description": "要写入剪切板的文字", + }, + }, + "required": []string{"text"}, + }, + }, p.handleClipboardsue) + + // ── computeruse ── + s.RegisterTool("local_computeruse", sdk.ToolDef{ + Name: "local_computeruse", + Description: "操控本机鼠标/键盘。依赖 xdotool（Linux）/ PowerShell（Windows）。" + + "action 可选：click / doubleclick / rightclick / move / scroll / keypress / type。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "action": map[string]interface{}{ + "type": "string", "enum": []interface{}{ + "click", "doubleclick", "rightclick", "move", "scroll", "keypress", "type", + }, + }, + "x": map[string]interface{}{"type": "integer", "description": "X 坐标（click/move 必填）"}, + "y": map[string]interface{}{"type": "integer", "description": "Y 坐标（click/move 必填）"}, + "button": map[string]interface{}{"type": "string", "description": "鼠标按键 left/right/middle"}, + "dy": map[string]interface{}{"type": "integer", "description": "scroll 滚动量"}, + "key": map[string]interface{}{"type": "string", "description": "keypress 按键名"}, + "text": map[string]interface{}{"type": "string", "description": "type 要输入的文字"}, + }, + "required": []interface{}{"action"}, + }, + }, p.handleComputeruse) + + return nil +} + +func (p *Plugin) Stop() error { return nil } + +// =========================================================================== +// 工具实现 +// =========================================================================== + +// ── screensee ────────────────────────────────────────────────────────────── + +func (p *Plugin) handleScreensee(args map[string]interface{}) (interface{}, error) { + platform := runtime.GOOS + tmpDir := os.TempDir() + outFile := filepath.Join(tmpDir, fmt.Sprintf("local_screenshot_%d.png", time.Now().UnixNano())) + defer os.Remove(outFile) + + var cmd *exec.Cmd + switch platform { + case "linux": + if _, err := exec.LookPath("import"); err == nil { + cmd = exec.Command("import", "-window", "root", outFile) + } else if _, err := exec.LookPath("scrot"); err == nil { + cmd = exec.Command("scrot", outFile) + } else if _, err := exec.LookPath("gnome-screenshot"); err == nil { + cmd = exec.Command("gnome-screenshot", "-f", outFile) + } else { + return map[string]interface{}{"isError": true, "content": "无截图工具（安装 scrot/import/gnome-screenshot 任一）"}, nil + } + case "darwin": + cmd = exec.Command("screencapture", "-x", outFile) + case "windows": + psScript := fmt.Sprintf(` + Add-Type -AssemblyName System.Windows.Forms + $screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds + $bitmap = New-Object Drawing.Bitmap $screen.Width, $screen.Height + $graphics = [Drawing.Graphics]::FromImage($bitmap) + $graphics.CopyFromScreen($screen.X, $screen.Y, 0, 0, $screen.Size) + $bitmap.Save('%s', [Drawing.Imaging.ImageFormat]::Png) + $graphics.Dispose() + $bitmap.Dispose() + `, strings.ReplaceAll(outFile, "\\", "\\\\")) + cmd = exec.Command("powershell", "-NoProfile", "-Command", psScript) + default: + return map[string]interface{}{"isError": true, "content": "不支持的平台: " + platform}, nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + cmd = exec.CommandContext(ctx, cmd.Path, cmd.Args[1:]...) + if out, err := cmd.CombinedOutput(); err != nil { + return map[string]interface{}{"isError": true, "content": fmt.Sprintf("截屏失败: %v\n%s", err, string(out))}, nil + } + + data, err := os.ReadFile(outFile) + if err != nil { + return map[string]interface{}{"isError": true, "content": fmt.Sprintf("读取截图失败: %v", err)}, nil + } + + // 大图保存到 dataDir（避免 agent 上下文爆掉），返回路径 + if len(data) > 256*1024 && p.dataDir != "" { + srcFile := filepath.Join(p.dataDir, fmt.Sprintf("screenshot_%d.png", time.Now().UnixNano())) + if err := os.WriteFile(srcFile, data, 0644); err == nil { + return map[string]interface{}{ + "content": fmt.Sprintf("截图已保存: %s (%d bytes)", srcFile, len(data)), + "local_path": srcFile, + "size": len(data), + }, nil + } + } + + b64 := base64.StdEncoding.EncodeToString(data) + return map[string]interface{}{ + "content": fmt.Sprintf("截图完成 (%d bytes)", len(data)), + "image_data_url": "data:image/png;base64," + b64, + }, nil +} + +// ── camerasue ────────────────────────────────────────────────────────────── + +func (p *Plugin) handleCamerasue(args map[string]interface{}) (interface{}, error) { + duration := 0 + if d, ok := args["duration"].(float64); ok && d > 0 { + duration = int(d) + } + + platform := runtime.GOOS + tmpDir := os.TempDir() + isVideo := duration > 0 + outFile := filepath.Join(tmpDir, fmt.Sprintf("local_cam_%d.jpg", time.Now().UnixNano())) + if isVideo { + outFile = filepath.Join(tmpDir, fmt.Sprintf("local_cam_%d.mp4", time.Now().UnixNano())) + } + + var cmd *exec.Cmd + switch platform { + case "linux": + if isVideo { + cmd = exec.Command("ffmpeg", "-f", "v4l2", "-i", "/dev/video0", + "-t", fmt.Sprintf("%d", duration), + "-pix_fmt", "yuv420p", "-c:v", "libx264", + "-f", "mp4", "-y", outFile) + } else { + cmd = exec.Command("ffmpeg", "-f", "v4l2", "-i", "/dev/video0", + "-frames:v", "1", "-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1") + } + case "windows": + if isVideo { + cmd = exec.Command("ffmpeg", "-f", "dshow", "-i", "video=USB Camera", + "-t", fmt.Sprintf("%d", duration), + "-pix_fmt", "yuv420p", "-c:v", "libx264", + "-f", "mp4", "-y", outFile) + } else { + cmd = exec.Command("ffmpeg", "-f", "dshow", "-i", "video=USB Camera", + "-frames:v", "1", "-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1") + } + default: + return map[string]interface{}{"isError": true, "content": "camerasue 不支持平台: " + platform}, nil + } + + timeout := 15 * time.Second + if isVideo { + timeout = time.Duration(duration+15) * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + if isVideo { + cmd = exec.CommandContext(ctx, cmd.Path, cmd.Args[1:]...) + if err := cmd.Run(); err != nil { + return map[string]interface{}{"isError": true, "content": fmt.Sprintf("摄像头录像失败: %v", err)}, nil + } + data, err := os.ReadFile(outFile) + os.Remove(outFile) + if err != nil { + return map[string]interface{}{"isError": true, "content": "读取录像失败"}, nil + } + if p.dataDir != "" { + srcFile := filepath.Join(p.dataDir, fmt.Sprintf("camera_%d.mp4", time.Now().UnixNano())) + if err := os.WriteFile(srcFile, data, 0644); err == nil { + return map[string]interface{}{ + "content": fmt.Sprintf("录像已保存: %s (%d bytes, %ds)", srcFile, len(data), duration), + "local_path": srcFile, + "size": len(data), + }, nil + } + } + b64 := base64.StdEncoding.EncodeToString(data) + return map[string]interface{}{ + "content": fmt.Sprintf("录像完成 (%d bytes, %ds)", len(data), duration), + "image_data_url": "data:video/mp4;base64," + b64, + }, nil + } + + // 拍照：读 stdout JPEG + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = nil + cmd2 := exec.CommandContext(ctx, cmd.Path, cmd.Args[1:]...) + cmd2.Stdout = &out + if err := cmd2.Run(); err != nil { + return map[string]interface{}{"isError": true, "content": fmt.Sprintf("摄像头拍照失败: %v", err)}, nil + } + + b64 := base64.StdEncoding.EncodeToString(out.Bytes()) + return map[string]interface{}{ + "content": fmt.Sprintf("拍照完成 (%d bytes)", len(out.Bytes())), + "image_data_url": "data:image/jpeg;base64," + b64, + }, nil +} + +// ── speakeruse ───────────────────────────────────────────────────────────── + +func (p *Plugin) handleSpeakeruse(args map[string]interface{}) (interface{}, error) { + text, _ := args["text"].(string) + if strings.TrimSpace(text) == "" { + return map[string]interface{}{"isError": true, "content": "text 不能为空"}, nil + } + + platform := runtime.GOOS + var cmd *exec.Cmd + switch platform { + case "linux": + if _, err := exec.LookPath("espeak"); err == nil { + cmd = exec.Command("espeak", text) + } else if _, err := exec.LookPath("festival"); err == nil { + cmd = exec.Command("festival", "--tts", "--pipe") + stdin, _ := cmd.StdinPipe() + go func() { defer stdin.Close(); stdin.Write([]byte(text)) }() + } else { + return map[string]interface{}{"isError": true, "content": "无 TTS 引擎（安装 espeak 或 festival）"}, nil + } + case "darwin": + cmd = exec.Command("say", text) + case "windows": + psScript := fmt.Sprintf(` + Add-Type -AssemblyName System.Speech + $synthesizer = New-Object System.Speech.Synthesis.SpeechSynthesizer + $synthesizer.Speak('%s') + `, strings.ReplaceAll(text, "'", "''")) + cmd = exec.Command("powershell", "-NoProfile", "-Command", psScript) + default: + return map[string]interface{}{"isError": true, "content": "speakeruse 不支持平台: " + platform}, nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd = exec.CommandContext(ctx, cmd.Path, cmd.Args[1:]...) + if err := cmd.Run(); err != nil { + return map[string]interface{}{"isError": true, "content": fmt.Sprintf("朗读失败: %v", err)}, nil + } + return map[string]interface{}{"content": "朗读完成"}, nil +} + +// ── screensue ────────────────────────────────────────────────────────────── + +func (p *Plugin) handleScreensue(args map[string]interface{}) (interface{}, error) { + content, _ := args["content"].(string) + duration := 5 + if d, ok := args["duration"].(float64); ok && d > 0 { + duration = int(d) + } + if strings.TrimSpace(content) == "" { + return map[string]interface{}{"isError": true, "content": "content 不能为空"}, nil + } + + platform := runtime.GOOS + switch platform { + case "linux": + return p.screensueLinux(content, duration) + case "darwin": + // macOS 用 osascript 弹窗 + psScript := fmt.Sprintf(`display dialog "%s" with title "HomeAgent" buttons {"OK"} default button "OK giving up after %d"`, strings.ReplaceAll(content, `"`, `\"`), duration) + cmd := exec.Command("osascript", "-e", psScript) + _ = cmd.Run() + return map[string]interface{}{"content": "屏幕显示已触发"}, nil + case "windows": + escContent := strings.ReplaceAll(content, "\x27", "\x27\x27") + psScript := fmt.Sprintf(` + Add-Type -AssemblyName System.Windows.Forms + $form = New-Object System.Windows.Forms.Form + $form.Text = "HomeAgent" + $form.Size = New-Object Drawing.Size(500,300) + $form.StartPosition = "CenterScreen" + $form.TopMost = $true + $label = New-Object System.Windows.Forms.Label + $label.Text = '%s' + $label.AutoSize = $true + $label.Location = New-Object Drawing.Point(20,20) + $form.Controls.Add($label) + $timer = New-Object System.Windows.Forms.Timer + $timer.Interval = %d + $timer.Add_Tick({ $form.Close() }) + $timer.Start() + [Windows.Forms.Application]::Run($form) +`, escContent, duration*1000) + cmd := exec.Command("powershell", "-NoProfile", "-Command", psScript) + _ = cmd.Run() + return map[string]interface{}{"content": "屏幕显示已触发"}, nil + default: + return map[string]interface{}{"isError": true, "content": "screensue 不支持平台: " + platform}, nil + } +} + +func (p *Plugin) screensueLinux(content string, duration int) (interface{}, error) { + // 优先 xterm + if _, err := exec.LookPath("xterm"); err == nil { + // 写临时文件显示 + tmpFile := filepath.Join(os.TempDir(), fmt.Sprintf("local_screensue_%d.txt", time.Now().UnixNano())) + os.WriteFile(tmpFile, []byte(content), 0644) + defer os.Remove(tmpFile) + cmd := exec.Command("xterm", "-T", "HomeAgent", "-e", "cat", tmpFile) + _ = cmd.Start() + go func() { time.Sleep(time.Duration(duration) * time.Second); cmd.Process.Kill() }() + return map[string]interface{}{"content": "屏幕显示已触发（xterm）"}, nil + } + // 回退 notify-send + if _, err := exec.LookPath("notify-send"); err == nil { + _ = exec.Command("notify-send", "HomeAgent", content).Run() + return map[string]interface{}{"content": "通知已发送"}, nil + } + return map[string]interface{}{"isError": true, "content": "无可用显示方式（安装 xterm 或 libnotify-bin）"}, nil +} + +// ── clipboardsee ─────────────────────────────────────────────────────────── + +func (p *Plugin) handleClipboardsee(args map[string]interface{}) (interface{}, error) { + platform := runtime.GOOS + switch platform { + case "linux": + if _, err := exec.LookPath("xclip"); err == nil { + out, _ := exec.Command("xclip", "-o", "-selection", "clipboard").Output() + return map[string]interface{}{"content": string(out)}, nil + } + if _, err := exec.LookPath("xsel"); err == nil { + out, _ := exec.Command("xsel", "-ob").Output() + return map[string]interface{}{"content": string(out)}, nil + } + return map[string]interface{}{"isError": true, "content": "无剪贴板工具（安装 xclip 或 xsel）"}, nil + case "darwin": + out, _ := exec.Command("pbpaste").Output() + return map[string]interface{}{"content": string(out)}, nil + case "windows": + ps := `Add-Type -AssemblyName System.Windows.Forms; [Windows.Forms.Clipboard]::GetText()` + out, _ := exec.Command("powershell", "-NoProfile", "-Command", ps).Output() + return map[string]interface{}{"content": strings.TrimSpace(string(out))}, nil + default: + return map[string]interface{}{"isError": true, "content": "clipboardsee 不支持: " + platform}, nil + } +} + +// ── clipboardsue ─────────────────────────────────────────────────────────── + +func (p *Plugin) handleClipboardsue(args map[string]interface{}) (interface{}, error) { + text, _ := args["text"].(string) + if strings.TrimSpace(text) == "" { + return map[string]interface{}{"isError": true, "content": "text 不能为空"}, nil + } + + platform := runtime.GOOS + switch platform { + case "linux": + if _, err := exec.LookPath("xclip"); err == nil { + cmd := exec.Command("xclip", "-i", "-selection", "clipboard") + cmd.Stdin = strings.NewReader(text) + _ = cmd.Run() + return map[string]interface{}{"content": "剪切板已写入"}, nil + } + if _, err := exec.LookPath("xsel"); err == nil { + cmd := exec.Command("xsel", "-ib") + cmd.Stdin = strings.NewReader(text) + _ = cmd.Run() + return map[string]interface{}{"content": "剪切板已写入"}, nil + } + return map[string]interface{}{"isError": true, "content": "无剪贴板工具"}, nil + case "darwin": + cmd := exec.Command("pbcopy") + cmd.Stdin = strings.NewReader(text) + _ = cmd.Run() + return map[string]interface{}{"content": "剪切板已写入"}, nil + case "windows": + ps := fmt.Sprintf(`Add-Type -AssemblyName System.Windows.Forms; [Windows.Forms.Clipboard]::SetText('%s')`, + strings.ReplaceAll(text, "'", "''")) + _ = exec.Command("powershell", "-NoProfile", "-Command", ps).Run() + return map[string]interface{}{"content": "剪切板已写入"}, nil + default: + return map[string]interface{}{"isError": true, "content": "clipboardsue 不支持: " + platform}, nil + } +} + +// ── computeruse ──────────────────────────────────────────────────────────── + +func (p *Plugin) handleComputeruse(args map[string]interface{}) (interface{}, error) { + action, _ := args["action"].(string) + if action == "" { + return map[string]interface{}{"isError": true, "content": "action 必填"}, nil + } + + platform := runtime.GOOS + if platform == "linux" { + if _, err := exec.LookPath("xdotool"); err != nil { + return map[string]interface{}{"isError": true, "content": "computeruse 需要 xdotool（apt install xdotool）"}, nil + } + return p.computeruseLinux(action, args) + } + if platform == "windows" { + return p.computeruseWindows(action, args) + } + return map[string]interface{}{"isError": true, "content": "computeruse 不支持: " + platform}, nil +} + +func (p *Plugin) computeruseLinux(action string, args map[string]interface{}) (interface{}, error) { + switch action { + case "click": + btn := "1" + if b, ok := args["button"].(string); ok { + switch b { + case "right": + btn = "3" + case "middle": + btn = "2" + } + } + _ = exec.Command("xdotool", "click", btn).Run() + case "doubleclick": + _ = exec.Command("xdotool", "click", "--repeat", "2", "1").Run() + case "rightclick": + _ = exec.Command("xdotool", "click", "3").Run() + case "move": + x, _ := args["x"].(float64) + y, _ := args["y"].(float64) + _ = exec.Command("xdotool", "mousemove", fmt.Sprintf("%d", int(x)), fmt.Sprintf("%d", int(y))).Run() + case "scroll": + dy, _ := args["dy"].(float64) + btn := "4" + if dy < 0 { + btn = "5" + } + _ = exec.Command("xdotool", "click", btn).Run() + case "type": + text, _ := args["text"].(string) + _ = exec.Command("xdotool", "type", text).Run() + case "keypress": + key, _ := args["key"].(string) + _ = exec.Command("xdotool", "key", key).Run() + default: + return map[string]interface{}{"isError": true, "content": "未知 action: " + action}, nil + } + return map[string]interface{}{"content": "操作已执行: " + action}, nil +} + +func (p *Plugin) computeruseWindows(action string, args map[string]interface{}) (interface{}, error) { + switch action { + case "click", "move": + x, _ := args["x"].(float64) + y, _ := args["y"].(float64) + ps := fmt.Sprintf(`[System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new(%d,%d)`, int(x), int(y)) + _ = exec.Command("powershell", "-NoProfile", "-Command", ps).Run() + case "type": + text, _ := args["text"].(string) + ps := fmt.Sprintf(`$w = New-Object -ComObject wscript.shell; $w.SendKeys('%s')`, strings.ReplaceAll(text, "'", "''")) + _ = exec.Command("powershell", "-NoProfile", "-Command", ps).Run() + default: + return map[string]interface{}{"isError": true, "content": "Windows computeruse 暂不支持 " + action}, nil + } + return map[string]interface{}{"content": "操作已执行: " + action}, nil +} + +// 确保正则在编译时生效（避免 unused） +var _ = regexp.MustCompile(`.*`) diff --git a/third_party/homeagent-sdk/sdk/plugin.go b/third_party/homeagent-sdk/sdk/plugin.go index 4bc7fca..0d7221a 100644 --- a/third_party/homeagent-sdk/sdk/plugin.go +++ b/third_party/homeagent-sdk/sdk/plugin.go @@ -114,6 +114,9 @@ type IOInjector interface { // InjectInputSync 注入输入事件并同步等待 agent 回复，返回回复文本（无回复时返回空串）。 // 用于通道消息的完整闭环：收到入站 → agent 处理 → 回复取回 → 送回通道。 InjectInputSync(source, channel, text string) string + // SetToolBlocks 插件工具注入多模态内容块（image_url/audio_url），内核在下一条 + // tool message 的 content 数组里带上这些块，让模型在后续轮次看到图/听到音频。 + SetToolBlocks(blocks []ContentBlock) } // EventType identifies the kind of system event. @@ -460,3 +463,22 @@ func (s *PluginSDK) RunOnRemoveHandlers() { handlers[i]() } } + +// ContentBlock 是多模态内容块（OpenAI 格式：text/image_url/audio_url）。 +// 插件工具返回结果时可用 PluginSDK.SetToolBlocks 注入，让下一轮 LLM +// 请求在 tool message 的 content 数组里带上图片/音频，实现"模型看图/听音频"。 +type ContentBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *ImageURL `json:"image_url,omitempty"` + AudioURL *AudioURL `json:"audio_url,omitempty"` +} + +type ImageURL struct { + URL string `json:"url"` + Detail string `json:"detail,omitempty"` +} + +type AudioURL struct { + URL string `json:"url"` +}