From c3816dc6996759a7c29fc02bcac45b57c60bf061 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 4 Jul 2026 15:01:35 +0800 Subject: [PATCH] =?UTF-8?q?IO=20=E6=8A=BD=E8=B1=A1=E5=B1=82=E5=A2=9E?= =?UTF-8?q?=E5=BC=BA=EF=BC=9A=E9=9D=9E=E6=96=87=E6=9C=AC=E8=BE=93=E5=85=A5?= =?UTF-8?q?=E6=94=AF=E6=8C=81=20+=20waiter=20CLI=20=E9=87=8D=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PluginSDK: - 添加 InjectInput / InjectInputSync / InjectInterrupt 泛型接口 - 插件现在可注入 image/audio/file 等任意类型输入 Provider: - 添加 ContentBlock / ImageURL / AudioURL 类型 - Message 增加 Blocks 字段，Content 在非空 Blocks 时序列化为数组（多模态格式） Agent: - handleInput 新增 image/audio 类型分发 → processMediaInput - processMediaInput 将媒体数据附着到对话上下文，LLM 自主决策处理策略 - 新增内置工具：describe_image / transcribe_audio / ocr_image（pendingMedia 驱动） - 工具仅当有未处理媒体数据时注册，通过 Provider 直接调用多模态模型 Config: - 新增 InputProcessingConfig（image/audio 处理配置） - 含 fallback_provider / describe_prompt / ocr_enabled 等选项 Waiter CLI 重写: - 配置文件 ~/.config/homeagent/cli.yaml（自动发现 socket） - 原始终端行编辑 + 命令历史持久化 + 彩色输出 - 内置命令：/help /reconnect /connect /remote /local /prompt - 断线自动重连 --- Makefile | 4 +- README.md | 18 +- cmd/homed/main.go | 8 +- cmd/waiter/main.go | 862 ++++++++++++++++++++++++++++----- config/config.go | 13 + config/config.yaml | 11 + internal/agent/api/provider.go | 37 +- internal/agent/core/agent.go | 349 +++++++++++++ internal/sdk/plugin.go | 54 +-- pkg/types/types.go | 27 +- 10 files changed, 1209 insertions(+), 174 deletions(-) diff --git a/Makefile b/Makefile index 245ca81..6473422 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: all build clean install test run build-cli BINARY=homed -CLI_BINARY=homecli +CLI_BINARY=waiter GO=go GOCACHE=/tmp/gocache GOPATH=$(shell go env GOPATH) @@ -16,7 +16,7 @@ build: build-cli: @mkdir -p $(BUILD_DIR) - CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(CLI_BINARY) ./cmd/cli/ + CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(CLI_BINARY) ./cmd/waiter/ @echo "Built: $(BUILD_DIR)/$(CLI_BINARY)" build-static: diff --git a/README.md b/README.md index 73de51d..87e0f01 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,25 @@ ```bash # 构建 -make build +make build build-cli # 启动内核（需要 DeepSeek API 密钥） DEEPSEEK_API_KEY="sk-xxx" ./build/homed -data /tmp/ha -# 在另一个终端聊天 -echo "你好" | ./build/waiter -socket /tmp/ha/cli.sock +# 交互模式（自动发现 socket） +./build/waiter + +# 或单条消息 +echo "你好" | ./build/waiter +``` + +配置文件 `~/.config/homeagent/cli.yaml`： + +```yaml +mode: auto # auto / local / remote +colors: true +history_size: 1000 +prompt: "waiter> " ``` ## 架构一句话 diff --git a/cmd/homed/main.go b/cmd/homed/main.go index 64504ce..1420dc7 100644 --- a/cmd/homed/main.go +++ b/cmd/homed/main.go @@ -339,6 +339,11 @@ func main() { - timer_set — 设置定时提醒 - plgreload — 热重载插件 - spawn_child — 生成子 Agent 执行独立任务 +- describe_image — 描述用户上传的图片 +- transcribe_audio — 转写用户上传的音频 +- ocr_image — 识别图片中的文字 + +当用户上传图片或音频时，系统会自动附着媒体内容。如果模型不支持直接处理多媒体，请使用上述工具。 回复你的真实想法，用自然语言与用户交流。`, Provider: provider, @@ -359,7 +364,8 @@ func main() { ContextSavePath: filepath.Join(cfg.Daemon.DataDir, "memory", "context.json"), StageHost: stageHost, EventBus: evBus, - ThinkingEnabled: cfg.LLM.ThinkingEnabled, + ThinkingEnabled: cfg.LLM.ThinkingEnabled, + InputProcessing: cfg.InputProcessing, }) // 为内置插件注入内核依赖（各插件通过 init() 自注册工厂） diff --git a/cmd/waiter/main.go b/cmd/waiter/main.go index 58028ea..6fe1fdc 100644 --- a/cmd/waiter/main.go +++ b/cmd/waiter/main.go @@ -5,176 +5,778 @@ import ( "encoding/json" "flag" "fmt" - "log" + "io" "net" "net/http" "os" + "os/signal" + "path/filepath" "strings" "sync" + "syscall" "time" + "unsafe" + + "gopkg.in/yaml.v3" ) -func main() { - socket := flag.String("socket", "/var/lib/homeagent/cli.sock", "unix socket path") - remote := flag.String("remote", "", "remote webui URL (e.g. http://127.0.0.1:8080)") - say := flag.String("say", "", "send a message and print response (one-shot, no TUI)") - flag.Parse() +const ( + colorReset = "\033[0m" + colorGreen = "\033[32m" + colorRed = "\033[31m" + colorCyan = "\033[36m" + colorYellow = "\033[33m" + colorBold = "\033[1m" +) - if *say != "" { - if *remote != "" { - sayRemote(*remote, *say) - } else { - sayLocal(*socket, *say) - } - return - } +type Config struct { + Socket string `yaml:"socket"` + Remote string `yaml:"remote"` + Mode string `yaml:"mode"` + Colors bool `yaml:"colors"` + HistorySize int `yaml:"history_size"` + Prompt string `yaml:"prompt"` +} - if *remote != "" { - runRemote(*remote) - } else { - runLocal(*socket) +func defaultConfig() Config { + return Config{ + Mode: "auto", + Colors: true, + HistorySize: 1000, + Prompt: "waiter> ", } } -// sayLocal sends one message via Unix socket and prints the response -func sayLocal(socketPath, message string) { - conn, err := net.Dial("unix", socketPath) +func configPaths() []string { + home, _ := os.UserHomeDir() + xdgConfig := os.Getenv("XDG_CONFIG_HOME") + if xdgConfig == "" { + xdgConfig = filepath.Join(home, ".config") + } + return []string{ + filepath.Join(xdgConfig, "homeagent", "cli.yaml"), + filepath.Join(home, ".homeagent.yaml"), + ".homeagent.yaml", + } +} + +func historyPath() string { + home, _ := os.UserHomeDir() + xdgData := os.Getenv("XDG_DATA_HOME") + if xdgData == "" { + xdgData = filepath.Join(home, ".local", "share") + } + dir := filepath.Join(xdgData, "homeagent") + os.MkdirAll(dir, 0755) + return filepath.Join(dir, "cli_history") +} + +func loadConfig() Config { + cfg := defaultConfig() + for _, p := range configPaths() { + data, err := os.ReadFile(p) + if err != nil { + continue + } + yaml.Unmarshal(data, &cfg) + break + } + if cfg.HistorySize < 1 { + cfg.HistorySize = 100 + } + return cfg +} + +func saveConfig(cfg Config) { + for _, p := range configPaths() { + dir := filepath.Dir(p) + if err := os.MkdirAll(dir, 0755); err != nil { + continue + } + data, _ := yaml.Marshal(cfg) + os.WriteFile(p, data, 0644) + return + } +} + +func discoverSocket(configured string) string { + if configured != "" { + return configured + } + if s := os.Getenv("HOMEAGENT_SOCKET"); s != "" { + return s + } + home, _ := os.UserHomeDir() + candidates := []string{ + filepath.Join(home, ".homeagent", "cli.sock"), + "/var/lib/homeagent/cli.sock", + } + if xdg := os.Getenv("XDG_RUNTIME_DIR"); xdg != "" { + candidates = append([]string{filepath.Join(xdg, "homeagent", "cli.sock")}, candidates...) + } + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + return c + } + } + return candidates[0] +} + +func resolveEndpoint(cfg Config) (mode string, addr string) { + switch cfg.Mode { + case "local": + return "local", discoverSocket(cfg.Socket) + case "remote": + return "remote", cfg.Remote + default: + sock := discoverSocket(cfg.Socket) + if sock != "" { + if _, err := os.Stat(sock); err == nil { + return "local", sock + } + } + if cfg.Remote != "" { + return "remote", cfg.Remote + } + return "local", sock + } +} + +type respLine struct { + Type string `json:"type"` + Content string `json:"content"` + Error string `json:"error"` +} + +func printColored(cfg Config, color, msg string) { + if !cfg.Colors { + fmt.Println(msg) + return + } + fmt.Printf("%s%s%s\n", color, msg, colorReset) +} + +func main() { + socket := flag.String("socket", "", "unix socket path (overrides config)") + remote := flag.String("remote", "", "remote webui URL (overrides config)") + say := flag.String("say", "", "send a message and print response (one-shot)") + flag.Parse() + + cfg := loadConfig() + if *socket != "" { + cfg.Socket = *socket + } + if *remote != "" { + cfg.Remote = *remote + cfg.Mode = "remote" + } + mode, addr := resolveEndpoint(cfg) + + if *say != "" { + oneShot(cfg, mode, addr, *say) + return + } + + runInteractive(cfg, mode, addr) +} + +func oneShot(cfg Config, mode, addr, message string) { + if mode == "remote" { + resp, err := doRemoteOnce(addr, message) + if err != nil { + printColored(cfg, colorRed, fmt.Sprintf("error: %v", err)) + os.Exit(1) + } + fmt.Println(resp) + return + } + + conn, err := net.DialTimeout("unix", addr, 5*time.Second) if err != nil { - log.Fatalf("connect to %s: %v", socketPath, err) + printColored(cfg, colorRed, fmt.Sprintf("connect to %s: %v", addr, err)) + os.Exit(1) } defer conn.Close() - fmt.Fprintf(conn, "%s\n", message) scanner := bufio.NewScanner(conn) - scanner.Scan() - if err := scanner.Err(); err != nil { - log.Fatalf("read: %v", err) - } - - var resp struct { - Type string `json:"type"` - Content string `json:"content"` - Error string `json:"error"` - } - if err := json.Unmarshal(scanner.Bytes(), &resp); err != nil { - fmt.Println(scanner.Text()) - return - } - switch resp.Type { - case "response": - fmt.Println(resp.Content) - case "error": - log.Fatalf("error: %s", resp.Error) - default: - fmt.Println(scanner.Text()) + if scanner.Scan() { + var rl respLine + if err := json.Unmarshal(scanner.Bytes(), &rl); err != nil { + fmt.Println(scanner.Text()) + return + } + switch rl.Type { + case "response": + fmt.Println(rl.Content) + case "error": + printColored(cfg, colorRed, fmt.Sprintf("error: %s", rl.Error)) + os.Exit(1) + default: + fmt.Println(scanner.Text()) + } } } -// sayRemote sends one message via HTTP and prints the response -func sayRemote(baseURL, message string) { +func doRemoteOnce(baseURL, message string) (string, error) { baseURL = strings.TrimRight(baseURL, "/") body := fmt.Sprintf(`{"message":%q}`, message) resp, err := http.Post(baseURL+"/api/v1/chat", "application/json", strings.NewReader(body)) if err != nil { - log.Fatalf("http post: %v", err) + return "", err } defer resp.Body.Close() var result map[string]interface{} - json.NewDecoder(resp.Body).Decode(&result) - if content, ok := result["response"].(string); ok { - fmt.Println(content) + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err } + content, _ := result["response"].(string) + return content, nil } -// runLocal starts an interactive TUI via Unix socket -func runLocal(socketPath string) { - conn, err := net.Dial("unix", socketPath) - if err != nil { - log.Fatalf("connect to %s: %v", socketPath, err) - } - defer conn.Close() +func runInteractive(cfg Config, mode, addr string) { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - scanner := bufio.NewScanner(conn) - for scanner.Scan() { - var resp struct { - Type string `json:"type"` - Content string `json:"content"` - Error string `json:"error"` - } - if err := json.Unmarshal(scanner.Bytes(), &resp); err != nil { - fmt.Println(scanner.Text()) + history := newHistory(historyPath(), cfg.HistorySize) + history.load() + + line := newLineEditor(&history) + + restore, err := setRawMode(0) + if err != nil { + restore = func() {} + } + defer restore() + + if cfg.Colors { + fmt.Print(colorBold) + } + fmt.Printf("HomeAgent CLI — %s://%s\n", mode, addr) + if cfg.Colors { + fmt.Print(colorReset) + } + fmt.Println("Type /help for commands.") + + var conn io.ReadWriteCloser + var readerDone chan struct{} + connect := func() error { + if conn != nil { + conn.Close() + } + if readerDone != nil { + <-readerDone + } + c, err := dial(mode, addr) + if err != nil { + return err + } + conn = c + readerDone = make(chan struct{}) + go readLoop(cfg, conn, readerDone) + return nil + } + + reconnect := func() { + for i := 0; i < 30; i++ { + if err := connect(); err != nil { + if cfg.Colors { + fmt.Printf("\r\n%sreconnecting (%d/30): %v%s\n", colorYellow, i+1, err, colorReset) + } else { + fmt.Printf("\r\nreconnecting (%d/30): %v\n", i+1, err) + } + time.Sleep(2 * time.Second) continue } - switch resp.Type { - case "response": - fmt.Println(resp.Content) - case "error": - fmt.Fprintf(os.Stderr, "error: %s\n", resp.Error) - default: - fmt.Println(scanner.Text()) + if cfg.Colors { + fmt.Printf("\r%s%sreconnected%s\n", clearLine, colorGreen, colorReset) + } else { + fmt.Printf("\r%sreconnected\n", clearLine) + } + return + } + if cfg.Colors { + fmt.Printf("\r%s%sgiving up after 30 attempts%s\n", clearLine, colorRed, colorReset) + } else { + fmt.Printf("\r%sgiving up after 30 attempts\n", clearLine) + } + } + + // initial connect + for { + if err := connect(); err != nil { + if cfg.Colors { + fmt.Printf("%sconnect: %v, retrying in 2s...%s\n", colorYellow, err, colorReset) + } else { + fmt.Printf("connect: %v, retrying in 2s...\n", err) + } + time.Sleep(2 * time.Second) + continue + } + break + } + + prompt := cfg.Prompt + + for { + fmt.Print(prompt) + text, err := line.read() + if err != nil { + // EOF or error + break + } + line.clear() + + cmd := strings.TrimSpace(text) + if cmd == "" { + continue + } + + if cmd[0] == '/' { + if handleBuiltin(cfg, cmd, &mode, &addr, &prompt, reconnect, &history) { + continue + } + // unknown command falls through to send as message + } + + history.add(cmd) + history.save() + + _, err = fmt.Fprintf(conn, "%s\n", cmd) + if err != nil { + if cfg.Colors { + fmt.Printf("%sconnection lost, reconnecting...%s\n", colorYellow, colorReset) + } else { + fmt.Println("connection lost, reconnecting...") + } + line.redrawPending(cmd) + reconnect() + fmt.Fprintf(conn, "%s\n", cmd) + } + + select { + case <-sigCh: + goto exit + default: + } + } + +exit: + if conn != nil { + conn.Close() + } + if readerDone != nil { + <-readerDone + } +} + +func dial(mode, addr string) (io.ReadWriteCloser, error) { + if mode == "remote" { + return newHTTPConn(strings.TrimRight(addr, "/")), nil + } + return net.DialTimeout("unix", addr, 5*time.Second) +} + +const clearLine = "\033[2K\r" + +func readLoop(cfg Config, conn io.ReadWriteCloser, done chan struct{}) { + defer close(done) + scanner := bufio.NewScanner(conn) + for scanner.Scan() { + if !cfg.Colors { + fmt.Printf("%s%s\n", clearLine, scanner.Text()) + continue + } + var rl respLine + if err := json.Unmarshal(scanner.Bytes(), &rl); err != nil { + fmt.Printf("%s%s%s\n", clearLine, scanner.Text(), colorReset) + continue + } + switch rl.Type { + case "response": + fmt.Printf("%s%s%s%s\n", clearLine, colorGreen, rl.Content, colorReset) + case "error": + fmt.Printf("%s%s%s%s\n", clearLine, colorRed, rl.Error, colorReset) + default: + fmt.Printf("%s%s%s\n", clearLine, scanner.Text(), colorReset) + } + } +} + +// httpConn wraps an HTTP endpoint as a read/write/closer for chat. +type httpConn struct { + url string + mu sync.Mutex + buf []byte + closed bool +} + +func newHTTPConn(baseURL string) *httpConn { + return &httpConn{url: baseURL + "/api/v1/chat"} +} + +func (c *httpConn) Read(p []byte) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + for len(c.buf) == 0 && !c.closed { + c.mu.Unlock() + time.Sleep(100 * time.Millisecond) + c.mu.Lock() + } + if c.closed && len(c.buf) == 0 { + return 0, io.EOF + } + n := copy(p, c.buf) + c.buf = c.buf[n:] + return n, nil +} + +func (c *httpConn) Write(p []byte) (int, error) { + msg := strings.TrimSpace(string(p)) + resp, err := doRemoteOnce(c.url, msg) + if err != nil { + return 0, err + } + data, _ := json.Marshal(respLine{Type: "response", Content: resp}) + data = append(data, '\n') + c.mu.Lock() + c.buf = append(c.buf, data...) + c.mu.Unlock() + return len(p), nil +} + +func (c *httpConn) Close() error { + c.mu.Lock() + c.closed = true + c.mu.Unlock() + return nil +} + +func handleBuiltin(cfg Config, cmd string, mode, addr *string, prompt *string, reconnect func(), history *History) bool { + switch { + case cmd == "/help": + fmt.Println(`Built-in commands: + /help show this help + /exit, /quit exit waiter + /clear clear screen + /reconnect force reconnection + /connect switch to a different unix socket + /remote switch to remote HTTP mode + /local switch back to local socket mode + /prompt change the prompt + +Any other text is sent as a message to the agent.`) + return true + + case cmd == "/exit" || cmd == "/quit": + os.Exit(0) + return true + + case cmd == "/clear": + fmt.Print("\033[H\033[2J") + return true + + case cmd == "/reconnect": + if cfg.Colors { + fmt.Printf("%sreconnecting...%s\n", colorYellow, colorReset) + } else { + fmt.Println("reconnecting...") + } + reconnect() + return true + + case strings.HasPrefix(cmd, "/connect "): + *mode = "local" + *addr = strings.TrimSpace(cmd[9:]) + cfg.Socket = *addr + saveConfig(cfg) + reconnect() + return true + + case strings.HasPrefix(cmd, "/remote "): + *mode = "remote" + *addr = strings.TrimSpace(cmd[8:]) + cfg.Remote = *addr + saveConfig(cfg) + reconnect() + return true + + case cmd == "/local": + *mode = "local" + *addr = discoverSocket("") + reconnect() + return true + + case strings.HasPrefix(cmd, "/prompt "): + *prompt = strings.TrimSpace(cmd[8:]) + return true + + default: + return false + } +} + +type LineEditor struct { + buf []rune + pos int + hist *History + histI int + pending string +} + +func newLineEditor(h *History) *LineEditor { + return &LineEditor{ + hist: h, + histI: -1, + } +} + +func (e *LineEditor) clear() { + e.buf = e.buf[:0] + e.pos = 0 + e.histI = -1 +} + +func (e *LineEditor) redrawPending(text string) { + e.pending = text +} + +func (e *LineEditor) read() (string, error) { + if e.pending != "" { + t := e.pending + e.pending = "" + return t, nil + } + + e.buf = e.buf[:0] + e.pos = 0 + e.histI = -1 + + in := bufio.NewReader(os.Stdin) + for { + b := make([]byte, 1) + _, err := in.Read(b) + if err != nil { + return "", err + } + + switch b[0] { + case '\r', '\n': + fmt.Print("\n") + return string(e.buf), nil + + case 0x03: // Ctrl+C + fmt.Print("^C\n") + os.Exit(130) + return "", nil + + case 0x04: // Ctrl+D + if len(e.buf) == 0 { + return "", io.EOF + } + continue + + case 0x08, 0x7f: // Backspace + if e.pos > 0 { + e.pos-- + e.buf = append(e.buf[:e.pos], e.buf[e.pos+1:]...) + e.redraw() + } + + case 0x1b: // Escape sequence + seq := make([]byte, 2) + if _, err := io.ReadFull(in, seq); err != nil { + continue + } + if seq[0] != '[' { + continue + } + switch seq[1] { + case 'A': // Up + e.historyPrev() + case 'B': // Down + e.historyNext() + case 'C': // Right + if e.pos < len(e.buf) { + e.pos++ + e.redraw() + } + case 'D': // Left + if e.pos > 0 { + e.pos-- + e.redraw() + } + case 'H', '1': // Home (\x1b[H) or (\x1b[1~) + if seq[1] == '1' { + io.ReadFull(in, make([]byte, 1)) // consume ~ + } + e.pos = 0 + e.redraw() + case 'F', '4': // End (\x1b[F) or (\x1b[4~) + if seq[1] == '4' { + io.ReadFull(in, make([]byte, 1)) // consume ~ + } + e.pos = len(e.buf) + e.redraw() + case '3': // Delete (\x1b[3~) + io.ReadFull(in, make([]byte, 1)) // consume ~ + if e.pos < len(e.buf) { + e.buf = append(e.buf[:e.pos], e.buf[e.pos+1:]...) + e.redraw() + } + } + + case '\t': // Tab + e.doCompletion() + + default: + if b[0] >= 0x20 { + e.buf = append(e.buf, 0) + copy(e.buf[e.pos+1:], e.buf[e.pos:]) + e.buf[e.pos] = rune(b[0]) + e.pos++ + e.redraw() } } - }() - - scanner := bufio.NewScanner(os.Stdin) - fmt.Print("> ") - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - fmt.Print("> ") - continue - } - if line == "/exit" || line == "/quit" { - break - } - fmt.Fprintf(conn, "%s\n", line) - } - - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - select { - case <-done: - case <-time.After(60 * time.Second): } } -// runRemote starts an interactive TUI via HTTP -func runRemote(baseURL string) { - baseURL = strings.TrimRight(baseURL, "/") - scanner := bufio.NewScanner(os.Stdin) - fmt.Print("> ") - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - fmt.Print("> ") - continue +func (e *LineEditor) historyPrev() { + all := e.hist.all() + if len(all) == 0 { + return + } + if e.histI == -1 { + e.histI = len(all) - 1 + } else if e.histI > 0 { + e.histI-- + } + e.buf = []rune(all[e.histI]) + e.pos = len(e.buf) + e.redraw() +} + +func (e *LineEditor) historyNext() { + if e.histI == -1 { + return + } + all := e.hist.all() + e.histI++ + if e.histI >= len(all) { + e.histI = -1 + e.buf = e.buf[:0] + e.pos = 0 + } else { + e.buf = []rune(all[e.histI]) + e.pos = len(e.buf) + } + e.redraw() +} + +func (e *LineEditor) doCompletion() { + cmds := []string{"/help", "/exit", "/quit", "/clear", "/reconnect", "/connect ", "/remote ", "/local", "/prompt "} + prefix := string(e.buf) + for _, c := range cmds { + if strings.HasPrefix(c, prefix) && c != prefix { + e.buf = []rune(c) + e.pos = len(e.buf) + e.redraw() + return } - if line == "/exit" || line == "/quit" { - break - } - body := fmt.Sprintf(`{"message":%q}`, line) - resp, err := http.Post(baseURL+"/api/v1/chat", "application/json", strings.NewReader(body)) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - fmt.Print("> ") - continue - } - var result map[string]interface{} - if resp.StatusCode == http.StatusOK { - json.NewDecoder(resp.Body).Decode(&result) - } - resp.Body.Close() - if content, ok := result["response"].(string); ok { - fmt.Println(content) - } - fmt.Print("> ") } } + +func (e *LineEditor) redraw() { + fmt.Print("\r\033[K") // clear line + fmt.Print(string(e.buf)) + if e.pos < len(e.buf) { + // move cursor back + skip := len(e.buf) - e.pos + fmt.Printf("\033[%dD", skip) + } +} + +type History struct { + path string + lines []string + max int + mu sync.Mutex +} + +func newHistory(path string, max int) History { + return History{path: path, max: max} +} + +func (h *History) load() { + h.mu.Lock() + defer h.mu.Unlock() + data, err := os.ReadFile(h.path) + if err != nil { + return + } + h.lines = strings.Split(strings.TrimSpace(string(data)), "\n") + if len(h.lines) > h.max { + h.lines = h.lines[len(h.lines)-h.max:] + } +} + +func (h *History) save() { + h.mu.Lock() + defer h.mu.Unlock() + data := strings.Join(h.lines, "\n") + "\n" + os.WriteFile(h.path, []byte(data), 0644) +} + +func (h *History) add(line string) { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.lines) > 0 && h.lines[len(h.lines)-1] == line { + return + } + h.lines = append(h.lines, line) + if len(h.lines) > h.max { + h.lines = h.lines[len(h.lines)-h.max:] + } +} + +func (h *History) all() []string { + h.mu.Lock() + defer h.mu.Unlock() + r := make([]string, len(h.lines)) + copy(r, h.lines) + return r +} + +// setRawMode sets stdin to raw mode (non-canonical, no echo). +func setRawMode(fd int) (func(), error) { + if fd == 0 { + fd = int(os.Stdin.Fd()) + } + if !isTerminal(fd) { + return func() {}, nil + } + var oldState syscall.Termios + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCGETS, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); err != 0 { + return func() {}, fmt.Errorf("tcgets: %v", err) + } + newState := oldState + newState.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON + newState.Oflag &^= syscall.OPOST + newState.Lflag &^= syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN + newState.Cflag &^= syscall.CSIZE | syscall.PARENB + newState.Cflag |= syscall.CS8 + newState.Cc[syscall.VMIN] = 1 + newState.Cc[syscall.VTIME] = 0 + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCSETS, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 { + return func() {}, fmt.Errorf("tcset: %v", err) + } + return func() { + syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCSETS, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0) + }, nil +} + +func isTerminal(fd int) bool { + var t syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCGETS, uintptr(unsafe.Pointer(&t)), 0, 0, 0) + return err == 0 +} diff --git a/config/config.go b/config/config.go index 28ff885..3187bdc 100644 --- a/config/config.go +++ b/config/config.go @@ -40,6 +40,19 @@ func DefaultConfig() types.Config { {Name: "ollama", BaseURL: "http://localhost:11434", Model: "llama3", Adapter: "ollama", AdapterPath: "adapters/ollama.lua"}, }, }, + InputProcessing: types.InputProcessingConfig{ + Image: types.ImageProcessingConfig{ + FallbackProvider: "", + FallbackModel: "", + DescribePrompt: "请详细描述这张图片的内容", + OCREnabled: true, + }, + Audio: types.AudioProcessingConfig{ + FallbackProvider: "", + FallbackModel: "", + DescribePrompt: "请描述这段音频的内容", + }, + }, Defaults: types.AgentConfig{ Image: "homeagent/agent-base:latest", LLMEndpoints: []string{"https://api.openai.com/v1"}, diff --git a/config/config.yaml b/config/config.yaml index 810efc8..9d34c0a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -55,6 +55,17 @@ llm: adapter: "ollama" adapter_path: "adapters/ollama.lua" +input_processing: + image: + fallback_provider: "" + fallback_model: "" + describe_prompt: "请详细描述这张图片的内容" + ocr_enabled: true + audio: + fallback_provider: "" + fallback_model: "" + describe_prompt: "请描述这段音频的内容" + defaults: image: "homeagent/agent-base:latest" llm_endpoints: diff --git a/internal/agent/api/provider.go b/internal/agent/api/provider.go index 7c55903..7400484 100644 --- a/internal/agent/api/provider.go +++ b/internal/agent/api/provider.go @@ -14,18 +14,41 @@ import ( luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua" ) +// ContentBlock 定义多模态内容块，用于图片/音频等非文本输入。 +type ContentBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *ImageURL `json:"image_url,omitempty"` + AudioURL *AudioURL `json:"audio_url,omitempty"` +} + +type ImageURL struct { + URL string `json:"url"` + Detail string `json:"detail,omitempty"` +} + +type AudioURL struct { + URL string `json:"url"` +} + +// Message 表示对话消息。当 Blocks 不为空时 content 在 JSON 中序列化为数组（多模态格式）。 type Message struct { - Role string `json:"role"` - Content string `json:"content"` - ReasoningContent string `json:"reasoning_content,omitempty"` - ToolCallID string `json:"tool_call_id,omitempty"` - ToolCalls []ToolCall `json:"-"` + Role string `json:"role"` + Content string `json:"content,omitempty"` + Blocks []ContentBlock `json:"-"` + ReasoningContent string `json:"reasoning_content,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + ToolCalls []ToolCall `json:"-"` } func (m Message) MarshalJSON() ([]byte, error) { raw := map[string]interface{}{ - "role": m.Role, - "content": m.Content, + "role": m.Role, + } + if len(m.Blocks) > 0 { + raw["content"] = m.Blocks + } else { + raw["content"] = m.Content } if m.ReasoningContent != "" { raw["reasoning_content"] = m.ReasoningContent diff --git a/internal/agent/core/agent.go b/internal/agent/core/agent.go index c391358..da8afb3 100644 --- a/internal/agent/core/agent.go +++ b/internal/agent/core/agent.go @@ -96,6 +96,12 @@ type Agent struct { // 启动时间 startTime time.Time + + // 当前轮次的非文本媒体数据（图片/音频），供 describe_image 等工具访问 + pendingMedia map[string]interface{} + + // 非文本输入处理配置 + inputCfg types.InputProcessingConfig } type AgentConfig struct { @@ -123,6 +129,8 @@ type AgentConfig struct { StageHost *StageHost EventBus *events.Bus ThinkingEnabled bool + + InputProcessing types.InputProcessingConfig // 非文本输入处理配置 } func New(cfg AgentConfig) *Agent { @@ -166,6 +174,7 @@ func New(cfg AgentConfig) *Agent { childResults: make(map[string]string), interceptCh: make(chan string, 64), thinkingEnabled: cfg.ThinkingEnabled, + inputCfg: cfg.InputProcessing, } } @@ -268,6 +277,9 @@ func (a *Agent) handleInput(evt *agentIO.InputEvent) { } a.processTextInput(evt, input) + case "image", "audio": + a.processMediaInput(evt) + case "event": log.Printf("[agent] event from %s: %v", evt.Source, evt.Payload) @@ -280,6 +292,123 @@ func (a *Agent) handleInput(evt *agentIO.InputEvent) { } } +// processMediaInput 处理图片/音频等非文本输入。 +// 将媒体数据附着到对话中，LLM 可通过 describe_image / transcribe_audio 等工具自主处理。 +func (a *Agent) processMediaInput(evt *agentIO.InputEvent) { + start := time.Now() + a.pendingMedia = evt.Payload + defer func() { a.pendingMedia = nil }() + + a.currentOutputChannel = evt.OutputChannel + if a.currentOutputChannel == "" { + a.currentOutputChannel = evt.Source + } + + blocks, fallback := a.mediaToBlocks(evt.Payload, evt.Type) + + a.context.Append(ContextEvent{ + Timestamp: start, + Source: evt.Source, + Input: fallback, + }) + + // stage 上下文携带 blocks，process() 会将其附着到 user message 上 + stageCtx := a.stageCtxFromInput(fallback, evt.Source, "") + stageCtx.Extra = map[string]interface{}{ + "media_blocks": blocks, + "media_type": evt.Type, + } + + a.publishEvent(events.EventRawInput, map[string]interface{}{ + "content": evt.Payload, + "source": evt.Source, + }) + + if a.runStage(sdk.StageOnInput, stageCtx) { + a.emitResponse(evt, *stageCtx.Response) + return + } + + response, toolsUsed, err := a.process(fallback, stageCtx) + if err != nil { + log.Printf("[agent] process media error: %v", err) + resp := fmt.Sprintf("处理错误: %v", err) + a.emitResponse(evt, resp) + a.context.Append(ContextEvent{Timestamp: time.Now(), Source: "agent", Input: fallback, Response: resp}) + return + } + + elapsed := time.Since(start) + log.Printf("[agent] %s from %s → response (%dms, tools=%v)", evt.Type, evt.Source, elapsed.Milliseconds(), toolsUsed) + + a.context.Append(ContextEvent{ + Timestamp: time.Now(), + Source: "agent", + Input: fallback, + Response: response, + ToolsUsed: toolsUsed, + }) + + archived := a.context.Prune(response, a.maxContextSize, a.docStore) + if archived > 0 { + log.Printf("[agent] pruned %d low-relevance events to document memory", archived) + } + + a.emitResponse(evt, response) +} + +// mediaToBlocks 将媒体 payload 转为多模态 ContentBlock 数组和纯文本 fallback。 +func (a *Agent) mediaToBlocks(payload map[string]interface{}, mediaType string) ([]agentAPI.ContentBlock, string) { + data, _ := payload["data"].(string) + mime, _ := payload["mime"].(string) + url, _ := payload["url"].(string) + alt, _ := payload["alt"].(string) + if alt == "" { + alt = fmt.Sprintf("[用户上传了%s]", mediaType) + } + + var blocks []agentAPI.ContentBlock + + // 文本描述块 + desc := "" + switch mediaType { + case "image": + desc = a.inputCfg.Image.DescribePrompt + if desc == "" { + desc = "用户上传了一张图片，请使用 describe_image 工具查看详情。" + } + case "audio": + desc = a.inputCfg.Audio.DescribePrompt + if desc == "" { + desc = "用户上传了一段音频，请使用 transcribe_audio 工具查看内容。" + } + } + blocks = append(blocks, agentAPI.ContentBlock{Type: "text", Text: desc}) + + if data != "" || url != "" { + imgURL := url + if data != "" { + if mime == "" { + mime = "image/png" + } + imgURL = "data:" + mime + ";base64," + data + } + if mediaType == "image" { + blocks = append(blocks, agentAPI.ContentBlock{ + Type: "image_url", + ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: "auto"}, + }) + } else if mediaType == "audio" { + blocks = append(blocks, agentAPI.ContentBlock{ + Type: "audio_url", + AudioURL: &agentAPI.AudioURL{URL: imgURL}, + }) + } + } + + return blocks, alt +} + func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) { start := time.Now() @@ -418,6 +547,12 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri tools := a.buildToolDefs() msgs := a.buildMessages(sysPrompt, input) + // 如果 stageCtx 携带多模态 blocks，附着到 user message 上 + if blocks, ok := stageCtx.Extra["media_blocks"].([]agentAPI.ContentBlock); ok && len(blocks) > 0 { + if len(msgs) > 0 { + msgs[len(msgs)-1].Blocks = blocks + } + } log.Printf("[agent] tool call loop start, %d tools, %d context events, personality=%t, docs=%d", len(tools), a.context.Len(), @@ -616,6 +751,12 @@ func (a *Agent) executeToolCall(tc agentAPI.ToolCall) string { return a.executeChildResultTool(tc) case strings.HasPrefix(tc.Name, "llm_"): return a.executeLLMTool(tc) + case tc.Name == "describe_image": + return a.executeDescribeImage(tc) + case tc.Name == "transcribe_audio": + return a.executeTranscribeAudio(tc) + case tc.Name == "ocr_image": + return a.executeOCRImage(tc) } // 插件工具（通过 SDK RegisterTool 注册） @@ -1475,6 +1616,65 @@ func (a *Agent) buildToolDefs() []interface{} { }, }) + // 媒体处理工具：仅当本轮有未处理的媒体数据时注册 + if a.pendingMedia != nil { + tools = append(tools, map[string]interface{}{ + "type": "function", + "function": map[string]interface{}{ + "name": "describe_image", + "description": "描述当前用户上传的图片内容。使用配置的多模态模型或默认 LLM 进行识别。调用此工具后你将获得图片的详细文字描述。", + "parameters": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "provider": map[string]interface{}{ + "type": "string", + "description": "可选：用于图片描述的 LLM 源名称，不填则使用默认模型", + }, + "detail": map[string]interface{}{ + "type": "string", + "description": "描述详细程度: high / low / auto", + "default": "high", + }, + }, + }, + }, + }) + tools = append(tools, map[string]interface{}{ + "type": "function", + "function": map[string]interface{}{ + "name": "transcribe_audio", + "description": "转写当前用户上传的音频内容为文字。使用配置的多模态模型或默认 LLM 进行语音识别。", + "parameters": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "provider": map[string]interface{}{ + "type": "string", + "description": "可选：用于音频转写的 LLM 源名称，不填则使用默认模型", + }, + }, + }, + }, + }) + if a.inputCfg.Image.OCREnabled { + tools = append(tools, map[string]interface{}{ + "type": "function", + "function": map[string]interface{}{ + "name": "ocr_image", + "description": "对当前用户上传的图片执行 OCR 文字识别，提取图片中的文字内容。适用于截图、文档照片、菜单等场景。", + "parameters": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "language": map[string]interface{}{ + "type": "string", + "description": "OCR 语言（如 chi_sim+eng），默认自动", + }, + }, + }, + }, + }) + } + } + return tools } @@ -2091,6 +2291,155 @@ func (a *Agent) executeLLMTool(tc agentAPI.ToolCall) string { } } +// executeDescribeImage 调用多模态模型描述当前图片。 +func (a *Agent) executeDescribeImage(tc agentAPI.ToolCall) string { + if a.pendingMedia == nil { + return "没有待处理的图片数据" + } + data, _ := a.pendingMedia["data"].(string) + mime, _ := a.pendingMedia["mime"].(string) + url, _ := a.pendingMedia["url"].(string) + if data == "" && url == "" { + return "图片数据为空" + } + + providerName, _ := tc.Arguments["provider"].(string) + detail, _ := tc.Arguments["detail"].(string) + if detail == "" { + detail = "high" + } + + p := a.providerManager.Get(providerName) + if p == nil { + p = a.provider + } + + prompt := a.inputCfg.Image.DescribePrompt + if prompt == "" { + prompt = "请详细描述这张图片的内容，包括其中的文字、物体、人物、场景等信息。" + } + + imgURL := url + if data != "" { + if mime == "" { + mime = "image/png" + } + imgURL = "data:" + mime + ";base64," + data + } + + msg := agentAPI.Message{ + Role: "user", + Blocks: []agentAPI.ContentBlock{ + {Type: "text", Text: prompt}, + {Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: detail}}, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + resp, err := p.Chat(ctx, &agentAPI.CompletionRequest{ + Messages: []agentAPI.Message{msg}, + MaxTokens: 2048, + }) + if err != nil { + return fmt.Sprintf("图片描述失败: %v", err) + } + return fmt.Sprintf("[图片描述] %s", resp.Content) +} + +// executeTranscribeAudio 调用多模态模型转写/描述当前音频。 +func (a *Agent) executeTranscribeAudio(tc agentAPI.ToolCall) string { + if a.pendingMedia == nil { + return "没有待处理的音频数据" + } + data, _ := a.pendingMedia["data"].(string) + mime, _ := a.pendingMedia["mime"].(string) + url, _ := a.pendingMedia["url"].(string) + if data == "" && url == "" { + return "音频数据为空" + } + + providerName, _ := tc.Arguments["provider"].(string) + p := a.providerManager.Get(providerName) + if p == nil { + p = a.provider + } + + prompt := a.inputCfg.Audio.DescribePrompt + if prompt == "" { + prompt = "请转写这段音频的内容。" + } + + audURL := url + if data != "" { + if mime == "" { + mime = "audio/wav" + } + audURL = "data:" + mime + ";base64," + data + } + + msg := agentAPI.Message{ + Role: "user", + Blocks: []agentAPI.ContentBlock{ + {Type: "text", Text: prompt}, + {Type: "audio_url", AudioURL: &agentAPI.AudioURL{URL: audURL}}, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + resp, err := p.Chat(ctx, &agentAPI.CompletionRequest{ + Messages: []agentAPI.Message{msg}, + MaxTokens: 2048, + }) + if err != nil { + return fmt.Sprintf("音频转写失败: %v", err) + } + return fmt.Sprintf("[音频转写] %s", resp.Content) +} + +// executeOCRImage 对图片执行 OCR 文字识别（通过多模态模型实现）。 +func (a *Agent) executeOCRImage(tc agentAPI.ToolCall) string { + if a.pendingMedia == nil { + return "没有待处理的图片数据" + } + data, _ := a.pendingMedia["data"].(string) + mime, _ := a.pendingMedia["mime"].(string) + url, _ := a.pendingMedia["url"].(string) + if data == "" && url == "" { + return "图片数据为空" + } + + p := a.provider + + imgURL := url + if data != "" { + if mime == "" { + mime = "image/png" + } + imgURL = "data:" + mime + ";base64," + data + } + + msg := agentAPI.Message{ + Role: "user", + Blocks: []agentAPI.ContentBlock{ + {Type: "text", Text: "请识别这张图片中的所有文字内容，按原文输出。仅输出文字本身，不要添加额外描述。"}, + {Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: "high"}}, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + resp, err := p.Chat(ctx, &agentAPI.CompletionRequest{ + Messages: []agentAPI.Message{msg}, + MaxTokens: 4096, + }) + if err != nil { + return fmt.Sprintf("OCR 识别失败: %v", err) + } + return fmt.Sprintf("[OCR 结果] %s", resp.Content) +} + // runStage — 运行阶段管道，若插件 Response 被设置则返回 true（短路） func (a *Agent) runStage(stage sdk.Stage, ctx *sdk.StageContext) bool { if a.stageHost == nil { diff --git a/internal/sdk/plugin.go b/internal/sdk/plugin.go index 838015a..1f62f3a 100644 --- a/internal/sdk/plugin.go +++ b/internal/sdk/plugin.go @@ -120,57 +120,57 @@ func New(name string, iom *agentIO.IOManager, eventBus *events.Bus, mem MemoryAP // === IO 双通道 === -func (s *PluginSDK) InjectInput(source, channel string, payload map[string]interface{}) { +// InjectInput 注入任意类型的输入事件。 +// eventType 可选值: "text", "event", "command", 或插件自定义类型。 +// payload 可包含 "content" (文本), "image" (图片), "file" (文件), "audio" (音频) 等字段。 +func (s *PluginSDK) InjectInput(source, channel, eventType string, payload map[string]interface{}) { if s.iom != nil { - s.iom.InjectInputTo(source, channel, "text", payload) + s.iom.InjectInputTo(source, channel, eventType, payload) } } -func (s *PluginSDK) InjectInterrupt(source, channel string, payload map[string]interface{}) { +// InjectInputSync 注入任意类型输入并同步等待响应。 +func (s *PluginSDK) InjectInputSync(source, channel, eventType string, payload map[string]interface{}) *agentIO.OutputEvent { if s.iom != nil { - p := payload - if p == nil { - p = map[string]interface{}{} + return s.iom.InjectInputSyncTo(source, channel, eventType, payload) + } + return nil +} + +// InjectInterrupt 向中断通道注入任意类型的输入事件，可打断当前 LLM 处理。 +// eventType 会被写入 payload["type"]。 +func (s *PluginSDK) InjectInterrupt(source, channel, eventType string, payload map[string]interface{}) { + if s.iom != nil { + if payload == nil { + payload = map[string]interface{}{} } - if _, ok := p["type"]; !ok { - p["type"] = "text" - } - s.iom.InjectInterrupt(source, channel, p) + payload["type"] = eventType + s.iom.InjectInterrupt(source, channel, payload) } } +// 以下 InjectText* / InjectInterruptText 为快捷方式，等价于调用对应的泛型方法并传入 "text" 类型。 + func (s *PluginSDK) InjectText(source, channel, text string) { - if s.iom != nil { - s.iom.InjectTextTo(source, channel, text) - } + s.InjectInput(source, channel, "text", map[string]interface{}{"content": text}) } // InjectTextNoMemory 注入文本输入（不产生记忆）。适用于健康检查等无需记忆碎片的场景。 func (s *PluginSDK) InjectTextNoMemory(source, channel, text string) { - if s.iom != nil { - s.iom.InjectTextNoMemoryTo(source, channel, text) - } + s.InjectInput(source, channel, "text", map[string]interface{}{"content": text, "no_memory": true}) } func (s *PluginSDK) InjectTextSync(source, channel, text string) *agentIO.OutputEvent { - if s.iom != nil { - return s.iom.InjectTextSyncTo(source, channel, text) - } - return nil + return s.InjectInputSync(source, channel, "text", map[string]interface{}{"content": text}) } // InjectTextSyncNoMemory 注入文本输入（同步等待，不产生记忆）。 func (s *PluginSDK) InjectTextSyncNoMemory(source, channel, text string) *agentIO.OutputEvent { - if s.iom != nil { - return s.iom.InjectTextSyncNoMemoryTo(source, channel, text) - } - return nil + return s.InjectInputSync(source, channel, "text", map[string]interface{}{"content": text, "no_memory": true}) } func (s *PluginSDK) InjectInterruptText(source, channel, text string) { - if s.iom != nil { - s.iom.InjectInterruptText(source, channel, text) - } + s.InjectInterrupt(source, channel, "text", map[string]interface{}{"content": text}) } func (s *PluginSDK) OutputChan() <-chan *agentIO.OutputEvent { diff --git a/pkg/types/types.go b/pkg/types/types.go index e64453f..cb40a31 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -117,11 +117,30 @@ type LLMConfig struct { Sources []LLMSource `json:"sources,omitempty"` } +type ImageProcessingConfig struct { + FallbackProvider string `json:"fallback_provider" yaml:"fallback_provider"` + FallbackModel string `json:"fallback_model" yaml:"fallback_model"` + DescribePrompt string `json:"describe_prompt" yaml:"describe_prompt"` + OCREnabled bool `json:"ocr_enabled" yaml:"ocr_enabled"` +} + +type AudioProcessingConfig struct { + FallbackProvider string `json:"fallback_provider" yaml:"fallback_provider"` + FallbackModel string `json:"fallback_model" yaml:"fallback_model"` + DescribePrompt string `json:"describe_prompt" yaml:"describe_prompt"` +} + +type InputProcessingConfig struct { + Image ImageProcessingConfig `json:"image" yaml:"image"` + Audio AudioProcessingConfig `json:"audio" yaml:"audio"` +} + type Config struct { - Daemon DaemonConfig `json:"daemon"` - LLM LLMConfig `json:"llm"` - Defaults AgentConfig `json:"defaults"` - Agents []AgentConfig `json:"agents"` + Daemon DaemonConfig `json:"daemon"` + LLM LLMConfig `json:"llm"` + InputProcessing InputProcessingConfig `json:"input_processing"` + Defaults AgentConfig `json:"defaults"` + Agents []AgentConfig `json:"agents"` } type DaemonConfig struct {