IO 抽象层增强：非文本输入支持 + waiter CLI 重写

PluginSDK:
- 添加 InjectInput / InjectInputSync / InjectInterrupt 泛型接口
- 插件现在可注入 image/audio/file 等任意类型输入

Provider:
- 添加 ContentBlock / ImageURL / AudioURL 类型
- Message 增加 Blocks 字段，Content 在非空 Blocks 时序列化为数组（多模态格式）

Agent:
- handleInput 新增 image/audio 类型分发 → processMediaInput
- processMediaInput 将媒体数据附着到对话上下文，LLM 自主决策处理策略
- 新增内置工具：describe_image / transcribe_audio / ocr_image（pendingMedia 驱动）
- 工具仅当有未处理媒体数据时注册，通过 Provider 直接调用多模态模型

Config:
- 新增 InputProcessingConfig（image/audio 处理配置）
- 含 fallback_provider / describe_prompt / ocr_enabled 等选项

Waiter CLI 重写:
- 配置文件 ~/.config/homeagent/cli.yaml（自动发现 socket）
- 原始终端行编辑 + 命令历史持久化 + 彩色输出
- 内置命令：/help /reconnect /connect /remote /local /prompt
- 断线自动重连

Makefile

@@ -1,7 +1,7 @@
 .PHONY: all build clean install test run build-cli
 
 BINARY=homed
-CLI_BINARY=homecli
+CLI_BINARY=waiter
 GO=go
 GOCACHE=/tmp/gocache
 GOPATH=$(shell go env GOPATH)
@@ -16,7 +16,7 @@ build:
 
 build-cli:
 	@mkdir -p $(BUILD_DIR)
-	CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(CLI_BINARY) ./cmd/cli/
+	CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(CLI_BINARY) ./cmd/waiter/
 	@echo "Built: $(BUILD_DIR)/$(CLI_BINARY)"
 
 build-static:

README.md

@@ -11,13 +11,25 @@
 
 ```bash
 # 构建
-make build
+make build build-cli
 
 # 启动内核（需要 DeepSeek API 密钥）
 DEEPSEEK_API_KEY="sk-xxx" ./build/homed -data /tmp/ha
 
-# 在另一个终端聊天
-echo "你好" | ./build/waiter -socket /tmp/ha/cli.sock
+# 交互模式（自动发现 socket）
+./build/waiter
+
+# 或单条消息
+echo "你好" | ./build/waiter
+```
+
+配置文件 `~/.config/homeagent/cli.yaml`：
+
+```yaml
+mode: auto          # auto / local / remote
+colors: true
+history_size: 1000
+prompt: "waiter> "
 ```
 
 ## 架构一句话

cmd/homed/main.go

@@ -339,6 +339,11 @@ func main() {
 - timer_set — 设置定时提醒
 - plgreload — 热重载插件
 - spawn_child — 生成子 Agent 执行独立任务
+- describe_image — 描述用户上传的图片
+- transcribe_audio — 转写用户上传的音频
+- ocr_image — 识别图片中的文字
+
+当用户上传图片或音频时，系统会自动附着媒体内容。如果模型不支持直接处理多媒体，请使用上述工具。
 
 回复你的真实想法，用自然语言与用户交流。`,
 		Provider:        provider,
@@ -359,7 +364,8 @@ func main() {
 		ContextSavePath: filepath.Join(cfg.Daemon.DataDir, "memory", "context.json"),
 		StageHost:       stageHost,
 		EventBus:        evBus,
-		ThinkingEnabled: cfg.LLM.ThinkingEnabled,
+		ThinkingEnabled:  cfg.LLM.ThinkingEnabled,
+		InputProcessing:  cfg.InputProcessing,
 	})
 
 	// 为内置插件注入内核依赖（各插件通过 init() 自注册工厂）

cmd/waiter/main.go

@@ -5,176 +5,778 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"log"
+	"io"
 	"net"
 	"net/http"
 	"os"
+	"os/signal"
+	"path/filepath"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
+	"unsafe"
+
+	"gopkg.in/yaml.v3"
 )
 
-func main() {
-	socket := flag.String("socket", "/var/lib/homeagent/cli.sock", "unix socket path")
-	remote := flag.String("remote", "", "remote webui URL (e.g. http://127.0.0.1:8080)")
-	say := flag.String("say", "", "send a message and print response (one-shot, no TUI)")
-	flag.Parse()
+const (
+	colorReset  = "\033[0m"
+	colorGreen  = "\033[32m"
+	colorRed    = "\033[31m"
+	colorCyan   = "\033[36m"
+	colorYellow = "\033[33m"
+	colorBold   = "\033[1m"
+)
 
-	if *say != "" {
-		if *remote != "" {
-			sayRemote(*remote, *say)
-		} else {
-			sayLocal(*socket, *say)
-		}
-		return
-	}
+type Config struct {
+	Socket      string `yaml:"socket"`
+	Remote      string `yaml:"remote"`
+	Mode        string `yaml:"mode"`
+	Colors      bool   `yaml:"colors"`
+	HistorySize int    `yaml:"history_size"`
+	Prompt      string `yaml:"prompt"`
+}
 
-	if *remote != "" {
-		runRemote(*remote)
-	} else {
-		runLocal(*socket)
+func defaultConfig() Config {
+	return Config{
+		Mode:        "auto",
+		Colors:      true,
+		HistorySize: 1000,
+		Prompt:      "waiter> ",
 	}
 }
 
-// sayLocal sends one message via Unix socket and prints the response
-func sayLocal(socketPath, message string) {
-	conn, err := net.Dial("unix", socketPath)
+func configPaths() []string {
+	home, _ := os.UserHomeDir()
+	xdgConfig := os.Getenv("XDG_CONFIG_HOME")
+	if xdgConfig == "" {
+		xdgConfig = filepath.Join(home, ".config")
+	}
+	return []string{
+		filepath.Join(xdgConfig, "homeagent", "cli.yaml"),
+		filepath.Join(home, ".homeagent.yaml"),
+		".homeagent.yaml",
+	}
+}
+
+func historyPath() string {
+	home, _ := os.UserHomeDir()
+	xdgData := os.Getenv("XDG_DATA_HOME")
+	if xdgData == "" {
+		xdgData = filepath.Join(home, ".local", "share")
+	}
+	dir := filepath.Join(xdgData, "homeagent")
+	os.MkdirAll(dir, 0755)
+	return filepath.Join(dir, "cli_history")
+}
+
+func loadConfig() Config {
+	cfg := defaultConfig()
+	for _, p := range configPaths() {
+		data, err := os.ReadFile(p)
+		if err != nil {
+			continue
+		}
+		yaml.Unmarshal(data, &cfg)
+		break
+	}
+	if cfg.HistorySize < 1 {
+		cfg.HistorySize = 100
+	}
+	return cfg
+}
+
+func saveConfig(cfg Config) {
+	for _, p := range configPaths() {
+		dir := filepath.Dir(p)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			continue
+		}
+		data, _ := yaml.Marshal(cfg)
+		os.WriteFile(p, data, 0644)
+		return
+	}
+}
+
+func discoverSocket(configured string) string {
+	if configured != "" {
+		return configured
+	}
+	if s := os.Getenv("HOMEAGENT_SOCKET"); s != "" {
+		return s
+	}
+	home, _ := os.UserHomeDir()
+	candidates := []string{
+		filepath.Join(home, ".homeagent", "cli.sock"),
+		"/var/lib/homeagent/cli.sock",
+	}
+	if xdg := os.Getenv("XDG_RUNTIME_DIR"); xdg != "" {
+		candidates = append([]string{filepath.Join(xdg, "homeagent", "cli.sock")}, candidates...)
+	}
+	for _, c := range candidates {
+		if _, err := os.Stat(c); err == nil {
+			return c
+		}
+	}
+	return candidates[0]
+}
+
+func resolveEndpoint(cfg Config) (mode string, addr string) {
+	switch cfg.Mode {
+	case "local":
+		return "local", discoverSocket(cfg.Socket)
+	case "remote":
+		return "remote", cfg.Remote
+	default:
+		sock := discoverSocket(cfg.Socket)
+		if sock != "" {
+			if _, err := os.Stat(sock); err == nil {
+				return "local", sock
+			}
+		}
+		if cfg.Remote != "" {
+			return "remote", cfg.Remote
+		}
+		return "local", sock
+	}
+}
+
+type respLine struct {
+	Type    string `json:"type"`
+	Content string `json:"content"`
+	Error   string `json:"error"`
+}
+
+func printColored(cfg Config, color, msg string) {
+	if !cfg.Colors {
+		fmt.Println(msg)
+		return
+	}
+	fmt.Printf("%s%s%s\n", color, msg, colorReset)
+}
+
+func main() {
+	socket := flag.String("socket", "", "unix socket path (overrides config)")
+	remote := flag.String("remote", "", "remote webui URL (overrides config)")
+	say := flag.String("say", "", "send a message and print response (one-shot)")
+	flag.Parse()
+
+	cfg := loadConfig()
+	if *socket != "" {
+		cfg.Socket = *socket
+	}
+	if *remote != "" {
+		cfg.Remote = *remote
+		cfg.Mode = "remote"
+	}
+	mode, addr := resolveEndpoint(cfg)
+
+	if *say != "" {
+		oneShot(cfg, mode, addr, *say)
+		return
+	}
+
+	runInteractive(cfg, mode, addr)
+}
+
+func oneShot(cfg Config, mode, addr, message string) {
+	if mode == "remote" {
+		resp, err := doRemoteOnce(addr, message)
+		if err != nil {
+			printColored(cfg, colorRed, fmt.Sprintf("error: %v", err))
+			os.Exit(1)
+		}
+		fmt.Println(resp)
+		return
+	}
+
+	conn, err := net.DialTimeout("unix", addr, 5*time.Second)
 	if err != nil {
-		log.Fatalf("connect to %s: %v", socketPath, err)
+		printColored(cfg, colorRed, fmt.Sprintf("connect to %s: %v", addr, err))
+		os.Exit(1)
 	}
 	defer conn.Close()
-
 	fmt.Fprintf(conn, "%s\n", message)
 
 	scanner := bufio.NewScanner(conn)
-	scanner.Scan()
-	if err := scanner.Err(); err != nil {
-		log.Fatalf("read: %v", err)
-	}
-
-	var resp struct {
-		Type    string `json:"type"`
-		Content string `json:"content"`
-		Error   string `json:"error"`
-	}
-	if err := json.Unmarshal(scanner.Bytes(), &resp); err != nil {
-		fmt.Println(scanner.Text())
-		return
-	}
-	switch resp.Type {
-	case "response":
-		fmt.Println(resp.Content)
-	case "error":
-		log.Fatalf("error: %s", resp.Error)
-	default:
-		fmt.Println(scanner.Text())
+	if scanner.Scan() {
+		var rl respLine
+		if err := json.Unmarshal(scanner.Bytes(), &rl); err != nil {
+			fmt.Println(scanner.Text())
+			return
+		}
+		switch rl.Type {
+		case "response":
+			fmt.Println(rl.Content)
+		case "error":
+			printColored(cfg, colorRed, fmt.Sprintf("error: %s", rl.Error))
+			os.Exit(1)
+		default:
+			fmt.Println(scanner.Text())
+		}
 	}
 }
 
-// sayRemote sends one message via HTTP and prints the response
-func sayRemote(baseURL, message string) {
+func doRemoteOnce(baseURL, message string) (string, error) {
 	baseURL = strings.TrimRight(baseURL, "/")
 	body := fmt.Sprintf(`{"message":%q}`, message)
 	resp, err := http.Post(baseURL+"/api/v1/chat", "application/json", strings.NewReader(body))
 	if err != nil {
-		log.Fatalf("http post: %v", err)
+		return "", err
 	}
 	defer resp.Body.Close()
 	var result map[string]interface{}
-	json.NewDecoder(resp.Body).Decode(&result)
-	if content, ok := result["response"].(string); ok {
-		fmt.Println(content)
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return "", err
 	}
+	content, _ := result["response"].(string)
+	return content, nil
 }
 
-// runLocal starts an interactive TUI via Unix socket
-func runLocal(socketPath string) {
-	conn, err := net.Dial("unix", socketPath)
-	if err != nil {
-		log.Fatalf("connect to %s: %v", socketPath, err)
-	}
-	defer conn.Close()
+func runInteractive(cfg Config, mode, addr string) {
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
 
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		scanner := bufio.NewScanner(conn)
-		for scanner.Scan() {
-			var resp struct {
-				Type    string `json:"type"`
-				Content string `json:"content"`
-				Error   string `json:"error"`
-			}
-			if err := json.Unmarshal(scanner.Bytes(), &resp); err != nil {
-				fmt.Println(scanner.Text())
+	history := newHistory(historyPath(), cfg.HistorySize)
+	history.load()
+
+	line := newLineEditor(&history)
+
+	restore, err := setRawMode(0)
+	if err != nil {
+		restore = func() {}
+	}
+	defer restore()
+
+	if cfg.Colors {
+		fmt.Print(colorBold)
+	}
+	fmt.Printf("HomeAgent CLI — %s://%s\n", mode, addr)
+	if cfg.Colors {
+		fmt.Print(colorReset)
+	}
+	fmt.Println("Type /help for commands.")
+
+	var conn io.ReadWriteCloser
+	var readerDone chan struct{}
+	connect := func() error {
+		if conn != nil {
+			conn.Close()
+		}
+		if readerDone != nil {
+			<-readerDone
+		}
+		c, err := dial(mode, addr)
+		if err != nil {
+			return err
+		}
+		conn = c
+		readerDone = make(chan struct{})
+		go readLoop(cfg, conn, readerDone)
+		return nil
+	}
+
+	reconnect := func() {
+		for i := 0; i < 30; i++ {
+			if err := connect(); err != nil {
+				if cfg.Colors {
+					fmt.Printf("\r\n%sreconnecting (%d/30): %v%s\n", colorYellow, i+1, err, colorReset)
+				} else {
+					fmt.Printf("\r\nreconnecting (%d/30): %v\n", i+1, err)
+				}
+				time.Sleep(2 * time.Second)
 				continue
 			}
-			switch resp.Type {
-			case "response":
-				fmt.Println(resp.Content)
-			case "error":
-				fmt.Fprintf(os.Stderr, "error: %s\n", resp.Error)
-			default:
-				fmt.Println(scanner.Text())
+			if cfg.Colors {
+				fmt.Printf("\r%s%sreconnected%s\n", clearLine, colorGreen, colorReset)
+			} else {
+				fmt.Printf("\r%sreconnected\n", clearLine)
+			}
+			return
+		}
+		if cfg.Colors {
+			fmt.Printf("\r%s%sgiving up after 30 attempts%s\n", clearLine, colorRed, colorReset)
+		} else {
+			fmt.Printf("\r%sgiving up after 30 attempts\n", clearLine)
+		}
+	}
+
+	// initial connect
+	for {
+		if err := connect(); err != nil {
+			if cfg.Colors {
+				fmt.Printf("%sconnect: %v, retrying in 2s...%s\n", colorYellow, err, colorReset)
+			} else {
+				fmt.Printf("connect: %v, retrying in 2s...\n", err)
+			}
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		break
+	}
+
+	prompt := cfg.Prompt
+
+	for {
+		fmt.Print(prompt)
+		text, err := line.read()
+		if err != nil {
+			// EOF or error
+			break
+		}
+		line.clear()
+
+		cmd := strings.TrimSpace(text)
+		if cmd == "" {
+			continue
+		}
+
+		if cmd[0] == '/' {
+			if handleBuiltin(cfg, cmd, &mode, &addr, &prompt, reconnect, &history) {
+				continue
+			}
+			// unknown command falls through to send as message
+		}
+
+		history.add(cmd)
+		history.save()
+
+		_, err = fmt.Fprintf(conn, "%s\n", cmd)
+		if err != nil {
+			if cfg.Colors {
+				fmt.Printf("%sconnection lost, reconnecting...%s\n", colorYellow, colorReset)
+			} else {
+				fmt.Println("connection lost, reconnecting...")
+			}
+			line.redrawPending(cmd)
+			reconnect()
+			fmt.Fprintf(conn, "%s\n", cmd)
+		}
+
+		select {
+		case <-sigCh:
+			goto exit
+		default:
+		}
+	}
+
+exit:
+	if conn != nil {
+		conn.Close()
+	}
+	if readerDone != nil {
+		<-readerDone
+	}
+}
+
+func dial(mode, addr string) (io.ReadWriteCloser, error) {
+	if mode == "remote" {
+		return newHTTPConn(strings.TrimRight(addr, "/")), nil
+	}
+	return net.DialTimeout("unix", addr, 5*time.Second)
+}
+
+const clearLine = "\033[2K\r"
+
+func readLoop(cfg Config, conn io.ReadWriteCloser, done chan struct{}) {
+	defer close(done)
+	scanner := bufio.NewScanner(conn)
+	for scanner.Scan() {
+		if !cfg.Colors {
+			fmt.Printf("%s%s\n", clearLine, scanner.Text())
+			continue
+		}
+		var rl respLine
+		if err := json.Unmarshal(scanner.Bytes(), &rl); err != nil {
+			fmt.Printf("%s%s%s\n", clearLine, scanner.Text(), colorReset)
+			continue
+		}
+		switch rl.Type {
+		case "response":
+			fmt.Printf("%s%s%s%s\n", clearLine, colorGreen, rl.Content, colorReset)
+		case "error":
+			fmt.Printf("%s%s%s%s\n", clearLine, colorRed, rl.Error, colorReset)
+		default:
+			fmt.Printf("%s%s%s\n", clearLine, scanner.Text(), colorReset)
+		}
+	}
+}
+
+// httpConn wraps an HTTP endpoint as a read/write/closer for chat.
+type httpConn struct {
+	url    string
+	mu     sync.Mutex
+	buf    []byte
+	closed bool
+}
+
+func newHTTPConn(baseURL string) *httpConn {
+	return &httpConn{url: baseURL + "/api/v1/chat"}
+}
+
+func (c *httpConn) Read(p []byte) (int, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for len(c.buf) == 0 && !c.closed {
+		c.mu.Unlock()
+		time.Sleep(100 * time.Millisecond)
+		c.mu.Lock()
+	}
+	if c.closed && len(c.buf) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, c.buf)
+	c.buf = c.buf[n:]
+	return n, nil
+}
+
+func (c *httpConn) Write(p []byte) (int, error) {
+	msg := strings.TrimSpace(string(p))
+	resp, err := doRemoteOnce(c.url, msg)
+	if err != nil {
+		return 0, err
+	}
+	data, _ := json.Marshal(respLine{Type: "response", Content: resp})
+	data = append(data, '\n')
+	c.mu.Lock()
+	c.buf = append(c.buf, data...)
+	c.mu.Unlock()
+	return len(p), nil
+}
+
+func (c *httpConn) Close() error {
+	c.mu.Lock()
+	c.closed = true
+	c.mu.Unlock()
+	return nil
+}
+
+func handleBuiltin(cfg Config, cmd string, mode, addr *string, prompt *string, reconnect func(), history *History) bool {
+	switch {
+	case cmd == "/help":
+		fmt.Println(`Built-in commands:
+  /help            show this help
+  /exit, /quit     exit waiter
+  /clear           clear screen
+  /reconnect       force reconnection
+  /connect <path>  switch to a different unix socket
+  /remote <url>    switch to remote HTTP mode
+  /local           switch back to local socket mode
+  /prompt <text>   change the prompt
+
+Any other text is sent as a message to the agent.`)
+		return true
+
+	case cmd == "/exit" || cmd == "/quit":
+		os.Exit(0)
+		return true
+
+	case cmd == "/clear":
+		fmt.Print("\033[H\033[2J")
+		return true
+
+	case cmd == "/reconnect":
+		if cfg.Colors {
+			fmt.Printf("%sreconnecting...%s\n", colorYellow, colorReset)
+		} else {
+			fmt.Println("reconnecting...")
+		}
+		reconnect()
+		return true
+
+	case strings.HasPrefix(cmd, "/connect "):
+		*mode = "local"
+		*addr = strings.TrimSpace(cmd[9:])
+		cfg.Socket = *addr
+		saveConfig(cfg)
+		reconnect()
+		return true
+
+	case strings.HasPrefix(cmd, "/remote "):
+		*mode = "remote"
+		*addr = strings.TrimSpace(cmd[8:])
+		cfg.Remote = *addr
+		saveConfig(cfg)
+		reconnect()
+		return true
+
+	case cmd == "/local":
+		*mode = "local"
+		*addr = discoverSocket("")
+		reconnect()
+		return true
+
+	case strings.HasPrefix(cmd, "/prompt "):
+		*prompt = strings.TrimSpace(cmd[8:])
+		return true
+
+	default:
+		return false
+	}
+}
+
+type LineEditor struct {
+	buf     []rune
+	pos     int
+	hist    *History
+	histI   int
+	pending string
+}
+
+func newLineEditor(h *History) *LineEditor {
+	return &LineEditor{
+		hist:  h,
+		histI: -1,
+	}
+}
+
+func (e *LineEditor) clear() {
+	e.buf = e.buf[:0]
+	e.pos = 0
+	e.histI = -1
+}
+
+func (e *LineEditor) redrawPending(text string) {
+	e.pending = text
+}
+
+func (e *LineEditor) read() (string, error) {
+	if e.pending != "" {
+		t := e.pending
+		e.pending = ""
+		return t, nil
+	}
+
+	e.buf = e.buf[:0]
+	e.pos = 0
+	e.histI = -1
+
+	in := bufio.NewReader(os.Stdin)
+	for {
+		b := make([]byte, 1)
+		_, err := in.Read(b)
+		if err != nil {
+			return "", err
+		}
+
+		switch b[0] {
+		case '\r', '\n':
+			fmt.Print("\n")
+			return string(e.buf), nil
+
+		case 0x03: // Ctrl+C
+			fmt.Print("^C\n")
+			os.Exit(130)
+			return "", nil
+
+		case 0x04: // Ctrl+D
+			if len(e.buf) == 0 {
+				return "", io.EOF
+			}
+			continue
+
+		case 0x08, 0x7f: // Backspace
+			if e.pos > 0 {
+				e.pos--
+				e.buf = append(e.buf[:e.pos], e.buf[e.pos+1:]...)
+				e.redraw()
+			}
+
+		case 0x1b: // Escape sequence
+			seq := make([]byte, 2)
+			if _, err := io.ReadFull(in, seq); err != nil {
+				continue
+			}
+			if seq[0] != '[' {
+				continue
+			}
+			switch seq[1] {
+			case 'A': // Up
+				e.historyPrev()
+			case 'B': // Down
+				e.historyNext()
+			case 'C': // Right
+				if e.pos < len(e.buf) {
+					e.pos++
+					e.redraw()
+				}
+			case 'D': // Left
+				if e.pos > 0 {
+					e.pos--
+					e.redraw()
+				}
+			case 'H', '1': // Home (\x1b[H) or (\x1b[1~)
+				if seq[1] == '1' {
+					io.ReadFull(in, make([]byte, 1)) // consume ~
+				}
+				e.pos = 0
+				e.redraw()
+			case 'F', '4': // End (\x1b[F) or (\x1b[4~)
+				if seq[1] == '4' {
+					io.ReadFull(in, make([]byte, 1)) // consume ~
+				}
+				e.pos = len(e.buf)
+				e.redraw()
+			case '3': // Delete (\x1b[3~)
+				io.ReadFull(in, make([]byte, 1)) // consume ~
+				if e.pos < len(e.buf) {
+					e.buf = append(e.buf[:e.pos], e.buf[e.pos+1:]...)
+					e.redraw()
+				}
+			}
+
+		case '\t': // Tab
+			e.doCompletion()
+
+		default:
+			if b[0] >= 0x20 {
+				e.buf = append(e.buf, 0)
+				copy(e.buf[e.pos+1:], e.buf[e.pos:])
+				e.buf[e.pos] = rune(b[0])
+				e.pos++
+				e.redraw()
 			}
 		}
-	}()
-
-	scanner := bufio.NewScanner(os.Stdin)
-	fmt.Print("> ")
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" {
-			fmt.Print("> ")
-			continue
-		}
-		if line == "/exit" || line == "/quit" {
-			break
-		}
-		fmt.Fprintf(conn, "%s\n", line)
-	}
-
-	done := make(chan struct{})
-	go func() {
-		wg.Wait()
-		close(done)
-	}()
-	select {
-	case <-done:
-	case <-time.After(60 * time.Second):
 	}
 }
 
-// runRemote starts an interactive TUI via HTTP
-func runRemote(baseURL string) {
-	baseURL = strings.TrimRight(baseURL, "/")
-	scanner := bufio.NewScanner(os.Stdin)
-	fmt.Print("> ")
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" {
-			fmt.Print("> ")
-			continue
+func (e *LineEditor) historyPrev() {
+	all := e.hist.all()
+	if len(all) == 0 {
+		return
+	}
+	if e.histI == -1 {
+		e.histI = len(all) - 1
+	} else if e.histI > 0 {
+		e.histI--
+	}
+	e.buf = []rune(all[e.histI])
+	e.pos = len(e.buf)
+	e.redraw()
+}
+
+func (e *LineEditor) historyNext() {
+	if e.histI == -1 {
+		return
+	}
+	all := e.hist.all()
+	e.histI++
+	if e.histI >= len(all) {
+		e.histI = -1
+		e.buf = e.buf[:0]
+		e.pos = 0
+	} else {
+		e.buf = []rune(all[e.histI])
+		e.pos = len(e.buf)
+	}
+	e.redraw()
+}
+
+func (e *LineEditor) doCompletion() {
+	cmds := []string{"/help", "/exit", "/quit", "/clear", "/reconnect", "/connect ", "/remote ", "/local", "/prompt "}
+	prefix := string(e.buf)
+	for _, c := range cmds {
+		if strings.HasPrefix(c, prefix) && c != prefix {
+			e.buf = []rune(c)
+			e.pos = len(e.buf)
+			e.redraw()
+			return
 		}
-		if line == "/exit" || line == "/quit" {
-			break
-		}
-		body := fmt.Sprintf(`{"message":%q}`, line)
-		resp, err := http.Post(baseURL+"/api/v1/chat", "application/json", strings.NewReader(body))
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "error: %v\n", err)
-			fmt.Print("> ")
-			continue
-		}
-		var result map[string]interface{}
-		if resp.StatusCode == http.StatusOK {
-			json.NewDecoder(resp.Body).Decode(&result)
-		}
-		resp.Body.Close()
-		if content, ok := result["response"].(string); ok {
-			fmt.Println(content)
-		}
-		fmt.Print("> ")
 	}
 }
+
+func (e *LineEditor) redraw() {
+	fmt.Print("\r\033[K") // clear line
+	fmt.Print(string(e.buf))
+	if e.pos < len(e.buf) {
+		// move cursor back
+		skip := len(e.buf) - e.pos
+		fmt.Printf("\033[%dD", skip)
+	}
+}
+
+type History struct {
+	path   string
+	lines  []string
+	max    int
+	mu     sync.Mutex
+}
+
+func newHistory(path string, max int) History {
+	return History{path: path, max: max}
+}
+
+func (h *History) load() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	data, err := os.ReadFile(h.path)
+	if err != nil {
+		return
+	}
+	h.lines = strings.Split(strings.TrimSpace(string(data)), "\n")
+	if len(h.lines) > h.max {
+		h.lines = h.lines[len(h.lines)-h.max:]
+	}
+}
+
+func (h *History) save() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	data := strings.Join(h.lines, "\n") + "\n"
+	os.WriteFile(h.path, []byte(data), 0644)
+}
+
+func (h *History) add(line string) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	if len(h.lines) > 0 && h.lines[len(h.lines)-1] == line {
+		return
+	}
+	h.lines = append(h.lines, line)
+	if len(h.lines) > h.max {
+		h.lines = h.lines[len(h.lines)-h.max:]
+	}
+}
+
+func (h *History) all() []string {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	r := make([]string, len(h.lines))
+	copy(r, h.lines)
+	return r
+}
+
+// setRawMode sets stdin to raw mode (non-canonical, no echo).
+func setRawMode(fd int) (func(), error) {
+	if fd == 0 {
+		fd = int(os.Stdin.Fd())
+	}
+	if !isTerminal(fd) {
+		return func() {}, nil
+	}
+	var oldState syscall.Termios
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCGETS, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); err != 0 {
+		return func() {}, fmt.Errorf("tcgets: %v", err)
+	}
+	newState := oldState
+	newState.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON
+	newState.Oflag &^= syscall.OPOST
+	newState.Lflag &^= syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN
+	newState.Cflag &^= syscall.CSIZE | syscall.PARENB
+	newState.Cflag |= syscall.CS8
+	newState.Cc[syscall.VMIN] = 1
+	newState.Cc[syscall.VTIME] = 0
+	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCSETS, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 {
+		return func() {}, fmt.Errorf("tcset: %v", err)
+	}
+	return func() {
+		syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCSETS, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0)
+	}, nil
+}
+
+func isTerminal(fd int) bool {
+	var t syscall.Termios
+	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TCGETS, uintptr(unsafe.Pointer(&t)), 0, 0, 0)
+	return err == 0
+}

config/config.go

@@ -40,6 +40,19 @@ func DefaultConfig() types.Config {
 				{Name: "ollama", BaseURL: "http://localhost:11434", Model: "llama3", Adapter: "ollama", AdapterPath: "adapters/ollama.lua"},
 			},
 		},
+		InputProcessing: types.InputProcessingConfig{
+			Image: types.ImageProcessingConfig{
+				FallbackProvider: "",
+				FallbackModel:    "",
+				DescribePrompt:   "请详细描述这张图片的内容",
+				OCREnabled:       true,
+			},
+			Audio: types.AudioProcessingConfig{
+				FallbackProvider: "",
+				FallbackModel:    "",
+				DescribePrompt:   "请描述这段音频的内容",
+			},
+		},
 		Defaults: types.AgentConfig{
 			Image:  "homeagent/agent-base:latest",
 			LLMEndpoints: []string{"https://api.openai.com/v1"},

config/config.yaml

@@ -55,6 +55,17 @@ llm:
       adapter: "ollama"
       adapter_path: "adapters/ollama.lua"
 
+input_processing:
+  image:
+    fallback_provider: ""
+    fallback_model: ""
+    describe_prompt: "请详细描述这张图片的内容"
+    ocr_enabled: true
+  audio:
+    fallback_provider: ""
+    fallback_model: ""
+    describe_prompt: "请描述这段音频的内容"
+
 defaults:
   image: "homeagent/agent-base:latest"
   llm_endpoints:

internal/agent/api/provider.go

@@ -14,18 +14,41 @@ import (
 	luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
 )
 
+// ContentBlock 定义多模态内容块，用于图片/音频等非文本输入。
+type ContentBlock struct {
+	Type     string    `json:"type"`
+	Text     string    `json:"text,omitempty"`
+	ImageURL *ImageURL `json:"image_url,omitempty"`
+	AudioURL *AudioURL `json:"audio_url,omitempty"`
+}
+
+type ImageURL struct {
+	URL    string `json:"url"`
+	Detail string `json:"detail,omitempty"`
+}
+
+type AudioURL struct {
+	URL string `json:"url"`
+}
+
+// Message 表示对话消息。当 Blocks 不为空时 content 在 JSON 中序列化为数组（多模态格式）。
 type Message struct {
-	Role             string     `json:"role"`
-	Content          string     `json:"content"`
-	ReasoningContent string     `json:"reasoning_content,omitempty"`
-	ToolCallID       string     `json:"tool_call_id,omitempty"`
-	ToolCalls        []ToolCall `json:"-"`
+	Role             string         `json:"role"`
+	Content          string         `json:"content,omitempty"`
+	Blocks           []ContentBlock `json:"-"`
+	ReasoningContent string         `json:"reasoning_content,omitempty"`
+	ToolCallID       string         `json:"tool_call_id,omitempty"`
+	ToolCalls        []ToolCall     `json:"-"`
 }
 
 func (m Message) MarshalJSON() ([]byte, error) {
 	raw := map[string]interface{}{
-		"role":    m.Role,
-		"content": m.Content,
+		"role": m.Role,
+	}
+	if len(m.Blocks) > 0 {
+		raw["content"] = m.Blocks
+	} else {
+		raw["content"] = m.Content
 	}
 	if m.ReasoningContent != "" {
 		raw["reasoning_content"] = m.ReasoningContent

internal/agent/core/agent.go

@@ -96,6 +96,12 @@ type Agent struct {
 
 	// 启动时间
 	startTime time.Time
+
+	// 当前轮次的非文本媒体数据（图片/音频），供 describe_image 等工具访问
+	pendingMedia map[string]interface{}
+
+	// 非文本输入处理配置
+	inputCfg types.InputProcessingConfig
 }
 
 type AgentConfig struct {
@@ -123,6 +129,8 @@ type AgentConfig struct {
 	StageHost     *StageHost
 	EventBus      *events.Bus
 	ThinkingEnabled bool
+
+	InputProcessing types.InputProcessingConfig // 非文本输入处理配置
 }
 
 func New(cfg AgentConfig) *Agent {
@@ -166,6 +174,7 @@ func New(cfg AgentConfig) *Agent {
 		childResults:     make(map[string]string),
 		interceptCh:      make(chan string, 64),
 		thinkingEnabled:  cfg.ThinkingEnabled,
+		inputCfg:         cfg.InputProcessing,
 	}
 }
 
@@ -268,6 +277,9 @@ func (a *Agent) handleInput(evt *agentIO.InputEvent) {
 		}
 		a.processTextInput(evt, input)
 
+	case "image", "audio":
+		a.processMediaInput(evt)
+
 	case "event":
 		log.Printf("[agent] event from %s: %v", evt.Source, evt.Payload)
 
@@ -280,6 +292,123 @@ func (a *Agent) handleInput(evt *agentIO.InputEvent) {
 	}
 }
 
+// processMediaInput 处理图片/音频等非文本输入。
+// 将媒体数据附着到对话中，LLM 可通过 describe_image / transcribe_audio 等工具自主处理。
+func (a *Agent) processMediaInput(evt *agentIO.InputEvent) {
+	start := time.Now()
+	a.pendingMedia = evt.Payload
+	defer func() { a.pendingMedia = nil }()
+
+	a.currentOutputChannel = evt.OutputChannel
+	if a.currentOutputChannel == "" {
+		a.currentOutputChannel = evt.Source
+	}
+
+	blocks, fallback := a.mediaToBlocks(evt.Payload, evt.Type)
+
+	a.context.Append(ContextEvent{
+		Timestamp: start,
+		Source:    evt.Source,
+		Input:     fallback,
+	})
+
+	// stage 上下文携带 blocks，process() 会将其附着到 user message 上
+	stageCtx := a.stageCtxFromInput(fallback, evt.Source, "")
+	stageCtx.Extra = map[string]interface{}{
+		"media_blocks": blocks,
+		"media_type":   evt.Type,
+	}
+
+	a.publishEvent(events.EventRawInput, map[string]interface{}{
+		"content": evt.Payload,
+		"source":  evt.Source,
+	})
+
+	if a.runStage(sdk.StageOnInput, stageCtx) {
+		a.emitResponse(evt, *stageCtx.Response)
+		return
+	}
+
+	response, toolsUsed, err := a.process(fallback, stageCtx)
+	if err != nil {
+		log.Printf("[agent] process media error: %v", err)
+		resp := fmt.Sprintf("处理错误: %v", err)
+		a.emitResponse(evt, resp)
+		a.context.Append(ContextEvent{Timestamp: time.Now(), Source: "agent", Input: fallback, Response: resp})
+		return
+	}
+
+	elapsed := time.Since(start)
+	log.Printf("[agent] %s from %s → response (%dms, tools=%v)", evt.Type, evt.Source, elapsed.Milliseconds(), toolsUsed)
+
+	a.context.Append(ContextEvent{
+		Timestamp: time.Now(),
+		Source:    "agent",
+		Input:     fallback,
+		Response:  response,
+		ToolsUsed: toolsUsed,
+	})
+
+	archived := a.context.Prune(response, a.maxContextSize, a.docStore)
+	if archived > 0 {
+		log.Printf("[agent] pruned %d low-relevance events to document memory", archived)
+	}
+
+	a.emitResponse(evt, response)
+}
+
+// mediaToBlocks 将媒体 payload 转为多模态 ContentBlock 数组和纯文本 fallback。
+func (a *Agent) mediaToBlocks(payload map[string]interface{}, mediaType string) ([]agentAPI.ContentBlock, string) {
+	data, _ := payload["data"].(string)
+	mime, _ := payload["mime"].(string)
+	url, _ := payload["url"].(string)
+	alt, _ := payload["alt"].(string)
+	if alt == "" {
+		alt = fmt.Sprintf("[用户上传了%s]", mediaType)
+	}
+
+	var blocks []agentAPI.ContentBlock
+
+	// 文本描述块
+	desc := ""
+	switch mediaType {
+	case "image":
+		desc = a.inputCfg.Image.DescribePrompt
+		if desc == "" {
+			desc = "用户上传了一张图片，请使用 describe_image 工具查看详情。"
+		}
+	case "audio":
+		desc = a.inputCfg.Audio.DescribePrompt
+		if desc == "" {
+			desc = "用户上传了一段音频，请使用 transcribe_audio 工具查看内容。"
+		}
+	}
+	blocks = append(blocks, agentAPI.ContentBlock{Type: "text", Text: desc})
+
+	if data != "" || url != "" {
+		imgURL := url
+		if data != "" {
+			if mime == "" {
+				mime = "image/png"
+			}
+			imgURL = "data:" + mime + ";base64," + data
+		}
+		if mediaType == "image" {
+			blocks = append(blocks, agentAPI.ContentBlock{
+				Type: "image_url",
+				ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: "auto"},
+			})
+		} else if mediaType == "audio" {
+			blocks = append(blocks, agentAPI.ContentBlock{
+				Type: "audio_url",
+				AudioURL: &agentAPI.AudioURL{URL: imgURL},
+			})
+		}
+	}
+
+	return blocks, alt
+}
+
 func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
 	start := time.Now()
 
@@ -418,6 +547,12 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
 	tools := a.buildToolDefs()
 
 	msgs := a.buildMessages(sysPrompt, input)
+	// 如果 stageCtx 携带多模态 blocks，附着到 user message 上
+	if blocks, ok := stageCtx.Extra["media_blocks"].([]agentAPI.ContentBlock); ok && len(blocks) > 0 {
+		if len(msgs) > 0 {
+			msgs[len(msgs)-1].Blocks = blocks
+		}
+	}
 
 	log.Printf("[agent] tool call loop start, %d tools, %d context events, personality=%t, docs=%d",
 		len(tools), a.context.Len(),
@@ -616,6 +751,12 @@ func (a *Agent) executeToolCall(tc agentAPI.ToolCall) string {
 		return a.executeChildResultTool(tc)
 	case strings.HasPrefix(tc.Name, "llm_"):
 		return a.executeLLMTool(tc)
+	case tc.Name == "describe_image":
+		return a.executeDescribeImage(tc)
+	case tc.Name == "transcribe_audio":
+		return a.executeTranscribeAudio(tc)
+	case tc.Name == "ocr_image":
+		return a.executeOCRImage(tc)
 	}
 
 	// 插件工具（通过 SDK RegisterTool 注册）
@@ -1475,6 +1616,65 @@ func (a *Agent) buildToolDefs() []interface{} {
 		},
 	})
 
+	// 媒体处理工具：仅当本轮有未处理的媒体数据时注册
+	if a.pendingMedia != nil {
+		tools = append(tools, map[string]interface{}{
+			"type": "function",
+			"function": map[string]interface{}{
+				"name":        "describe_image",
+				"description": "描述当前用户上传的图片内容。使用配置的多模态模型或默认 LLM 进行识别。调用此工具后你将获得图片的详细文字描述。",
+				"parameters": map[string]interface{}{
+					"type": "object",
+					"properties": map[string]interface{}{
+						"provider": map[string]interface{}{
+							"type":        "string",
+							"description": "可选：用于图片描述的 LLM 源名称，不填则使用默认模型",
+						},
+						"detail": map[string]interface{}{
+							"type":        "string",
+							"description": "描述详细程度: high / low / auto",
+							"default":     "high",
+						},
+					},
+				},
+			},
+		})
+		tools = append(tools, map[string]interface{}{
+			"type": "function",
+			"function": map[string]interface{}{
+				"name":        "transcribe_audio",
+				"description": "转写当前用户上传的音频内容为文字。使用配置的多模态模型或默认 LLM 进行语音识别。",
+				"parameters": map[string]interface{}{
+					"type": "object",
+					"properties": map[string]interface{}{
+						"provider": map[string]interface{}{
+							"type":        "string",
+							"description": "可选：用于音频转写的 LLM 源名称，不填则使用默认模型",
+						},
+					},
+				},
+			},
+		})
+		if a.inputCfg.Image.OCREnabled {
+			tools = append(tools, map[string]interface{}{
+				"type": "function",
+				"function": map[string]interface{}{
+					"name":        "ocr_image",
+					"description": "对当前用户上传的图片执行 OCR 文字识别，提取图片中的文字内容。适用于截图、文档照片、菜单等场景。",
+					"parameters": map[string]interface{}{
+						"type": "object",
+						"properties": map[string]interface{}{
+							"language": map[string]interface{}{
+								"type":        "string",
+								"description": "OCR 语言（如 chi_sim+eng），默认自动",
+							},
+						},
+					},
+				},
+			})
+		}
+	}
+
 	return tools
 }
 
@@ -2091,6 +2291,155 @@ func (a *Agent) executeLLMTool(tc agentAPI.ToolCall) string {
 	}
 }
 
+// executeDescribeImage 调用多模态模型描述当前图片。
+func (a *Agent) executeDescribeImage(tc agentAPI.ToolCall) string {
+	if a.pendingMedia == nil {
+		return "没有待处理的图片数据"
+	}
+	data, _ := a.pendingMedia["data"].(string)
+	mime, _ := a.pendingMedia["mime"].(string)
+	url, _ := a.pendingMedia["url"].(string)
+	if data == "" && url == "" {
+		return "图片数据为空"
+	}
+
+	providerName, _ := tc.Arguments["provider"].(string)
+	detail, _ := tc.Arguments["detail"].(string)
+	if detail == "" {
+		detail = "high"
+	}
+
+	p := a.providerManager.Get(providerName)
+	if p == nil {
+		p = a.provider
+	}
+
+	prompt := a.inputCfg.Image.DescribePrompt
+	if prompt == "" {
+		prompt = "请详细描述这张图片的内容，包括其中的文字、物体、人物、场景等信息。"
+	}
+
+	imgURL := url
+	if data != "" {
+		if mime == "" {
+			mime = "image/png"
+		}
+		imgURL = "data:" + mime + ";base64," + data
+	}
+
+	msg := agentAPI.Message{
+		Role: "user",
+		Blocks: []agentAPI.ContentBlock{
+			{Type: "text", Text: prompt},
+			{Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: detail}},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+	resp, err := p.Chat(ctx, &agentAPI.CompletionRequest{
+		Messages:  []agentAPI.Message{msg},
+		MaxTokens: 2048,
+	})
+	if err != nil {
+		return fmt.Sprintf("图片描述失败: %v", err)
+	}
+	return fmt.Sprintf("[图片描述] %s", resp.Content)
+}
+
+// executeTranscribeAudio 调用多模态模型转写/描述当前音频。
+func (a *Agent) executeTranscribeAudio(tc agentAPI.ToolCall) string {
+	if a.pendingMedia == nil {
+		return "没有待处理的音频数据"
+	}
+	data, _ := a.pendingMedia["data"].(string)
+	mime, _ := a.pendingMedia["mime"].(string)
+	url, _ := a.pendingMedia["url"].(string)
+	if data == "" && url == "" {
+		return "音频数据为空"
+	}
+
+	providerName, _ := tc.Arguments["provider"].(string)
+	p := a.providerManager.Get(providerName)
+	if p == nil {
+		p = a.provider
+	}
+
+	prompt := a.inputCfg.Audio.DescribePrompt
+	if prompt == "" {
+		prompt = "请转写这段音频的内容。"
+	}
+
+	audURL := url
+	if data != "" {
+		if mime == "" {
+			mime = "audio/wav"
+		}
+		audURL = "data:" + mime + ";base64," + data
+	}
+
+	msg := agentAPI.Message{
+		Role: "user",
+		Blocks: []agentAPI.ContentBlock{
+			{Type: "text", Text: prompt},
+			{Type: "audio_url", AudioURL: &agentAPI.AudioURL{URL: audURL}},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+	resp, err := p.Chat(ctx, &agentAPI.CompletionRequest{
+		Messages:  []agentAPI.Message{msg},
+		MaxTokens: 2048,
+	})
+	if err != nil {
+		return fmt.Sprintf("音频转写失败: %v", err)
+	}
+	return fmt.Sprintf("[音频转写] %s", resp.Content)
+}
+
+// executeOCRImage 对图片执行 OCR 文字识别（通过多模态模型实现）。
+func (a *Agent) executeOCRImage(tc agentAPI.ToolCall) string {
+	if a.pendingMedia == nil {
+		return "没有待处理的图片数据"
+	}
+	data, _ := a.pendingMedia["data"].(string)
+	mime, _ := a.pendingMedia["mime"].(string)
+	url, _ := a.pendingMedia["url"].(string)
+	if data == "" && url == "" {
+		return "图片数据为空"
+	}
+
+	p := a.provider
+
+	imgURL := url
+	if data != "" {
+		if mime == "" {
+			mime = "image/png"
+		}
+		imgURL = "data:" + mime + ";base64," + data
+	}
+
+	msg := agentAPI.Message{
+		Role: "user",
+		Blocks: []agentAPI.ContentBlock{
+			{Type: "text", Text: "请识别这张图片中的所有文字内容，按原文输出。仅输出文字本身，不要添加额外描述。"},
+			{Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: "high"}},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+	resp, err := p.Chat(ctx, &agentAPI.CompletionRequest{
+		Messages:  []agentAPI.Message{msg},
+		MaxTokens: 4096,
+	})
+	if err != nil {
+		return fmt.Sprintf("OCR 识别失败: %v", err)
+	}
+	return fmt.Sprintf("[OCR 结果] %s", resp.Content)
+}
+
 // runStage — 运行阶段管道，若插件 Response 被设置则返回 true（短路）
 func (a *Agent) runStage(stage sdk.Stage, ctx *sdk.StageContext) bool {
 	if a.stageHost == nil {

internal/sdk/plugin.go

@@ -120,57 +120,57 @@ func New(name string, iom *agentIO.IOManager, eventBus *events.Bus, mem MemoryAP
 
 // === IO 双通道 ===
 
-func (s *PluginSDK) InjectInput(source, channel string, payload map[string]interface{}) {
+// InjectInput 注入任意类型的输入事件。
+// eventType 可选值: "text", "event", "command", 或插件自定义类型。
+// payload 可包含 "content" (文本), "image" (图片), "file" (文件), "audio" (音频) 等字段。
+func (s *PluginSDK) InjectInput(source, channel, eventType string, payload map[string]interface{}) {
 	if s.iom != nil {
-		s.iom.InjectInputTo(source, channel, "text", payload)
+		s.iom.InjectInputTo(source, channel, eventType, payload)
 	}
 }
 
-func (s *PluginSDK) InjectInterrupt(source, channel string, payload map[string]interface{}) {
+// InjectInputSync 注入任意类型输入并同步等待响应。
+func (s *PluginSDK) InjectInputSync(source, channel, eventType string, payload map[string]interface{}) *agentIO.OutputEvent {
 	if s.iom != nil {
-		p := payload
-		if p == nil {
-			p = map[string]interface{}{}
+		return s.iom.InjectInputSyncTo(source, channel, eventType, payload)
+	}
+	return nil
+}
+
+// InjectInterrupt 向中断通道注入任意类型的输入事件，可打断当前 LLM 处理。
+// eventType 会被写入 payload["type"]。
+func (s *PluginSDK) InjectInterrupt(source, channel, eventType string, payload map[string]interface{}) {
+	if s.iom != nil {
+		if payload == nil {
+			payload = map[string]interface{}{}
 		}
-		if _, ok := p["type"]; !ok {
-			p["type"] = "text"
-		}
-		s.iom.InjectInterrupt(source, channel, p)
+		payload["type"] = eventType
+		s.iom.InjectInterrupt(source, channel, payload)
 	}
 }
 
+// 以下 InjectText* / InjectInterruptText 为快捷方式，等价于调用对应的泛型方法并传入 "text" 类型。
+
 func (s *PluginSDK) InjectText(source, channel, text string) {
-	if s.iom != nil {
-		s.iom.InjectTextTo(source, channel, text)
-	}
+	s.InjectInput(source, channel, "text", map[string]interface{}{"content": text})
 }
 
 // InjectTextNoMemory 注入文本输入（不产生记忆）。适用于健康检查等无需记忆碎片的场景。
 func (s *PluginSDK) InjectTextNoMemory(source, channel, text string) {
-	if s.iom != nil {
-		s.iom.InjectTextNoMemoryTo(source, channel, text)
-	}
+	s.InjectInput(source, channel, "text", map[string]interface{}{"content": text, "no_memory": true})
 }
 
 func (s *PluginSDK) InjectTextSync(source, channel, text string) *agentIO.OutputEvent {
-	if s.iom != nil {
-		return s.iom.InjectTextSyncTo(source, channel, text)
-	}
-	return nil
+	return s.InjectInputSync(source, channel, "text", map[string]interface{}{"content": text})
 }
 
 // InjectTextSyncNoMemory 注入文本输入（同步等待，不产生记忆）。
 func (s *PluginSDK) InjectTextSyncNoMemory(source, channel, text string) *agentIO.OutputEvent {
-	if s.iom != nil {
-		return s.iom.InjectTextSyncNoMemoryTo(source, channel, text)
-	}
-	return nil
+	return s.InjectInputSync(source, channel, "text", map[string]interface{}{"content": text, "no_memory": true})
 }
 
 func (s *PluginSDK) InjectInterruptText(source, channel, text string) {
-	if s.iom != nil {
-		s.iom.InjectInterruptText(source, channel, text)
-	}
+	s.InjectInterrupt(source, channel, "text", map[string]interface{}{"content": text})
 }
 
 func (s *PluginSDK) OutputChan() <-chan *agentIO.OutputEvent {

pkg/types/types.go

@@ -117,11 +117,30 @@ type LLMConfig struct {
 	Sources         []LLMSource `json:"sources,omitempty"`
 }
 
+type ImageProcessingConfig struct {
+	FallbackProvider string `json:"fallback_provider" yaml:"fallback_provider"`
+	FallbackModel    string `json:"fallback_model" yaml:"fallback_model"`
+	DescribePrompt   string `json:"describe_prompt" yaml:"describe_prompt"`
+	OCREnabled       bool   `json:"ocr_enabled" yaml:"ocr_enabled"`
+}
+
+type AudioProcessingConfig struct {
+	FallbackProvider string `json:"fallback_provider" yaml:"fallback_provider"`
+	FallbackModel    string `json:"fallback_model" yaml:"fallback_model"`
+	DescribePrompt   string `json:"describe_prompt" yaml:"describe_prompt"`
+}
+
+type InputProcessingConfig struct {
+	Image ImageProcessingConfig `json:"image" yaml:"image"`
+	Audio AudioProcessingConfig `json:"audio" yaml:"audio"`
+}
+
 type Config struct {
-	Daemon    DaemonConfig    `json:"daemon"`
-	LLM       LLMConfig       `json:"llm"`
-	Defaults  AgentConfig     `json:"defaults"`
-	Agents    []AgentConfig   `json:"agents"`
+	Daemon          DaemonConfig          `json:"daemon"`
+	LLM             LLMConfig             `json:"llm"`
+	InputProcessing InputProcessingConfig `json:"input_processing"`
+	Defaults        AgentConfig           `json:"defaults"`
+	Agents          []AgentConfig         `json:"agents"`
 }
 
 type DaemonConfig struct {