feat(multimodal): 内置多模态感知插件 + process.go 原生支持 tool message 多模态块

【新插件 internal/plugins/multimodal】
- see_picture(path): 读取本地图片/URL，base64 注入 image_url block，
  模型在下一轮 LLM 请求的 tool message 里直接看到图（1024×1024 图约 8500 token）。
  自动识别 MIME，限 3MB 防爆 context。
- see_video(path, frames): ffmpeg 提取关键帧，多帧作为 image_url block 注入。
  默认 4 帧，最大 10 帧，每帧限 2MB。
- listen(path): 读取音频文件，转为 audio_url block 注入，支持 mp3/wav/ogg/m4a。
  限 5MB。

【内核多模态 tool message 支持】
- agent/api 新增 ToolOutput 类型（为后续 handler 直接返回 blocks 预留）
- SDK 公共层新增 ContentBlock/ImageURL/AudioURL（OpenAI 多模态格式）
- IOManager 新增 SetToolBlocks/ConsumeToolBlocks（interface{} 避免循环依赖）
- PluginSDK.SetToolBlocks(blocks) 插件工具调用后注入 blocks
- ioAdapter 桥接 IOInjector.SetToolBlocks
- process.go 工具执行后消费 pending blocks → 追加到 tool message 的 Blocks 字段
  → MarshalJSON 输出 content 数组格式 → LLM 看到图/音频

【验证】
multimodal_see_picture 注入 1024×1024 PNG 后 llmsproxy 统计：
  prompt_tokens=44407（含 ~8500 image token），模型正确描述了图片内容。

internal/agent/api/provider.go

@@ -1230,6 +1230,15 @@ func getFloat(m map[string]interface{}, key string) float64 {
 	return 0
 }
 
+// ToolOutput 是工具 handler 返回的结构化结果，支持多模态内容。
+// 返回 string 时等价于 ToolOutput{Text: result}。
+type ToolOutput struct {
+	Text   string         `json:"text"`                         // LLM 看到的文字描述
+	Blocks []ContentBlock `json:"blocks,omitempty"`             // 附加的多模态块（image_url/audio_url），追加到 tool message
+}
+
+func (t ToolOutput) String() string { return t.Text }
+
 // truncateForLog 诊断日志用截断。
 func truncateForLog(s string, n int) string {
 	if len(s) <= n {

internal/agent/core/agent.go

@@ -98,6 +98,8 @@ type Agent struct {
 	// 模型思考模式（thinking/reasoning）
 	thinkingEnabled bool
 
+	pendingToolBlocks []interface{} // 插件工具通过 SetToolBlocks 注入的多模态块（type agentAPI.ContentBlock），process.go 消费后追加到 tool message
+
 	// 启动时间
 	startTime time.Time
 

internal/agent/core/process.go

@@ -12,6 +12,7 @@ import (
 	agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
 	"gitcode.com/JianFeeeee/HomeAgent/internal/events"
 	sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
+	pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
 )
 
 func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response string, toolsUsed []string, toolResults []ToolResultItem, err error) {
@@ -311,7 +312,31 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
 				contentOnce = false
 			}
 			msgs = append(msgs, agentAPI.Message{Role: "assistant", Content: msgContent, ReasoningContent: resp.ReasoningContent, ToolCalls: []agentAPI.ToolCall{tc}})
-			msgs = append(msgs, agentAPI.Message{Role: "tool", ToolCallID: tc.ID, Content: result})
+
+			// 多模态工具结果：插件通过 SDK.SetToolBlocks 注入 image_url/audio_url block，
+			// process.go 拾起并追加到 tool message 的 content 数组（OpenAI 多模态格式），
+			// 让下一轮 LLM 请求在 tool message 里看到图/音频。
+			toolMsg := agentAPI.Message{Role: "tool", ToolCallID: tc.ID, Content: result}
+			if rawBlocks := a.io.ConsumeToolBlocks(); len(rawBlocks) > 0 {
+				var blocks []agentAPI.ContentBlock
+				for _, b := range rawBlocks {
+					if cb, ok := b.(pubsdk.ContentBlock); ok {
+						// 跨包类型拷贝（pubsdk.ContentBlock → agentAPI.ContentBlock）
+						block := agentAPI.ContentBlock{Type: cb.Type, Text: cb.Text}
+						if cb.ImageURL != nil {
+							block.ImageURL = &agentAPI.ImageURL{URL: cb.ImageURL.URL, Detail: cb.ImageURL.Detail}
+						}
+						if cb.AudioURL != nil {
+							block.AudioURL = &agentAPI.AudioURL{URL: cb.AudioURL.URL}
+						}
+						blocks = append(blocks, block)
+					}
+				}
+				if len(blocks) > 0 {
+					toolMsg.Blocks = blocks
+				}
+			}
+			msgs = append(msgs, toolMsg)
 
 			a.publishEvent(events.EventToolCall, map[string]interface{}{
 				"tool":    tc.Name,

internal/agent/io/channel.go

@@ -103,6 +103,11 @@ type IOManager struct {
 	outputCh      chan *OutputEvent
 	nextReqID     int64
 	inputChannels map[string]ChannelDef
+
+	// toolBlocks：插件工具注入多模态内容块，process.go 在下一条 tool message 时消费。
+	// 用 interface{}[] 避免 import api.ContentBlock 导致的循环依赖。
+	toolBlocksMu      sync.Mutex
+	toolPendingBlocks []interface{}
 }
 
 func NewIOManager() *IOManager {
@@ -669,3 +674,20 @@ func (d *GPIODevice) Tools() []ToolDef {
 func (d *GPIODevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
 	return map[string]interface{}{"device": d.name, "tool": tool, "status": "ok"}, nil
 }
+
+// SetToolBlocks 插件工具调用时注入多模态内容块（image_url/audio_url 等），
+// 下一条 tool message 追加这些块到 content 数组（OpenAI 多模态格式）。
+func (m *IOManager) SetToolBlocks(blocks []interface{}) {
+	m.toolBlocksMu.Lock()
+	m.toolPendingBlocks = blocks
+	m.toolBlocksMu.Unlock()
+}
+
+// ConsumeToolBlocks 返回并清空 pending blocks，process.go 在 append tool message 时调用。
+func (m *IOManager) ConsumeToolBlocks() []interface{} {
+	m.toolBlocksMu.Lock()
+	blocks := m.toolPendingBlocks
+	m.toolPendingBlocks = nil
+	m.toolBlocksMu.Unlock()
+	return blocks
+}

internal/plugins/agentcli/plugin_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
+	sdkpub "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
 )
 
 type toolCapture struct {
@@ -415,6 +416,10 @@ func (c *injectCapture) InjectTextNoMemory(source, channel, text string) {
 	c.texts = append(c.texts, text)
 	c.mu.Unlock()
 }
+
+func (c *injectCapture) SetToolBlocks(blocks []sdkpub.ContentBlock) {
+	// 测试桩：忽略多模态块
+}
 func (c *injectCapture) InjectInputSync(source, channel, text string) string { return "" }
 
 func (c *injectCapture) snapshot() []string {

internal/plugins/all.go

@@ -10,6 +10,7 @@ import (
 	_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/files"
 	_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/healthcheck"
 	_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/mcp"
+	_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/multimodal"
 	_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/pluginmgr"
 	_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/remotedevice"
 	_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/skillmgr"

internal/plugins/multimodal/plugin.go

@@ -0,0 +1,279 @@
+package multimodal
+
+import (
+	"encoding/base64"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
+	sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
+	pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
+)
+
+func init() {
+	plugin.RegisterPluginMeta("multimodal", "多模态感知", "Multimodal Perception")
+	plugin.RegisterFactory("multimodal", NewPluginFactory)
+}
+
+func NewPluginFactory(name string, config map[string]interface{}) (sdk.Plugin, error) {
+	return &Plugin{name: name}, nil
+}
+
+type Plugin struct {
+	name string
+	sdk  *sdk.PluginSDK
+}
+
+func (p *Plugin) Name() string { return p.name }
+
+func (p *Plugin) Start(s *sdk.PluginSDK) error {
+	p.sdk = s
+	tp := p.name + "_"
+
+	s.RegisterTool(tp+"see_picture", sdk.ToolDef{
+		Name:        tp + "see_picture",
+		Description: "让模型看到一张图片。输入文件路径或 URL，图片以 image_url 格式注入后续对话，模型可看到并描述/分析图片内容。",
+		Parameters: map[string]interface{}{
+			"type": "object",
+			"properties": map[string]interface{}{
+				"path": map[string]interface{}{
+					"type":        "string",
+					"description": "图片的本地文件路径或 HTTP URL",
+				},
+			},
+			"required": []string{"path"},
+		},
+	}, p.handleSeePicture)
+
+	s.RegisterTool(tp+"see_video", sdk.ToolDef{
+		Name:        tp + "see_video",
+		Description: "让模型看到一段视频的关键帧。输入视频文件路径，ffmpeg 提取 N 帧作为 image_url 注入后续对话，模型可分析视频内容。需要 ffmpeg 已安装。",
+		Parameters: map[string]interface{}{
+			"type": "object",
+			"properties": map[string]interface{}{
+				"path": map[string]interface{}{
+					"type":        "string",
+					"description": "视频的本地文件路径",
+				},
+				"frames": map[string]interface{}{
+					"type":        "integer",
+					"description": "提取关键帧数量（默认 4，最大 10）",
+				},
+			},
+			"required": []string{"path"},
+		},
+	}, p.handleSeeVideo)
+
+	s.RegisterTool(tp+"listen", sdk.ToolDef{
+		Name:        tp + "listen",
+		Description: "让模型听到一段音频。输入音频文件路径（mp3/wav/ogg/m4a），音频注入后续对话，支持音频的模型可识别语音/声音内容。",
+		Parameters: map[string]interface{}{
+			"type": "object",
+			"properties": map[string]interface{}{
+				"path": map[string]interface{}{
+					"type":        "string",
+					"description": "音频文件路径",
+				},
+			},
+			"required": []string{"path"},
+		},
+	}, p.handleListen)
+
+	log.Printf("[%s] multimodal perception tools registered", p.name)
+	return nil
+}
+
+func (p *Plugin) Stop() error { return nil }
+
+// ── see_picture ──────────────────────────────────────────────────
+
+func (p *Plugin) handleSeePicture(args map[string]interface{}) (interface{}, error) {
+	path := getArgStr(args, "path")
+	if path == "" {
+		return "path is required", nil
+	}
+
+	var dataURL string
+	var mime string
+
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		// 远程 URL：直接用作 image URL，不下载
+		dataURL = path
+		mime = "image/png"
+	} else {
+		// 本地文件
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			return fmt.Sprintf("文件不存在: %s", path), nil
+		}
+		ext := strings.ToLower(filepath.Ext(path))
+		switch ext {
+		case ".jpg", ".jpeg":
+			mime = "image/jpeg"
+		case ".gif":
+			mime = "image/gif"
+		case ".webp":
+			mime = "image/webp"
+		default:
+			mime = "image/png"
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Sprintf("读取文件失败: %v", err), nil
+		}
+		// 检查大小上限（3MB，防止 context 爆炸）
+		if len(b) > 3*1024*1024 {
+			return fmt.Sprintf("图片过大（%d bytes，超过 3MB），无法注入上下文", len(b)), nil
+		}
+		dataURL = "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b)
+	}
+
+	// 注入多模态块：模型下一轮可看到图片
+	p.sdk.SetToolBlocks([]pubsdk.ContentBlock{
+		{Type: "image_url", ImageURL: &pubsdk.ImageURL{URL: dataURL, Detail: "auto"}},
+	})
+
+	text := fmt.Sprintf("[已将图片注入后续对话] %s", path)
+	return text, nil
+}
+
+// ── see_video ────────────────────────────────────────────────────
+
+func (p *Plugin) handleSeeVideo(args map[string]interface{}) (interface{}, error) {
+	path := getArgStr(args, "path")
+	if path == "" {
+		return "path is required", nil
+	}
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return fmt.Sprintf("文件不存在: %s", path), nil
+	}
+
+	// 检查 ffmpeg
+	ffmpegPath := ""
+	for _, c := range []string{"ffmpeg", "/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg"} {
+		if _, err := os.Stat(c); err == nil {
+			ffmpegPath = c
+			break
+		}
+	}
+	if ffmpegPath == "" {
+		if _, err := exec.LookPath("ffmpeg"); err == nil {
+			ffmpegPath = "ffmpeg"
+		} else {
+			return "ffmpeg 未安装，无法提取视频关键帧。请先安装: apt install ffmpeg", nil
+		}
+	}
+
+	nFrames := 4
+	if n, ok := args["frames"].(float64); ok && n > 0 {
+		nFrames = int(n)
+		if nFrames > 10 {
+			nFrames = 10
+		}
+	}
+
+	// 用 ffmpeg 提取关键帧
+	tmpDir, err := os.MkdirTemp("", "mm_video_*")
+	if err != nil {
+		return fmt.Sprintf("创建临时目录失败: %v", err), nil
+	}
+	defer os.RemoveAll(tmpDir)
+
+	outPattern := filepath.Join(tmpDir, "frame_%03d.jpg")
+	cmd := exec.Command(ffmpegPath, "-i", path, "-vf", fmt.Sprintf("fps=1/%d", nFrames),
+		"-q:v", "5", outPattern)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Sprintf("ffmpeg 提取帧失败: %v\n%s", err, string(out)), nil
+	}
+
+	// 读取提取的帧
+	entries, _ := os.ReadDir(tmpDir)
+	var blocks []pubsdk.ContentBlock
+	for i, entry := range entries {
+		if strings.HasSuffix(entry.Name(), ".jpg") {
+			b, err := os.ReadFile(filepath.Join(tmpDir, entry.Name()))
+			if err != nil {
+				continue
+			}
+			if len(b) > 2*1024*1024 {
+				continue // 跳过过大帧
+			}
+			dURL := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(b)
+			blocks = append(blocks, pubsdk.ContentBlock{
+				Type: "image_url",
+				ImageURL: &pubsdk.ImageURL{URL: dURL, Detail: "low"},
+			})
+			if i >= 9 { // 最多 10 帧
+				break
+			}
+		}
+	}
+
+	if len(blocks) == 0 {
+		return "视频中未提取到有效帧", nil
+	}
+
+	// 全部帧注入（一次 SetToolBlocks 调用，下一轮 LLM 可看到）
+	p.sdk.SetToolBlocks(blocks)
+
+	text := fmt.Sprintf("[已将 %d 个视频关键帧注入后续对话] %s", len(blocks), path)
+	return text, nil
+}
+
+// ── listen ───────────────────────────────────────────────────────
+
+func (p *Plugin) handleListen(args map[string]interface{}) (interface{}, error) {
+	path := getArgStr(args, "path")
+	if path == "" {
+		return "path is required", nil
+	}
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return fmt.Sprintf("文件不存在: %s", path), nil
+	}
+
+	ext := strings.ToLower(filepath.Ext(path))
+	var mime string
+	switch ext {
+	case ".mp3":
+		mime = "audio/mpeg"
+	case ".wav":
+		mime = "audio/wav"
+	case ".ogg":
+		mime = "audio/ogg"
+	case ".m4a", ".aac":
+		mime = "audio/mp4"
+	default:
+		mime = "audio/ogg" // 默认
+	}
+
+	// 检查大小（5MB 限制，避免上下文爆炸）
+	info, _ := os.Stat(path)
+	if info != nil && info.Size() > 5*1024*1024 {
+		return fmt.Sprintf("音频过大（%d bytes，超过 5MB），无法注入上下文", info.Size()), nil
+	}
+
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Sprintf("读取音频文件失败: %v", err), nil
+	}
+	dataURL := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b)
+
+	p.sdk.SetToolBlocks([]pubsdk.ContentBlock{
+		{Type: "audio_url", AudioURL: &pubsdk.AudioURL{URL: dataURL}},
+	})
+
+	text := fmt.Sprintf("[已将音频注入后续对话] %s（%s，%.1fKB）", path, mime, float64(len(b))/1024)
+	return text, nil
+}
+
+// ── helpers ──────────────────────────────────────────────────────
+
+func getArgStr(args map[string]interface{}, key string) string {
+	if v, ok := args[key].(string); ok {
+		return v
+	}
+	return ""
+}

internal/sdk/plugin.go

@@ -339,3 +339,27 @@ func (s *PluginSDK) Subscribe(eventType events.EventType, handler events.Handler
 	}
 	return func() {}
 }
+
+// SetToolBlocks 桥接到 IOManager：插件工具注入多模态块，process.go 消费。
+func (a ioAdapter) SetToolBlocks(blocks []pubsdk.ContentBlock) {
+	if a.iom == nil {
+		return
+	}
+	ifaces := make([]interface{}, len(blocks))
+	for i, b := range blocks {
+		ifaces[i] = b
+	}
+	a.iom.SetToolBlocks(ifaces)
+}
+
+// SetToolBlocks 注入多模态内容块（图片/音频），内核在下一条 tool message
+// 的 content 数组里带上这些块，让模型在后续轮次看到图/听到音频。
+func (s *PluginSDK) SetToolBlocks(blocks []pubsdk.ContentBlock) {
+	if s.iom != nil {
+		ifaces := make([]interface{}, len(blocks))
+		for i, b := range blocks {
+			ifaces[i] = b
+		}
+		s.iom.SetToolBlocks(ifaces)
+	}
+}