From b777322b9562e7ae6a9dac47bdbf48b476cc6491 Mon Sep 17 00:00:00 2001 From: JianFeeeee Date: Thu, 27 Aug 2026 08:39:21 +0800 Subject: [PATCH] =?UTF-8?q?feat(multimodal):=20=E5=86=85=E7=BD=AE=E5=A4=9A?= =?UTF-8?q?=E6=A8=A1=E6=80=81=E6=84=9F=E7=9F=A5=E6=8F=92=E4=BB=B6=20+=20pr?= =?UTF-8?q?ocess.go=20=E5=8E=9F=E7=94=9F=E6=94=AF=E6=8C=81=20tool=20messag?= =?UTF-8?q?e=20=E5=A4=9A=E6=A8=A1=E6=80=81=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 【新插件 internal/plugins/multimodal】 - see_picture(path): 读取本地图片/URL，base64 注入 image_url block， 模型在下一轮 LLM 请求的 tool message 里直接看到图（1024×1024 图约 8500 token）。 自动识别 MIME，限 3MB 防爆 context。 - see_video(path, frames): ffmpeg 提取关键帧，多帧作为 image_url block 注入。 默认 4 帧，最大 10 帧，每帧限 2MB。 - listen(path): 读取音频文件，转为 audio_url block 注入，支持 mp3/wav/ogg/m4a。 限 5MB。 【内核多模态 tool message 支持】 - agent/api 新增 ToolOutput 类型（为后续 handler 直接返回 blocks 预留） - SDK 公共层新增 ContentBlock/ImageURL/AudioURL（OpenAI 多模态格式） - IOManager 新增 SetToolBlocks/ConsumeToolBlocks（interface{} 避免循环依赖） - PluginSDK.SetToolBlocks(blocks) 插件工具调用后注入 blocks - ioAdapter 桥接 IOInjector.SetToolBlocks - process.go 工具执行后消费 pending blocks → 追加到 tool message 的 Blocks 字段 → MarshalJSON 输出 content 数组格式 → LLM 看到图/音频 【验证】 multimodal_see_picture 注入 1024×1024 PNG 后 llmsproxy 统计： prompt_tokens=44407（含 ~8500 image token），模型正确描述了图片内容。 --- internal/agent/api/provider.go | 9 + internal/agent/core/agent.go | 2 + internal/agent/core/process.go | 27 ++- internal/agent/io/channel.go | 22 ++ internal/plugins/agentcli/plugin_test.go | 5 + internal/plugins/all.go | 1 + internal/plugins/multimodal/plugin.go | 279 +++++++++++++++++++++++ internal/sdk/plugin.go | 24 ++ 8 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 internal/plugins/multimodal/plugin.go diff --git a/internal/agent/api/provider.go b/internal/agent/api/provider.go index 034eaf0..4953f25 100644 --- a/internal/agent/api/provider.go +++ b/internal/agent/api/provider.go @@ -1230,6 +1230,15 @@ func getFloat(m map[string]interface{}, key string) float64 { return 0 } +// ToolOutput 是工具 handler 返回的结构化结果，支持多模态内容。 +// 返回 string 时等价于 ToolOutput{Text: result}。 +type ToolOutput struct { + Text string `json:"text"` // LLM 看到的文字描述 + Blocks []ContentBlock `json:"blocks,omitempty"` // 附加的多模态块（image_url/audio_url），追加到 tool message +} + +func (t ToolOutput) String() string { return t.Text } + // truncateForLog 诊断日志用截断。 func truncateForLog(s string, n int) string { if len(s) <= n { diff --git a/internal/agent/core/agent.go b/internal/agent/core/agent.go index 646bb3b..1aa3881 100644 --- a/internal/agent/core/agent.go +++ b/internal/agent/core/agent.go @@ -98,6 +98,8 @@ type Agent struct { // 模型思考模式（thinking/reasoning） thinkingEnabled bool + pendingToolBlocks []interface{} // 插件工具通过 SetToolBlocks 注入的多模态块（type agentAPI.ContentBlock），process.go 消费后追加到 tool message + // 启动时间 startTime time.Time diff --git a/internal/agent/core/process.go b/internal/agent/core/process.go index ea24617..5d451d8 100644 --- a/internal/agent/core/process.go +++ b/internal/agent/core/process.go @@ -12,6 +12,7 @@ import ( agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api" "gitcode.com/JianFeeeee/HomeAgent/internal/events" sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk" + pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" ) func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response string, toolsUsed []string, toolResults []ToolResultItem, err error) { @@ -311,7 +312,31 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri contentOnce = false } msgs = append(msgs, agentAPI.Message{Role: "assistant", Content: msgContent, ReasoningContent: resp.ReasoningContent, ToolCalls: []agentAPI.ToolCall{tc}}) - msgs = append(msgs, agentAPI.Message{Role: "tool", ToolCallID: tc.ID, Content: result}) + + // 多模态工具结果：插件通过 SDK.SetToolBlocks 注入 image_url/audio_url block， + // process.go 拾起并追加到 tool message 的 content 数组（OpenAI 多模态格式）， + // 让下一轮 LLM 请求在 tool message 里看到图/音频。 + toolMsg := agentAPI.Message{Role: "tool", ToolCallID: tc.ID, Content: result} + if rawBlocks := a.io.ConsumeToolBlocks(); len(rawBlocks) > 0 { + var blocks []agentAPI.ContentBlock + for _, b := range rawBlocks { + if cb, ok := b.(pubsdk.ContentBlock); ok { + // 跨包类型拷贝（pubsdk.ContentBlock → agentAPI.ContentBlock） + block := agentAPI.ContentBlock{Type: cb.Type, Text: cb.Text} + if cb.ImageURL != nil { + block.ImageURL = &agentAPI.ImageURL{URL: cb.ImageURL.URL, Detail: cb.ImageURL.Detail} + } + if cb.AudioURL != nil { + block.AudioURL = &agentAPI.AudioURL{URL: cb.AudioURL.URL} + } + blocks = append(blocks, block) + } + } + if len(blocks) > 0 { + toolMsg.Blocks = blocks + } + } + msgs = append(msgs, toolMsg) a.publishEvent(events.EventToolCall, map[string]interface{}{ "tool": tc.Name, diff --git a/internal/agent/io/channel.go b/internal/agent/io/channel.go index 779388b..2f6e6bd 100644 --- a/internal/agent/io/channel.go +++ b/internal/agent/io/channel.go @@ -103,6 +103,11 @@ type IOManager struct { outputCh chan *OutputEvent nextReqID int64 inputChannels map[string]ChannelDef + + // toolBlocks：插件工具注入多模态内容块，process.go 在下一条 tool message 时消费。 + // 用 interface{}[] 避免 import api.ContentBlock 导致的循环依赖。 + toolBlocksMu sync.Mutex + toolPendingBlocks []interface{} } func NewIOManager() *IOManager { @@ -669,3 +674,20 @@ func (d *GPIODevice) Tools() []ToolDef { func (d *GPIODevice) Execute(tool string, args map[string]interface{}) (interface{}, error) { return map[string]interface{}{"device": d.name, "tool": tool, "status": "ok"}, nil } + +// SetToolBlocks 插件工具调用时注入多模态内容块（image_url/audio_url 等）， +// 下一条 tool message 追加这些块到 content 数组（OpenAI 多模态格式）。 +func (m *IOManager) SetToolBlocks(blocks []interface{}) { + m.toolBlocksMu.Lock() + m.toolPendingBlocks = blocks + m.toolBlocksMu.Unlock() +} + +// ConsumeToolBlocks 返回并清空 pending blocks，process.go 在 append tool message 时调用。 +func (m *IOManager) ConsumeToolBlocks() []interface{} { + m.toolBlocksMu.Lock() + blocks := m.toolPendingBlocks + m.toolPendingBlocks = nil + m.toolBlocksMu.Unlock() + return blocks +} diff --git a/internal/plugins/agentcli/plugin_test.go b/internal/plugins/agentcli/plugin_test.go index 66b1fd9..7dc4bd7 100644 --- a/internal/plugins/agentcli/plugin_test.go +++ b/internal/plugins/agentcli/plugin_test.go @@ -11,6 +11,7 @@ import ( "time" sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk" + sdkpub "gitcode.com/JianFeeeee/homeagent-sdk/sdk" ) type toolCapture struct { @@ -415,6 +416,10 @@ func (c *injectCapture) InjectTextNoMemory(source, channel, text string) { c.texts = append(c.texts, text) c.mu.Unlock() } + +func (c *injectCapture) SetToolBlocks(blocks []sdkpub.ContentBlock) { + // 测试桩：忽略多模态块 +} func (c *injectCapture) InjectInputSync(source, channel, text string) string { return "" } func (c *injectCapture) snapshot() []string { diff --git a/internal/plugins/all.go b/internal/plugins/all.go index 157de63..2245528 100644 --- a/internal/plugins/all.go +++ b/internal/plugins/all.go @@ -10,6 +10,7 @@ import ( _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/files" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/healthcheck" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/mcp" + _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/multimodal" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/pluginmgr" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/remotedevice" _ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/skillmgr" diff --git a/internal/plugins/multimodal/plugin.go b/internal/plugins/multimodal/plugin.go new file mode 100644 index 0000000..4ff2c92 --- /dev/null +++ b/internal/plugins/multimodal/plugin.go @@ -0,0 +1,279 @@ +package multimodal + +import ( + "encoding/base64" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "gitcode.com/JianFeeeee/HomeAgent/internal/plugin" + sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk" + pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" +) + +func init() { + plugin.RegisterPluginMeta("multimodal", "多模态感知", "Multimodal Perception") + plugin.RegisterFactory("multimodal", NewPluginFactory) +} + +func NewPluginFactory(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name}, nil +} + +type Plugin struct { + name string + sdk *sdk.PluginSDK +} + +func (p *Plugin) Name() string { return p.name } + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + p.sdk = s + tp := p.name + "_" + + s.RegisterTool(tp+"see_picture", sdk.ToolDef{ + Name: tp + "see_picture", + Description: "让模型看到一张图片。输入文件路径或 URL，图片以 image_url 格式注入后续对话，模型可看到并描述/分析图片内容。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "path": map[string]interface{}{ + "type": "string", + "description": "图片的本地文件路径或 HTTP URL", + }, + }, + "required": []string{"path"}, + }, + }, p.handleSeePicture) + + s.RegisterTool(tp+"see_video", sdk.ToolDef{ + Name: tp + "see_video", + Description: "让模型看到一段视频的关键帧。输入视频文件路径，ffmpeg 提取 N 帧作为 image_url 注入后续对话，模型可分析视频内容。需要 ffmpeg 已安装。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "path": map[string]interface{}{ + "type": "string", + "description": "视频的本地文件路径", + }, + "frames": map[string]interface{}{ + "type": "integer", + "description": "提取关键帧数量（默认 4，最大 10）", + }, + }, + "required": []string{"path"}, + }, + }, p.handleSeeVideo) + + s.RegisterTool(tp+"listen", sdk.ToolDef{ + Name: tp + "listen", + Description: "让模型听到一段音频。输入音频文件路径（mp3/wav/ogg/m4a），音频注入后续对话，支持音频的模型可识别语音/声音内容。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "path": map[string]interface{}{ + "type": "string", + "description": "音频文件路径", + }, + }, + "required": []string{"path"}, + }, + }, p.handleListen) + + log.Printf("[%s] multimodal perception tools registered", p.name) + return nil +} + +func (p *Plugin) Stop() error { return nil } + +// ── see_picture ────────────────────────────────────────────────── + +func (p *Plugin) handleSeePicture(args map[string]interface{}) (interface{}, error) { + path := getArgStr(args, "path") + if path == "" { + return "path is required", nil + } + + var dataURL string + var mime string + + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + // 远程 URL：直接用作 image URL，不下载 + dataURL = path + mime = "image/png" + } else { + // 本地文件 + if _, err := os.Stat(path); os.IsNotExist(err) { + return fmt.Sprintf("文件不存在: %s", path), nil + } + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".jpg", ".jpeg": + mime = "image/jpeg" + case ".gif": + mime = "image/gif" + case ".webp": + mime = "image/webp" + default: + mime = "image/png" + } + b, err := os.ReadFile(path) + if err != nil { + return fmt.Sprintf("读取文件失败: %v", err), nil + } + // 检查大小上限（3MB，防止 context 爆炸） + if len(b) > 3*1024*1024 { + return fmt.Sprintf("图片过大（%d bytes，超过 3MB），无法注入上下文", len(b)), nil + } + dataURL = "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b) + } + + // 注入多模态块：模型下一轮可看到图片 + p.sdk.SetToolBlocks([]pubsdk.ContentBlock{ + {Type: "image_url", ImageURL: &pubsdk.ImageURL{URL: dataURL, Detail: "auto"}}, + }) + + text := fmt.Sprintf("[已将图片注入后续对话] %s", path) + return text, nil +} + +// ── see_video ──────────────────────────────────────────────────── + +func (p *Plugin) handleSeeVideo(args map[string]interface{}) (interface{}, error) { + path := getArgStr(args, "path") + if path == "" { + return "path is required", nil + } + if _, err := os.Stat(path); os.IsNotExist(err) { + return fmt.Sprintf("文件不存在: %s", path), nil + } + + // 检查 ffmpeg + ffmpegPath := "" + for _, c := range []string{"ffmpeg", "/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg"} { + if _, err := os.Stat(c); err == nil { + ffmpegPath = c + break + } + } + if ffmpegPath == "" { + if _, err := exec.LookPath("ffmpeg"); err == nil { + ffmpegPath = "ffmpeg" + } else { + return "ffmpeg 未安装，无法提取视频关键帧。请先安装: apt install ffmpeg", nil + } + } + + nFrames := 4 + if n, ok := args["frames"].(float64); ok && n > 0 { + nFrames = int(n) + if nFrames > 10 { + nFrames = 10 + } + } + + // 用 ffmpeg 提取关键帧 + tmpDir, err := os.MkdirTemp("", "mm_video_*") + if err != nil { + return fmt.Sprintf("创建临时目录失败: %v", err), nil + } + defer os.RemoveAll(tmpDir) + + outPattern := filepath.Join(tmpDir, "frame_%03d.jpg") + cmd := exec.Command(ffmpegPath, "-i", path, "-vf", fmt.Sprintf("fps=1/%d", nFrames), + "-q:v", "5", outPattern) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Sprintf("ffmpeg 提取帧失败: %v\n%s", err, string(out)), nil + } + + // 读取提取的帧 + entries, _ := os.ReadDir(tmpDir) + var blocks []pubsdk.ContentBlock + for i, entry := range entries { + if strings.HasSuffix(entry.Name(), ".jpg") { + b, err := os.ReadFile(filepath.Join(tmpDir, entry.Name())) + if err != nil { + continue + } + if len(b) > 2*1024*1024 { + continue // 跳过过大帧 + } + dURL := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(b) + blocks = append(blocks, pubsdk.ContentBlock{ + Type: "image_url", + ImageURL: &pubsdk.ImageURL{URL: dURL, Detail: "low"}, + }) + if i >= 9 { // 最多 10 帧 + break + } + } + } + + if len(blocks) == 0 { + return "视频中未提取到有效帧", nil + } + + // 全部帧注入（一次 SetToolBlocks 调用，下一轮 LLM 可看到） + p.sdk.SetToolBlocks(blocks) + + text := fmt.Sprintf("[已将 %d 个视频关键帧注入后续对话] %s", len(blocks), path) + return text, nil +} + +// ── listen ─────────────────────────────────────────────────────── + +func (p *Plugin) handleListen(args map[string]interface{}) (interface{}, error) { + path := getArgStr(args, "path") + if path == "" { + return "path is required", nil + } + if _, err := os.Stat(path); os.IsNotExist(err) { + return fmt.Sprintf("文件不存在: %s", path), nil + } + + ext := strings.ToLower(filepath.Ext(path)) + var mime string + switch ext { + case ".mp3": + mime = "audio/mpeg" + case ".wav": + mime = "audio/wav" + case ".ogg": + mime = "audio/ogg" + case ".m4a", ".aac": + mime = "audio/mp4" + default: + mime = "audio/ogg" // 默认 + } + + // 检查大小（5MB 限制，避免上下文爆炸） + info, _ := os.Stat(path) + if info != nil && info.Size() > 5*1024*1024 { + return fmt.Sprintf("音频过大（%d bytes，超过 5MB），无法注入上下文", info.Size()), nil + } + + b, err := os.ReadFile(path) + if err != nil { + return fmt.Sprintf("读取音频文件失败: %v", err), nil + } + dataURL := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b) + + p.sdk.SetToolBlocks([]pubsdk.ContentBlock{ + {Type: "audio_url", AudioURL: &pubsdk.AudioURL{URL: dataURL}}, + }) + + text := fmt.Sprintf("[已将音频注入后续对话] %s（%s，%.1fKB）", path, mime, float64(len(b))/1024) + return text, nil +} + +// ── helpers ────────────────────────────────────────────────────── + +func getArgStr(args map[string]interface{}, key string) string { + if v, ok := args[key].(string); ok { + return v + } + return "" +} diff --git a/internal/sdk/plugin.go b/internal/sdk/plugin.go index d51cd16..a76789a 100644 --- a/internal/sdk/plugin.go +++ b/internal/sdk/plugin.go @@ -339,3 +339,27 @@ func (s *PluginSDK) Subscribe(eventType events.EventType, handler events.Handler } return func() {} } + +// SetToolBlocks 桥接到 IOManager：插件工具注入多模态块，process.go 消费。 +func (a ioAdapter) SetToolBlocks(blocks []pubsdk.ContentBlock) { + if a.iom == nil { + return + } + ifaces := make([]interface{}, len(blocks)) + for i, b := range blocks { + ifaces[i] = b + } + a.iom.SetToolBlocks(ifaces) +} + +// SetToolBlocks 注入多模态内容块（图片/音频），内核在下一条 tool message +// 的 content 数组里带上这些块，让模型在后续轮次看到图/听到音频。 +func (s *PluginSDK) SetToolBlocks(blocks []pubsdk.ContentBlock) { + if s.iom != nil { + ifaces := make([]interface{}, len(blocks)) + for i, b := range blocks { + ifaces[i] = b + } + s.iom.SetToolBlocks(ifaces) + } +}