mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 09:28:14 +00:00
b777322b95
【新插件 internal/plugins/multimodal】
- see_picture(path): 读取本地图片/URL，base64 注入 image_url block，
模型在下一轮 LLM 请求的 tool message 里直接看到图（1024×1024 图约 8500 token）。
自动识别 MIME，限 3MB 防爆 context。
- see_video(path, frames): ffmpeg 提取关键帧，多帧作为 image_url block 注入。
默认 4 帧，最大 10 帧，每帧限 2MB。
- listen(path): 读取音频文件，转为 audio_url block 注入，支持 mp3/wav/ogg/m4a。
限 5MB。
【内核多模态 tool message 支持】
- agent/api 新增 ToolOutput 类型（为后续 handler 直接返回 blocks 预留）
- SDK 公共层新增 ContentBlock/ImageURL/AudioURL（OpenAI 多模态格式）
- IOManager 新增 SetToolBlocks/ConsumeToolBlocks（interface{} 避免循环依赖）
- PluginSDK.SetToolBlocks(blocks) 插件工具调用后注入 blocks
- ioAdapter 桥接 IOInjector.SetToolBlocks
- process.go 工具执行后消费 pending blocks → 追加到 tool message 的 Blocks 字段
→ MarshalJSON 输出 content 数组格式 → LLM 看到图/音频
【验证】
multimodal_see_picture 注入 1024×1024 PNG 后 llmsproxy 统计：
prompt_tokens=44407（含 ~8500 image token），模型正确描述了图片内容。
280 lines
8.2 KiB
Go
280 lines
8.2 KiB
Go
package multimodal
|
||
|
||
import (
|
||
"encoding/base64"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"strings"
|
||
|
||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||
)
|
||
|
||
func init() {
|
||
plugin.RegisterPluginMeta("multimodal", "多模态感知", "Multimodal Perception")
|
||
plugin.RegisterFactory("multimodal", NewPluginFactory)
|
||
}
|
||
|
||
func NewPluginFactory(name string, config map[string]interface{}) (sdk.Plugin, error) {
|
||
return &Plugin{name: name}, nil
|
||
}
|
||
|
||
type Plugin struct {
|
||
name string
|
||
sdk *sdk.PluginSDK
|
||
}
|
||
|
||
func (p *Plugin) Name() string { return p.name }
|
||
|
||
func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||
p.sdk = s
|
||
tp := p.name + "_"
|
||
|
||
s.RegisterTool(tp+"see_picture", sdk.ToolDef{
|
||
Name: tp + "see_picture",
|
||
Description: "让模型看到一张图片。输入文件路径或 URL，图片以 image_url 格式注入后续对话，模型可看到并描述/分析图片内容。",
|
||
Parameters: map[string]interface{}{
|
||
"type": "object",
|
||
"properties": map[string]interface{}{
|
||
"path": map[string]interface{}{
|
||
"type": "string",
|
||
"description": "图片的本地文件路径或 HTTP URL",
|
||
},
|
||
},
|
||
"required": []string{"path"},
|
||
},
|
||
}, p.handleSeePicture)
|
||
|
||
s.RegisterTool(tp+"see_video", sdk.ToolDef{
|
||
Name: tp + "see_video",
|
||
Description: "让模型看到一段视频的关键帧。输入视频文件路径，ffmpeg 提取 N 帧作为 image_url 注入后续对话，模型可分析视频内容。需要 ffmpeg 已安装。",
|
||
Parameters: map[string]interface{}{
|
||
"type": "object",
|
||
"properties": map[string]interface{}{
|
||
"path": map[string]interface{}{
|
||
"type": "string",
|
||
"description": "视频的本地文件路径",
|
||
},
|
||
"frames": map[string]interface{}{
|
||
"type": "integer",
|
||
"description": "提取关键帧数量（默认 4，最大 10）",
|
||
},
|
||
},
|
||
"required": []string{"path"},
|
||
},
|
||
}, p.handleSeeVideo)
|
||
|
||
s.RegisterTool(tp+"listen", sdk.ToolDef{
|
||
Name: tp + "listen",
|
||
Description: "让模型听到一段音频。输入音频文件路径（mp3/wav/ogg/m4a），音频注入后续对话，支持音频的模型可识别语音/声音内容。",
|
||
Parameters: map[string]interface{}{
|
||
"type": "object",
|
||
"properties": map[string]interface{}{
|
||
"path": map[string]interface{}{
|
||
"type": "string",
|
||
"description": "音频文件路径",
|
||
},
|
||
},
|
||
"required": []string{"path"},
|
||
},
|
||
}, p.handleListen)
|
||
|
||
log.Printf("[%s] multimodal perception tools registered", p.name)
|
||
return nil
|
||
}
|
||
|
||
func (p *Plugin) Stop() error { return nil }
|
||
|
||
// ── see_picture ──────────────────────────────────────────────────
|
||
|
||
func (p *Plugin) handleSeePicture(args map[string]interface{}) (interface{}, error) {
|
||
path := getArgStr(args, "path")
|
||
if path == "" {
|
||
return "path is required", nil
|
||
}
|
||
|
||
var dataURL string
|
||
var mime string
|
||
|
||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||
// 远程 URL：直接用作 image URL，不下载
|
||
dataURL = path
|
||
mime = "image/png"
|
||
} else {
|
||
// 本地文件
|
||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||
return fmt.Sprintf("文件不存在: %s", path), nil
|
||
}
|
||
ext := strings.ToLower(filepath.Ext(path))
|
||
switch ext {
|
||
case ".jpg", ".jpeg":
|
||
mime = "image/jpeg"
|
||
case ".gif":
|
||
mime = "image/gif"
|
||
case ".webp":
|
||
mime = "image/webp"
|
||
default:
|
||
mime = "image/png"
|
||
}
|
||
b, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return fmt.Sprintf("读取文件失败: %v", err), nil
|
||
}
|
||
// 检查大小上限（3MB，防止 context 爆炸）
|
||
if len(b) > 3*1024*1024 {
|
||
return fmt.Sprintf("图片过大（%d bytes，超过 3MB），无法注入上下文", len(b)), nil
|
||
}
|
||
dataURL = "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b)
|
||
}
|
||
|
||
// 注入多模态块：模型下一轮可看到图片
|
||
p.sdk.SetToolBlocks([]pubsdk.ContentBlock{
|
||
{Type: "image_url", ImageURL: &pubsdk.ImageURL{URL: dataURL, Detail: "auto"}},
|
||
})
|
||
|
||
text := fmt.Sprintf("[已将图片注入后续对话] %s", path)
|
||
return text, nil
|
||
}
|
||
|
||
// ── see_video ────────────────────────────────────────────────────
|
||
|
||
func (p *Plugin) handleSeeVideo(args map[string]interface{}) (interface{}, error) {
|
||
path := getArgStr(args, "path")
|
||
if path == "" {
|
||
return "path is required", nil
|
||
}
|
||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||
return fmt.Sprintf("文件不存在: %s", path), nil
|
||
}
|
||
|
||
// 检查 ffmpeg
|
||
ffmpegPath := ""
|
||
for _, c := range []string{"ffmpeg", "/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg"} {
|
||
if _, err := os.Stat(c); err == nil {
|
||
ffmpegPath = c
|
||
break
|
||
}
|
||
}
|
||
if ffmpegPath == "" {
|
||
if _, err := exec.LookPath("ffmpeg"); err == nil {
|
||
ffmpegPath = "ffmpeg"
|
||
} else {
|
||
return "ffmpeg 未安装，无法提取视频关键帧。请先安装: apt install ffmpeg", nil
|
||
}
|
||
}
|
||
|
||
nFrames := 4
|
||
if n, ok := args["frames"].(float64); ok && n > 0 {
|
||
nFrames = int(n)
|
||
if nFrames > 10 {
|
||
nFrames = 10
|
||
}
|
||
}
|
||
|
||
// 用 ffmpeg 提取关键帧
|
||
tmpDir, err := os.MkdirTemp("", "mm_video_*")
|
||
if err != nil {
|
||
return fmt.Sprintf("创建临时目录失败: %v", err), nil
|
||
}
|
||
defer os.RemoveAll(tmpDir)
|
||
|
||
outPattern := filepath.Join(tmpDir, "frame_%03d.jpg")
|
||
cmd := exec.Command(ffmpegPath, "-i", path, "-vf", fmt.Sprintf("fps=1/%d", nFrames),
|
||
"-q:v", "5", outPattern)
|
||
if out, err := cmd.CombinedOutput(); err != nil {
|
||
return fmt.Sprintf("ffmpeg 提取帧失败: %v\n%s", err, string(out)), nil
|
||
}
|
||
|
||
// 读取提取的帧
|
||
entries, _ := os.ReadDir(tmpDir)
|
||
var blocks []pubsdk.ContentBlock
|
||
for i, entry := range entries {
|
||
if strings.HasSuffix(entry.Name(), ".jpg") {
|
||
b, err := os.ReadFile(filepath.Join(tmpDir, entry.Name()))
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if len(b) > 2*1024*1024 {
|
||
continue // 跳过过大帧
|
||
}
|
||
dURL := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(b)
|
||
blocks = append(blocks, pubsdk.ContentBlock{
|
||
Type: "image_url",
|
||
ImageURL: &pubsdk.ImageURL{URL: dURL, Detail: "low"},
|
||
})
|
||
if i >= 9 { // 最多 10 帧
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
if len(blocks) == 0 {
|
||
return "视频中未提取到有效帧", nil
|
||
}
|
||
|
||
// 全部帧注入（一次 SetToolBlocks 调用，下一轮 LLM 可看到）
|
||
p.sdk.SetToolBlocks(blocks)
|
||
|
||
text := fmt.Sprintf("[已将 %d 个视频关键帧注入后续对话] %s", len(blocks), path)
|
||
return text, nil
|
||
}
|
||
|
||
// ── listen ───────────────────────────────────────────────────────
|
||
|
||
func (p *Plugin) handleListen(args map[string]interface{}) (interface{}, error) {
|
||
path := getArgStr(args, "path")
|
||
if path == "" {
|
||
return "path is required", nil
|
||
}
|
||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||
return fmt.Sprintf("文件不存在: %s", path), nil
|
||
}
|
||
|
||
ext := strings.ToLower(filepath.Ext(path))
|
||
var mime string
|
||
switch ext {
|
||
case ".mp3":
|
||
mime = "audio/mpeg"
|
||
case ".wav":
|
||
mime = "audio/wav"
|
||
case ".ogg":
|
||
mime = "audio/ogg"
|
||
case ".m4a", ".aac":
|
||
mime = "audio/mp4"
|
||
default:
|
||
mime = "audio/ogg" // 默认
|
||
}
|
||
|
||
// 检查大小（5MB 限制，避免上下文爆炸）
|
||
info, _ := os.Stat(path)
|
||
if info != nil && info.Size() > 5*1024*1024 {
|
||
return fmt.Sprintf("音频过大（%d bytes，超过 5MB），无法注入上下文", info.Size()), nil
|
||
}
|
||
|
||
b, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return fmt.Sprintf("读取音频文件失败: %v", err), nil
|
||
}
|
||
dataURL := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b)
|
||
|
||
p.sdk.SetToolBlocks([]pubsdk.ContentBlock{
|
||
{Type: "audio_url", AudioURL: &pubsdk.AudioURL{URL: dataURL}},
|
||
})
|
||
|
||
text := fmt.Sprintf("[已将音频注入后续对话] %s（%s，%.1fKB）", path, mime, float64(len(b))/1024)
|
||
return text, nil
|
||
}
|
||
|
||
// ── helpers ──────────────────────────────────────────────────────
|
||
|
||
func getArgStr(args map[string]interface{}, key string) string {
|
||
if v, ok := args[key].(string); ok {
|
||
return v
|
||
}
|
||
return ""
|
||
}
|