mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-04 00:03:59 +00:00
feat(vector): pluggable multimodal vector space
核心暴露 MultimodalEmbedder 接口,两条路径共享同一套 L0/L2/L3 向量缓存、media.Store 坐标、QueryMemoryMediaScored 检索: - onnx:内嵌 ONNX 模型(CLIP 等),通过 build tag 编译 - http:外部向量 API 服务(Jina v5 / OpenAI / 自建) 跨模态融合权重改为 CrossModalFusionConfig 可配置结构体, 移除所有模型特定硬编码(CLIP/Jina),版本切换只需改配置。 模型切换自动迁移: - StaleVecDigestsAll 支持全模态(image+audio+video) - 启动时并发重算(ONNX 4 workers / API 8 workers) - 修复 SQL 运算符优先级导致 kind 过滤失效的 bug 实测对比(492 篇生产文档 + 3 张真实图片): - TF-IDF:MRR 0.457(精确匹配快,语义差) - fastText:MRR 0.530(语义中等,延迟 8ms) - Jina v5-omni:MRR 0.900(全面领先,延迟 40ms) - 中文文本→图片:Jina MRR 0.833 vs CLIP 0.611 See docs/embedding-comparison.md for full benchmark.
This commit is contained in:
@ -13,11 +13,11 @@ import (
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/events"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/knowledge"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/clip"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||
@ -145,8 +145,12 @@ type Agent struct {
|
||||
// 词嵌入模型,用于实体语义相似度计算
|
||||
embedder *memory.StaticEmbedder
|
||||
|
||||
// clipEmb 是 CLIP 多模态嵌入(可选),nil 时跳过视觉向量计算。
|
||||
clipEmb *clip.Embedder
|
||||
// multimodalSpace 是统一多模态向量空间(可选)。实现可以是内嵌 ONNX,
|
||||
// 也可以是外部 API 客户端;两者共享同一套 L0/L2/L3 向量缓存与检索基础设施。
|
||||
multimodalSpace vector.MultimodalEmbedder
|
||||
|
||||
// fusionCfg 控制文本路与视觉路的跨模态融合权重,可按模型实测结果配置。
|
||||
fusionCfg CrossModalFusionConfig
|
||||
|
||||
// 技能索引提供者:由 skillmgr 插件实现,向 system prompt 注入轻量技能索引
|
||||
skillIndex SkillIndexProvider
|
||||
@ -178,7 +182,8 @@ type AgentConfig struct {
|
||||
MediaGCInterval time.Duration
|
||||
MediaGCMinAge time.Duration
|
||||
MediaDescribe bool
|
||||
ClipEmbedder *clip.Embedder
|
||||
MultimodalSpace vector.MultimodalEmbedder
|
||||
FusionCfg CrossModalFusionConfig // 跨模态融合权重;零值用默认
|
||||
Personality *agentPkg.Personality
|
||||
PluginReg *plugin.Registry
|
||||
PluginDir string
|
||||
@ -284,7 +289,8 @@ func New(cfg AgentConfig) *Agent {
|
||||
thinkingEnabled: cfg.ThinkingEnabled,
|
||||
inputCfg: cfg.InputProcessing,
|
||||
embedder: embedder,
|
||||
clipEmb: cfg.ClipEmbedder,
|
||||
multimodalSpace: cfg.MultimodalSpace,
|
||||
fusionCfg: cfg.FusionCfg,
|
||||
noMergeMarkers: make(map[string]int),
|
||||
lastInput: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user