mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 09:28:14 +00:00
feat(clip): 多模态向量器(CLIP ONNX)——文本/图像 512 维共享空间 + 媒体向量写入与重算
- internal/memory/clip:CLIP ONNX 向量器(onnxruntime 构建标签控制,默认构建不链接 ONNX) - clip.New(modelDir) 加载 text.onnx/vision.onnx(输出 text_embed/image_embed [batch,512]) - 实现 vector.Vectorizer + vector.MultimodalEmbedder(Vectorize/EmbedImage + Dense 变体) - 词级 BPE tokenizer:merges 合并后词末片段带 </w> 查 vocab,与官方 encode 逐 id 对齐 - EmbedImage:解码→resize 224→NCHW→normalize→vision session - Fingerprint(text+vision 文件 sha256)供模型切换检测 - stub 版(无 onnxruntime 标签)保持默认构建行为不变 - vector/store.go:新增 MultimodalEmbedder 接口 - media.Store:新增 StaleVecDigests(currentModel)——查 vec_model 不匹配/缺失的图片 - agent core:AgentConfig.ClipEmbedder + Agent.clipEmb 接线; describePendingMedia 描述成功后 EmbedImageDense→SetVec; 新增 reembedStaleMedia 启动补算历史无向量图片 - config:core.memory.media.clip_model_dir(未配置退化为现有 fastText/TF-IDF 行为) - cmd/homed:读 clip_model_dir 加载 CLIP,失败仅记日志不阻塞启动 测试:TestSmokeLoadAndEncode(文本语义 cat>dog 0.914>physics 0.740)、 TestCrossModalAlignment(red-image vs red-text 0.063>blue -0.009,与 Python 一致)、 TestTokEnd(与官方 encode 逐 id 对齐)、TestStaleVecDigests,含 -race 全绿
This commit is contained in:
@ -24,6 +24,7 @@ import (
|
||||
logpkg "gitcode.com/JianFeeeee/HomeAgent/internal/log"
|
||||
luapkg "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/clip"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/pipeline"
|
||||
@ -346,6 +347,21 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// 多模态嵌入(CLIP ONNX,可选):配置 clip_model_dir 时启用,
|
||||
// 图片入库/描述时计算视觉向量,供跨模态检索;未配置则退回纯 fastText 文本路径。
|
||||
var clipEmbedder *clip.Embedder
|
||||
if clipDir := cfgReg.GetString("core.memory.media.clip_model_dir", ""); clipDir != "" {
|
||||
e, err := clip.New(clipDir)
|
||||
if err != nil {
|
||||
// 配置了但加载失败:记日志降级,不阻塞启动(媒体记忆是增强项)
|
||||
log.Printf("[homed] warning: clip embedder load failed: %v(多模态向量检索已禁用)", err)
|
||||
} else {
|
||||
clipEmbedder = e
|
||||
defer e.Close()
|
||||
log.Printf("[homed] clip embedder active: dim=%d fp=%s", e.Dim(), e.Fingerprint()[:min(12, len(e.Fingerprint()))])
|
||||
}
|
||||
}
|
||||
|
||||
ks := knowledge.NewStore(filepath.Join(cfg.Daemon.DataDir, "knowledge"))
|
||||
if err := ks.Start(); err != nil {
|
||||
log.Printf("[homed] warning: knowledge store: %v", err)
|
||||
@ -468,6 +484,7 @@ func main() {
|
||||
ContextSavePath: filepath.Join(cfg.Daemon.DataDir, "memory", "context.json"),
|
||||
EmbeddingModelPath: cfgReg.GetString("core.agent.embedding_model_path", ""),
|
||||
Embedder: embedder,
|
||||
ClipEmbedder: clipEmbedder,
|
||||
StageHost: stageHost,
|
||||
EventBus: evBus,
|
||||
ThinkingEnabled: cfg.LLM.ThinkingEnabled,
|
||||
|
||||
Reference in New Issue
Block a user