mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 09:28:14 +00:00
feat(clip): 多模态向量器(CLIP ONNX)——文本/图像 512 维共享空间 + 媒体向量写入与重算
- internal/memory/clip:CLIP ONNX 向量器(onnxruntime 构建标签控制,默认构建不链接 ONNX) - clip.New(modelDir) 加载 text.onnx/vision.onnx(输出 text_embed/image_embed [batch,512]) - 实现 vector.Vectorizer + vector.MultimodalEmbedder(Vectorize/EmbedImage + Dense 变体) - 词级 BPE tokenizer:merges 合并后词末片段带 </w> 查 vocab,与官方 encode 逐 id 对齐 - EmbedImage:解码→resize 224→NCHW→normalize→vision session - Fingerprint(text+vision 文件 sha256)供模型切换检测 - stub 版(无 onnxruntime 标签)保持默认构建行为不变 - vector/store.go:新增 MultimodalEmbedder 接口 - media.Store:新增 StaleVecDigests(currentModel)——查 vec_model 不匹配/缺失的图片 - agent core:AgentConfig.ClipEmbedder + Agent.clipEmb 接线; describePendingMedia 描述成功后 EmbedImageDense→SetVec; 新增 reembedStaleMedia 启动补算历史无向量图片 - config:core.memory.media.clip_model_dir(未配置退化为现有 fastText/TF-IDF 行为) - cmd/homed:读 clip_model_dir 加载 CLIP,失败仅记日志不阻塞启动 测试:TestSmokeLoadAndEncode(文本语义 cat>dog 0.914>physics 0.740)、 TestCrossModalAlignment(red-image vs red-text 0.063>blue -0.009,与 Python 一致)、 TestTokEnd(与官方 encode 逐 id 对齐)、TestStaleVecDigests,含 -race 全绿
This commit is contained in:
@ -13,6 +13,7 @@ import (
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/events"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/knowledge"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/clip"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
||||
@ -144,6 +145,9 @@ type Agent struct {
|
||||
// 词嵌入模型,用于实体语义相似度计算
|
||||
embedder *memory.StaticEmbedder
|
||||
|
||||
// clipEmb 是 CLIP 多模态嵌入(可选),nil 时跳过视觉向量计算。
|
||||
clipEmb *clip.Embedder
|
||||
|
||||
// 技能索引提供者:由 skillmgr 插件实现,向 system prompt 注入轻量技能索引
|
||||
skillIndex SkillIndexProvider
|
||||
}
|
||||
@ -174,6 +178,7 @@ type AgentConfig struct {
|
||||
MediaGCInterval time.Duration
|
||||
MediaGCMinAge time.Duration
|
||||
MediaDescribe bool
|
||||
ClipEmbedder *clip.Embedder
|
||||
Personality *agentPkg.Personality
|
||||
PluginReg *plugin.Registry
|
||||
PluginDir string
|
||||
@ -279,6 +284,7 @@ func New(cfg AgentConfig) *Agent {
|
||||
thinkingEnabled: cfg.ThinkingEnabled,
|
||||
inputCfg: cfg.InputProcessing,
|
||||
embedder: embedder,
|
||||
clipEmb: cfg.ClipEmbedder,
|
||||
noMergeMarkers: make(map[string]int),
|
||||
lastInput: make(map[string]time.Time),
|
||||
}
|
||||
@ -296,6 +302,7 @@ func (a *Agent) Start() {
|
||||
go a.reviewLoop()
|
||||
go a.mediaGCLoop()
|
||||
go a.mediaDescribeLoop()
|
||||
a.reembedStaleMedia()
|
||||
log.Printf("[agent] %s started, waiting for IO interrupts", a.id)
|
||||
}
|
||||
|
||||
|
||||
@ -179,5 +179,64 @@ func (a *Agent) describePendingMedia() {
|
||||
continue
|
||||
}
|
||||
log.Printf("[media] 已描述 %s (%s, %d 字, 源=%s)", shortDigest(it.Digest), kind, len([]rune(desc)), srcName)
|
||||
|
||||
// 描述成功后,若 CLIP 嵌入器可用且是图片,计算视觉向量。
|
||||
// 这是"描述 + 向量"两步同步完成的路径;对于历史已有描述但无向量的媒体,
|
||||
// 由启动时的 reembedStaleMedia 补算。
|
||||
if a.clipEmb != nil && kind == "image" && it.Kind == media.KindImage {
|
||||
vec, err := a.clipEmb.EmbedImageDense(data, mime)
|
||||
if err != nil {
|
||||
log.Printf("[media] 视觉嵌入失败 %s: %v", shortDigest(it.Digest), err)
|
||||
} else if err := a.mediaStore.SetVec(it.Digest, vec, a.clipEmb.Fingerprint()); err != nil {
|
||||
log.Printf("[media] 写向量失败 %s: %v", shortDigest(it.Digest), err)
|
||||
} else {
|
||||
log.Printf("[media] 已嵌入 %s (dim=%d)", shortDigest(it.Digest), len(vec))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reembedStaleMedia 在启动时为历史已有描述但无 CLIP 向量的图片补算视觉向量。
|
||||
// 避免安装 CLIP 后,旧图片永远只有描述文本、没有视觉向量,直到下次 Describe 才能写入。
|
||||
func (a *Agent) reembedStaleMedia() {
|
||||
if a.clipEmb == nil || a.mediaStore == nil {
|
||||
return
|
||||
}
|
||||
fp := a.clipEmb.Fingerprint()
|
||||
digests, err := a.mediaStore.StaleVecDigests(fp)
|
||||
if err != nil {
|
||||
log.Printf("[media] 查询需重算向量的媒体失败: %v", err)
|
||||
return
|
||||
}
|
||||
if len(digests) == 0 {
|
||||
log.Printf("[media] 无历史媒体需要补算视觉向量")
|
||||
return
|
||||
}
|
||||
log.Printf("[media] 启动补算视觉向量: %d 条 (fp=%s...)", len(digests), fp[:min(12, len(fp))])
|
||||
done := 0
|
||||
for _, d := range digests {
|
||||
it, err := a.mediaStore.Stat(d)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
data, err := a.mediaStore.Get(d)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mime := it.MIME
|
||||
if mime == "" {
|
||||
mime = "image/png"
|
||||
}
|
||||
vec, err := a.clipEmb.EmbedImageDense(data, mime)
|
||||
if err != nil {
|
||||
log.Printf("[media] 启动补算失败 %s: %v", shortDigest(d), err)
|
||||
continue
|
||||
}
|
||||
if err := a.mediaStore.SetVec(d, vec, fp); err != nil {
|
||||
log.Printf("[media] 启动写入向量失败 %s: %v", shortDigest(d), err)
|
||||
continue
|
||||
}
|
||||
done++
|
||||
}
|
||||
log.Printf("[media] 启动补算完成: %d/%d", done, len(digests))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user