feat(clip): 多模态向量器(CLIP ONNX)——文本/图像 512 维共享空间 + 媒体向量写入与重算

- internal/memory/clip:CLIP ONNX 向量器(onnxruntime 构建标签控制,默认构建不链接 ONNX)
  - clip.New(modelDir) 加载 text.onnx/vision.onnx(输出 text_embed/image_embed [batch,512])
  - 实现 vector.Vectorizer + vector.MultimodalEmbedder(Vectorize/EmbedImage + Dense 变体)
  - 词级 BPE tokenizer:merges 合并后词末片段带 </w> 查 vocab,与官方 encode 逐 id 对齐
  - EmbedImage:解码→resize 224→NCHW→normalize→vision session
  - Fingerprint(text+vision 文件 sha256)供模型切换检测
  - stub 版(无 onnxruntime 标签)保持默认构建行为不变
- vector/store.go:新增 MultimodalEmbedder 接口
- media.Store:新增 StaleVecDigests(currentModel)——查 vec_model 不匹配/缺失的图片
- agent core:AgentConfig.ClipEmbedder + Agent.clipEmb 接线;
  describePendingMedia 描述成功后 EmbedImageDense→SetVec;
  新增 reembedStaleMedia 启动补算历史无向量图片
- config:core.memory.media.clip_model_dir(未配置退化为现有 fastText/TF-IDF 行为)
- cmd/homed:读 clip_model_dir 加载 CLIP,失败仅记日志不阻塞启动

测试:TestSmokeLoadAndEncode(文本语义 cat>dog 0.914>physics 0.740)、
TestCrossModalAlignment(red-image vs red-text 0.063>blue -0.009,与 Python 一致)、
TestTokEnd(与官方 encode 逐 id 对齐)、TestStaleVecDigests,含 -race 全绿
This commit is contained in:
JianFeeeee
2026-09-09 10:23:31 +08:00
parent 8c9d96e065
commit 98365f3a55
10 changed files with 863 additions and 2 deletions

View File

@ -0,0 +1,154 @@
//go:build onnxruntime
package clip
import (
"bytes"
"fmt"
"image"
"image/color"
"image/png"
"os"
"testing"
)
func TestSmokeLoadAndEncode(t *testing.T) {
modelDir := os.Getenv("CLIP_MODEL_DIR")
if modelDir == "" {
modelDir = "/home/newqqagent/models/clip-vit-b32"
}
if _, err := os.Stat(modelDir + "/text.onnx"); err != nil {
t.Skipf("模型目录不存在: %v", err)
}
emb, err := New(modelDir)
if err != nil {
t.Fatalf("New: %v", err)
}
defer emb.Close()
if !emb.Loaded() {
t.Fatal("loaded should be true")
}
if emb.Dim() != 512 {
t.Fatalf("dim = %d, want 512", emb.Dim())
}
if emb.Fingerprint() == "" {
t.Fatal("fingerprint should not be empty")
}
// 文本编码
textVec, err := emb.VectorizeDense("a photo of a cat")
if err != nil {
t.Fatalf("VectorizeDense: %v", err)
}
if len(textVec) != 512 {
t.Fatalf("text vec len = %d, want 512", len(textVec))
}
fmt.Printf("text vec[:5] = %v\n", textVec[:5])
// 同义文本应比远义文本更相似
textVec2, _ := emb.VectorizeDense("a photograph of a dog")
textVec3, _ := emb.VectorizeDense("quantum physics equations")
sim12 := cosineSim(textVec, textVec2)
sim13 := cosineSim(textVec, textVec3)
fmt.Printf("cat vs dog = %.4f, cat vs physics = %.4f\n", sim12, sim13)
if sim12 <= sim13 {
t.Errorf("cat-dog sim (%.4f) should be > cat-physics sim (%.4f)", sim12, sim13)
}
// 通过 Vectorizer 接口(稀疏 map)
sparseVec := emb.Vectorize("hello world")
if len(sparseVec) == 0 {
t.Error("sparse Vectorize should return non-empty")
}
fmt.Printf("sparse len = %d\n", len(sparseVec))
}
// TestCrossModalAlignment 验证图文在同一向量空间可比:
// 红底图的向量应与 "a red image" 更相似,而非 "a blue image"。
func TestCrossModalAlignment(t *testing.T) {
modelDir := os.Getenv("CLIP_MODEL_DIR")
if modelDir == "" {
modelDir = "/home/newqqagent/models/clip-vit-b32"
}
emb, err := New(modelDir)
if err != nil {
t.Fatalf("New: %v", err)
}
defer emb.Close()
// 生成 224x224 纯红底 PNG
img := image.NewRGBA(image.Rect(0, 0, 224, 224))
red := color.RGBA{220, 40, 40, 255}
for y := 0; y < 224; y++ {
for x := 0; x < 224; x++ {
img.Set(x, y, red)
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatal(err)
}
imgVec, err := emb.EmbedImageDense(buf.Bytes(), "image/png")
if err != nil {
t.Fatalf("EmbedImageDense: %v", err)
}
if len(imgVec) != 512 {
t.Fatalf("img vec len = %d, want 512", len(imgVec))
}
redText, _ := emb.VectorizeDense("a red image")
blueText, _ := emb.VectorizeDense("a blue image")
redSim := cosineSim(imgVec, redText)
blueSim := cosineSim(imgVec, blueText)
fmt.Printf("red-image vs red-text = %.4f, vs blue-text = %.4f\n", redSim, blueSim)
if redSim <= blueSim {
t.Errorf("red image should align better with red text (%.4f) than blue (%.4f)", redSim, blueSim)
}
// 同图应比异图更相似:存两张不同颜色,query 用红底图应召回红图
d1 := imgVec
blueImg := image.NewRGBA(image.Rect(0, 0, 224, 224))
blue := color.RGBA{40, 40, 220, 255}
for y := 0; y < 224; y++ {
for x := 0; x < 224; x++ {
blueImg.Set(x, y, blue)
}
}
var buf2 bytes.Buffer
png.Encode(&buf2, blueImg)
d2, _ := emb.EmbedImageDense(buf2.Bytes(), "image/png")
if cosineSim(d1, d2) >= 0.99 {
t.Errorf("red and blue images should differ (got sim %.4f)", cosineSim(d1, d2))
}
}
func cosineSim(a, b []float64) float64 {
if len(a) != len(b) {
return 0
}
var dot, na, nb float64
for i := range a {
dot += a[i] * b[i]
na += a[i] * a[i]
nb += b[i] * b[i]
}
if na == 0 || nb == 0 {
return 0
}
return dot / (sqrt(na) * sqrt(nb))
}
func sqrt(x float64) float64 {
if x <= 0 {
return 0
}
z := x
for i := 0; i < 50; i++ {
z = (z + x/z) / 2
}
return z
}