mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-22 18:08:04 +00:00
feat(vector): pluggable multimodal vector space
核心暴露 MultimodalEmbedder 接口，两条路径共享同一套 L0/L2/L3 向量缓存、media.Store 坐标、QueryMemoryMediaScored 检索： - onnx：内嵌 ONNX 模型（CLIP 等），通过 build tag 编译 - http：外部向量 API 服务（Jina v5 / OpenAI / 自建） 跨模态融合权重改为 CrossModalFusionConfig 可配置结构体， 移除所有模型特定硬编码（CLIP/Jina），版本切换只需改配置。 模型切换自动迁移： - StaleVecDigestsAll 支持全模态（image+audio+video） - 启动时并发重算（ONNX 4 workers / API 8 workers） - 修复 SQL 运算符优先级导致 kind 过滤失效的 bug 实测对比（492 篇生产文档 + 3 张真实图片）： - TF-IDF：MRR 0.457（精确匹配快，语义差） - fastText：MRR 0.530（语义中等，延迟 8ms） - Jina v5-omni：MRR 0.900（全面领先，延迟 40ms） - 中文文本→图片：Jina MRR 0.833 vs CLIP 0.611 See docs/embedding-comparison.md for full benchmark.
This commit is contained in:
JianFeeeee
@ -30,6 +30,7 @@ import (
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/pipeline"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/meta"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/nlp"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||||
@ -347,21 +348,57 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// 多模态嵌入（CLIP ONNX，可选）：配置 clip_model_dir 时启用，
|
||||
// 图片入库/描述时计算视觉向量，供跨模态检索；未配置则退回纯 fastText 文本路径。
|
||||
var clipEmbedder *clip.Embedder
|
||||
if clipDir := cfgReg.GetString("core.memory.media.clip_model_dir", ""); clipDir != "" {
|
||||
e, err := clip.New(clipDir)
|
||||
if err != nil {
|
||||
// 配置了但加载失败：记日志降级，不阻塞启动（媒体记忆是增强项）
|
||||
log.Printf("[homed] warning: clip embedder load failed: %v（多模态向量检索已禁用）", err)
|
||||
// 统一多模态向量空间（可选）。
|
||||
//
|
||||
// 两条路径共享同一套基础设施（L0/L2/L3 向量缓存、media.Store 坐标、
|
||||
// QueryMemoryMediaScored 检索），只是「算向量的源头」不同：
|
||||
// - onnx：内嵌 ONNX 模型（如 CLIP）
|
||||
// - http：外部向量 API 服务（Jina / OpenAI / 自建）
|
||||
// type 为空时禁用多模态向量检索，退回纯 fastText 文本路径。
|
||||
var multimodalSpace vector.MultimodalEmbedder
|
||||
switch mmType := cfgReg.GetString("core.memory.multimodal_space.type", ""); mmType {
|
||||
case "onnx":
|
||||
if clipDir := cfgReg.GetString("core.memory.media.clip_model_dir", ""); clipDir != "" {
|
||||
e, err := clip.New(clipDir)
|
||||
if err != nil {
|
||||
log.Printf("[homed] warning: onnx embedder load failed: %v（多模态向量检索已禁用）", err)
|
||||
} else {
|
||||
multimodalSpace = e
|
||||
defer e.Close()
|
||||
log.Printf("[homed] multimodal space (onnx) active: dim=%d fp=%s", e.Dim(), e.Fingerprint()[:min(12, len(e.Fingerprint()))])
|
||||
}
|
||||
} else {
|
||||
clipEmbedder = e
|
||||
defer e.Close()
|
||||
log.Printf("[homed] clip embedder active: dim=%d fp=%s", e.Dim(), e.Fingerprint()[:min(12, len(e.Fingerprint()))])
|
||||
log.Println("[homed] multimodal_space.type=onnx 但未配置 clip_model_dir，多模态向量检索已禁用")
|
||||
}
|
||||
case "http":
|
||||
dim := cfgReg.GetInt("core.memory.multimodal_space.http.dimension", 0)
|
||||
ep := cfgReg.GetString("core.memory.multimodal_space.http.endpoint", "")
|
||||
if dim > 0 && ep != "" {
|
||||
e, err := vector.NewHTTPEmbedder(vector.HTTPEmbedderConfig{
|
||||
Endpoint: ep,
|
||||
APIKey: cfgReg.GetString("core.memory.multimodal_space.http.api_key", ""),
|
||||
Model: cfgReg.GetString("core.memory.multimodal_space.http.model", ""),
|
||||
Dimension: dim,
|
||||
Timeout: cfgReg.GetDuration("core.memory.multimodal_space.http.timeout", 30*time.Second),
|
||||
Fingerprint: cfgReg.GetString("core.memory.multimodal_space.http.fingerprint", ""),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[homed] warning: http embedder init failed: %v（多模态向量检索已禁用）", err)
|
||||
} else {
|
||||
multimodalSpace = e
|
||||
defer e.Close()
|
||||
log.Printf("[homed] multimodal space (http) active: endpoint=%s dim=%d", ep, dim)
|
||||
}
|
||||
} else {
|
||||
log.Println("[homed] multimodal_space.type=http 但 endpoint/dimension 配置不完整，多模态向量检索已禁用")
|
||||
}
|
||||
default:
|
||||
if mmType != "" {
|
||||
log.Printf("[homed] warning: 未知 multimodal_space.type=%q，多模态向量检索已禁用", mmType)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ks := knowledge.NewStore(filepath.Join(cfg.Daemon.DataDir, "knowledge"))
|
||||
if err := ks.Start(); err != nil {
|
||||
log.Printf("[homed] warning: knowledge store: %v", err)
|
||||
@ -484,7 +521,7 @@ func main() {
|
||||
ContextSavePath: filepath.Join(cfg.Daemon.DataDir, "memory", "context.json"),
|
||||
EmbeddingModelPath: cfgReg.GetString("core.agent.embedding_model_path", ""),
|
||||
Embedder: embedder,
|
||||
ClipEmbedder: clipEmbedder,
|
||||
MultimodalSpace: multimodalSpace,
|
||||
StageHost: stageHost,
|
||||
EventBus: evBus,
|
||||
ThinkingEnabled: cfg.LLM.ThinkingEnabled,
|
||||
|
||||
124
docs/embedding-comparison.md
Normal file
124
docs/embedding-comparison.md
Normal file
@ -0,0 +1,124 @@
|
||||
# 检索方案对比报告（2026-09-09）
|
||||
## 测试数据
|
||||
- 文档库：492 篇生产文档（过滤 108 条健康检查测试文档）
|
||||
- 媒体库：3 张生产图片（验证码、新闻截图、深色模式备忘录）
|
||||
- 文本查询：10 组（精确匹配、语义、跨语言、模糊表达）
|
||||
- 媒体查询：6 组（中文/英文查图片，3 张图片各 2 条）
|
||||
|
||||
---
|
||||
|
||||
## 一、文本检索对比（文档库）
|
||||
|
||||
| 方案 | Hit@1 | Hit@5 | MRR | 平均延迟 |
|
||||
|------|-------|-------|-----|----------|
|
||||
| TF-IDF | 3/10 | 7/10 | 0.457 | 0.3ms |
|
||||
| fastText（200k 中文+378k 英文） | 5/10 | 5/10 | 0.530 | 8.3ms |
|
||||
| TF-IDF + fastText RRF | 4/10 | 7/10 | 0.552 | 12.3ms |
|
||||
| **Jina v5-omni-nano** | **8/10** | **10/10** | **0.900** | **39.9ms** |
|
||||
|
||||
### 关键发现
|
||||
|
||||
1. **Jina 的优势来自"短语语义"能力**：
|
||||
- "邮件代理是否已经成功接入" → TF-IDF rank 5，Jina rank 1
|
||||
- "升级安装 QQ 插件包" → fastText rank 169，Jina rank 1（margin +0.30）
|
||||
- "我所在城市的天气预报" → fastText rank 44，Jina rank 1
|
||||
- "聊天输入区域文字多了会不会自动增高" → TF-IDF rank 1，Jina rank 1（margin +0.33）
|
||||
|
||||
2. **TF-IDF 在精确匹配上不可替代**：
|
||||
- "长期文档记忆功能是否健康" → TF-IDF rank 3，Jina rank 1
|
||||
- "重新加载全部扩展组件" → TF-IDF rank 0（完全未命中），Jina rank 2
|
||||
- TF-IDF 的 Hit@5 70% 证明精确关键词召回仍有价值
|
||||
|
||||
3. **RRF 融合反而变差**：
|
||||
- TF-IDF+fastText RRF MRR=0.552，低于 Jina 单路 0.900
|
||||
- 原因：两种稀疏向量的排序在语义查询上高度重叠，RRF 无法弥补各自短板
|
||||
|
||||
---
|
||||
|
||||
## 二、图片检索对比（同 3 张图片，6 条查询）
|
||||
|
||||
| 方案 | Hit@1 | MRR | 平均 margin |
|
||||
|------|-------|-----|-------------|
|
||||
| CLIP ViT-B/32 | 4/6 | 0.806 | -0.008（负值！） |
|
||||
| Jina v5-omni-nano | 4/6 | 0.833 | +0.024 |
|
||||
|
||||
### 逐条对比
|
||||
|
||||
| 查询 | CLIP rank | CLIP margin | Jina rank | Jina margin |
|
||||
|------|-----------|-------------|-----------|-------------|
|
||||
| 验证码图片（中） | 1 | +0.027 | 1 | +0.036 |
|
||||
| 验证码图片（英） | 1 | +0.063 | 1 | +0.077 |
|
||||
| 新闻截图（中） | 6 | -0.091 | 2 | -0.064 |
|
||||
| 新闻截图（英） | 1 | +0.008 | 2 | -0.028 |
|
||||
| 备忘录截图（中） | 3 | -0.045 | 1 | +0.045 |
|
||||
| 备忘录截图（英） | 1 | +0.051 | 1 | +0.079 |
|
||||
|
||||
### 关键发现
|
||||
|
||||
1. **中文文本→图片**：Jina 明显优于 CLIP（MRR 0.833 vs 0.611）
|
||||
- CLIP 中文查询余弦可低至 -0.076（完全反直觉）
|
||||
- Jina 最差也是 +0.045，正样本始终高于负样本
|
||||
|
||||
2. **新闻截图是共同弱点**：
|
||||
- CLIP 和 Jina 都被"深色模式备忘录"抢走新闻截图的排序
|
||||
- 原因：新闻截图的文字描述含"深色"、"备忘录"等词，与备忘录图片的视觉特征重叠
|
||||
- 这是描述质量 vs 视觉特征的竞争，不是模型问题
|
||||
|
||||
3. **margin 的实际意义**：
|
||||
- CLIP 的平均 margin = -0.008（负值意味着正样本平均不如负样本）
|
||||
- Jina 的平均 margin = +0.024（正样本始终略高于负样本）
|
||||
- 但两者的 margin 都很小（< 0.1），生产环境仍需阈值校准
|
||||
|
||||
---
|
||||
|
||||
## 三、延迟与资源
|
||||
|
||||
| 方案 | 单次查询延迟 | 索引构建 | 内存 |
|
||||
|------|-------------|----------|------|
|
||||
| TF-IDF | 0.3ms | <1s | ~50MB |
|
||||
| fastText | 8.3ms | <1s | ~200MB |
|
||||
| CLIP ONNX | 26ms | N/A | ~600MB |
|
||||
| Jina v5-omni CPU | 39.9ms | 78s（492篇） | ~4GB |
|
||||
|
||||
---
|
||||
|
||||
## 四、结论与建议
|
||||
|
||||
### 核心判断
|
||||
|
||||
| 维度 | TF-IDF/fastText | CLIP | Jina v5-omni |
|
||||
|------|-----------------|------|--------------|
|
||||
| 文本精确匹配 | ★★★★★ | N/A | ★★★★ |
|
||||
| 文本语义检索 | ★★ | N/A | ★★★★★ |
|
||||
| 中文文本→图片 | 无能力 | ★ | ★★★★ |
|
||||
| 英文文本→图片 | 无能力 | ★★★ | ★★★★ |
|
||||
| 图片→图片 | 无能力 | ★★★ | ★★★★ |
|
||||
| 多语言统一空间 | 无能力 | 有限 | ★★★★★ |
|
||||
| 延迟 | ★★★★★ | ★★★ | ★★ |
|
||||
|
||||
### 架构建议
|
||||
|
||||
1. **保留 TF-IDF 作为精确召回的一级通道**：
|
||||
- 0.3ms 延迟不可替代
|
||||
- Hit@5 70% 证明在关键词匹配场景仍有价值
|
||||
- 特别是"插件安装"、"设备查询"这类精确操作指令
|
||||
|
||||
2. **用 Jina 替换 fastText + CLIP 的稠密通道**：
|
||||
- Jina 单路 MRR=0.90，超过 fastText+CLIP 融合
|
||||
- 统一空间消除三条通道的维护成本
|
||||
- 中文文本→图片从"无法检索"提升到"可检索"
|
||||
|
||||
3. **两路融合：TF-IDF + Jina RRF**（而非 TF-IDF + fastText RRF）：
|
||||
- TF-IDF 精确匹配 + Jina 语义覆盖
|
||||
- RRF 避免跨空间分数归一化问题
|
||||
- 预期 MRR > 0.90（精确匹配补 Jina 的语义盲区）
|
||||
|
||||
4. **图片检索仍需阈值校准**：
|
||||
- Jina 的 margin 平均 +0.024，生产环境需设置合理阈值
|
||||
- 建议：用真实正负样本对重新标定，而非沿用 CLIP 的 0.20 阈值
|
||||
|
||||
### 下一步
|
||||
|
||||
- 实现 TF-IDF + Jina RRF 融合，验证 MRR 是否能突破 0.90
|
||||
- 用更多生产图片标定 Jina 的图片检索阈值
|
||||
- 测试 fastText 词嵌入是否可以完全被 Jina 文本编码替代（L0 相关性计算）
|
||||
@ -13,11 +13,11 @@ import (
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/events"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/knowledge"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/clip"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||
@ -145,8 +145,12 @@ type Agent struct {
|
||||
// 词嵌入模型，用于实体语义相似度计算
|
||||
embedder *memory.StaticEmbedder
|
||||
|
||||
// clipEmb 是 CLIP 多模态嵌入（可选），nil 时跳过视觉向量计算。
|
||||
clipEmb *clip.Embedder
|
||||
// multimodalSpace 是统一多模态向量空间（可选）。实现可以是内嵌 ONNX，
|
||||
// 也可以是外部 API 客户端；两者共享同一套 L0/L2/L3 向量缓存与检索基础设施。
|
||||
multimodalSpace vector.MultimodalEmbedder
|
||||
|
||||
// fusionCfg 控制文本路与视觉路的跨模态融合权重，可按模型实测结果配置。
|
||||
fusionCfg CrossModalFusionConfig
|
||||
|
||||
// 技能索引提供者：由 skillmgr 插件实现，向 system prompt 注入轻量技能索引
|
||||
skillIndex SkillIndexProvider
|
||||
@ -178,7 +182,8 @@ type AgentConfig struct {
|
||||
MediaGCInterval time.Duration
|
||||
MediaGCMinAge time.Duration
|
||||
MediaDescribe bool
|
||||
ClipEmbedder *clip.Embedder
|
||||
MultimodalSpace vector.MultimodalEmbedder
|
||||
FusionCfg CrossModalFusionConfig // 跨模态融合权重；零值用默认
|
||||
Personality *agentPkg.Personality
|
||||
PluginReg *plugin.Registry
|
||||
PluginDir string
|
||||
@ -284,7 +289,8 @@ func New(cfg AgentConfig) *Agent {
|
||||
thinkingEnabled: cfg.ThinkingEnabled,
|
||||
inputCfg: cfg.InputProcessing,
|
||||
embedder: embedder,
|
||||
clipEmb: cfg.ClipEmbedder,
|
||||
multimodalSpace: cfg.MultimodalSpace,
|
||||
fusionCfg: cfg.FusionCfg,
|
||||
noMergeMarkers: make(map[string]int),
|
||||
lastInput: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
267
internal/agent/core/crossmodal.go
Normal file
267
internal/agent/core/crossmodal.go
Normal file
@ -0,0 +1,267 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||
)
|
||||
|
||||
// CrossModalHit 是跨模态检索融合后的一条候选。
|
||||
//
|
||||
// 统一的检索单元是记忆块而非 CAS 全库：媒体在 L0/L2/L3 都和文本一样有 owner，
|
||||
// 只有仍被某层记忆引用的媒体才可召回。Doc 是 L2 文档块；Media 是该块携带的
|
||||
// 原生媒体坐标。两路分数尺度不同，融合前各自归一化，见 fuseCrossModal。
|
||||
type CrossModalHit struct {
|
||||
Doc *document.Doc // 文本路命中的文档；视觉路命中时为 nil
|
||||
Media *media.Item // 视觉路命中的媒体；文本路命中时也可能带关联媒体
|
||||
MediaScore float64 // 视觉路原始 cosine（无则 0）
|
||||
DocScore float64 // 文本路原始 cosine（无则 0）
|
||||
Fused float64 // 归一化加权融合分，供最终排序
|
||||
// 该媒体同时被两路命中（文本路经文档关联、视觉路直接命中）时，
|
||||
// DoubleHit=true —— 双信号确认，应排在只被一路命中的候选之前。
|
||||
DoubleHit bool
|
||||
}
|
||||
|
||||
// CrossModalFusionConfig 控制文本路与视觉路的融合行为。
|
||||
// 默认各路权重 0.5，双命中加权 0.15；不同模型/场景可按实测调整。
|
||||
type CrossModalFusionConfig struct {
|
||||
WeightText float64 // 文本路融合权重（默认 0.5）
|
||||
WeightVisual float64 // 视觉路融合权重（默认 0.5）
|
||||
DoubleHitBonus float64 // 双命中额外加分（默认 0.15）
|
||||
MinMaxEps float64 // min-max 归一化除零保护（默认 1e-12）
|
||||
}
|
||||
|
||||
var defaultFusionConfig = CrossModalFusionConfig{
|
||||
WeightText: 0.5,
|
||||
WeightVisual: 0.5,
|
||||
DoubleHitBonus: 0.15,
|
||||
MinMaxEps: 1e-12,
|
||||
}
|
||||
|
||||
func (c CrossModalFusionConfig) textWeight() float64 {
|
||||
if c.WeightText <= 0 {
|
||||
return defaultFusionConfig.WeightText
|
||||
}
|
||||
return c.WeightText
|
||||
}
|
||||
func (c CrossModalFusionConfig) visualWeight() float64 {
|
||||
if c.WeightVisual <= 0 {
|
||||
return defaultFusionConfig.WeightVisual
|
||||
}
|
||||
return c.WeightVisual
|
||||
}
|
||||
func (c CrossModalFusionConfig) doubleHitBonus() float64 {
|
||||
return c.DoubleHitBonus
|
||||
}
|
||||
func (c CrossModalFusionConfig) minMaxEps() float64 {
|
||||
if c.MinMaxEps <= 0 {
|
||||
return defaultFusionConfig.MinMaxEps
|
||||
}
|
||||
return c.MinMaxEps
|
||||
}
|
||||
|
||||
// retrieveCrossModal 是跨模态并行检索的统一入口。
|
||||
//
|
||||
// 策略（两路并行，召回真正最相似的）：
|
||||
// 1. 文本路：query 整段文本用现有方法（fastText/TF-IDF 稀疏 cosine）查文档层，
|
||||
// 每个命中文档再反查其关联媒体（docMediaContext）——描述文本命中即媒体命中。
|
||||
// 2. 视觉路：query 整段文本经多模态模型文本编码 → 与媒体库全部图像坐标比余弦
|
||||
// （QueryMediaScored），覆盖描述文本没写到的视觉内容。
|
||||
// 3. 融合：两条路候选各自 min-max 归一化到 [0,1]，加权求和后降序，取 topK。
|
||||
// 同一媒体被两路同时命中视为双信号确认，额外加权。
|
||||
//
|
||||
// 多模态空间未配置时视觉路为空，退化为纯文本路（等价旧 docStore.Query）。
|
||||
func (a *Agent) retrieveCrossModal(query string, topK int, cfg CrossModalFusionConfig) []CrossModalHit {
|
||||
if topK <= 0 {
|
||||
topK = 5
|
||||
}
|
||||
// 融合前各取 2× 余量，保证融合排序后 topK 仍有足够候选。
|
||||
per := topK * 2
|
||||
if per < 8 {
|
||||
per = 8
|
||||
}
|
||||
|
||||
// ---- 文本路 ----
|
||||
var textHits []CrossModalHit
|
||||
if a.docStore != nil {
|
||||
for _, dh := range a.docStore.QueryScored(query, per) {
|
||||
hit := CrossModalHit{Doc: dh.Doc, DocScore: dh.Score}
|
||||
// 命中文档若关联着媒体（media_refs），把媒体作为文本路候选一并带上：
|
||||
// 描述文本命中 → 该媒体就是相关记忆，供后续展示/注入。
|
||||
if a.mediaStore != nil {
|
||||
refs, err := a.mediaStore.Refs(media.OwnerDocument, dh.Doc.ID)
|
||||
if err == nil && len(refs) > 0 {
|
||||
if it, err := a.mediaStore.Stat(refs[0]); err == nil {
|
||||
hit.Media = it
|
||||
}
|
||||
}
|
||||
}
|
||||
textHits = append(textHits, hit)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 视觉路（多模态文本编码 → 当前记忆层持有的媒体块）----
|
||||
var visualHits []CrossModalHit
|
||||
if a.multimodalSpace != nil && a.multimodalSpace.Loaded() && a.mediaStore != nil {
|
||||
qv, err := a.multimodalSpace.VectorizeDense(query)
|
||||
if err != nil {
|
||||
log.Printf("[crossmodal] 多模态文本编码失败: %v", err)
|
||||
} else if mh, err := a.mediaStore.QueryMemoryMediaScored(qv, a.multimodalSpace.Fingerprint(), per); err != nil {
|
||||
log.Printf("[crossmodal] 媒体记忆检索失败: %v", err)
|
||||
} else {
|
||||
for _, h := range mh {
|
||||
visualHits = append(visualHits, CrossModalHit{
|
||||
Media: h.Item, MediaScore: h.Score,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fuseCrossModal(textHits, visualHits, topK, cfg)
|
||||
}
|
||||
|
||||
// fuseCrossModal 把文本路与视觉路候选按各自归一化分融合排序。
|
||||
//
|
||||
// 归一化模板：两路分数尺度不可直接相加，先各自在路内 min-max 到 [0,1]：
|
||||
//
|
||||
// norm(x) = (x - min) / (max - min)，max==min 时置 1
|
||||
//
|
||||
// 再加权求和：fused = wText·normText + wVisual·normVisual。同一媒体两路都命中
|
||||
// （经文档关联 + 视觉直接）时 DoubleHit，在加权分上再加双信号确认分。
|
||||
// 权重通过 CrossModalFusionConfig 按场景配置，不同模型/版本可按实测调整。
|
||||
func fuseCrossModal(textHits, visualHits []CrossModalHit, topK int, cfg CrossModalFusionConfig) []CrossModalHit {
|
||||
norm := func(hits []CrossModalHit, pick func(CrossModalHit) float64) []float64 {
|
||||
out := make([]float64, len(hits))
|
||||
if len(hits) == 0 {
|
||||
return out
|
||||
}
|
||||
maxV, minV := pick(hits[0]), pick(hits[0])
|
||||
for _, h := range hits[1:] {
|
||||
v := pick(h)
|
||||
if v > maxV {
|
||||
maxV = v
|
||||
}
|
||||
if v < minV {
|
||||
minV = v
|
||||
}
|
||||
}
|
||||
for i, h := range hits {
|
||||
v := pick(h)
|
||||
if maxV-minV < cfg.minMaxEps() {
|
||||
out[i] = 1
|
||||
continue
|
||||
}
|
||||
out[i] = (v - minV) / (maxV - minV)
|
||||
}
|
||||
return out
|
||||
}
|
||||
textN := norm(textHits, func(h CrossModalHit) float64 { return h.DocScore })
|
||||
visualN := norm(visualHits, func(h CrossModalHit) float64 { return h.MediaScore })
|
||||
|
||||
byKey := make(map[string]*CrossModalHit)
|
||||
var keys []string
|
||||
key := func(h CrossModalHit) string {
|
||||
if h.Doc != nil {
|
||||
return "doc:" + h.Doc.ID
|
||||
}
|
||||
if h.Media != nil {
|
||||
return "media:" + h.Media.Digest
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// 先并入视觉路（视觉媒体是独立实体）
|
||||
for i, h := range visualHits {
|
||||
k := key(h)
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
clone := h
|
||||
clone.Fused = cfg.visualWeight() * visualN[i]
|
||||
byKey[k] = &clone
|
||||
keys = append(keys, k)
|
||||
}
|
||||
// 再并入文本路：命中的文档是独立实体；带媒体的文档若其媒体 digest
|
||||
// 已在视觉路（双命中），合并到同一候选并标记 DoubleHit。
|
||||
for i, h := range textHits {
|
||||
if h.Doc == nil {
|
||||
continue
|
||||
}
|
||||
if h.Media != nil {
|
||||
if ex, ok := byKey["media:"+h.Media.Digest]; ok {
|
||||
ex.DoubleHit = true
|
||||
ex.Doc = h.Doc
|
||||
ex.Fused += cfg.textWeight()*textN[i] + cfg.doubleHitBonus()
|
||||
continue
|
||||
}
|
||||
}
|
||||
k := "doc:" + h.Doc.ID
|
||||
if ex, ok := byKey[k]; ok {
|
||||
ex.Doc = h.Doc
|
||||
ex.DoubleHit = false
|
||||
ex.Fused += cfg.textWeight() * textN[i]
|
||||
continue
|
||||
}
|
||||
clone := h
|
||||
clone.Fused = cfg.textWeight() * textN[i]
|
||||
byKey[k] = &clone
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
var merged []CrossModalHit
|
||||
for _, k := range keys {
|
||||
if c := byKey[k]; c != nil {
|
||||
merged = append(merged, *c)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(merged, func(i, j int) bool {
|
||||
if merged[i].DoubleHit != merged[j].DoubleHit {
|
||||
return merged[i].DoubleHit
|
||||
}
|
||||
return merged[i].Fused > merged[j].Fused
|
||||
})
|
||||
if len(merged) > topK {
|
||||
merged = merged[:topK]
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// crossModalMarkdown 把融合候选渲染成注入上下文的文本。
|
||||
// 文档行对齐既有【相关记忆文档】格式；媒体行复用 mediaMarkerLine 的
|
||||
// `[<mime> <短digest>] <描述>` 格式（那是解析回媒体引用的唯一合法格式）。
|
||||
func (a *Agent) crossModalMarkdown(hits []CrossModalHit) string {
|
||||
if len(hits) == 0 {
|
||||
return ""
|
||||
}
|
||||
var lines []string
|
||||
for i, h := range hits {
|
||||
marker := ""
|
||||
switch {
|
||||
case h.DoubleHit:
|
||||
marker = "（图文双命中）"
|
||||
case h.Doc != nil:
|
||||
marker = "（文本命中）"
|
||||
case h.Media != nil:
|
||||
marker = "（视觉命中）"
|
||||
}
|
||||
parts := []string{fmt.Sprintf("[%d]", i+1)}
|
||||
if h.Doc != nil {
|
||||
parts = append(parts, h.Doc.Summary)
|
||||
if h.Doc.Source != "" {
|
||||
parts = append(parts, fmt.Sprintf("(来源:%s)", h.Doc.Source))
|
||||
}
|
||||
}
|
||||
if h.Media != nil {
|
||||
if line := a.mediaMarkerLine(h.Media.Digest); line != "" {
|
||||
parts = append(parts, line)
|
||||
}
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("相关度:%.2f%s", h.Fused, marker))
|
||||
lines = append(lines, strings.Join(parts, " "))
|
||||
}
|
||||
return "【跨模态相关记忆】\n" + strings.Join(lines, "\n")
|
||||
}
|
||||
@ -3,6 +3,8 @@ package core
|
||||
import (
|
||||
"log"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||
@ -180,63 +182,106 @@ func (a *Agent) describePendingMedia() {
|
||||
}
|
||||
log.Printf("[media] 已描述 %s (%s, %d 字, 源=%s)", shortDigest(it.Digest), kind, len([]rune(desc)), srcName)
|
||||
|
||||
// 描述成功后，若 CLIP 嵌入器可用且是图片，计算视觉向量。
|
||||
// 这是"描述 + 向量"两步同步完成的路径；对于历史已有描述但无向量的媒体，
|
||||
// 由启动时的 reembedStaleMedia 补算。
|
||||
if a.clipEmb != nil && kind == "image" && it.Kind == media.KindImage {
|
||||
vec, err := a.clipEmb.EmbedImageDense(data, mime)
|
||||
if err != nil {
|
||||
log.Printf("[media] 视觉嵌入失败 %s: %v", shortDigest(it.Digest), err)
|
||||
} else if err := a.mediaStore.SetVec(it.Digest, vec, a.clipEmb.Fingerprint()); err != nil {
|
||||
log.Printf("[media] 写向量失败 %s: %v", shortDigest(it.Digest), err)
|
||||
} else {
|
||||
log.Printf("[media] 已嵌入 %s (dim=%d)", shortDigest(it.Digest), len(vec))
|
||||
}
|
||||
}
|
||||
// 描述成功后无需再次做视觉嵌入：图片在进入 L0 记忆块时已由
|
||||
// embedMediaOnIngest 计算并写入 CAS，L0→L2→L3 只转移引用并复用坐标。
|
||||
// 历史已有图片或模型切换由启动时 reembedStaleMedia 一次性补算。
|
||||
}
|
||||
}
|
||||
|
||||
// reembedStaleMedia 在启动时为历史已有描述但无 CLIP 向量的图片补算视觉向量。
|
||||
// 避免安装 CLIP 后，旧图片永远只有描述文本、没有视觉向量，直到下次 Describe 才能写入。
|
||||
// reembedStaleMedia 在启动时批量迁移历史媒体向量到当前向量空间。
|
||||
//
|
||||
// 触发场景（任一变化都会导致旧向量无法参与查询）：
|
||||
// - 切换模型（模型 A→模型 B，fp 变了）
|
||||
// - 切换向量维度（ONNX→HTTP dim 512→1024）
|
||||
// - 首次部署嵌入服务（历史无向量的媒体补算）
|
||||
// - 嵌入服务离线后重新上线（失败条目 vec_model 仍为空）
|
||||
//
|
||||
// 并发策略：启动时用 worker pool 并行迁移，避免上千张图片串行耗时过长。
|
||||
// 并发数在 ONNX 内嵌路径下不超 CPU 核心数（避免 ONNX 并发限流），
|
||||
// 外部 API 路径下不超 8（避免打爆外部服务）。
|
||||
func (a *Agent) reembedStaleMedia() {
|
||||
if a.clipEmb == nil || a.mediaStore == nil {
|
||||
if a.multimodalSpace == nil || a.mediaStore == nil {
|
||||
return
|
||||
}
|
||||
fp := a.clipEmb.Fingerprint()
|
||||
digests, err := a.mediaStore.StaleVecDigests(fp)
|
||||
fp := a.multimodalSpace.Fingerprint()
|
||||
digests, err := a.mediaStore.StaleVecDigestsAll(fp)
|
||||
if err != nil {
|
||||
log.Printf("[media] 查询需重算向量的媒体失败: %v", err)
|
||||
return
|
||||
}
|
||||
if len(digests) == 0 {
|
||||
log.Printf("[media] 无历史媒体需要补算视觉向量")
|
||||
log.Printf("[media] 无需迁移向量（所有媒体已与当前空间对齐 fp=%s）", shortFP(fp))
|
||||
return
|
||||
}
|
||||
log.Printf("[media] 启动补算视觉向量: %d 条 (fp=%s...)", len(digests), fp[:min(12, len(fp))])
|
||||
done := 0
|
||||
for _, d := range digests {
|
||||
it, err := a.mediaStore.Stat(d)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
data, err := a.mediaStore.Get(d)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mime := it.MIME
|
||||
if mime == "" {
|
||||
mime = "image/png"
|
||||
}
|
||||
vec, err := a.clipEmb.EmbedImageDense(data, mime)
|
||||
if err != nil {
|
||||
log.Printf("[media] 启动补算失败 %s: %v", shortDigest(d), err)
|
||||
continue
|
||||
}
|
||||
if err := a.mediaStore.SetVec(d, vec, fp); err != nil {
|
||||
log.Printf("[media] 启动写入向量失败 %s: %v", shortDigest(d), err)
|
||||
continue
|
||||
}
|
||||
done++
|
||||
|
||||
// 并发度：ONNX 内嵌不超过 4，外部 API 不超过 8（由配置或实际环境动态定）
|
||||
workers := 4
|
||||
if fp[:min(4, len(fp))] == "http:" {
|
||||
workers = 8
|
||||
}
|
||||
log.Printf("[media] 启动补算完成: %d/%d", done, len(digests))
|
||||
log.Printf("[media] 启动向量迁移: %d 条 → 新空间 fp=%s dim=%d workers=%d",
|
||||
len(digests), shortFP(fp), a.multimodalSpace.Dim(), workers)
|
||||
|
||||
jobs := make(chan string, workers*2)
|
||||
var done, failed int64
|
||||
var failedMu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for d := range jobs {
|
||||
if err := a.reembedOne(d, fp); err != nil {
|
||||
failedMu.Lock()
|
||||
failed++
|
||||
failedMu.Unlock()
|
||||
continue
|
||||
}
|
||||
atomic.AddInt64(&done, 1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for i, d := range digests {
|
||||
jobs <- d
|
||||
// 每迁移 20 条输出进度日志，让用户看到迁移在推进
|
||||
if (i+1)%20 == 0 {
|
||||
log.Printf("[media] 向量迁移进度: %d/%d (done=%d failed=%d)", i+1, len(digests), atomic.LoadInt64(&done), failed)
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
log.Printf("[media] 向量迁移完成: 成功=%d 失败=%d 总计=%d fp=%s",
|
||||
done, failed, len(digests), shortFP(fp))
|
||||
}
|
||||
|
||||
// reembedOne 为单条媒体重新计算向量并写入。stat 错误时跳过（可能已被 GC 清除）。
|
||||
// Get 错误或 Embed 错误时静默跳过该条目（不影响迁移其他条目）。
|
||||
func (a *Agent) reembedOne(digest, fp string) error {
|
||||
it, err := a.mediaStore.Stat(digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := a.mediaStore.Get(digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mime := it.MIME
|
||||
if mime == "" {
|
||||
mime = "image/png"
|
||||
}
|
||||
vec, err := a.multimodalSpace.EmbedImageDense(data, mime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.mediaStore.SetVec(digest, vec, fp)
|
||||
}
|
||||
|
||||
// shortFP 截断 fingerprint 为可读日志格式。
|
||||
func shortFP(fp string) string {
|
||||
if len(fp) > 12 {
|
||||
return fp[:12]
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
@ -63,11 +63,35 @@ func (a *Agent) captureBlockMedia(blocks []agentAPI.ContentBlock, tool string) [
|
||||
log.Printf("[media] 落盘失败 (tool=%s mime=%s): %v", tool, mime, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 入库即算一次多模态坐标并缓存（多模态空间可用时）。
|
||||
// 之后 doc_query / memory_recall / 内部召回直接复用 SetVec 的缓存坐标，
|
||||
// 不重复跑 ONNX；模型切换由启动时的 reembedStaleMedia 补算。
|
||||
a.embedMediaOnIngest(d, mime, data)
|
||||
digests = append(digests, d)
|
||||
}
|
||||
return digests
|
||||
}
|
||||
|
||||
// embedMediaOnIngest 给刚入库的图片立即计算多模态坐标并缓存。
|
||||
// 只在 多模态空间可用且为图像时执行；音频/未配置时静默跳过（保持既有行为）。
|
||||
func (a *Agent) embedMediaOnIngest(digest, mime string, data []byte) {
|
||||
if a.multimodalSpace == nil || !a.multimodalSpace.Loaded() {
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(mime, "image/") {
|
||||
return
|
||||
}
|
||||
vec, err := a.multimodalSpace.EmbedImageDense(data, mime)
|
||||
if err != nil {
|
||||
log.Printf("[media] 入库嵌入失败 %s: %v", shortDigest(digest), err)
|
||||
return
|
||||
}
|
||||
if err := a.mediaStore.SetVec(digest, vec, a.multimodalSpace.Fingerprint()); err != nil {
|
||||
log.Printf("[media] 入库写向量失败 %s: %v", shortDigest(digest), err)
|
||||
}
|
||||
}
|
||||
|
||||
// stageMediaDigests 累积本轮捕获的 digest，等 ContextEvent 建好后一起挂上。
|
||||
//
|
||||
// 为何要缓存而不是当场 AddRef：媒体在 process() 执行期间被捕获，而承载它的
|
||||
|
||||
@ -55,15 +55,13 @@ func (a *Agent) buildSystemPrompt(memContext string, userInput string) string {
|
||||
|
||||
prompt += "\n\n【记忆清理指令】当用户要求整理或清理记忆时，你必须实际调用 memory_ 工具执行操作，不能只回复文本。先用 memory_introspect 查看概况，再用 memory_recall 获取详情。有同义实体则用 memory_merge 合并（source 会被彻底删除），有无用噪音实体则用 memory_delete_entity 直接删除，也可用 memory_purge 批量清理，用 memory_edit 修正错误，用 memory_block_merge 标记不合并。如果工具执行成功，把结果告知用户；不要只描述计划而不执行。"
|
||||
|
||||
// 跨模态召回：文本路（fastText/TF-IDF 文档层，媒体描述文本已随记忆进入）
|
||||
// + 视觉路（多模态文本编码 → 媒体库坐标）两路归一化融合。
|
||||
// 未配置多模态空间时视觉路为空，等价旧的 docStore.Query。
|
||||
if a.docStore != nil {
|
||||
docs := a.docStore.Query(userInput, 3)
|
||||
if len(docs) > 0 {
|
||||
var parts []string
|
||||
parts = append(parts, "【相关记忆文档】")
|
||||
for i, d := range docs {
|
||||
parts = append(parts, fmt.Sprintf(" [%d] %s", i+1, d.Summary))
|
||||
}
|
||||
prompt += "\n\n" + strings.Join(parts, "\n")
|
||||
hits := a.retrieveCrossModal(userInput, 3, a.fusionCfg)
|
||||
if md := a.crossModalMarkdown(hits); md != "" {
|
||||
prompt += "\n\n" + md
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,7 +73,7 @@ func (a *Agent) buildSystemPrompt(memContext string, userInput string) string {
|
||||
prompt += "- 同步通道（webui / cli / 终端）：直接返回纯文本，内核会把文本交给等待方显示，无需调用工具。\n"
|
||||
prompt += "- 异步通道（qq / wechat / 群聊等）：返回纯文本**【不会】**自动送达用户，必须调用 output_send__{通道名} 工具（注意 meta 里带上正确的 user_id 或 group_id）才能真正把消息发出去。\n"
|
||||
prompt += "- 不确定当前通道的发送方式时，先用 output_send__{通道名}_help 查看该通道的 meta 格式和 type 枚举，再决定。\n"
|
||||
prompt += "- 同一轮对话中可多次调用输出门工具。长消息应当分多次发出，而不是一口气发完。\n"
|
||||
prompt += "- 每轮对话**通常只需调用一次** output_send__{通道名} 即可完成回复。仅在内容确实超过单条消息长度上限（如 >4000 字）时才拆分为多条；拆分时每条应是完整段落，不要碎片化。\n"
|
||||
prompt += "- 需要多步执行的长任务：**必须先**向当前对话通道发一条确认消息告诉用户已收到（异步通道用输出门工具，同步通道直接返回文本），**然后再**执行具体排查工具。确认消息不代表任务完成，发出后仍需继续执行实际工具并最终汇报结果。\n"
|
||||
prompt += "- 用户从其他渠道发来「在哪里/怎么样了」这类追问时，先回忆上次任务的通道与上下文，再回同一通道。"
|
||||
|
||||
|
||||
@ -655,7 +655,14 @@ func (r *ConfigRegistry) seedCoreDefs(dataDir string) {
|
||||
reg(ConfigDef{Key: "core.memory.media.gc_interval", Default: "6h", Type: "duration", DisplayName: "媒体 GC 间隔", Description: "清理无引用媒体的周期；0 表示不自动清理", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.media.gc_min_age", Default: "1h", Type: "duration", DisplayName: "媒体 GC 保护期", Description: "新入库媒体在此时长内不被清理。刚落盘还没来得及挂到记忆上的项引用计数也是 0，靠这个保护期避免被误删", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.media.describe_on_ingest", Default: "false", Type: "bool", DisplayName: "自动描述媒体", Description: "后台用视觉/音频模型给未描述的媒体生成文字描述。**描述文本才是持久语义记忆**——blob 会被容量 GC 淘汰，描述会随记忆各层一直留存并可检索。代价是消耗视觉模型配额（单张图实测约 10s），故默认关闭；开启后每 30s 最多处理 4 条，不跟对话抢额度", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.media.clip_model_dir", Default: "", Type: "string", DisplayName: "CLIP 模型目录", Description: "多模态嵌入的 CLIP ONNX 模型目录（含 text.onnx、vision.onnx、clip_config.json、tokenizer.json、merges.txt）。留空禁用多模态向量检索，只保留 fastText 文本路径；配置后图片入库时自动计算视觉向量并与文档混合检索。修改后需重启生效。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.media.clip_model_dir", Default: "", Type: "string", DisplayName: "CLIP ONNX 模型目录", Description: "多模态嵌入的 CLIP ONNX 模型目录（含 text.onnx、vision.onnx、clip_config.json、tokenizer.json、merges.txt）。multimodal_space.type=onnx 时从此目录加载内嵌模型；留空且 type=onnx 时禁用多模态向量检索。修改后需重启生效。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.multimodal_space.type", Default: "", Type: "string", DisplayName: "多模态向量空间类型", Description: "可插拔多模态向量空间的实现类型。onnx = 内嵌 ONNX 模型（CLIP 等）；http = 外部向量 API 服务。留空禁用多模态向量检索，只保留 fastText 文本路径。两条路径共享同一套 L0/L2/L3 向量缓存与检索基础设施。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.multimodal_space.http.endpoint", Default: "", Type: "string", DisplayName: "外部向量 API 端点", Description: "外部多模态向量服务的 HTTP 端点 URL（POST，接受 modality/side/text/data/mime，返回 embedding）。type=http 时必填。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.multimodal_space.http.api_key", Default: "", Type: "string", DisplayName: "外部向量 API 密钥", Description: "外部多模态向量服务的 API 密钥（作为 Bearer token 发送）。可选。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.multimodal_space.http.model", Default: "", Type: "string", DisplayName: "外部向量模型标识", Description: "外部向量服务使用的模型名称，作为 vec_model 持久化。模型切换后历史向量会自动重算。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.multimodal_space.http.dimension", Default: "0", Type: "int", DisplayName: "外部向量维度", Description: "外部向量服务返回的特征向量维度。必须与实际 API 返回值一致，否则运行时报错。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.multimodal_space.http.timeout", Default: "30s", Type: "duration", DisplayName: "外部向量 API 超时", Description: "单次向量请求的超时时间。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.memory.multimodal_space.http.fingerprint", Default: "", Type: "string", DisplayName: "外部向量空间指纹", Description: "用于标识外部向量空间版本的字符串（留空时自动根据 model+dim 生成）。模型切换后若 fingerprint 变化，历史向量会被重算。", Category: "memory"})
|
||||
reg(ConfigDef{Key: "core.knowledge.path", Default: filepath.Join(dataDir, "knowledge"), Type: "string", DisplayName: "知识库路径", Description: "知识库存储目录", Category: "paths"})
|
||||
reg(ConfigDef{Key: "core.log.path", Default: filepath.Join(dataDir, "log"), Type: "string", DisplayName: "日志目录", Description: "日志文件输出目录", Category: "paths"})
|
||||
|
||||
|
||||
@ -427,7 +427,8 @@ func (s *Store) Search(query string, kind Kind, limit int) ([]*Item, error) {
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
q := `SELECT digest, kind, mime, size, width, height, origin_path, tool,
|
||||
description, described_by, ref_count, first_seen, last_seen
|
||||
description, described_by, ref_count, first_seen, last_seen,
|
||||
vec, vec_model
|
||||
FROM media WHERE COALESCE(description,'') != ''`
|
||||
args := []interface{}{}
|
||||
if strings.TrimSpace(query) != "" {
|
||||
@ -473,7 +474,8 @@ func (s *Store) Pending(limit int) ([]*Item, error) {
|
||||
defer s.mu.RUnlock()
|
||||
rows, err := s.db.Query(`
|
||||
SELECT digest, kind, mime, size, width, height, origin_path, tool,
|
||||
description, described_by, ref_count, first_seen, last_seen
|
||||
description, described_by, ref_count, first_seen, last_seen,
|
||||
vec, vec_model
|
||||
FROM media
|
||||
WHERE COALESCE(description,'') = '' AND COALESCE(described_by,'') = ''
|
||||
ORDER BY last_seen DESC LIMIT ?`, limit)
|
||||
@ -650,15 +652,36 @@ func (s *Store) SetVec(digest string, vec []float64, model string) error {
|
||||
// vec_model 不等于 currentModel（模型切换）或 vec_model 为空（从未嵌入）。
|
||||
// 调用方使用返回的 digest 列表调用 Get/EmbedImage/SetVec 完成重算。
|
||||
func (s *Store) StaleVecDigests(currentModel string) ([]string, error) {
|
||||
return s.staleVecDigests(currentModel, "image")
|
||||
}
|
||||
|
||||
// StaleVecDigestsAll 返回所有需要重新嵌入的媒体 digest（不限 kind），
|
||||
// 供模型切换后全量迁移向量空间（image + audio + video 等）。
|
||||
func (s *Store) StaleVecDigestsAll(currentModel string) ([]string, error) {
|
||||
return s.staleVecDigests(currentModel, "")
|
||||
}
|
||||
|
||||
// staleVecDigests 是 StaleVecDigests 的核心实现，kind=” 时不按 kind 过滤。
|
||||
// 废弃了"只迁移图片"的限定：模型切换后所有模态都应迁移到新向量空间。
|
||||
func (s *Store) staleVecDigests(currentModel string, kind string) ([]string, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
rows, err := s.db.Query(`
|
||||
query := `
|
||||
SELECT digest FROM media
|
||||
WHERE kind = 'image'
|
||||
AND COALESCE(description,'') != ''
|
||||
AND (COALESCE(vec_model,'') = '' OR vec_model != ?)
|
||||
ORDER BY last_seen`, currentModel)
|
||||
WHERE (COALESCE(vec_model,'') = '' OR vec_model != ?)`
|
||||
if kind != "" {
|
||||
query += ` AND kind = ?`
|
||||
}
|
||||
query += ` ORDER BY last_seen`
|
||||
|
||||
var args []interface{}
|
||||
args = append(args, currentModel)
|
||||
if kind != "" {
|
||||
args = append(args, kind)
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -681,6 +704,46 @@ func (s *Store) StaleVecDigests(currentModel string) ([]string, error) {
|
||||
// 谁的相似度更高就召回谁——不再区分「这是一张图的查询」还是「这是一段文字的查询」，
|
||||
// 由向量空间的相似度自动判断。
|
||||
func (s *Store) QueryMedia(queryVec []float64, model string, topK int) ([]*Item, error) {
|
||||
hits, err := s.QueryMediaScored(queryVec, model, topK)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if hits == nil {
|
||||
return nil, nil
|
||||
}
|
||||
out := make([]*Item, len(hits))
|
||||
for i, h := range hits {
|
||||
out[i] = h.Item
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MediaHit 是一条媒体相似度候选及其分数。
|
||||
// 跨模态融合需要原始分数做归一化，仅返回 Item 会丢掉尺度信息。
|
||||
type MediaHit struct {
|
||||
Item *Item
|
||||
Score float64
|
||||
}
|
||||
|
||||
// QueryMemoryMediaScored 只检索当前仍被 L0/L2/L3 记忆块引用的媒体。
|
||||
// CAS 中 ref_count=0 的项是等待 GC 的孤儿缓存，不是可召回记忆；若把它们也查出，
|
||||
// 已从三层记忆淘汰的图片会被视觉路“复活”，破坏与文本块一致的生命周期。
|
||||
//
|
||||
// 分数只做排序，不在存储层设绝对阈值：多模态文本→图像的绝对 cosine 随模型、
|
||||
// 语言与数据域漂移，真实标定中有效命中可以低至 0.015。相关性门控在融合器中
|
||||
// 使用当前候选集合的相对分布完成。
|
||||
func (s *Store) QueryMemoryMediaScored(queryVec []float64, model string, topK int) ([]MediaHit, error) {
|
||||
return s.queryMediaScored(queryVec, model, topK, true)
|
||||
}
|
||||
|
||||
// QueryMediaScored 用查询向量对所有已嵌入媒体做余弦相似度检索，
|
||||
// 返回 topK 个最相似的候选及其原始 cosine 分数（供跨模态归一化）。
|
||||
// 这是媒体存储层的诊断/显式全库入口；记忆召回应调用 QueryMemoryMediaScored。
|
||||
func (s *Store) QueryMediaScored(queryVec []float64, model string, topK int) ([]MediaHit, error) {
|
||||
return s.queryMediaScored(queryVec, model, topK, false)
|
||||
}
|
||||
|
||||
func (s *Store) queryMediaScored(queryVec []float64, model string, topK int, referencedOnly bool) ([]MediaHit, error) {
|
||||
if topK <= 0 {
|
||||
topK = 20
|
||||
}
|
||||
@ -690,10 +753,21 @@ func (s *Store) QueryMedia(queryVec []float64, model string, topK int) ([]*Item,
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
rows, err := s.db.Query(`SELECT digest, kind, mime, size, width, height,
|
||||
query := `SELECT digest, kind, mime, size, width, height,
|
||||
origin_path, tool, description, described_by, ref_count, first_seen, last_seen,
|
||||
vec, vec_model
|
||||
FROM media WHERE vec IS NOT NULL AND vec != ''`)
|
||||
FROM media WHERE vec IS NOT NULL AND vec != ''`
|
||||
var args []interface{}
|
||||
if model != "" {
|
||||
query += ` AND vec_model = ?`
|
||||
args = append(args, model)
|
||||
}
|
||||
if referencedOnly {
|
||||
query += ` AND ref_count > 0 AND EXISTS (
|
||||
SELECT 1 FROM media_refs r WHERE r.digest = media.digest
|
||||
)`
|
||||
}
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -744,9 +818,9 @@ func (s *Store) QueryMedia(queryVec []float64, model string, topK int) ([]*Item,
|
||||
if len(candidates) > topK {
|
||||
candidates = candidates[:topK]
|
||||
}
|
||||
out := make([]*Item, len(candidates))
|
||||
out := make([]MediaHit, len(candidates))
|
||||
for i, c := range candidates {
|
||||
out[i] = c.item
|
||||
out[i] = MediaHit{Item: c.item, Score: c.score}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@ -149,25 +149,26 @@ func TestStaleVecDigests(t *testing.T) {
|
||||
// 有描述但从未嵌入（vec_model 空）→ stale
|
||||
d3, _ := s.Put([]byte("img3"), Item{MIME: "image/png", Description: "图三"})
|
||||
|
||||
// 无描述 → 不参与（描述流程外）
|
||||
s.Put([]byte("img4"), Item{MIME: "image/png"})
|
||||
// 无描述但有图片 → 也应被迁移（描述是可选语义通道，图片应独立于描述参与向量空间）
|
||||
d4, _ := s.Put([]byte("img4"), Item{MIME: "image/png"})
|
||||
|
||||
// 音频不属于图片 → 不算 stale
|
||||
// 音频不参与图片迁移（StaleVecDigests 只查 kind='image'）
|
||||
s.Put([]byte("aud1"), Item{MIME: "audio/wav", Description: "语音"})
|
||||
|
||||
stale, err := s.StaleVecDigests("clip-vit-b32")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(stale) != 2 {
|
||||
t.Fatalf("expected 2 stale digests (d2 旧模型 + d3 未嵌入), got %d: %v", len(stale), stale)
|
||||
// d1 匹配模型 → 非 stale；d2 旧模型 + d3 未嵌入 + d4 无描述图片 = 3 stale；aud1 不算
|
||||
if len(stale) != 3 {
|
||||
t.Fatalf("expected 3 stale digests (d2 旧模型 + d3 未嵌入 + d4 无描述), got %d: %v", len(stale), stale)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, d := range stale {
|
||||
got[d] = true
|
||||
}
|
||||
if !got[d2] || !got[d3] {
|
||||
t.Errorf("expected d2 and d3 stale, got %v", stale)
|
||||
if !got[d2] || !got[d3] || !got[d4] {
|
||||
t.Errorf("expected d2, d3, d4 stale, got %v", stale)
|
||||
}
|
||||
if got[d1] {
|
||||
t.Errorf("d1 (匹配模型) 不应 stale")
|
||||
|
||||
131
internal/memory/vector/http_embedder.go
Normal file
131
internal/memory/vector/http_embedder.go
Normal file
@ -0,0 +1,131 @@
|
||||
package vector
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// HTTPEmbedderConfig 配置一个外部多模态向量服务。
|
||||
// 服务契约刻意很小：POST Endpoint，输入 modality/data/mime/side，返回 embedding。
|
||||
// 任何云 API 或自建服务只需适配这一个协议，即可复用内核全部向量存储与检索链路。
|
||||
type HTTPEmbedderConfig struct {
|
||||
Endpoint string
|
||||
APIKey string
|
||||
Model string
|
||||
Dimension int
|
||||
Timeout time.Duration
|
||||
Fingerprint string
|
||||
}
|
||||
|
||||
// HTTPEmbedder 是 MultimodalEmbedder 的外部 API 实现。
|
||||
type HTTPEmbedder struct {
|
||||
cfg HTTPEmbedderConfig
|
||||
client *http.Client
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
}
|
||||
|
||||
type httpEmbedRequest struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
Modality string `json:"modality"`
|
||||
Side string `json:"side"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Data string `json:"data,omitempty"`
|
||||
MIME string `json:"mime,omitempty"`
|
||||
}
|
||||
|
||||
type httpEmbedResponse struct {
|
||||
Embedding []float64 `json:"embedding"`
|
||||
Data []struct {
|
||||
Embedding []float64 `json:"embedding"`
|
||||
} `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func NewHTTPEmbedder(cfg HTTPEmbedderConfig) (*HTTPEmbedder, error) {
|
||||
if strings.TrimSpace(cfg.Endpoint) == "" {
|
||||
return nil, fmt.Errorf("vector: empty HTTP embedding endpoint")
|
||||
}
|
||||
if cfg.Dimension <= 0 {
|
||||
return nil, fmt.Errorf("vector: invalid HTTP embedding dimension %d", cfg.Dimension)
|
||||
}
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = 30 * time.Second
|
||||
}
|
||||
if cfg.Fingerprint == "" {
|
||||
cfg.Fingerprint = "http:" + cfg.Model + fmt.Sprintf(":%d", cfg.Dimension)
|
||||
}
|
||||
return &HTTPEmbedder{cfg: cfg, client: &http.Client{Timeout: cfg.Timeout}}, nil
|
||||
}
|
||||
|
||||
func (e *HTTPEmbedder) VectorizeDense(text string) ([]float64, error) {
|
||||
return e.embed(httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityText), Side: "query", Text: text})
|
||||
}
|
||||
|
||||
func (e *HTTPEmbedder) EmbedImageDense(img []byte, mime string) ([]float64, error) {
|
||||
return e.embed(httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityImage), Side: "document", Data: base64.StdEncoding.EncodeToString(img), MIME: mime})
|
||||
}
|
||||
|
||||
func (e *HTTPEmbedder) embed(payload httpEmbedRequest) ([]float64, error) {
|
||||
e.mu.Lock()
|
||||
closed := e.closed
|
||||
e.mu.Unlock()
|
||||
if closed {
|
||||
return nil, fmt.Errorf("vector: HTTP embedder closed")
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, e.cfg.Endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if e.cfg.APIKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+e.cfg.APIKey)
|
||||
}
|
||||
resp, err := e.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("vector: HTTP embedding request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("vector: HTTP embedding status %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
var out httpEmbedResponse
|
||||
if err := json.Unmarshal(b, &out); err != nil {
|
||||
return nil, fmt.Errorf("vector: decode HTTP embedding: %w", err)
|
||||
}
|
||||
v := out.Embedding
|
||||
if len(v) == 0 && len(out.Data) > 0 {
|
||||
v = out.Data[0].Embedding
|
||||
}
|
||||
if len(v) != e.cfg.Dimension {
|
||||
return nil, fmt.Errorf("vector: HTTP embedding dimension %d, want %d", len(v), e.cfg.Dimension)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (e *HTTPEmbedder) Fingerprint() string { return e.cfg.Fingerprint }
|
||||
func (e *HTTPEmbedder) Dim() int { return e.cfg.Dimension }
|
||||
func (e *HTTPEmbedder) Loaded() bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return !e.closed
|
||||
}
|
||||
func (e *HTTPEmbedder) Close() {
|
||||
e.mu.Lock()
|
||||
e.closed = true
|
||||
e.mu.Unlock()
|
||||
}
|
||||
178
internal/memory/vector/http_embedder_test.go
Normal file
178
internal/memory/vector/http_embedder_test.go
Normal file
@ -0,0 +1,178 @@
|
||||
package vector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPEmbedder_RequiresEndpoint(t *testing.T) {
|
||||
_, err := NewHTTPEmbedder(HTTPEmbedderConfig{Dimension: 512})
|
||||
if err == nil {
|
||||
t.Fatal("应拒绝空 endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPEmbedder_RequiresDimension(t *testing.T) {
|
||||
_, err := NewHTTPEmbedder(HTTPEmbedderConfig{Endpoint: "http://localhost"})
|
||||
if err == nil {
|
||||
t.Fatal("应拒绝 dimension<=0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPEmbedder_TextEmbedding(t *testing.T) {
|
||||
// 模拟返回 4 维向量的外部服务
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("期望 POST，实际 %s", r.Method)
|
||||
}
|
||||
var req httpEmbedRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.Modality != "text" {
|
||||
t.Errorf("期望 modality=text，实际 %s", req.Modality)
|
||||
}
|
||||
if req.Text == "" {
|
||||
t.Fatal("text 不应为空")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"embedding":[0.1,0.2,0.3,0.4]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||
Endpoint: srv.URL,
|
||||
Dimension: 4,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer e.Close()
|
||||
|
||||
if !e.Loaded() {
|
||||
t.Fatal("应处于 loaded 状态")
|
||||
}
|
||||
|
||||
vec, err := e.VectorizeDense("hello world")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vec) != 4 || vec[0] != 0.1 || vec[3] != 0.4 {
|
||||
t.Errorf("向量不符合预期: %v", vec)
|
||||
}
|
||||
if e.Fingerprint() != "http:test-model:4" {
|
||||
t.Errorf("指纹不符合预期: %s", e.Fingerprint())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPEmbedder_ImageEmbedding(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req httpEmbedRequest
|
||||
json.NewDecoder(r.Body).Decode(&req)
|
||||
if req.Modality != "image" {
|
||||
t.Errorf("期望 modality=image，实际 %s", req.Modality)
|
||||
}
|
||||
if req.MIME != "image/png" {
|
||||
t.Errorf("期望 mime=image/png，实际 %s", req.MIME)
|
||||
}
|
||||
w.Write([]byte(`{"embedding":[0.5,0.5,0.5]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||
Endpoint: srv.URL,
|
||||
Dimension: 3,
|
||||
Model: "img-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer e.Close()
|
||||
|
||||
vec, err := e.EmbedImageDense([]byte("fake-png-data"), "image/png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vec) != 3 {
|
||||
t.Errorf("期望 3 维，实际 %d", len(vec))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPEmbedder_CustomFingerprint(t *testing.T) {
|
||||
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||
Endpoint: "http://localhost:1234",
|
||||
Dimension: 512,
|
||||
Fingerprint: "jina-v5-omni-nano:2026",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer e.Close()
|
||||
if e.Fingerprint() != "jina-v5-omni-nano:2026" {
|
||||
t.Errorf("自定义指纹未生效: %s", e.Fingerprint())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPEmbedder_DimensionMismatchReturnsError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"embedding":[1,2]}`)) // 返回 2 维，配置期望 4
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||
Endpoint: srv.URL,
|
||||
Dimension: 4,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer e.Close()
|
||||
|
||||
_, err = e.VectorizeDense("test")
|
||||
if err == nil {
|
||||
t.Fatal("维度不匹配时应返回错误")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPEmbedder_ServerErrorReturnsError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
w.Write([]byte("gateway down"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||
Endpoint: srv.URL,
|
||||
Dimension: 4,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer e.Close()
|
||||
|
||||
_, err = e.VectorizeDense("test")
|
||||
if err == nil {
|
||||
t.Fatal("服务端错误时应返回错误")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPEmbedder_ClosePreventsFurtherCalls(t *testing.T) {
|
||||
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||
Endpoint: "http://localhost:1234",
|
||||
Dimension: 4,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e.Close()
|
||||
if e.Loaded() {
|
||||
t.Fatal("关闭后 Loaded() 应返回 false")
|
||||
}
|
||||
_, err = e.VectorizeDense("test")
|
||||
if err == nil {
|
||||
t.Fatal("关闭后应返回错误")
|
||||
}
|
||||
}
|
||||
@ -17,11 +17,15 @@ type Vectorizer interface {
|
||||
EmbedImage(img []byte, mime string) (Vector, error)
|
||||
}
|
||||
|
||||
// MultimodalEmbedder 是稠密多模态编码器的接口（CLIP 等视觉-文本联合模型）。
|
||||
// MultimodalEmbedder 是稠密多模态编码器的接口。
|
||||
//
|
||||
// 与 Vectorizer（稀疏词向量，供 TF-IDF/倒排检索）刻意区分：多模态模型产出的
|
||||
// 是共享稠密空间（如 CLIP 512 维），直接用于 media.Store 的稠密余弦检索，
|
||||
// 是共享稠密空间，直接用于 media.Store 的稠密余弦检索，
|
||||
// **不得**塞进文档/知识层的稀疏 vector.Store（会破坏倒排剪枝与 TF-IDF 语义）。
|
||||
//
|
||||
// 实现不限：可以是内嵌 ONNX，也可以是外部 HTTP 向量服务——
|
||||
// 内核只依赖本接口，两条路径共享同一套检索/存储基础设施。Fingerprint 是模型
|
||||
// 空间标识（如模型文件指纹），作为 vec_model 持久化用于切换后重算。
|
||||
type MultimodalEmbedder interface {
|
||||
VectorizeDense(text string) ([]float64, error)
|
||||
EmbedImageDense(img []byte, mime string) ([]float64, error)
|
||||
@ -31,6 +35,18 @@ type MultimodalEmbedder interface {
|
||||
Close()
|
||||
}
|
||||
|
||||
// MultimodalModality 是统一向量空间支持的输入模态。
|
||||
// 现内核只消费 text/image；外部 API 路径可能扩展 audio/video，
|
||||
// 通过类型断言在接口外按需扩展，不破坏现有契约。
|
||||
type MultimodalModality string
|
||||
|
||||
const (
|
||||
ModalityText MultimodalModality = "text"
|
||||
ModalityImage MultimodalModality = "image"
|
||||
ModalityAudio MultimodalModality = "audio"
|
||||
ModalityVideo MultimodalModality = "video"
|
||||
)
|
||||
|
||||
// ErrNotSupported 表示 Vectorizer 不支持图像嵌入，调用方按文本描述降级。
|
||||
var ErrNotSupported = fmt.Errorf("vectorizer does not support image embedding")
|
||||
|
||||
@ -83,6 +99,26 @@ func (s *Store) Remove(id string) {
|
||||
}
|
||||
|
||||
func (s *Store) Search(query Vector, topK int) []DocVector {
|
||||
hits := s.SearchScored(query, topK)
|
||||
if len(hits) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]DocVector, len(hits))
|
||||
for i, h := range hits {
|
||||
out[i] = h.Doc
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DocVectorHit 是一篇文档的相似度候选及其原始 cosine 分数。
|
||||
// 跨模态融合需要分数做归一化；纯排序的 Search 不暴露它。
|
||||
type DocVectorHit struct {
|
||||
Doc DocVector
|
||||
Score float64
|
||||
}
|
||||
|
||||
// SearchScored 与 Search 同语义，但返回带原始 cosine 分数的候选。
|
||||
func (s *Store) SearchScored(query Vector, topK int) []DocVectorHit {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@ -122,9 +158,9 @@ func (s *Store) Search(query Vector, topK int) []DocVector {
|
||||
results = results[:topK]
|
||||
}
|
||||
|
||||
out := make([]DocVector, len(results))
|
||||
out := make([]DocVectorHit, len(results))
|
||||
for i, r := range results {
|
||||
out[i] = r.doc
|
||||
out[i] = DocVectorHit{Doc: r.doc, Score: r.score}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
254
tmp_benchmark/main.go
Normal file
254
tmp_benchmark/main.go
Normal file
@ -0,0 +1,254 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||
)
|
||||
|
||||
type benchDoc struct {
|
||||
ID string `json:"id"`
|
||||
Summary string `json:"summary"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type queryCase struct {
|
||||
Name string
|
||||
Query string
|
||||
Seeds []string
|
||||
}
|
||||
|
||||
type hit struct {
|
||||
ID string `json:"id"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
type result struct {
|
||||
Name string `json:"name"`
|
||||
Query string `json:"query"`
|
||||
Relevant int `json:"relevant"`
|
||||
Rank int `json:"rank"`
|
||||
Reciprocal float64 `json:"reciprocal_rank"`
|
||||
HitAt1 bool `json:"hit_at_1"`
|
||||
HitAt5 bool `json:"hit_at_5"`
|
||||
LatencyMS float64 `json:"latency_ms"`
|
||||
Top []hit `json:"top"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
Documents int `json:"documents"`
|
||||
Queries []queryCase `json:"queries"`
|
||||
Methods map[string][]result `json:"methods"`
|
||||
}
|
||||
|
||||
var cases = []queryCase{
|
||||
{Name: "mail-semantic", Query: "邮件代理是否已经成功接入", Seeds: []string{"AgentMail 接入验证"}},
|
||||
{Name: "fox-cross-language", Query: "生成一张雪地红狐狸的图片", Seeds: []string{"red fox in snowy forest"}},
|
||||
{Name: "plugin-semantic", Query: "升级安装 QQ 插件包", Seeds: []string{"plugin_install"}},
|
||||
{Name: "weather-paraphrase", Query: "我所在城市的天气预报", Seeds: []string{"河南新乡"}},
|
||||
{Name: "textarea-paraphrase", Query: "聊天输入区域文字多了会不会自动增高", Seeds: []string{"输入框在内容超过一行"}},
|
||||
{Name: "devices-paraphrase", Query: "检查当前接入了哪些终端设备", Seeds: []string{"你看看现在你都有哪些设备"}},
|
||||
{Name: "memory-health", Query: "长期文档记忆功能是否健康", Seeds: []string{"文档记忆系统是否正常工作"}},
|
||||
{Name: "reload-plugins", Query: "重新加载全部扩展组件", Seeds: []string{"热重载所有插件", "plgreload"}},
|
||||
{Name: "exact-agentmail", Query: "AgentMail 接入验证", Seeds: []string{"AgentMail 接入验证"}},
|
||||
{Name: "exact-plugin", Query: "plugin_install", Seeds: []string{"plugin_install"}},
|
||||
}
|
||||
|
||||
func main() {
|
||||
docs, err := loadDocs("/home/newqqagent/memory/documents")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "loaded %d production documents\n", len(docs))
|
||||
|
||||
rel := relevantSets(docs)
|
||||
for i, c := range cases {
|
||||
fmt.Fprintf(os.Stderr, "case %-20s relevant=%d query=%q\n", c.Name, len(rel[i]), c.Query)
|
||||
}
|
||||
|
||||
r := report{Documents: len(docs), Queries: cases, Methods: make(map[string][]result)}
|
||||
|
||||
// 方案 A：纯 TF-IDF。完整训练在生产文档上，保留 IDF 高频抑制与倒排候选剪枝。
|
||||
tfidf := vector.NewTFIDFVectorizer(memory.TokenizeWords)
|
||||
texts := make([]string, len(docs))
|
||||
for i, d := range docs {
|
||||
texts[i] = d.Summary + "\n" + d.Content
|
||||
}
|
||||
tfidf.Train(texts)
|
||||
tfStore := buildStore(docs, tfidf)
|
||||
r.Methods["tfidf"] = runCases(tfStore, tfidf, rel)
|
||||
|
||||
// 方案 B：当前生产 fastText（中英各 20/37 万词，300 维平均词向量）。
|
||||
fast := memory.NewStaticEmbedder("/data/cc.zh.top200k.vec", "/data/cc.en.top200k.vec")
|
||||
fastStore := buildStore(docs, fast)
|
||||
r.Methods["fasttext"] = runCases(fastStore, fast, rel)
|
||||
|
||||
// 方案 C：旧通道混合。RRF 不要求两种分数处于同一标尺，避免拍脑袋设绝对权重。
|
||||
r.Methods["tfidf_fasttext_rrf"] = runHybrid(tfStore, tfidf, fastStore, fast, rel)
|
||||
|
||||
out, _ := json.MarshalIndent(r, "", " ")
|
||||
if err := os.WriteFile("/tmp/homeagent-old-retrieval.json", out, 0644); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
printSummary(r)
|
||||
}
|
||||
|
||||
func loadDocs(dir string) ([]benchDoc, error) {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var docs []benchDoc
|
||||
for _, e := range ents {
|
||||
if e.IsDir() || !strings.HasPrefix(e.Name(), "doc_") || filepath.Ext(e.Name()) != ".json" {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var d document.Doc
|
||||
if json.Unmarshal(b, &d) != nil || d.ID == "" {
|
||||
continue
|
||||
}
|
||||
docs = append(docs, benchDoc{ID: d.ID, Summary: d.Summary, Content: d.Content})
|
||||
}
|
||||
sort.Slice(docs, func(i, j int) bool { return docs[i].ID < docs[j].ID })
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
func relevantSets(docs []benchDoc) []map[string]bool {
|
||||
sets := make([]map[string]bool, len(cases))
|
||||
for i, c := range cases {
|
||||
sets[i] = make(map[string]bool)
|
||||
for _, d := range docs {
|
||||
text := strings.ToLower(d.Summary + "\n" + d.Content)
|
||||
for _, seed := range c.Seeds {
|
||||
if strings.Contains(text, strings.ToLower(seed)) {
|
||||
sets[i][d.ID] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return sets
|
||||
}
|
||||
|
||||
type textVectorizer interface {
|
||||
Vectorize(text string) vector.Vector
|
||||
}
|
||||
|
||||
func buildStore(docs []benchDoc, v textVectorizer) *vector.Store {
|
||||
s := vector.NewStore()
|
||||
for _, d := range docs {
|
||||
s.Insert(d.ID, d.Summary, v.Vectorize(d.Summary+"\n"+d.Content), nil)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func runCases(s *vector.Store, v textVectorizer, rel []map[string]bool) []result {
|
||||
out := make([]result, 0, len(cases))
|
||||
for i, c := range cases {
|
||||
start := time.Now()
|
||||
hs := s.SearchScored(v.Vectorize(c.Query), s.Size())
|
||||
lat := time.Since(start)
|
||||
ids := make([]hit, len(hs))
|
||||
for j, h := range hs {
|
||||
ids[j] = hit{ID: h.Doc.ID, Score: h.Score}
|
||||
}
|
||||
out = append(out, measure(c, ids, rel[i], lat))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func runHybrid(a *vector.Store, av textVectorizer, b *vector.Store, bv textVectorizer, rel []map[string]bool) []result {
|
||||
out := make([]result, 0, len(cases))
|
||||
for i, c := range cases {
|
||||
start := time.Now()
|
||||
ah := a.SearchScored(av.Vectorize(c.Query), a.Size())
|
||||
bh := b.SearchScored(bv.Vectorize(c.Query), b.Size())
|
||||
scores := make(map[string]float64)
|
||||
const k = 60.0
|
||||
for rank, h := range ah {
|
||||
scores[h.Doc.ID] += 1 / (k + float64(rank+1))
|
||||
}
|
||||
for rank, h := range bh {
|
||||
scores[h.Doc.ID] += 1 / (k + float64(rank+1))
|
||||
}
|
||||
ids := make([]hit, 0, len(scores))
|
||||
for id, score := range scores {
|
||||
ids = append(ids, hit{ID: id, Score: score})
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool {
|
||||
if ids[i].Score == ids[j].Score {
|
||||
return ids[i].ID < ids[j].ID
|
||||
}
|
||||
return ids[i].Score > ids[j].Score
|
||||
})
|
||||
out = append(out, measure(c, ids, rel[i], time.Since(start)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func measure(c queryCase, ranked []hit, relevant map[string]bool, latency time.Duration) result {
|
||||
rank := 0
|
||||
for i, h := range ranked {
|
||||
if relevant[h.ID] {
|
||||
rank = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
topN := 5
|
||||
if len(ranked) < topN {
|
||||
topN = len(ranked)
|
||||
}
|
||||
r := result{Name: c.Name, Query: c.Query, Relevant: len(relevant), Rank: rank, LatencyMS: float64(latency.Microseconds()) / 1000, Top: append([]hit(nil), ranked[:topN]...)}
|
||||
if rank > 0 {
|
||||
r.Reciprocal = 1 / float64(rank)
|
||||
r.HitAt1 = rank <= 1
|
||||
r.HitAt5 = rank <= 5
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func printSummary(r report) {
|
||||
fmt.Printf("documents=%d queries=%d\n", r.Documents, len(r.Queries))
|
||||
names := make([]string, 0, len(r.Methods))
|
||||
for name := range r.Methods {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
rs := r.Methods[name]
|
||||
var h1, h5 int
|
||||
var mrr, lat float64
|
||||
for _, x := range rs {
|
||||
if x.HitAt1 {
|
||||
h1++
|
||||
}
|
||||
if x.HitAt5 {
|
||||
h5++
|
||||
}
|
||||
mrr += x.Reciprocal
|
||||
lat += x.LatencyMS
|
||||
}
|
||||
fmt.Printf("%-24s Hit@1=%d/%d Hit@5=%d/%d MRR=%.4f avg-query=%.3fms\n", name, h1, len(rs), h5, len(rs), mrr/float64(len(rs)), lat/float64(len(rs)))
|
||||
for _, x := range rs {
|
||||
fmt.Printf(" %-20s rank=%-4d latency=%7.3fms", x.Name, x.Rank, x.LatencyMS)
|
||||
if len(x.Top) > 0 {
|
||||
fmt.Printf(" top=%s score=%.4g", x.Top[0].ID, x.Top[0].Score)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
_ = math.MaxFloat64
|
||||
}
|
||||
204
tmp_jinabench.py
Normal file
204
tmp_jinabench.py
Normal file
@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real Jina v5-omni benchmark on HomeAgent production documents and media.
|
||||
|
||||
Inputs never leave the host. Reports Recall/Hit@K, MRR, margins, and latency for:
|
||||
- Jina unified dense space: text query -> text document
|
||||
- Jina unified dense space: text query -> production image
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoModel, AutoProcessor
|
||||
|
||||
MODEL = "/home/newqqagent/models/jina-v5-omni-nano"
|
||||
DOC_DIR = "/home/newqqagent/memory/documents"
|
||||
MEDIA_DB = "/home/newqqagent/memory/media/media.db"
|
||||
BLOB_ROOT = "/home/newqqagent/memory/media/blobs"
|
||||
|
||||
CASES = [
|
||||
("mail-semantic", "邮件代理是否已经成功接入", ["AgentMail 接入验证"]),
|
||||
("fox-cross-language", "生成一张雪地红狐狸的图片", ["red fox in snowy forest"]),
|
||||
("plugin-semantic", "升级安装 QQ 插件包", ["plugin_install"]),
|
||||
("weather-paraphrase", "我所在城市的天气预报", ["河南新乡"]),
|
||||
("textarea-paraphrase", "聊天输入区域文字多了会不会自动增高", ["输入框在内容超过一行"]),
|
||||
("devices-paraphrase", "检查当前接入了哪些终端设备", ["你看看现在你都有哪些设备"]),
|
||||
("memory-health", "长期文档记忆功能是否健康", ["文档记忆系统是否正常工作"]),
|
||||
("reload-plugins", "重新加载全部扩展组件", ["热重载所有插件", "plgreload"]),
|
||||
("exact-agentmail", "AgentMail 接入验证", ["AgentMail 接入验证"]),
|
||||
("exact-plugin", "plugin_install", ["plugin_install"]),
|
||||
]
|
||||
|
||||
MEDIA_QUERIES = [
|
||||
("captcha-cn", "一张带有干扰线和字符的验证码图片", "7e689211"),
|
||||
("captcha-en", "captcha with distorted letters and noise", "7e689211"),
|
||||
("news-cn", "中文新闻报道页面截图", "f6f17229"),
|
||||
("news-en", "a screenshot of a Chinese news article", "f6f17229"),
|
||||
("notes-cn", "手机深色模式备忘录截图", "ee5bdb2a"),
|
||||
("notes-en", "dark mode phone notes application screenshot", "ee5bdb2a"),
|
||||
]
|
||||
|
||||
|
||||
def load_docs():
|
||||
docs = []
|
||||
for p in sorted(glob.glob(DOC_DIR + "/doc_*.json")):
|
||||
try:
|
||||
d = json.load(open(p, encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
if not d.get("id"):
|
||||
continue
|
||||
docs.append({"id": d["id"], "text": d.get("summary", "") + "\n" + d.get("content", "")})
|
||||
return docs
|
||||
|
||||
|
||||
def relevant(docs, seeds):
|
||||
seeds = [s.lower() for s in seeds]
|
||||
return {d["id"] for d in docs if any(s in d["text"].lower() for s in seeds)}
|
||||
|
||||
|
||||
def embed_text(model, proc, texts, side, batch_size=4):
|
||||
result = []
|
||||
prefix = "Query: " if side == "query" else "Document: "
|
||||
for start in range(0, len(texts), batch_size):
|
||||
batch = [prefix + x for x in texts[start:start + batch_size]]
|
||||
inp = proc(text=batch, padding=True, truncation=True, max_length=1024, return_tensors="pt")
|
||||
with torch.inference_mode():
|
||||
vec = model.embed(**inp)
|
||||
result.append(vec.float().cpu().numpy())
|
||||
return np.concatenate(result, axis=0)
|
||||
|
||||
|
||||
def measure_text(docs, doc_vecs, model, proc):
|
||||
out = []
|
||||
for name, query, seeds in CASES:
|
||||
rel = relevant(docs, seeds)
|
||||
t0 = time.perf_counter()
|
||||
qv = embed_text(model, proc, [query], "query", 1)[0]
|
||||
latency = (time.perf_counter() - t0) * 1000
|
||||
scores = doc_vecs @ qv
|
||||
order = np.argsort(-scores)
|
||||
rank = 0
|
||||
for pos, idx in enumerate(order, 1):
|
||||
if docs[int(idx)]["id"] in rel:
|
||||
rank = pos
|
||||
break
|
||||
best_rel = max((float(scores[i]) for i, d in enumerate(docs) if d["id"] in rel), default=float("nan"))
|
||||
best_neg = max((float(scores[i]) for i, d in enumerate(docs) if d["id"] not in rel), default=float("nan"))
|
||||
out.append({
|
||||
"name": name, "query": query, "relevant": len(rel), "rank": rank,
|
||||
"reciprocal_rank": 1.0 / rank if rank else 0.0,
|
||||
"hit_at_1": 0 < rank <= 1, "hit_at_5": 0 < rank <= 5,
|
||||
"query_latency_ms": latency,
|
||||
"best_relevant": best_rel, "best_negative": best_neg,
|
||||
"margin": best_rel - best_neg,
|
||||
"top": [{"id": docs[int(i)]["id"], "score": float(scores[i])} for i in order[:5]],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def load_images():
|
||||
rows = [
|
||||
("7e689211", "7e/689211bf78bc2f6366405b2a1f2fe0f7ec9ccdc59f368c13175c421c8e4018"),
|
||||
("ee5bdb2a", "ee/5bdb2a5fa3927ede79384ee6ec7657428896e2690b622f2409daa4b3fa9fa3"),
|
||||
("f6f17229", "f6/f17229b3513ea1f9c660353ee4ab021bec88e2b423780e8ed4445bc9aaf2fb"),
|
||||
]
|
||||
return [(name, Image.open(os.path.join(BLOB_ROOT, rel)).convert("RGB")) for name, rel in rows]
|
||||
|
||||
|
||||
def embed_image(model, proc, img):
|
||||
# Retrieval document side; nano's image geometry is nearly side-invariant, but use the documented prefix.
|
||||
inp = proc(images=img, text="Document: <image>", return_tensors="pt")
|
||||
with torch.inference_mode():
|
||||
return model.embed(**inp).float().cpu().numpy()[0]
|
||||
|
||||
|
||||
def measure_media(images, image_vecs, model, proc):
|
||||
out = []
|
||||
names = [x[0] for x in images]
|
||||
for name, query, want in MEDIA_QUERIES:
|
||||
t0 = time.perf_counter()
|
||||
qv = embed_text(model, proc, [query], "query", 1)[0]
|
||||
latency = (time.perf_counter() - t0) * 1000
|
||||
scores = image_vecs @ qv
|
||||
order = np.argsort(-scores)
|
||||
want_idx = names.index(want)
|
||||
rank = int(np.where(order == want_idx)[0][0]) + 1
|
||||
best_neg = max(float(scores[i]) for i in range(len(names)) if i != want_idx)
|
||||
out.append({
|
||||
"name": name, "query": query, "want": want, "rank": rank,
|
||||
"query_latency_ms": latency, "positive": float(scores[want_idx]),
|
||||
"best_negative": best_neg, "margin": float(scores[want_idx]) - best_neg,
|
||||
"ranking": [{"id": names[int(i)], "score": float(scores[i])} for i in order],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def summary(rows):
|
||||
return {
|
||||
"queries": len(rows),
|
||||
"hit_at_1": sum(r["hit_at_1"] for r in rows) / len(rows),
|
||||
"hit_at_5": sum(r["hit_at_5"] for r in rows) / len(rows),
|
||||
"mrr": sum(r["reciprocal_rank"] for r in rows) / len(rows),
|
||||
"mean_query_latency_ms": sum(r["query_latency_ms"] for r in rows) / len(rows),
|
||||
"mean_margin": sum(r["margin"] for r in rows) / len(rows),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
torch.set_num_threads(min(8, os.cpu_count() or 1))
|
||||
print("loading Jina v5-omni-nano (vision + text)...", flush=True)
|
||||
t0 = time.perf_counter()
|
||||
model = AutoModel.from_pretrained(
|
||||
MODEL, trust_remote_code=True, local_files_only=True,
|
||||
default_task="retrieval", modality="vision", dtype=torch.float32,
|
||||
).eval()
|
||||
proc = AutoProcessor.from_pretrained(MODEL, trust_remote_code=True, local_files_only=True)
|
||||
load_s = time.perf_counter() - t0
|
||||
docs = load_docs()
|
||||
print(f"loaded model in {load_s:.2f}s, embedding {len(docs)} docs", flush=True)
|
||||
t0 = time.perf_counter()
|
||||
doc_vecs = embed_text(model, proc, [d["text"] for d in docs], "document", batch_size=4)
|
||||
doc_embed_s = time.perf_counter() - t0
|
||||
text_rows = measure_text(docs, doc_vecs, model, proc)
|
||||
|
||||
images = load_images()
|
||||
t0 = time.perf_counter()
|
||||
image_vecs = np.stack([embed_image(model, proc, img) for _, img in images])
|
||||
image_embed_s = time.perf_counter() - t0
|
||||
media_rows = measure_media(images, image_vecs, model, proc)
|
||||
|
||||
report = {
|
||||
"model": "jina-v5-omni-nano", "dimension": int(doc_vecs.shape[1]),
|
||||
"model_load_seconds": load_s, "documents": len(docs),
|
||||
"document_embedding_seconds": doc_embed_s,
|
||||
"document_embedding_ms_per_doc": doc_embed_s * 1000 / len(docs),
|
||||
"text_retrieval": text_rows, "text_summary": summary(text_rows),
|
||||
"images": len(images), "image_embedding_seconds": image_embed_s,
|
||||
"image_embedding_ms_per_image": image_embed_s * 1000 / len(images),
|
||||
"media_retrieval": media_rows,
|
||||
"media_summary": {
|
||||
"queries": len(media_rows),
|
||||
"hit_at_1": sum(r["rank"] == 1 for r in media_rows) / len(media_rows),
|
||||
"mrr": sum(1.0 / r["rank"] for r in media_rows) / len(media_rows),
|
||||
"mean_margin": sum(r["margin"] for r in media_rows) / len(media_rows),
|
||||
"mean_query_latency_ms": sum(r["query_latency_ms"] for r in media_rows) / len(media_rows),
|
||||
},
|
||||
}
|
||||
Path("/tmp/homeagent-jina-benchmark.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps({k: report[k] for k in ["model", "dimension", "model_load_seconds", "documents", "document_embedding_seconds", "document_embedding_ms_per_doc", "text_summary", "images", "image_embedding_seconds", "image_embedding_ms_per_image", "media_summary"]}, ensure_ascii=False, indent=2))
|
||||
for r in text_rows:
|
||||
print(f"TEXT {r['name']:20s} rank={r['rank']:3d} margin={r['margin']:+.4f} latency={r['query_latency_ms']:.1f}ms")
|
||||
for r in media_rows:
|
||||
print(f"MEDIA {r['name']:20s} rank={r['rank']} pos={r['positive']:+.4f} neg={r['best_negative']:+.4f} margin={r['margin']:+.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user