mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-23 10:28:06 +00:00
refactor(memory): 核心不再适配具体模型——公共 embedding provider SPI + 注册表
问题：cmd/homed 里 `case "onnx": qwen.New(modelDir)` 把模型适配写进了核心， `type=onnx` 名义上是格式、实际写死了一个模型家族；2117 行 Qwen 专属代码 （BPE、chat template、M-RoPE、Vision_gN 命名）住在内核树里，还带着一对 `//go:build onnxruntime` 的 stub。加任何新模型都要改内核。 现在核心只认一个模型无关的公共契约（pkg/embedding）： - 输入是不透明的 Data+MIME，解码/预处理/时序分组全归 provider - 能力是数据（Info.Modalities），不是接口方法——新增模态无需改核心接口 - 不支持的模态返回 embedding.ErrUnsupportedModality（可 errors.Is 识别） - 按名字注册，重复注册 panic；Options 是 provider 私有命名空间，核心不解释 改动： - 新增 pkg/embedding：Modality/Purpose/Input/Info/Provider/Config + 注册表 （Open 校验 Info，ValidateVector 在入库前拦下维度错与非有限值） - providers/qwen3vl：Qwen 实现整体移出内核（git mv），实现公共 SPI 并自注册 - internal/memory/vector：新增 ProviderAdapter（公共 SPI → 内部小接口）； ErrModalityUnsupported 改为公共哨兵别名；删除 VideoEmbedder 可选接口 （那正是「核心为每个新模态长方法」的坏味道） - http embedder 也变成普通 provider（注册名 http） - cmd/homed：删除 qwen import 与 onnx/http 分支，改为按 provider 名打开 + 透传 options.*；provider 打开失败只警告并禁用多模态检索，不影响启动 - config：multimodal_space.type/onnx./http.* → provider + options.* - 删除 internal/memory/qwen（整体搬迁） 测试： - pkg/embedding：注册表隔离/未知名字/非法 Info 自动关闭/ValidateVector - vector：适配器原样透传字节与 MIME、维度错被拦、Close 幂等且停止使用、 两个哨兵 errors.Is 互通 - providers/qwen3vl：新增公共 SPI 全链路集成测试（Open→Info→Embed→ 未知模态哨兵），并明确断言 Info 不声明 video 已知未完成（不得当作已验证）： - 视频冻结回归 TestEmbedderVideoMatchesONNXReference **显式跳过**：Go 侧 video 模板缺少 processor 按时间组插入的字面时间戳文本 （<0.0 seconds>/<1.0 seconds>），同一输入 Python seq=1190(1152+38)、 Go 只有 22 个文本 token。时间戳也占 M-RoPE 位置，故现有 M-RoPE 自洽断言 通过不能证明与官方实现一致。修复属 provider 内部工作。 - 视觉侧三档已导出并逐档校验通过（cos 1.000000119/1.000000119/1.000000000） 验证：go build ./... ；go vet -tags onnxruntime ./... ； go test -short ./internal/memory/... ./internal/agent/core/... ./internal/sdk/... ./pkg/... ；onnxruntime 下 providers/qwen3vl 全绿（视频为显式 skip）
This commit is contained in:
JianFeeeee
@ -2,6 +2,7 @@ package vector
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@ -10,14 +11,50 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// http 也是一个普通 provider：核心只按名字打开它，不知道它背后是云 API、
|
||||
// 自建服务还是别的语言写的模型。
|
||||
embedding.Register("http", func(cfg embedding.Config) (embedding.Provider, error) {
|
||||
return NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||
Endpoint: cfg.Options["endpoint"],
|
||||
APIKey: cfg.Options["api_key"],
|
||||
Model: cfg.Options["model"],
|
||||
Fingerprint: cfg.Options["fingerprint"],
|
||||
Dimension: atoiOrZero(cfg.Options["dimension"]),
|
||||
Timeout: durationOrZero(cfg.Options["timeout"]),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func atoiOrZero(s string) int {
|
||||
n := 0
|
||||
for _, r := range strings.TrimSpace(s) {
|
||||
if r < '0' || r > '9' {
|
||||
return 0
|
||||
}
|
||||
n = n*10 + int(r-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func durationOrZero(s string) time.Duration {
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// HTTPEmbedderConfig 配置一个外部多模态向量服务。
|
||||
// 服务契约刻意很小：POST Endpoint，输入 modality/data/mime/side，返回 embedding。
|
||||
// 任何云 API 或自建服务只需适配这一个协议，即可复用内核全部向量存储与检索链路。
|
||||
type HTTPEmbedderConfig struct {
|
||||
Endpoint string
|
||||
APIKey string
|
||||
APIKey string
|
||||
Model string
|
||||
Dimension int
|
||||
Timeout time.Duration
|
||||
@ -65,14 +102,14 @@ func NewHTTPEmbedder(cfg HTTPEmbedderConfig) (*HTTPEmbedder, error) {
|
||||
}
|
||||
|
||||
func (e *HTTPEmbedder) VectorizeDense(text string) ([]float64, error) {
|
||||
return e.embed(httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityText), Side: "query", Text: text})
|
||||
return e.embed(context.Background(), httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityText), Side: "query", Text: text})
|
||||
}
|
||||
|
||||
func (e *HTTPEmbedder) EmbedImageDense(img []byte, mime string) ([]float64, error) {
|
||||
return e.embed(httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityImage), Side: "document", Data: base64.StdEncoding.EncodeToString(img), MIME: mime})
|
||||
return e.embed(context.Background(), httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityImage), Side: "document", Data: base64.StdEncoding.EncodeToString(img), MIME: mime})
|
||||
}
|
||||
|
||||
func (e *HTTPEmbedder) embed(payload httpEmbedRequest) ([]float64, error) {
|
||||
func (e *HTTPEmbedder) embed(ctx context.Context, payload httpEmbedRequest) ([]float64, error) {
|
||||
e.mu.Lock()
|
||||
closed := e.closed
|
||||
e.mu.Unlock()
|
||||
@ -83,7 +120,7 @@ func (e *HTTPEmbedder) embed(payload httpEmbedRequest) ([]float64, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, e.cfg.Endpoint, bytes.NewReader(body))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.cfg.Endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -119,6 +156,32 @@ func (e *HTTPEmbedder) embed(payload httpEmbedRequest) ([]float64, error) {
|
||||
|
||||
func (e *HTTPEmbedder) Fingerprint() string { return e.cfg.Fingerprint }
|
||||
func (e *HTTPEmbedder) Dim() int { return e.cfg.Dimension }
|
||||
|
||||
// Embed 实现公共 provider 契约：核心只传模态与不透明字节，本实现负责把它
|
||||
// 翻译成外部服务的协议。
|
||||
func (e *HTTPEmbedder) Embed(ctx context.Context, in embedding.Input) ([]float64, error) {
|
||||
req := httpEmbedRequest{
|
||||
Model: e.cfg.Model,
|
||||
Modality: string(in.Modality),
|
||||
Side: string(in.Purpose),
|
||||
Text: in.Text,
|
||||
MIME: in.MIME,
|
||||
}
|
||||
if in.Modality != embedding.ModalityText {
|
||||
req.Data = base64.StdEncoding.EncodeToString(in.Data)
|
||||
}
|
||||
return e.embed(ctx, req)
|
||||
}
|
||||
|
||||
// Info 声明本 provider 的向量空间身份。外部服务的支持模态无法在本地探测，
|
||||
// 因此只声明 text/image 这两条内核真正会走到的路径。
|
||||
func (e *HTTPEmbedder) Info() embedding.Info {
|
||||
return embedding.Info{
|
||||
Dimension: e.cfg.Dimension,
|
||||
Fingerprint: e.cfg.Fingerprint,
|
||||
Modalities: []embedding.Modality{embedding.ModalityText, embedding.ModalityImage},
|
||||
}
|
||||
}
|
||||
func (e *HTTPEmbedder) Loaded() bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
|
||||
83
internal/memory/vector/provider_adapter.go
Normal file
83
internal/memory/vector/provider_adapter.go
Normal file
@ -0,0 +1,83 @@
|
||||
package vector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||
)
|
||||
|
||||
// ProviderAdapter translates the public model-neutral embedding.Provider SPI
|
||||
// to the small internal interface used by the existing memory consumers.
|
||||
// Model selection, media decoding, preprocessing, and runtime details remain
|
||||
// entirely inside the selected provider.
|
||||
type ProviderAdapter struct {
|
||||
provider embedding.Provider
|
||||
info embedding.Info
|
||||
|
||||
mu sync.RWMutex
|
||||
closed bool
|
||||
}
|
||||
|
||||
// AdaptProvider validates and wraps a public provider for internal memory use.
|
||||
func AdaptProvider(provider embedding.Provider) (*ProviderAdapter, error) {
|
||||
info := provider.Info()
|
||||
if err := embedding.ValidateInfo(info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ProviderAdapter{provider: provider, info: info}, nil
|
||||
}
|
||||
|
||||
func (a *ProviderAdapter) VectorizeDense(text string) ([]float64, error) {
|
||||
return a.embed(embedding.Input{
|
||||
Modality: embedding.ModalityText,
|
||||
Purpose: embedding.PurposeQuery,
|
||||
Text: text,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *ProviderAdapter) EmbedImageDense(data []byte, mime string) ([]float64, error) {
|
||||
return a.embed(embedding.Input{
|
||||
Modality: embedding.ModalityImage,
|
||||
Purpose: embedding.PurposeDocument,
|
||||
Data: data,
|
||||
MIME: mime,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *ProviderAdapter) embed(input embedding.Input) ([]float64, error) {
|
||||
a.mu.RLock()
|
||||
closed := a.closed
|
||||
a.mu.RUnlock()
|
||||
if closed {
|
||||
return nil, context.Canceled
|
||||
}
|
||||
vec, err := a.provider.Embed(context.Background(), input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := embedding.ValidateVector(vec, a.info.Dimension); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return vec, nil
|
||||
}
|
||||
|
||||
func (a *ProviderAdapter) Fingerprint() string { return a.info.Fingerprint }
|
||||
func (a *ProviderAdapter) Dim() int { return a.info.Dimension }
|
||||
|
||||
func (a *ProviderAdapter) Loaded() bool {
|
||||
a.mu.RLock()
|
||||
defer a.mu.RUnlock()
|
||||
return !a.closed
|
||||
}
|
||||
|
||||
func (a *ProviderAdapter) Close() {
|
||||
a.mu.Lock()
|
||||
if a.closed {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.closed = true
|
||||
a.mu.Unlock()
|
||||
a.provider.Close()
|
||||
}
|
||||
115
internal/memory/vector/provider_adapter_test.go
Normal file
115
internal/memory/vector/provider_adapter_test.go
Normal file
@ -0,0 +1,115 @@
|
||||
package vector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||
)
|
||||
|
||||
// recordingProvider 记录核心传给 provider 的原始请求，用来断言
|
||||
// 「核心不解释内容、只搬字节」这一契约。
|
||||
type recordingProvider struct {
|
||||
got []embedding.Input
|
||||
dim int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (p *recordingProvider) Embed(_ context.Context, in embedding.Input) ([]float64, error) {
|
||||
p.got = append(p.got, in)
|
||||
return make([]float64, p.dim), nil
|
||||
}
|
||||
|
||||
func (p *recordingProvider) Info() embedding.Info {
|
||||
return embedding.Info{Dimension: p.dim, Fingerprint: "recording:1"}
|
||||
}
|
||||
|
||||
func (p *recordingProvider) Close() { p.closed = true }
|
||||
|
||||
func TestProviderAdapterPassesOpaqueDataUnchanged(t *testing.T) {
|
||||
inner := &recordingProvider{dim: 3}
|
||||
adapted, err := AdaptProvider(inner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer adapted.Close()
|
||||
|
||||
// 核心把媒体当作不透明字节搬运：既不解码也不改字节。
|
||||
raw := []byte{0x89, 'P', 'N', 'G', 0x00, 0xff}
|
||||
if _, err := adapted.EmbedImageDense(raw, "image/png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := inner.got[0]
|
||||
if string(got.Data) != string(raw) {
|
||||
t.Fatalf("provider 收到的字节被改动: %v", got.Data)
|
||||
}
|
||||
if got.Modality != embedding.ModalityImage || got.MIME != "image/png" {
|
||||
t.Fatalf("模态/MIME 未原样传递: %+v", got)
|
||||
}
|
||||
if got.Purpose != embedding.PurposeDocument {
|
||||
t.Fatalf("用途应为 document: %q", got.Purpose)
|
||||
}
|
||||
|
||||
if _, err := adapted.VectorizeDense("hello"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inner.got[1].Modality != embedding.ModalityText || inner.got[1].Text != "hello" {
|
||||
t.Fatalf("文本请求不正确: %+v", inner.got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderAdapterRejectsWrongDimensionFromProvider(t *testing.T) {
|
||||
// provider 声明 3 维却返回 2 维：必须在进入存储前被拦下，
|
||||
// 否则一个维度错的向量会污染整个余弦检索。
|
||||
bad := &badDimProvider{}
|
||||
adapted, err := AdaptProvider(bad)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer adapted.Close()
|
||||
if _, err := adapted.VectorizeDense("x"); err == nil {
|
||||
t.Fatal("维度不符时应返回错误")
|
||||
}
|
||||
}
|
||||
|
||||
type badDimProvider struct{}
|
||||
|
||||
func (badDimProvider) Embed(context.Context, embedding.Input) ([]float64, error) {
|
||||
return []float64{1, 2}, nil
|
||||
}
|
||||
func (badDimProvider) Info() embedding.Info {
|
||||
return embedding.Info{Dimension: 3, Fingerprint: "bad:1"}
|
||||
}
|
||||
func (badDimProvider) Close() {}
|
||||
|
||||
func TestProviderAdapterCloseIsIdempotentAndStopsUse(t *testing.T) {
|
||||
inner := &recordingProvider{dim: 2}
|
||||
adapted, err := AdaptProvider(inner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adapted.Close()
|
||||
adapted.Close() // 重复关闭不应 panic 或二次 Close provider
|
||||
if !inner.closed {
|
||||
t.Fatal("Close 未传递到 provider")
|
||||
}
|
||||
if _, err := adapted.VectorizeDense("x"); err == nil {
|
||||
t.Fatal("关闭后应拒绝调用")
|
||||
}
|
||||
if adapted.Loaded() {
|
||||
t.Fatal("关闭后 Loaded() 应为 false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestModalityUnsupportedSentinelIsShared(t *testing.T) {
|
||||
// 内核侧的哨兵与公共契约的哨兵必须是同一个：provider 返回公共哨兵时，
|
||||
// 内核仍能用自己原有的名字识别。
|
||||
if !errors.Is(ErrModalityUnsupported, embedding.ErrUnsupportedModality) {
|
||||
t.Fatal("vector.ErrModalityUnsupported 与 embedding.ErrUnsupportedModality 未打通")
|
||||
}
|
||||
wrapped := errors.Join(embedding.ErrUnsupportedModality, errors.New("audio/wav"))
|
||||
if !errors.Is(wrapped, ErrModalityUnsupported) {
|
||||
t.Fatal("包装后的错误无法用内核哨兵识别")
|
||||
}
|
||||
}
|
||||
@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||
)
|
||||
|
||||
// Vectorizer 接口：将文本转为向量
|
||||
@ -56,8 +58,15 @@ var ErrNotSupported = fmt.Errorf("vectorizer does not support image embedding")
|
||||
// 而不是「这次失败了、下次重试」。绝不能拿另一个模型的向量顶替——那会把
|
||||
// 两套坐标系混进同一空间，检索出来的相似度没有任何意义。
|
||||
//
|
||||
// 例：Qwen3-VL 能原生编码文本/图像，音频需要未来接入真正的统一音频模型。
|
||||
var ErrModalityUnsupported = fmt.Errorf("modality not supported by this embedding space")
|
||||
// 它是公共 provider 契约里那个哨兵值的别名，两者 errors.Is 互通：
|
||||
// provider 在自己的包内返回 embedding.ErrUnsupportedModality 即可，
|
||||
// 内核侧的判断无需改变。
|
||||
var ErrModalityUnsupported = embedding.ErrUnsupportedModality
|
||||
|
||||
// 注：曾经这里还有一个可选的 VideoEmbedder 接口（用类型断言探测视频能力）。
|
||||
// 已删除：那让核心为每一个新模态长出一套模型专属方法，正是“核心适配模型”的
|
||||
// 坏味道。模态能力现在是数据（embedding.Info.Modalities），输入是不透明的
|
||||
// Data+MIME（见 pkg/embedding）。
|
||||
|
||||
// Vector 是带权特征映射：feature → weight
|
||||
type Vector map[string]float64
|
||||
|
||||
Reference in New Issue
Block a user