mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-15 12:15:16 +00:00
refactor(memory)!: 块 union 化,彻底淘汰 CAS 第一阶段(生产代码)
块自带负载(MemoryBlock.Data),删 PayloadDigest;检索纯向量,媒体随块 同生灭。生产代码全绿;CAS 专属测试将在下一提交随判据重写。 - memory: Data BLOB + migration9;blockColumns 不含 data(召回只比向量); MediaBlockID 内容派生 ID;BlockByIDPrefix/BlockData 按需加载 - sdk: memory_impl 去 media.Store;MemoryBlockView PayloadDigest→HasData - core: mediaref/graphmedia/crossmodal/medialoop 全部改块语义; captureBlockMedia 直接产块;resolveMediaRefs 按块 ID 解析 - knowledge: 媒体字节随条目目录自带(File+Data),删 mediaGet/mediaPut - webui: graphBlockView/pulse payload_digest→has_data - bootstrap/registry: 去 MediaStore 接线(SetMediaStore 空实现,待⑥删包)
This commit is contained in:
@ -22,12 +22,25 @@ const (
|
||||
//
|
||||
// 它不携带 Layer、Owner 或 RefCount:块当前由哪个层的容器持有,哪个层就是
|
||||
// 唯一事实源。Context→Document→Graph 迁移的是这个值本身,不建立平行保活账本。
|
||||
// PayloadDigest 仅用于定位内容寻址的原始字节,不表示另一条逻辑记忆。
|
||||
//
|
||||
// ★★ 负载是 **union**:一个块要么是文本块(负载在 Text),要么是媒体块
|
||||
// (负载在 Data,配 MIME/Width/Height)。两者是同一个 MemoryBlock 的两种取值,
|
||||
// 不是「块 + 外挂字节层」。
|
||||
//
|
||||
// 为何不存在独立的字节存储:块已经是内容的载体(文本块自己带正文),媒体块
|
||||
// 自当同理。把媒体内容拆到块外、只留一个 digest 指向外部 CAS,会凭空多出
|
||||
// 一层需单独维护生命周期的东西(第二套向量、第二套召回路、引用账本、自己的
|
||||
// db),并与「块是唯一记忆载体」直接冲突。内容随块同生同灭才是正解。
|
||||
type MemoryBlock struct {
|
||||
ID string `json:"id"`
|
||||
Modality BlockModality `json:"modality"`
|
||||
Text string `json:"text,omitempty"`
|
||||
PayloadDigest string `json:"payload_digest,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Modality BlockModality `json:"modality"`
|
||||
// Text 是文本块(Modality=text)的负载。
|
||||
Text string `json:"text,omitempty"`
|
||||
// Data 是媒体块(Modality=image/video/audio)的负载:内容的原始字节。
|
||||
//
|
||||
// 落盘形态随所在层而定:L0/L1 的块内联在 JSON 里(Data 序列化为 base64),
|
||||
// L3 的块落 SQLite 的 BLOB 列。无论哪层,字节都不脱离块单独存在。
|
||||
Data []byte `json:"data,omitempty"`
|
||||
MIME string `json:"mime,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
Width int `json:"width,omitempty"`
|
||||
@ -120,13 +133,13 @@ func (g *GraphDB) PutMemoryBlocks(blocks []MemoryBlock) error {
|
||||
block.CreatedAt = now
|
||||
}
|
||||
_, err = tx.Exec(`INSERT INTO memory_blocks (
|
||||
id, modality, text_content, payload_digest, mime, size, width, height,
|
||||
id, modality, text_content, data, mime, size, width, height,
|
||||
vector, fingerprint, source, tool, scene, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
modality = excluded.modality,
|
||||
text_content = excluded.text_content,
|
||||
payload_digest = excluded.payload_digest,
|
||||
data = excluded.data,
|
||||
mime = excluded.mime,
|
||||
size = excluded.size,
|
||||
width = excluded.width,
|
||||
@ -143,7 +156,7 @@ func (g *GraphDB) PutMemoryBlocks(blocks []MemoryBlock) error {
|
||||
-- 若让它覆盖已有类型,一次不带标注的写入就会抹掉语义类别。
|
||||
semantic_type = CASE WHEN excluded.semantic_type != '' THEN excluded.semantic_type ELSE memory_blocks.semantic_type END,
|
||||
updated_at = excluded.updated_at`,
|
||||
block.ID, block.Modality, block.Text, block.PayloadDigest, block.MIME,
|
||||
block.ID, block.Modality, block.Text, block.Data, block.MIME,
|
||||
block.Size, block.Width, block.Height, string(vectorJSON), block.Fingerprint,
|
||||
block.Source, block.Tool, block.Scene, block.CreatedAt, now)
|
||||
if err != nil {
|
||||
@ -279,14 +292,21 @@ func (g *GraphDB) MemoryBlocks() ([]MemoryBlock, error) {
|
||||
// 结果媒体与召回测试大面积变红(列数不匹配 ⇒ 读失败)。
|
||||
//
|
||||
// 单靠“记得同步”不可靠;提成常量后新增查询直接复用。
|
||||
const blockColumns = `id, modality, text_content, payload_digest, mime, size, width, height,
|
||||
// blockColumns 是 memory_blocks 的**元数据**列清单,供所有常规读取复用。
|
||||
//
|
||||
// ★ 为何**不含 data**:媒体负载动辄数 MB(生产实测:图均 495KB、最大 2.7MB),
|
||||
// 而 RecallBlocks 要全表扫描有向量的块。若把 data 列进来,每次召回都会把
|
||||
// 几十 MB 字节读进内存。块拥有自己的负载(同一行、同一生命周期),
|
||||
// 但负载**按需加载**(见 BlockData)——这与被淘汰的 CAS 有本质区别:
|
||||
// 那里字节归另一个存储层所有,这里归块自己。
|
||||
const blockColumns = `id, modality, text_content, mime, size, width, height,
|
||||
vector, fingerprint, source, tool, scene, semantic_type, created_at, updated_at`
|
||||
|
||||
func scanBlockRow(rows *sql.Rows) (MemoryBlock, error) {
|
||||
var b MemoryBlock
|
||||
var vectorJSON string
|
||||
var scene sql.NullString
|
||||
if err := rows.Scan(&b.ID, &b.Modality, &b.Text, &b.PayloadDigest,
|
||||
if err := rows.Scan(&b.ID, &b.Modality, &b.Text,
|
||||
&b.MIME, &b.Size, &b.Width, &b.Height, &vectorJSON,
|
||||
&b.Fingerprint, &b.Source, &b.Tool, &scene, &b.SemanticType,
|
||||
&b.CreatedAt, &b.UpdatedAt); err != nil {
|
||||
@ -420,7 +440,7 @@ func (g *GraphDB) MemoryBlockEdges() ([]MemoryBlockEdge, error) {
|
||||
func (g *GraphDB) BlocksForNode(nodeKind, nodeID string) ([]MemoryBlock, error) {
|
||||
g.mu.RLock()
|
||||
defer g.mu.RUnlock()
|
||||
rows, err := g.db.Query(`SELECT b.id, b.modality, b.text_content, b.payload_digest,
|
||||
rows, err := g.db.Query(`SELECT b.id, b.modality, b.text_content,
|
||||
b.mime, b.size, b.width, b.height, b.vector, b.fingerprint, b.source, b.tool,
|
||||
b.created_at, b.updated_at
|
||||
FROM memory_block_edges e
|
||||
@ -436,7 +456,7 @@ func (g *GraphDB) BlocksForNode(nodeKind, nodeID string) ([]MemoryBlock, error)
|
||||
for rows.Next() {
|
||||
var block MemoryBlock
|
||||
var vectorJSON string
|
||||
if err := rows.Scan(&block.ID, &block.Modality, &block.Text, &block.PayloadDigest,
|
||||
if err := rows.Scan(&block.ID, &block.Modality, &block.Text,
|
||||
&block.MIME, &block.Size, &block.Width, &block.Height, &vectorJSON,
|
||||
&block.Fingerprint, &block.Source, &block.Tool, &block.CreatedAt,
|
||||
&block.UpdatedAt); err != nil {
|
||||
|
||||
@ -97,21 +97,12 @@ func (g *GraphDB) RecallBlocks(q BlockRecallQuery) ([]BlockHit, error) {
|
||||
var hits []BlockHit
|
||||
var skippedDim, skippedFP, scanned int
|
||||
for rows.Next() {
|
||||
var b MemoryBlock
|
||||
var vectorJSON string
|
||||
if err := rows.Scan(&b.ID, &b.Modality, &b.Text, &b.PayloadDigest,
|
||||
&b.MIME, &b.Size, &b.Width, &b.Height, &vectorJSON,
|
||||
&b.Fingerprint, &b.Source, &b.Tool, &b.Scene, &b.SemanticType,
|
||||
&b.CreatedAt, &b.UpdatedAt); err != nil {
|
||||
b, err := scanBlockRow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scanned++
|
||||
vec, err := decodeVector(vectorJSON)
|
||||
if err != nil {
|
||||
// 单条坏数据不该让整次召回失败 —— 但要记日志,否则会静默缺失。
|
||||
log.Printf("[graph] recall blocks: skip %s: %v", b.ID, err)
|
||||
continue
|
||||
}
|
||||
vec := b.Vector
|
||||
// 维度不一致:跳过并计数。**不截断、不填充**——那会造出无意义的分数。
|
||||
if len(vec) != dim {
|
||||
skippedDim++
|
||||
|
||||
@ -309,7 +309,9 @@ func (g *GraphDB) initSchema() error {
|
||||
id TEXT PRIMARY KEY,
|
||||
modality TEXT NOT NULL,
|
||||
text_content TEXT DEFAULT '',
|
||||
payload_digest TEXT DEFAULT '',
|
||||
-- data 是媒体块的负载(union 的另一半):文本块用 text_content,
|
||||
-- 媒体块用 data。字节随块同生同灭,块外不存在独立的字节存储层。
|
||||
data BLOB,
|
||||
mime TEXT DEFAULT '',
|
||||
size INTEGER DEFAULT 0,
|
||||
width INTEGER DEFAULT 0,
|
||||
@ -436,6 +438,12 @@ func (g *GraphDB) initSchema() error {
|
||||
// `type` 在 SQLite 里合法但与很多工具的保留字冲突
|
||||
// (且 memory_blocks 已有 modality 列表达「模态」,语义类型是另一回事)。
|
||||
tx.Exec(`ALTER TABLE memory_blocks ADD COLUMN semantic_type TEXT DEFAULT ''`)
|
||||
// ★★★ 迁移9:媒体负载内联进块(2026-10-11)
|
||||
//
|
||||
// 淘汰独立的字节层(media.Store/CAS):媒体内容成为块自己的负载,
|
||||
// 与文本块的 text_content 对称——一个块要么带文本、要么带媒体字节。
|
||||
// 旧库的 payload_digest 列保留(迁移脚本据此回填 data),但不参与新写入。
|
||||
tx.Exec(`ALTER TABLE memory_blocks ADD COLUMN data BLOB`)
|
||||
|
||||
// ★★★ 迁移8:memory_block_edges 升格为「边是独立单位」
|
||||
//
|
||||
@ -1100,7 +1108,7 @@ func (g *GraphDB) recallSortedLocked(keywords []string, seedEntities []string, d
|
||||
var rank int
|
||||
// rank 是第 1 列,先扫掉,剩下 15 列正好对上 scanBlockRow。
|
||||
if err := rows.Scan(&rank, &bScratch.ID, &bScratch.Modality,
|
||||
&bScratch.Text, &bScratch.PayloadDigest, &bScratch.MIME,
|
||||
&bScratch.Text, &bScratch.MIME,
|
||||
&bScratch.Size, &bScratch.Width, &bScratch.Height, &vecJSON,
|
||||
&bScratch.Fingerprint, &bScratch.Source, &bScratch.Tool,
|
||||
&sceneNull, &bScratch.SemanticType,
|
||||
@ -1642,7 +1650,7 @@ func (g *GraphDB) GraphData() (map[string]interface{}, error) {
|
||||
// ★ 这条 Scan 历史上就少扫了 scene(SELECT 用 blockConstants 改了,
|
||||
// Scan 却没跟上)—— 与 semantic_type 是同一类错,
|
||||
// 报错形如「expected 16 destination arguments in Scan, not 14」。
|
||||
if err := brows.Scan(&block.ID, &block.Modality, &block.Text, &block.PayloadDigest,
|
||||
if err := brows.Scan(&block.ID, &block.Modality, &block.Text,
|
||||
&block.MIME, &block.Size, &block.Width, &block.Height, &vectorJSON,
|
||||
&block.Fingerprint, &block.Source, &block.Tool, &block.Scene,
|
||||
&block.SemanticType, &block.CreatedAt,
|
||||
|
||||
@ -520,7 +520,7 @@ func TestMemoryBlocksAreFirstClassGraphNodes(t *testing.T) {
|
||||
|
||||
image := MemoryBlock{
|
||||
ID: "block_image_1", Modality: BlockImage,
|
||||
PayloadDigest: "0123456789abcdef", MIME: "image/png", Size: 1234,
|
||||
Data: []byte("fake-png"), MIME: "image/png", Size: 1234,
|
||||
Width: 768, Height: 512, Vector: []float64{0.1, 0.2, 0.3},
|
||||
Fingerprint: "qwen:test", Source: "qq", Tool: "upload",
|
||||
}
|
||||
@ -545,7 +545,7 @@ func TestMemoryBlocksAreFirstClassGraphNodes(t *testing.T) {
|
||||
gotImage = &blocks[i]
|
||||
}
|
||||
}
|
||||
if gotImage == nil || gotImage.Modality != BlockImage || gotImage.PayloadDigest != image.PayloadDigest || gotImage.Fingerprint != image.Fingerprint || len(gotImage.Vector) != 3 {
|
||||
if gotImage == nil || gotImage.Modality != BlockImage || string(gotImage.Data) != string(image.Data) || gotImage.Fingerprint != image.Fingerprint || len(gotImage.Vector) != 3 {
|
||||
t.Fatalf("image block not round-tripped: %+v", gotImage)
|
||||
}
|
||||
|
||||
|
||||
@ -368,7 +368,7 @@ func (idx *Indexer) FormatContext(ctx *InjectedContext) string {
|
||||
if blk.Text != "" {
|
||||
b.WriteString(truncateRunes(blk.Text, sceneSentenceMaxRunes))
|
||||
} else {
|
||||
b.WriteString(shortBlockDigest(blk.PayloadDigest))
|
||||
b.WriteString(shortBlockDigest(blk.ID))
|
||||
}
|
||||
}
|
||||
b.WriteString("\n")
|
||||
@ -419,6 +419,10 @@ const sceneSentenceMaxRunes = 60
|
||||
const maxSceneRecallBlocks = 3
|
||||
|
||||
// shortBlockDigest 取 digest 前 12 位做展示(与 L3 里引用媒体的写法一致)。
|
||||
// shortBlockDigest 把块 ID 截短,供提示词里指认媒体块。
|
||||
//
|
||||
// 参数曾是内容 digest;块自带负载后,指认单位改成**块 ID**(模型按块 ID
|
||||
// 取回该块的负载,不再经内容寻址层中转)。
|
||||
func shortBlockDigest(d string) string {
|
||||
if len(d) <= 12 {
|
||||
return d
|
||||
|
||||
@ -87,20 +87,21 @@ func (g *GraphDB) MigrateLegacyMediaEntities(resolve LegacyMediaResolver) (block
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// 同一份字节可能被多个旧实体引用(重复注入的同一张图),
|
||||
// 迁移后应指向同一个块:块的身份是内容,不是实体行。
|
||||
blockIDForDigest := make(map[string]string)
|
||||
// 同一份内容可能被多个旧实体引用(重复注入的同一张图),迁移后应指向
|
||||
// 同一个块。旧实体名里的**短 digest 就是内容身份**(同内容⇒同短 digest),
|
||||
// 因此以它为去重键——块自带负载后不再有独立的 payload_digest 字段可依赖。
|
||||
blockIDForShort := make(map[string]string)
|
||||
|
||||
for _, e := range legacy {
|
||||
short := LegacyMediaEntityDigest(e.name)
|
||||
if short != "" {
|
||||
if block, ok := resolve(short); ok && block.PayloadDigest != "" {
|
||||
id, seen := blockIDForDigest[block.PayloadDigest]
|
||||
if block, ok := resolve(short); ok {
|
||||
id, seen := blockIDForShort[short]
|
||||
if !seen {
|
||||
if err := insertMigratedBlock(tx, block); err != nil {
|
||||
return 0, 0, fmt.Errorf("migrate legacy media %s: %w", short, err)
|
||||
}
|
||||
blockIDForDigest[block.PayloadDigest] = block.ID
|
||||
blockIDForShort[short] = block.ID
|
||||
id = block.ID
|
||||
blocks++
|
||||
}
|
||||
@ -145,11 +146,11 @@ func insertMigratedBlock(tx *sql.Tx, block MemoryBlock) error {
|
||||
created = time.Now()
|
||||
}
|
||||
_, err = tx.Exec(`INSERT INTO memory_blocks (
|
||||
id, modality, text_content, payload_digest, mime, size, width, height,
|
||||
id, modality, text_content, data, mime, size, width, height,
|
||||
vector, fingerprint, source, tool, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO NOTHING`,
|
||||
block.ID, block.Modality, block.Text, block.PayloadDigest, block.MIME,
|
||||
block.ID, block.Modality, block.Text, block.Data, block.MIME,
|
||||
block.Size, block.Width, block.Height, string(vectorJSON), block.Fingerprint,
|
||||
block.Source, block.Tool, created, time.Now())
|
||||
return err
|
||||
|
||||
@ -487,7 +487,7 @@ func putBlockTx(tx *sql.Tx, b MemoryBlock) (bool, error) {
|
||||
}
|
||||
|
||||
_, err := tx.Exec(`INSERT INTO memory_blocks
|
||||
(id, modality, text_content, payload_digest, mime, size, width, height,
|
||||
(id, modality, text_content, data, mime, size, width, height,
|
||||
vector, fingerprint, source, tool, scene, semantic_type, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
@ -499,7 +499,7 @@ func putBlockTx(tx *sql.Tx, b MemoryBlock) (bool, error) {
|
||||
semantic_type = CASE WHEN excluded.semantic_type != ''
|
||||
THEN excluded.semantic_type ELSE memory_blocks.semantic_type END,
|
||||
`+tsUpdate,
|
||||
b.ID, b.Modality, b.Text, b.PayloadDigest, b.MIME, b.Size, b.Width, b.Height,
|
||||
b.ID, b.Modality, b.Text, b.Data, b.MIME, b.Size, b.Width, b.Height,
|
||||
vectorJSON, b.Fingerprint, b.Source, b.Tool, b.Scene, b.SemanticType,
|
||||
created, updated)
|
||||
if err != nil {
|
||||
|
||||
@ -471,7 +471,7 @@ func (g *GraphDB) RecallByScene(scenes []string, limit int) (*SceneRecall, error
|
||||
}
|
||||
}
|
||||
|
||||
blockQuery := `SELECT b.id, b.modality, b.text_content, b.payload_digest, b.mime,
|
||||
blockQuery := `SELECT b.id, b.modality, b.text_content, b.mime,
|
||||
b.size, b.width, b.height, b.fingerprint, b.source, b.tool,
|
||||
COALESCE(b.scene, ''), COALESCE(b.semantic_type, ''),
|
||||
b.created_at, b.updated_at, MAX(sr.weight) AS w
|
||||
@ -491,7 +491,7 @@ func (g *GraphDB) RecallByScene(scenes []string, limit int) (*SceneRecall, error
|
||||
for brows.Next() {
|
||||
var b MemoryBlock
|
||||
var w float64
|
||||
if err := brows.Scan(&b.ID, &b.Modality, &b.Text, &b.PayloadDigest, &b.MIME,
|
||||
if err := brows.Scan(&b.ID, &b.Modality, &b.Text, &b.MIME,
|
||||
&b.Size, &b.Width, &b.Height, &b.Fingerprint, &b.Source, &b.Tool,
|
||||
&b.Scene, &b.SemanticType, &b.CreatedAt, &b.UpdatedAt, &w); err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -344,7 +344,7 @@ func TestSceneBlocks(t *testing.T) {
|
||||
defer g.Close()
|
||||
|
||||
blocks := []MemoryBlock{
|
||||
{ID: "blk_a", Modality: "image", Text: "老大发的排班表截图", PayloadDigest: "aaaa1111bbbb2222", Scene: "chan:qq"},
|
||||
{ID: "blk_a", Modality: "image", Text: "老大发的排班表截图", Data: []byte("png-bytes"), Scene: "chan:qq"},
|
||||
{ID: "blk_b", Modality: "text", Text: "无关场面的转写"},
|
||||
}
|
||||
if err := g.PutMemoryBlocks(blocks); err != nil {
|
||||
@ -375,7 +375,7 @@ func TestSceneBlocks(t *testing.T) {
|
||||
|
||||
// 无场景重写不得抹掉已挂的场景(静默降级防护)
|
||||
if err := g.PutMemoryBlocks([]MemoryBlock{
|
||||
{ID: "blk_a", Modality: "image", Text: "重新描述", PayloadDigest: "aaaa1111bbbb2222"},
|
||||
{ID: "blk_a", Modality: "image", Text: "重新描述", Data: []byte("png-bytes")},
|
||||
}); err != nil {
|
||||
t.Fatalf("重写块: %v", err)
|
||||
}
|
||||
@ -468,7 +468,7 @@ func TestFormatContextSceneBlocks(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := g.PutMemoryBlocks([]MemoryBlock{
|
||||
{ID: "blk_x", Modality: "image", Text: "老大发的排班表截图", PayloadDigest: "cccc3333dddd4444", Scene: "chan:qq"},
|
||||
{ID: "blk_x", Modality: "image", Text: "老大发的排班表截图", Data: []byte("png-bytes"), Scene: "chan:qq"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@ -226,3 +227,126 @@ func putTripleBlocksTx(tx *sql.Tx, t Triple, sessionID string, turnID int) (stri
|
||||
edgeID, err := res.LastInsertId()
|
||||
return src.ID, dst.ID, edgeID, created, err
|
||||
}
|
||||
|
||||
// ── 媒体块 ──────────────────────────────────────────────
|
||||
|
||||
// MediaBlockID 由**媒体内容字节**派生媒体块的 ID。
|
||||
//
|
||||
// ★ 为什么媒体块的 ID 也必须内容派生
|
||||
// ---------------------------------
|
||||
// 块 ID 是本仓的记忆身份,既有约定一律内容派生:
|
||||
// - 原句块 blk_src_<sha256(text)[:12]>
|
||||
// - 实体块 blk_ent_<sha256(name)[:12]>
|
||||
// - 蒸馏块 blk_<sha256(sentence+subject+dim+value)[:12]>
|
||||
// 而媒体块此前用时间戳+序号(blk_<unixnano>_<n>),是唯一的例外。
|
||||
//
|
||||
// 淘汰 CAS 之后,块自己承载内容,于是「同一份内容只应有一个块」这条
|
||||
// 不再由外部的内容寻址层保证,而必须由**块 ID 本身**保证:
|
||||
// 同内容 ⇒ 同 ID ⇒ PutMemoryBlocks 的 ON CONFLICT 天然去重。
|
||||
// 顺带让跨插件边界的引用(只带 ID 前缀)也能唯一定位。
|
||||
//
|
||||
// ★ 为何把 MIME 也掺进去:同样字节在不同 MIME 下是不同的媒体
|
||||
// (同一段字节按 image/png 与 application/octet-stream 解读得到不同结果),
|
||||
// 且 MIME 参与向量计算,掺入可避免「先入者定 MIME」的静默错配。
|
||||
func MediaBlockID(mime string, data []byte) string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte(strings.TrimSpace(mime)))
|
||||
h.Write([]byte{0})
|
||||
h.Write(data)
|
||||
return "blk_media_" + hex.EncodeToString(h.Sum(nil)[:12])
|
||||
}
|
||||
|
||||
// BlockByIDPrefix 按块 ID 的**前缀**取块(前缀唯一时命中)。
|
||||
//
|
||||
// 用途:跨进程/跨层的轻量引用(如插件只带块 ID 的前 12 位)。
|
||||
// 前缀匹配到多条时返回 false —— 宁可不解析也不猜一个(猜错=挂错媒体)。
|
||||
func (g *GraphDB) BlockByIDPrefix(prefix string) (MemoryBlock, bool, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return MemoryBlock{}, false, nil
|
||||
}
|
||||
g.mu.RLock()
|
||||
defer g.mu.RUnlock()
|
||||
rows, err := g.db.Query(`SELECT `+blockColumns+`
|
||||
FROM memory_blocks WHERE id LIKE ? LIMIT 2`, prefix+"%")
|
||||
if err != nil {
|
||||
return MemoryBlock{}, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var got []MemoryBlock
|
||||
for rows.Next() {
|
||||
b, err := scanBlockRow(rows)
|
||||
if err != nil {
|
||||
return MemoryBlock{}, false, err
|
||||
}
|
||||
got = append(got, b)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return MemoryBlock{}, false, err
|
||||
}
|
||||
if len(got) != 1 {
|
||||
return MemoryBlock{}, false, nil // 0 条或歧义(≥2)都不解析
|
||||
}
|
||||
return got[0], true, nil
|
||||
}
|
||||
|
||||
// BlockData 取单个块的负载字节(按需加载)。
|
||||
//
|
||||
// ★ 为何单独一个方法:blockColumns 刻意不含 data(召回只比向量,
|
||||
// 带上负载会让每次全表召回读入几十 MB)。需要字节的路径显式调本方法。
|
||||
func (g *GraphDB) BlockData(blockID string) ([]byte, error) {
|
||||
g.mu.RLock()
|
||||
defer g.mu.RUnlock()
|
||||
var data []byte
|
||||
err := g.db.QueryRow(`SELECT data FROM memory_blocks WHERE id = ?`, blockID).Scan(&data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// StaleBlockVectors 返回向量为空、或向量 fingerprint 与当前空间不符的块。
|
||||
//
|
||||
// 用途:启动时把历史媒体块/文档块的向量重算到当前多模态空间(换模型、
|
||||
// 换维度、首次接入嵌入服务、嵌入服务离线后重上线)。
|
||||
//
|
||||
// 为何需要它:向量是召回的唯一依据,而历史块可能是在另一个空间里算的。
|
||||
// fingerprint 不一致的向量不能参与余弦比较(两个坐标系的值比大小无意义),
|
||||
// 必须重算而不是留着当"看起来还行"的噪声。
|
||||
//
|
||||
// 只取非文本块:文本块的向量由文本路径维护,不在多模态空间迁移范围内。
|
||||
func (g *GraphDB) StaleBlockVectors(currentFP string) ([]MemoryBlock, error) {
|
||||
g.mu.RLock()
|
||||
defer g.mu.RUnlock()
|
||||
rows, err := g.db.Query(`SELECT `+blockColumns+`
|
||||
FROM memory_blocks
|
||||
WHERE modality != ?
|
||||
AND (vector IS NULL OR vector = '' OR vector = 'null' OR fingerprint != ?)
|
||||
ORDER BY created_at`, string(BlockText), currentFP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []MemoryBlock
|
||||
for rows.Next() {
|
||||
b, err := scanBlockRow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetBlockVector 给单个块写入新向量与指纹(迁移用)。只改向量列,不碰负载。
|
||||
func (g *GraphDB) SetBlockVector(blockID string, vec []float64, fp string) error {
|
||||
vectorJSON, err := json.Marshal(vec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
_, err = g.db.Exec(`UPDATE memory_blocks SET vector = ?, fingerprint = ?, updated_at = ?
|
||||
WHERE id = ?`, string(vectorJSON), fp, time.Now(), blockID)
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user