mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-03 15:53:56 +00:00
refactor: P0-P3 fixes, C1 cleanup, architecture diagrams, go.work upgrade
- P0-1: ProviderError type + ReportStatus for precise 401/403 detection - P0-2: Remove -config flag from deploy/homeagent.service - P2-1: 5s debounce on context.go Save() - P2-2→C1: Delete output_set_channel entirely - P2-3: Extract mediaDataURL/mediaChat helpers - P2-4: Dedup defaultSources var - P3: Delete dead packages (embed/tokenizer/container/snapshot) - P3: Delete dead functions (messagesToMap, RunStageAll) - CL: Update .gitignore, docs, Makefile, gojieba removal - Config: Delete config/config.yaml, update docs - Arch: Remove EmitOutputTo from emitResponse - CL-1: go.work 1.19→1.21 - Docs: Add Mermaid architecture diagrams to README - Docs: Add kernel-rebuild requires plugin-rebuild note to PLUGIN_DEV.md
This commit is contained in:
@ -37,11 +37,13 @@ type Store struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
docs map[string]*Doc
|
||||
summaries []string // 用于训练向量化器
|
||||
summaries []string // 用于训练向量化器,最大 10000 条
|
||||
|
||||
dirty bool
|
||||
}
|
||||
|
||||
const maxSummaries = 10000
|
||||
|
||||
func NewStore(dir string) *Store {
|
||||
return &Store{
|
||||
dir: dir,
|
||||
@ -83,11 +85,15 @@ func (s *Store) Insert(doc *Doc) error {
|
||||
|
||||
s.docs[doc.ID] = doc
|
||||
|
||||
// 增量训练向量化器并加入向量索引
|
||||
s.addSummary(doc.Summary)
|
||||
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
|
||||
|
||||
// 更新训练集
|
||||
s.summaries = append(s.summaries, doc.Summary)
|
||||
// 立即写盘
|
||||
path := filepath.Join(s.dir, doc.ID+".json")
|
||||
data, _ := json.MarshalIndent(doc, "", " ")
|
||||
os.WriteFile(path, data, 0644)
|
||||
|
||||
s.dirty = true
|
||||
return nil
|
||||
@ -110,12 +116,13 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
|
||||
content := strings.Join(parts, "\n")
|
||||
contentHash := simpleHash(content)
|
||||
|
||||
// 去重:检查是否已有相同 hash 的文档(在锁内完成创建/更新)
|
||||
summary := summarizeEntries(entries)
|
||||
tags := extractTags(entries)
|
||||
entities := extractEntities(entries)
|
||||
|
||||
s.mu.Lock()
|
||||
|
||||
// 去重
|
||||
for _, d := range s.docs {
|
||||
if d.Meta != nil && d.Meta["content_hash"] == contentHash {
|
||||
d.UpdatedAt = time.Now()
|
||||
@ -146,8 +153,20 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
|
||||
Meta: map[string]string{"content_hash": contentHash},
|
||||
}
|
||||
s.docs[id] = doc
|
||||
|
||||
// 增量训练向量化器并加入向量索引
|
||||
s.addSummary(summary)
|
||||
vec := s.veczer.Vectorize(summary + " " + content)
|
||||
s.vec.Insert(id, summary, vec, nil)
|
||||
|
||||
s.dirty = true
|
||||
s.mu.Unlock()
|
||||
|
||||
// 立即写盘
|
||||
path := filepath.Join(s.dir, id+".json")
|
||||
data, _ := json.MarshalIndent(doc, "", " ")
|
||||
os.WriteFile(path, data, 0644)
|
||||
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
@ -166,8 +185,7 @@ func (s *Store) Consume(text string, topK int) []*Doc {
|
||||
var docs []*Doc
|
||||
for _, r := range results {
|
||||
if d, ok := s.docs[r.ID]; ok {
|
||||
delete(s.docs, r.ID)
|
||||
s.vec.Remove(r.ID)
|
||||
s.removeDoc(r.ID)
|
||||
s.dirty = true
|
||||
docs = append(docs, d)
|
||||
}
|
||||
@ -266,14 +284,39 @@ func (s *Store) Remove(id string) {
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if _, ok := s.docs[id]; ok {
|
||||
delete(s.docs, id)
|
||||
s.vec.Remove(id)
|
||||
s.removeDoc(id)
|
||||
s.dirty = true
|
||||
}
|
||||
}
|
||||
|
||||
// ——— internal ———
|
||||
|
||||
// addSummary 添加一条摘要到训练集,超限时截断并触发重索引。
|
||||
// 调用方必须已持有 s.mu 写锁。
|
||||
func (s *Store) addSummary(summary string) {
|
||||
s.summaries = append(s.summaries, summary)
|
||||
if len(s.summaries) > maxSummaries {
|
||||
n := maxSummaries / 2
|
||||
copy(s.summaries, s.summaries[len(s.summaries)-n:])
|
||||
s.summaries = s.summaries[:n]
|
||||
s.veczer.Train(s.summaries)
|
||||
s.vec = vector.NewStore()
|
||||
for _, doc := range s.docs {
|
||||
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
s.vec.Insert(doc.ID, doc.Summary, vec, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// removeDoc 从内存索引和磁盘删除文档。
|
||||
// 调用方必须已持有 s.mu 写锁。
|
||||
func (s *Store) removeDoc(id string) {
|
||||
delete(s.docs, id)
|
||||
s.vec.Remove(id)
|
||||
path := filepath.Join(s.dir, id+".json")
|
||||
os.Remove(path)
|
||||
}
|
||||
|
||||
func (s *Store) loadAll() error {
|
||||
entries, err := os.ReadDir(s.dir)
|
||||
if err != nil {
|
||||
|
||||
@ -526,7 +526,7 @@ func (g *GraphDB) Introspect() (map[string]interface{}, error) {
|
||||
// MergeEntities 合并两个实体:将 sourceName 的所有信息合并到 targetName
|
||||
// 1. sourceName 的所有关系重新指向 targetName
|
||||
// 2. targetName 的 mention_count 增加 sourceName 的计数
|
||||
// 3. sourceName 标记为 merged
|
||||
// 3. sourceName 彻底删除(不再残留 @merged_ 实体)
|
||||
// 返回 (关系的重定向数, error)
|
||||
func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
|
||||
g.mu.Lock()
|
||||
@ -554,7 +554,7 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
|
||||
return 0, fmt.Errorf("cannot merge entity with itself")
|
||||
}
|
||||
|
||||
// 重定向 source → target 的关系(作为 source)
|
||||
// 重定向 source → target 的活跃关系(作为 source)
|
||||
res, err := tx.Exec(
|
||||
`UPDATE relations SET source_id = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE source_id = ? AND status = 'active'`,
|
||||
@ -565,7 +565,7 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
|
||||
}
|
||||
redirectedSource, _ := res.RowsAffected()
|
||||
|
||||
// 重定向 source → target 的关系(作为 target)
|
||||
// 重定向 source → target 的活跃关系(作为 target)
|
||||
res, err = tx.Exec(
|
||||
`UPDATE relations SET target_id = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE target_id = ? AND status = 'active'`,
|
||||
@ -586,6 +586,12 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// 清理 source 残留的非活跃关系(archived/deleted),否则外键约束阻止删除实体
|
||||
_, err = tx.Exec(`DELETE FROM relations WHERE source_id = ? OR target_id = ?`, sourceID, sourceID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// 更新 target 的 mention_count
|
||||
_, err = tx.Exec(
|
||||
`UPDATE entities SET mention_count = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`,
|
||||
@ -595,14 +601,8 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// 标记 source 为 merged(改名避免 UNIQUE 冲突)
|
||||
_, err = tx.Exec(
|
||||
`UPDATE entities SET name = ? || '@merged_' || ?,
|
||||
mention_count = 0,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
sourceName, time.Now().Format("20060102150405"), sourceID,
|
||||
)
|
||||
// 彻底删除 source 实体(所有关系已重定向,自引用已删除)
|
||||
_, err = tx.Exec(`DELETE FROM entities WHERE id = ?`, sourceID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@ -615,6 +615,36 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// DeleteEntity 彻底删除一个实体及其所有关联关系。
|
||||
func (g *GraphDB) DeleteEntity(name string) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
tx, err := g.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
var id int64
|
||||
err = tx.QueryRow("SELECT id FROM entities WHERE name = ?", name).Scan(&id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("entity '%s' not found: %w", name, err)
|
||||
}
|
||||
|
||||
_, err = tx.Exec(`DELETE FROM relations WHERE source_id = ? OR target_id = ?`, id, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = tx.Exec(`DELETE FROM entities WHERE id = ?`, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (g *GraphDB) Archive(days int) (int, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
@ -88,7 +88,7 @@ func (d *Distiller) flush() {
|
||||
if len(d.records) == 0 {
|
||||
return
|
||||
}
|
||||
path := filepath.Join(d.rawPath, fmt.Sprintf("raw_%d.jsonl", time.Now().UnixNano()))
|
||||
path := filepath.Join(d.rawPath, fmt.Sprintf("raw_%d.tsv", time.Now().UnixNano()))
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
log.Printf("[memory] flush error: %v", err)
|
||||
@ -113,7 +113,8 @@ func (d *Distiller) loadExisting() {
|
||||
}
|
||||
var files []fileInfo
|
||||
for _, entry := range entries {
|
||||
if filepath.Ext(entry.Name()) != ".jsonl" {
|
||||
ext := filepath.Ext(entry.Name())
|
||||
if ext != ".tsv" && ext != ".jsonl" {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
@ -296,8 +297,8 @@ func extractName(s string) string {
|
||||
}{
|
||||
{"我叫", ""},
|
||||
{"我的名字是", ""},
|
||||
{"我是", ""},
|
||||
{"名字是", ""},
|
||||
{"我是", ""},
|
||||
}
|
||||
s = strings.TrimSpace(s)
|
||||
for _, p := range patterns {
|
||||
@ -313,6 +314,10 @@ func extractName(s string) string {
|
||||
candidate = candidate[:idx]
|
||||
}
|
||||
}
|
||||
// "我是张三"(姓名) vs "我是一个程序员"(职业):名字通常 ≤4 字符
|
||||
if p.prefix == "我是" && len([]rune(candidate)) > 4 {
|
||||
continue
|
||||
}
|
||||
if len(candidate) > 0 && len(candidate) < 20 {
|
||||
return candidate
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user