refactor: P0-P3 fixes, C1 cleanup, architecture diagrams, go.work upgrade

- P0-1: ProviderError type + ReportStatus for precise 401/403 detection
- P0-2: Remove -config flag from deploy/homeagent.service
- P2-1: 5s debounce on context.go Save()
- P2-2→C1: Delete output_set_channel entirely
- P2-3: Extract mediaDataURL/mediaChat helpers
- P2-4: Dedup defaultSources var
- P3: Delete dead packages (embed/tokenizer/container/snapshot)
- P3: Delete dead functions (messagesToMap, RunStageAll)
- CL: Update .gitignore, docs, Makefile, gojieba removal
- Config: Delete config/config.yaml, update docs
- Arch: Remove EmitOutputTo from emitResponse
- CL-1: go.work 1.19→1.21
- Docs: Add Mermaid architecture diagrams to README
- Docs: Add kernel-rebuild requires plugin-rebuild note to PLUGIN_DEV.md
This commit is contained in:
root
2026-07-12 11:42:56 +08:00
parent 152954dc96
commit 19e51e61e8
89 changed files with 1764 additions and 6578 deletions

View File

@ -37,11 +37,13 @@ type Store struct {
mu sync.RWMutex
docs map[string]*Doc
summaries []string // 用于训练向量化器
summaries []string // 用于训练向量化器,最大 10000 条
dirty bool
}
const maxSummaries = 10000
func NewStore(dir string) *Store {
return &Store{
dir: dir,
@ -83,11 +85,15 @@ func (s *Store) Insert(doc *Doc) error {
s.docs[doc.ID] = doc
// 增量训练向量化器并加入向量索引
s.addSummary(doc.Summary)
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
// 更新训练集
s.summaries = append(s.summaries, doc.Summary)
// 立即写盘
path := filepath.Join(s.dir, doc.ID+".json")
data, _ := json.MarshalIndent(doc, "", " ")
os.WriteFile(path, data, 0644)
s.dirty = true
return nil
@ -110,12 +116,13 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
content := strings.Join(parts, "\n")
contentHash := simpleHash(content)
// 去重:检查是否已有相同 hash 的文档(在锁内完成创建/更新)
summary := summarizeEntries(entries)
tags := extractTags(entries)
entities := extractEntities(entries)
s.mu.Lock()
// 去重
for _, d := range s.docs {
if d.Meta != nil && d.Meta["content_hash"] == contentHash {
d.UpdatedAt = time.Now()
@ -146,8 +153,20 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
Meta: map[string]string{"content_hash": contentHash},
}
s.docs[id] = doc
// 增量训练向量化器并加入向量索引
s.addSummary(summary)
vec := s.veczer.Vectorize(summary + " " + content)
s.vec.Insert(id, summary, vec, nil)
s.dirty = true
s.mu.Unlock()
// 立即写盘
path := filepath.Join(s.dir, id+".json")
data, _ := json.MarshalIndent(doc, "", " ")
os.WriteFile(path, data, 0644)
return doc, nil
}
@ -166,8 +185,7 @@ func (s *Store) Consume(text string, topK int) []*Doc {
var docs []*Doc
for _, r := range results {
if d, ok := s.docs[r.ID]; ok {
delete(s.docs, r.ID)
s.vec.Remove(r.ID)
s.removeDoc(r.ID)
s.dirty = true
docs = append(docs, d)
}
@ -266,14 +284,39 @@ func (s *Store) Remove(id string) {
defer s.mu.Unlock()
if _, ok := s.docs[id]; ok {
delete(s.docs, id)
s.vec.Remove(id)
s.removeDoc(id)
s.dirty = true
}
}
// ——— internal ———
// addSummary 添加一条摘要到训练集,超限时截断并触发重索引。
// 调用方必须已持有 s.mu 写锁。
func (s *Store) addSummary(summary string) {
s.summaries = append(s.summaries, summary)
if len(s.summaries) > maxSummaries {
n := maxSummaries / 2
copy(s.summaries, s.summaries[len(s.summaries)-n:])
s.summaries = s.summaries[:n]
s.veczer.Train(s.summaries)
s.vec = vector.NewStore()
for _, doc := range s.docs {
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
s.vec.Insert(doc.ID, doc.Summary, vec, nil)
}
}
}
// removeDoc 从内存索引和磁盘删除文档。
// 调用方必须已持有 s.mu 写锁。
func (s *Store) removeDoc(id string) {
delete(s.docs, id)
s.vec.Remove(id)
path := filepath.Join(s.dir, id+".json")
os.Remove(path)
}
func (s *Store) loadAll() error {
entries, err := os.ReadDir(s.dir)
if err != nil {

View File

@ -526,7 +526,7 @@ func (g *GraphDB) Introspect() (map[string]interface{}, error) {
// MergeEntities 合并两个实体:将 sourceName 的所有信息合并到 targetName
// 1. sourceName 的所有关系重新指向 targetName
// 2. targetName 的 mention_count 增加 sourceName 的计数
// 3. sourceName 标记为 merged
// 3. sourceName 彻底删除(不再残留 @merged_ 实体)
// 返回 (关系的重定向数, error)
func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
g.mu.Lock()
@ -554,7 +554,7 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
return 0, fmt.Errorf("cannot merge entity with itself")
}
// 重定向 source → target 的关系(作为 source)
// 重定向 source → target 的活跃关系(作为 source)
res, err := tx.Exec(
`UPDATE relations SET source_id = ?, updated_at = CURRENT_TIMESTAMP
WHERE source_id = ? AND status = 'active'`,
@ -565,7 +565,7 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
}
redirectedSource, _ := res.RowsAffected()
// 重定向 source → target 的关系(作为 target)
// 重定向 source → target 的活跃关系(作为 target)
res, err = tx.Exec(
`UPDATE relations SET target_id = ?, updated_at = CURRENT_TIMESTAMP
WHERE target_id = ? AND status = 'active'`,
@ -586,6 +586,12 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
return 0, err
}
// 清理 source 残留的非活跃关系(archived/deleted),否则外键约束阻止删除实体
_, err = tx.Exec(`DELETE FROM relations WHERE source_id = ? OR target_id = ?`, sourceID, sourceID)
if err != nil {
return 0, err
}
// 更新 target 的 mention_count
_, err = tx.Exec(
`UPDATE entities SET mention_count = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`,
@ -595,14 +601,8 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
return 0, err
}
// 标记 source 为 merged(改名避免 UNIQUE 冲突)
_, err = tx.Exec(
`UPDATE entities SET name = ? || '@merged_' || ?,
mention_count = 0,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?`,
sourceName, time.Now().Format("20060102150405"), sourceID,
)
// 彻底删除 source 实体(所有关系已重定向,自引用已删除)
_, err = tx.Exec(`DELETE FROM entities WHERE id = ?`, sourceID)
if err != nil {
return 0, err
}
@ -615,6 +615,36 @@ func (g *GraphDB) MergeEntities(sourceName, targetName string) (int, error) {
return total, nil
}
// DeleteEntity 彻底删除一个实体及其所有关联关系。
func (g *GraphDB) DeleteEntity(name string) error {
g.mu.Lock()
defer g.mu.Unlock()
tx, err := g.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
var id int64
err = tx.QueryRow("SELECT id FROM entities WHERE name = ?", name).Scan(&id)
if err != nil {
return fmt.Errorf("entity '%s' not found: %w", name, err)
}
_, err = tx.Exec(`DELETE FROM relations WHERE source_id = ? OR target_id = ?`, id, id)
if err != nil {
return err
}
_, err = tx.Exec(`DELETE FROM entities WHERE id = ?`, id)
if err != nil {
return err
}
return tx.Commit()
}
func (g *GraphDB) Archive(days int) (int, error) {
g.mu.Lock()
defer g.mu.Unlock()

View File

@ -88,7 +88,7 @@ func (d *Distiller) flush() {
if len(d.records) == 0 {
return
}
path := filepath.Join(d.rawPath, fmt.Sprintf("raw_%d.jsonl", time.Now().UnixNano()))
path := filepath.Join(d.rawPath, fmt.Sprintf("raw_%d.tsv", time.Now().UnixNano()))
f, err := os.Create(path)
if err != nil {
log.Printf("[memory] flush error: %v", err)
@ -113,7 +113,8 @@ func (d *Distiller) loadExisting() {
}
var files []fileInfo
for _, entry := range entries {
if filepath.Ext(entry.Name()) != ".jsonl" {
ext := filepath.Ext(entry.Name())
if ext != ".tsv" && ext != ".jsonl" {
continue
}
info, err := entry.Info()
@ -296,8 +297,8 @@ func extractName(s string) string {
}{
{"我叫", ""},
{"我的名字是", ""},
{"我是", ""},
{"名字是", ""},
{"我是", ""},
}
s = strings.TrimSpace(s)
for _, p := range patterns {
@ -313,6 +314,10 @@ func extractName(s string) string {
candidate = candidate[:idx]
}
}
// "我是张三"(姓名) vs "我是一个程序员"(职业):名字通常 ≤4 字符
if p.prefix == "我是" && len([]rune(candidate)) > 4 {
continue
}
if len(candidate) > 0 && len(candidate) < 20 {
return candidate
}