jianf/HomeAgent
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/HomeAgent.git synced 2026-09-22 01:48:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: 修复记忆系统自循环与计算层污染

Browse Source 
- 删 syncGraphToDocs(): Graph 快照不再写入 Document，避免污染向量索引和三层隔离
- 删 toolCallRing(): 已被工具 NoMemory/Cleaner 机制取代，不再需要独立环形缓冲
- 加 toolOutputClean 回调线程 Prune→ContextToDoc: 归档时按 NoMemory 跳过、Cleaner 清洗后再过 jieba，原文保留
- 加 eval_status 持久化 (RecallPending/UpdateEvalStatus/ResolveEvaluating): 避免重复 LLM 评估
This commit is contained in:
root
2026-07-26 19:29:01 +08:00
parent db4844c6c6
commit d19b7bd13e
7 changed files with 373 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
internal/memory/document/document.go
View File

@ -124,15 +124,17 @@ func (s *Store) Insert(doc *Doc) error {
}
// ContextToDoc — 将一段上下文对话历史提炼为文档（带内容去重）
// cleanFn 可选，用于在计算层（摘要/标签/实体提取）前过滤文本，不影响原文存储。
func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.Vectorizer, cleanFn ...func(string) string) (*Doc, error) {
// cleanFn 可选，在计算层前统一过滤文本，不影响原文存储。
// toolCleanFn 可选，func(name, output string) string，按工具名对输出进行过滤/清洗：
// - 返回 "" → 跳过该工具输出（NoMemory）
// - 返回清洗后文本 → 用于计算层（Cleaner），原文不受影响
func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.Vectorizer, cleanFn func(string) string, toolCleanFn func(name, output string) string) (*Doc, error) {
if len(entries) == 0 {
return nil, nil
}
cleanText := func(text string) string { return text }
if len(cleanFn) > 0 && cleanFn[0] != nil {
cleanText = cleanFn[0]
if cleanFn == nil {
cleanFn = func(text string) string { return text }
}
var parts []string
@ -149,9 +151,9 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.V
content := strings.Join(parts, "\n")
contentHash := simpleHash(content)
summary := summarizeEntries(entries, cleanText)
tags := extractTags(entries, cleanText)
entities := extractEntities(entries, cleanText)
summary := summarizeEntries(entries, cleanFn, toolCleanFn)
tags := extractTags(entries, cleanFn, toolCleanFn)
entities := extractEntities(entries, cleanFn, toolCleanFn)
s.mu.Lock()
@ -439,23 +441,26 @@ type ContextEntry struct {
ToolResults []ToolResultItem
}
func summarizeEntries(entries []ContextEntry, cleanText ...func(string) string) string {
func summarizeEntries(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string) string {
if len(entries) == 0 {
return ""
}
clean := func(text string) string { return text }
if len(cleanText) > 0 && cleanText[0] != nil {
clean = cleanText[0]
}
sources := make(map[string]int)
var topics []string
for _, e := range entries {
sources[e.Source]++
words := memory.ExtractKeywords(clean(e.Content))
words := memory.ExtractKeywords(cleanText(e.Content))
topics = append(topics, words...)
for _, tr := range e.ToolResults {
cleaned := clean(tr.Output)
toolWords := memory.ExtractKeywords(cleaned)
out := tr.Output
if toolCleanFn != nil {
if c := toolCleanFn(tr.Name, tr.Output); c == "" {
continue
} else {
out = c
}
}
toolWords := memory.ExtractKeywords(out)
topics = append(topics, toolWords...)
}
}
@ -485,18 +490,22 @@ func summarizeEntries(entries []ContextEntry, cleanText ...func(string) string)
return summary
}
func extractTags(entries []ContextEntry, cleanText ...func(string) string) []string {
clean := func(text string) string { return text }
if len(cleanText) > 0 && cleanText[0] != nil {
clean = cleanText[0]
}
func extractTags(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string) []string {
tagSet := make(map[string]bool)
for _, e := range entries {
for _, kw := range memory.ExtractKeywords(clean(e.Content)) {
for _, kw := range memory.ExtractKeywords(cleanText(e.Content)) {
tagSet[kw] = true
}
for _, tr := range e.ToolResults {
for _, kw := range memory.ExtractKeywords(clean(tr.Output)) {
out := tr.Output
if toolCleanFn != nil {
if c := toolCleanFn(tr.Name, tr.Output); c == "" {
continue
} else {
out = c
}
}
for _, kw := range memory.ExtractKeywords(out) {
tagSet[kw] = true
}
}
@ -511,23 +520,26 @@ func extractTags(entries []ContextEntry, cleanText ...func(string) string) []str
return tags
}
func extractEntities(entries []ContextEntry, cleanText ...func(string) string) []string {
// 简易实体提取：提取引号内的内容、粗体/标记词
clean := func(text string) string { return text }
if len(cleanText) > 0 && cleanText[0] != nil {
clean = cleanText[0]
}
func extractEntities(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string) []string {
var entities []string
seen := make(map[string]bool)
for _, e := range entries {
for _, kw := range memory.ExtractKeywords(clean(e.Content)) {
for _, kw := range memory.ExtractKeywords(cleanText(e.Content)) {
if len(kw) >= 2 && !seen[kw] {
seen[kw] = true
entities = append(entities, kw)
}
}
for _, tr := range e.ToolResults {
for _, kw := range memory.ExtractKeywords(clean(tr.Output)) {
out := tr.Output
if toolCleanFn != nil {
if c := toolCleanFn(tr.Name, tr.Output); c == "" {
continue
} else {
out = c
}
}
for _, kw := range memory.ExtractKeywords(out) {
if len(kw) >= 2 && !seen[kw] {
seen[kw] = true
entities = append(entities, kw)

130
internal/memory/document/document_test.go
View File

@ -82,7 +82,7 @@ func TestContextToDoc(t *testing.T) {
{Timestamp: time.Now(), Source: "user", Content: "特别是Go语言", Response: "Go很棒"},
}
doc, err := s.ContextToDoc("test", entries, nil)
doc, err := s.ContextToDoc("test", entries, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -178,7 +178,7 @@ func TestSummarizeEntries(t *testing.T) {
{Source: "user", Content: "今天天气如何"},
{Source: "user", Content: "明天会下雨吗"},
}
summary := summarizeEntries(entries)
summary := summarizeEntries(entries, func(s string) string { return s }, nil)
if summary == "" {
t.Error("summary should not be empty")
}
@ -198,7 +198,7 @@ func TestExtractTags(t *testing.T) {
entries := []ContextEntry{
{Content: "我喜欢喝咖啡和编程"},
}
tags := extractTags(entries)
tags := extractTags(entries, func(s string) string { return s }, nil)
if len(tags) == 0 {
t.Error("should extract tags")
}
@ -343,3 +343,127 @@ func TestRemoveNonexistent(t *testing.T) {
t.Errorf("expected 1 doc after remove nonexistent, got %d", stats["doc_count"])
}
}
func TestSummarizeEntriesWithToolCleanFn(t *testing.T) {
tests := []struct {
name string
toolCleanFn func(name, output string) string
wantTopics []string
notTopics []string
}{
{
name: "nil toolCleanFn uses raw output",
toolCleanFn: nil,
wantTopics: []string{"手机", "电脑"},
notTopics: nil,
},
{
name: "NoMemory returns empty skips tool output",
toolCleanFn: func(name, output string) string {
return ""
},
wantTopics: nil,
notTopics: []string{"手机", "电脑"},
},
{
name: "Cleaner applies filter",
toolCleanFn: func(name, output string) string {
return "电脑 编程"
},
wantTopics: []string{"电脑", "编程"},
notTopics: nil,
},
}
entry := ContextEntry{
Content: "今天天气",
ToolResults: []ToolResultItem{
{Name: "test_tool", Output: "手机 电脑"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
summary := summarizeEntries([]ContextEntry{entry}, func(s string) string { return s }, tc.toolCleanFn)
for _, w := range tc.wantTopics {
if !contains(summary, w) {
t.Errorf("summary should contain %q, got: %s", w, summary)
}
}
for _, n := range tc.notTopics {
if contains(summary, n) {
t.Errorf("summary should NOT contain %q, got: %s", n, summary)
}
}
})
}
}
func TestExtractTagsWithToolCleanFn(t *testing.T) {
entries := []ContextEntry{
{
Content: "对话",
ToolResults: []ToolResultItem{
{Name: "search", Output: "编程和咖啡"},
},
},
}
// toolCleanFn 返回 "" → NoMemory，工具输出被跳过
tagsSkip := extractTags(entries, func(s string) string { return s }, func(name, output string) string { return "" })
for _, tag := range tagsSkip {
if tag == "编程" || tag == "咖啡" {
t.Errorf("NoMemory tool should not contribute keywords, got tag: %s", tag)
}
}
// toolCleanFn 返回清洗文本 → 用清洗后内容提取关键词
tagsClean := extractTags(entries, func(s string) string { return s }, func(name, output string) string { return "咖啡 编程" })
found := false
for _, tag := range tagsClean {
if tag == "编程" {
found = true
}
}
if !found {
t.Errorf("cleaner output keywords should appear in tags, got: %v", tagsClean)
}
}
func TestContextToDocContentPreservesRawToolOutput(t *testing.T) {
dir, err := os.MkdirTemp("", "doc_toolclean_*")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
s := NewStore(dir)
s.Start()
defer s.Stop()
entries := []ContextEntry{
{
Timestamp: time.Now(),
Source: "user",
Content: "查天气",
ToolResults: []ToolResultItem{
{Name: "weather", Output: "{\"temp\": 25}"},
},
},
}
// toolCleanFn 返回清洗文本，但 Content 必须保留原始输出
cleaner := func(name, output string) string {
return "天气 温度"
}
doc, err := s.ContextToDoc("test", entries, nil, nil, cleaner)
if err != nil {
t.Fatal(err)
}
if !contains(doc.Content, "{\"temp\": 25}") {
t.Errorf("Content should preserve raw tool output, got: %s", doc.Content)
}
if doc.Summary == "" {
t.Error("summary should not be empty")
}
}

118
internal/memory/graph.go
View File

@ -31,6 +31,9 @@ type Relation struct {
TurnID int `json:"turn_id"`
CreatedAt time.Time `json:"created_at"`
DateBucket string `json:"date_bucket"`
EvalStatus string `json:"eval_status"`
EvalRound int `json:"eval_round"`
EvalAt time.Time `json:"eval_at,omitempty"`
}
type Triple struct {
@ -93,6 +96,9 @@ func (g *GraphDB) initSchema() error {
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
date_bucket TEXT,
eval_status TEXT DEFAULT 'pending',
eval_round INTEGER DEFAULT 0,
eval_at TIMESTAMP,
FOREIGN KEY (source_id) REFERENCES entities(id),
FOREIGN KEY (target_id) REFERENCES entities(id)
)`,
@ -111,7 +117,20 @@ func (g *GraphDB) initSchema() error {
}
}
return tx.Commit()
if err := tx.Commit(); err != nil {
return err
}
migrations := []string{
`ALTER TABLE relations ADD COLUMN eval_status TEXT DEFAULT 'pending'`,
`ALTER TABLE relations ADD COLUMN eval_round INTEGER DEFAULT 0`,
`ALTER TABLE relations ADD COLUMN eval_at TIMESTAMP`,
}
for _, m := range migrations {
g.db.Exec(m)
}
return nil
}
func (g *GraphDB) Commit(triples []Triple, sessionID string, turnID int) (int, int, error) {
@ -241,7 +260,7 @@ func (g *GraphDB) Recall(keywords []string, seedEntities []string, depth int, se
if len(keywords) == 0 && len(seedEntities) == 0 {
rows, err := g.db.Query(
`SELECT id, name, type, mention_count, created_at, updated_at
FROM entities ORDER BY mention_count DESC LIMIT 50`,
FROM entities ORDER BY mention_count DESC`,
)
if err != nil {
return nil, err
@ -259,7 +278,8 @@ func (g *GraphDB) Recall(keywords []string, seedEntities []string, depth int, se
relRows, err := g.db.Query(
`SELECT r.id, r.source_id, r.target_id, e1.name, e2.name,
r.relation_type, r.confidence, r.status, r.session_id,
r.turn_id, r.created_at, COALESCE(r.date_bucket, '')
r.turn_id, r.created_at, COALESCE(r.date_bucket, ''),
COALESCE(r.eval_status, 'pending'), COALESCE(r.eval_round, 0), r.eval_at
FROM relations r
JOIN entities e1 ON r.source_id = e1.id
JOIN entities e2 ON r.target_id = e2.id
@ -275,7 +295,8 @@ func (g *GraphDB) Recall(keywords []string, seedEntities []string, depth int, se
if err := relRows.Scan(&rel.ID, &rel.SourceID, &rel.TargetID,
&rel.SourceName, &rel.TargetName, &rel.RelationType,
&rel.Confidence, &rel.Status, &rel.SessionID,
&rel.TurnID, &rel.CreatedAt, &rel.DateBucket); err != nil {
&rel.TurnID, &rel.CreatedAt, &rel.DateBucket,
&rel.EvalStatus, &rel.EvalRound, &rel.EvalAt); err != nil {
return nil, err
}
result.Relations = append(result.Relations, rel)
@ -340,7 +361,8 @@ func (g *GraphDB) Recall(keywords []string, seedEntities []string, depth int, se
query := fmt.Sprintf(
`SELECT r.id, r.source_id, r.target_id, e1.name, e2.name,
r.relation_type, r.confidence, r.status, r.session_id,
r.turn_id, r.created_at, COALESCE(r.date_bucket, '')
r.turn_id, r.created_at, COALESCE(r.date_bucket, ''),
COALESCE(r.eval_status, 'pending'), COALESCE(r.eval_round, 0), r.eval_at
FROM relations r
JOIN entities e1 ON r.source_id = e1.id
JOIN entities e2 ON r.target_id = e2.id
@ -367,7 +389,8 @@ func (g *GraphDB) Recall(keywords []string, seedEntities []string, depth int, se
if err := relRows.Scan(&rel.ID, &rel.SourceID, &rel.TargetID,
&rel.SourceName, &rel.TargetName, &rel.RelationType,
&rel.Confidence, &rel.Status, &rel.SessionID,
&rel.TurnID, &rel.CreatedAt, &rel.DateBucket); err != nil {
&rel.TurnID, &rel.CreatedAt, &rel.DateBucket,
&rel.EvalStatus, &rel.EvalRound, &rel.EvalAt); err != nil {
relRows.Close()
return nil, err
}
@ -747,6 +770,89 @@ func (g *GraphDB) Archive(days int) (int, error) {
return int(n), nil
}
func (g *GraphDB) RecallPending(limit int) ([]Relation, error) {
g.mu.RLock()
defer g.mu.RUnlock()
rows, err := g.db.Query(
`SELECT r.id, r.source_id, r.target_id, e1.name, e2.name,
r.relation_type, r.confidence, r.status, r.session_id,
r.turn_id, r.created_at, COALESCE(r.date_bucket, ''),
COALESCE(r.eval_status, 'pending'), COALESCE(r.eval_round, 0), r.eval_at
FROM relations r
JOIN entities e1 ON r.source_id = e1.id
JOIN entities e2 ON r.target_id = e2.id
WHERE r.status = 'active'
AND (r.eval_status IS NULL OR r.eval_status = 'pending')
ORDER BY r.created_at DESC
LIMIT ?`, limit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var relations []Relation
for rows.Next() {
var rel Relation
if err := rows.Scan(&rel.ID, &rel.SourceID, &rel.TargetID,
&rel.SourceName, &rel.TargetName, &rel.RelationType,
&rel.Confidence, &rel.Status, &rel.SessionID,
&rel.TurnID, &rel.CreatedAt, &rel.DateBucket,
&rel.EvalStatus, &rel.EvalRound, &rel.EvalAt); err != nil {
return nil, err
}
relations = append(relations, rel)
}
return relations, rows.Err()
}
func (g *GraphDB) UpdateEvalStatus(id int64, status string) error {
g.mu.Lock()
defer g.mu.Unlock()
_, err := g.db.Exec(
`UPDATE relations SET eval_status = ?, eval_round = eval_round + 1, eval_at = CURRENT_TIMESTAMP WHERE id = ?`,
status, id,
)
return err
}
func (g *GraphDB) UpdateEvalStatusBatch(ids []int64, status string) error {
g.mu.Lock()
defer g.mu.Unlock()
if len(ids) == 0 {
return nil
}
for _, id := range ids {
_, err := g.db.Exec(
`UPDATE relations SET eval_status = ?, eval_round = eval_round + 1, eval_at = CURRENT_TIMESTAMP WHERE id = ?`,
status, id,
)
if err != nil {
return err
}
}
return nil
}
func (g *GraphDB) ResolveEvaluating() (int, error) {
g.mu.Lock()
defer g.mu.Unlock()
result, err := g.db.Exec(
`UPDATE relations SET eval_status = 'approved', eval_round = eval_round + 1, eval_at = CURRENT_TIMESTAMP
WHERE eval_status = 'evaluating' AND status = 'active'`,
)
if err != nil {
return 0, err
}
n, _ := result.RowsAffected()
return int(n), nil
}
func (g *GraphDB) Close() error {
return g.db.Close()
}