Files
HomeAgent/internal/agent/core/agent_helpers_test.go
JianFeeeee b37141f3f5 fix(memory): doc→graph 补回常用词闸门 + 存量噪音/孤立节点清理
问题:图记忆里堆着「结果(192) / 什么(59) / 哪个(99) / 待命(176) / 报告(174) /
context_archived(43)」这类节点,mention_count 冲到几百、度数只有 1~2——
占着热实体位、挤满召回预算,却不带任何结构。

根因:这层过滤原本存在,后来被换掉没补回。1cb3e87(NLP 三元组提取系统)
把 doc→graph 从「CutExact 滑窗词链」换成依存句法提取器时,CutExact
(去停用词 324 条 + validEntityName + 去重,注释至今还写着「用于 doc→graph
蒸馏」)失去了唯一生产调用点,只剩 cut_test.go 在调它。此后落库闸门只剩
validEntityName——它挡的是「不像名字的字符串」(2–50 字符、含字母),
完全不挡「像名字的常用词」。

改动:
- 新增 internal/memory/noise.go:IsNoiseEntity 收敛判定(停用词 /
  context_archived / 模板摘要回声),FilterNoiseTriples 给自动填充路用,
  PurgeNoise + PurgeOrphans + cmd/memgc 供存量清理(默认 dry-run)。
  判定刻意保守:只拦三类客观噪音,开放类词(报告/对话/处理)不拦——
  它们挡不挡是领域决策,见函数注释。
- docToTriples 接上闸门(用户点名的「文档常用词」路)。
  对话蒸馏路 extractKeyTriples **不接**:CutExact 当年也只挂 doc→graph,
  且 pipeline_test 明确断言「我 --读书--> 杭州」必须抽出(代词主语是该路
  既定行为),是否拦属行为决策,已在代码注释里写明并留给使用者定夺。
- cut.go 补全封闭类常用词:咱俩/咱们(我们/你们/他们 早有)、任何/此/本/
  其中/以及/那么/这样/那样/一样/还有/还要/只是/老是/全部/所有/有些/一些/
  别的/其他/其余/各自/本身/方位词/部分/方面。
  「不能/不会」试过又撤回:它们是 embedder tokenize 的实词路径,
  加进停用词会让 static_embedder 的领域聚类用例翻转(今天天气句与股票句
  的相似度大小关系反了),属真回归,不留。
- graph.go:CleanupOrphanedSentences 拆出 Locked 版供 PurgeNoise 复用。

验证:go build/vet 干净,go test -count=1 ./... 全绿。
新增用例:IsNoiseEntity 判定表、FilterNoiseTriples 顺序与边界、
PurgeNoise(统计/幂等/dry-run 不写库/不误删仍被媒体块边引用的句子)、
PurgeOrphans(识别/不误判有边实体/幂等)、docToTriples 噪音闸门与
模板锚点不被误杀。

存量清理(生产库 /home/newqqagent/memory/graph.db,先用 sqlite3 .backup 备份
到 graph.db.bak-20260915-073210):
- 噪音实体 29 个 + 其关系 59 条
- 零关系孤立实体 25 个
- 结果:实体 778→724,关系 639→580;sentences 3 条与 block_edges 3 条原样保留
  (媒体块引用不被误删),PRAGMA integrity_check=ok、无悬空关系/块边。
- 运行中的 homed 无需重启:下一个 archive 心跳会 Indexer.Sync 重建实体名向量索引。
2026-09-15 07:47:16 +08:00

293 lines
8.5 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package core
import (
"testing"
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
)
func needJieba() bool {
return memory.GetJieba() != nil
}
func TestDocToTriplesEmpty(t *testing.T) {
doc := &document.Doc{
Summary: "empty doc",
Content: "",
Source: "test",
}
triples := docToTriples(doc, nil)
if len(triples) < 2 {
t.Fatalf("expected at least 2 triples (主题+来源), got %d", len(triples))
}
if triples[0].Subject != "文档" || triples[0].Relation != "主题" || triples[0].Object != "empty doc" {
t.Errorf("first triple mismatch: %+v", triples[0])
}
last := triples[len(triples)-1]
if last.Subject != "文档" || last.Relation != "来源" || last.Object != "test" {
t.Errorf("last triple mismatch: %+v", last)
}
}
func TestDocToTriplesConversation(t *testing.T) {
doc := &document.Doc{
Summary: "测试对话 (qq) 涉及: 天气",
Content: "[15:04] qq: 今天天气怎么样\n[15:05] agent: 今天天气很好",
Source: "qq",
}
triples := docToTriples(doc, nil)
if len(triples) < 2 {
t.Errorf("expected at least 2 triples (主题+来源), got %d", len(triples))
}
for i, tr := range triples {
if tr.Subject == "" || tr.Relation == "" || tr.Object == "" {
t.Errorf("triple[%d] has empty field: %+v", i, tr)
}
if tr.Confidence <= 0 {
t.Errorf("triple[%d] has non-positive confidence: %+v", i, tr)
}
}
}
func TestDocToTriplesMultiLine(t *testing.T) {
doc := &document.Doc{
Summary: "多轮对话",
Content: "[10:00] user: 你好\n[10:01] agent: 你好,有什么可以帮助你的\n[10:02] user: 今天天气如何\n[10:03] agent: 今天天气很好",
Source: "qq",
}
triples := docToTriples(doc, nil)
if len(triples) < 2 {
t.Fatalf("expected at least 2 triples, got %d", len(triples))
}
if triples[0].Subject != "文档" || triples[0].Relation != "主题" {
t.Errorf("first triple should be 主题, got %+v", triples[0])
}
last := triples[len(triples)-1]
if last.Subject != "文档" || last.Relation != "来源" {
t.Errorf("last triple should be 来源, got %+v", last)
}
}
func TestDocToTriplesEmptyContent(t *testing.T) {
doc := &document.Doc{
Summary: "空内容",
Content: "",
Source: "test",
}
triples := docToTriples(doc, nil)
if len(triples) != 2 {
t.Fatalf("expected exactly 2 triples (主题+来源) for empty content, got %d", len(triples))
}
}
// Phase 2: 归档上下文文档不得产出模板垃圾context_archived 来源/主题模板三元组)
func TestDocToTriplesArchivedContext(t *testing.T) {
doc := &document.Doc{
Summary: "来自 2 个来源的 5 条对话 (qq, webui) 涉及: 天气, 测试",
Content: "[15:04] qq: 今天天气怎么样\n[15:05] agent: 今天天气很好",
Source: "context_archived",
Meta: map[string]string{"is_archived_context": "true"},
}
triples := docToTriples(doc, nil)
for _, tr := range triples {
if tr.Subject == "文档" && tr.Relation == "来源" && tr.Object == "context_archived" {
t.Errorf("archived context must not write 来源 triple: %+v", tr)
}
if tr.Subject == "文档" && tr.Relation == "主题" {
t.Errorf("archived context must not write 主题 template triple: %+v", tr)
}
}
}
// Phase 2: 模板化摘要summarizeEntries 生成)不得作为主题写入
func TestDocToTriplesTemplateSummary(t *testing.T) {
doc := &document.Doc{
Summary: "来自 3 个来源的 10 条对话 (a, b, c) 涉及: 关键词1, 关键词2, 关键词3",
Content: "[10:00] a: 你好",
Source: "manual",
}
triples := docToTriples(doc, nil)
for _, tr := range triples {
if tr.Subject == "文档" && tr.Relation == "主题" {
t.Errorf("template summary must not be written as 主题 triple: %+v", tr)
}
}
// 但非归档来源仍保留 来源 三元组
foundSource := false
for _, tr := range triples {
if tr.Subject == "文档" && tr.Relation == "来源" && tr.Object == "manual" {
foundSource = true
}
}
if !foundSource {
t.Errorf("non-archived source should still produce 来源 triple")
}
}
// Phase 2: 过长摘要不得写入主题
func TestDocToTriplesLongSummary(t *testing.T) {
long := ""
for i := 0; i < 100; i++ {
long += "很长的摘要内容片段重复拼接"
}
doc := &document.Doc{
Summary: long,
Content: "[10:00] a: 你好",
Source: "test",
}
triples := docToTriples(doc, nil)
for _, tr := range triples {
if tr.Subject == "文档" && tr.Relation == "主题" {
t.Errorf("overlong summary must not be written as 主题 triple")
}
}
}
func TestIsTemplateSummary(t *testing.T) {
if !isTemplateSummary("来自 2 个来源的 5 条对话 (qq, webui) 涉及: 天气") {
t.Errorf("template summary not recognized")
}
if isTemplateSummary("今天天气很好") {
t.Errorf("plain summary wrongly recognized as template")
}
if !isTemplateSummary("") {
t.Errorf("empty summary should be treated as template")
}
}
func TestTruncateStr(t *testing.T) {
tests := []struct {
input string
max int
want string
}{
{"hello", 10, "hello"},
{"hello world", 5, "hello..."},
{"你好世界", 2, "你好..."},
{"", 5, ""},
{"abc", 3, "abc"},
}
for _, tt := range tests {
got := truncateStr(tt.input, tt.max)
if got != tt.want {
t.Errorf("truncateStr(%q, %d) = %q, want %q", tt.input, tt.max, got, tt.want)
}
}
}
func TestGetString(t *testing.T) {
m := map[string]interface{}{
"name": "张三",
"age": 30,
}
if got := getString(m, "name"); got != "张三" {
t.Errorf("expected '张三', got %q", got)
}
if got := getString(m, "age"); got != "" {
t.Errorf("expected empty for int, got %q", got)
}
if got := getString(m, "nonexistent"); got != "" {
t.Errorf("expected empty for missing key, got %q", got)
}
if got := getString(nil, "key"); got != "" {
t.Errorf("expected empty for nil map, got %q", got)
}
}
func TestGetFloat(t *testing.T) {
m := map[string]interface{}{
"count": 42.5,
"score": 100,
"name": "test",
}
if got := getFloat(m, "count"); got != 42.5 {
t.Errorf("expected 42.5, got %f", got)
}
if got := getFloat(m, "score"); got != 100.0 {
t.Errorf("expected 100.0, got %f", got)
}
if got := getFloat(m, "name"); got != 0 {
t.Errorf("expected 0 for string, got %f", got)
}
if got := getFloat(m, "nonexistent"); got != 0 {
t.Errorf("expected 0 for missing key, got %f", got)
}
if got := getFloat(nil, "key"); got != 0 {
t.Errorf("expected 0 for nil map, got %f", got)
}
}
func TestGetFloatInt(t *testing.T) {
m := map[string]interface{}{
"top_k": float64(5),
}
if got := getFloat(m, "top_k"); got != 5.0 {
t.Errorf("expected 5.0, got %f", got)
}
}
// TestDocToTriplesDropsNoiseEntities 钉住 doc→graph 的噪音闸门。
//
// 背景doc→graph 在 1cb3e87 从「CutExact 滑窗词链」换成 NLP 依存提取器后,
// 唯一还拦常用词的那层CutExact去停用词 + validEntityName失去调用点
// 闸门只剩 validEntityName——它只管名字像不像名字不管名字是不是常用词。
// 实测生产库里因此攒下「文档 --主题--> 来自 N 个来源的 M 条对话 …」这类
// 模板回声,以及 context_archived 这个内部标记。
func TestDocToTriplesDropsNoiseEntities(t *testing.T) {
doc := &document.Doc{
Summary: "来自 1 个来源的 2 条对话 (agent) 涉及: qq, 通道",
Content: "",
Source: "context_archived",
}
triples := docToTriples(doc, nil)
for _, tr := range triples {
if memory.IsNoiseEntity(tr.Subject) || memory.IsNoiseEntity(tr.Object) {
t.Errorf("docToTriples 漏出噪音实体: %+v", tr)
}
}
// 模板摘要不当「主题」、context_archived 不当「来源」:两条模板三元组都该被拦下。
for _, tr := range triples {
if tr.Relation == "主题" {
t.Errorf("模板摘要被写成主题: %+v", tr)
}
if tr.Relation == "来源" && tr.Object == "context_archived" {
t.Errorf("归档内部标记被写成来源: %+v", tr)
}
}
}
// TestDocToTriplesKeepsTemplateAnchors 保证闸门没把正常的模板三元组一起误杀。
func TestDocToTriplesKeepsTemplateAnchors(t *testing.T) {
doc := &document.Doc{
Summary: "多轮对话",
Content: "",
Source: "qq",
}
triples := docToTriples(doc, nil)
var hasTopic, hasSource bool
for _, tr := range triples {
if tr.Subject == "文档" && tr.Relation == "主题" && tr.Object == "多轮对话" {
hasTopic = true
}
if tr.Subject == "文档" && tr.Relation == "来源" && tr.Object == "qq" {
hasSource = true
}
}
if !hasTopic || !hasSource {
t.Errorf("正常模板三元组被误杀: topic=%v source=%v, triples=%+v", hasTopic, hasSource, triples)
}
}