feat: 完整实现 NLP 三元组提取系统 + token budget 上下文分配

- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并
- parser.go: 分句循环 + TransE 向量验证(h+r≈t)
- fallback.go: jieba POS 降级解析器
- bridge.go: nlp.Triple ↔ memory.Triple 转换
- pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则
- distill.go: docToTriples 改用 NLP 提取器
- reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断
- Provider 接口加 MaxContextTokens() + 模型窗口映射表
- tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率)
- process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline
This commit is contained in:
root
2026-07-27 15:26:23 +08:00
parent d19b7bd13e
commit 1cb3e87dde
30 changed files with 1508 additions and 773 deletions

View File

@ -113,27 +113,13 @@ func TestExtractKeyTriples(t *testing.T) {
tests := []struct {
user string
assistant string
want int // expected number of triples
check func([]memory.Triple) bool
}{
{
user: "我叫张三",
want: 1,
check: func(triples []memory.Triple) bool {
for _, tr := range triples {
if tr.Subject == "用户" && tr.Relation == "姓名" && tr.Object == "张三" {
return true
}
}
return false
},
},
{
user: "我住在北京",
want: 1,
check: func(triples []memory.Triple) bool {
for _, tr := range triples {
if tr.Subject == "用户" && tr.Relation == "居住地" && tr.Object == "北京" {
if tr.Subject == "我" && tr.Relation == "住" && tr.Object == "北京" {
return true
}
}
@ -141,35 +127,11 @@ func TestExtractKeyTriples(t *testing.T) {
},
},
{
user: "我喜欢打篮球",
want: 1,
user: "我在杭州读书",
assistant: "好的",
check: func(triples []memory.Triple) bool {
for _, tr := range triples {
if tr.Subject == "用户" && tr.Relation == "喜好" && tr.Object == "打篮球" {
return true
}
}
return false
},
},
{
user: "我28岁",
want: 1,
check: func(triples []memory.Triple) bool {
for _, tr := range triples {
if tr.Subject == "用户" && tr.Relation == "年龄" && tr.Object == "28" {
return true
}
}
return false
},
},
{
user: "我的工作是程序员",
want: 1,
check: func(triples []memory.Triple) bool {
for _, tr := range triples {
if tr.Subject == "用户" && tr.Relation == "职业" && tr.Object == "程序员" {
if tr.Subject == "我" && tr.Relation == "读书" && tr.Object == "杭州" {
return true
}
}
@ -178,80 +140,20 @@ func TestExtractKeyTriples(t *testing.T) {
},
{
user: "今天天气真好",
want: 0, // 没有匹配任何规则
check: func(triples []memory.Triple) bool {
return true // any result is fine
return true // NLP 提取器可能不提取形容词谓语句,0 个也没关系
},
},
}
for _, tt := range tests {
triples := extractKeyTriples(tt.user, tt.assistant)
if len(triples) != tt.want {
t.Errorf("extractKeyTriples(%q) = %d triples, want %d", tt.user, len(triples), tt.want)
}
if tt.check != nil && !tt.check(triples) {
t.Errorf("extractKeyTriples(%q) = %v, check failed", tt.user, triples)
}
}
}
func TestExtractName(t *testing.T) {
tests := []struct{ input, want string }{
{"我叫张三", "张三"},
{"我的名字是李四", "李四"},
{"今天天气好", ""},
}
for _, tt := range tests {
got := extractName(tt.input)
if got != tt.want {
t.Errorf("extractName(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestExtractLocation(t *testing.T) {
tests := []struct{ input, want string }{
{"我住在北京", "北京"},
{"我家在上海", "上海"},
{"hello", ""},
}
for _, tt := range tests {
got := extractLocation(tt.input)
if got != tt.want {
t.Errorf("extractLocation(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestExtractLike(t *testing.T) {
tests := []struct{ input, want string }{
{"我喜欢打篮球", "打篮球"},
{"我最喜欢跑步", "跑步"},
{"nothing", ""},
}
for _, tt := range tests {
got := extractLike(tt.input)
if got != tt.want {
t.Errorf("extractLike(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestExtractAge(t *testing.T) {
tests := []struct{ input, want string }{
{"我28岁", "28"},
{"我的年龄是30", "30"},
{"hello", ""},
}
for _, tt := range tests {
got := extractAge(tt.input)
if got != tt.want {
t.Errorf("extractAge(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestDistillerGetRecentRecords(t *testing.T) {
d := NewDistiller(nil, t.TempDir(), DistillerConfig{})
d.Append("s1", "user", "a")