feat: 完整实现 NLP 三元组提取系统 + token budget 上下文分配

- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并
- parser.go: 分句循环 + TransE 向量验证(h+r≈t)
- fallback.go: jieba POS 降级解析器
- bridge.go: nlp.Triple ↔ memory.Triple 转换
- pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则
- distill.go: docToTriples 改用 NLP 提取器
- reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断
- Provider 接口加 MaxContextTokens() + 模型窗口映射表
- tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率)
- process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline
This commit is contained in:
root
2026-07-27 15:26:23 +08:00
parent d19b7bd13e
commit 1cb3e87dde
30 changed files with 1508 additions and 773 deletions

View File

@ -65,7 +65,7 @@ func TestCosineSimilarity(t *testing.T) {
}
func TestTFIDFVectorizer(t *testing.T) {
v := NewTFIDFVectorizer(2)
v := NewTFIDFVectorizer(NGramTokenizer(2))
docs := []string{"今天天气很好", "今天心情不错", "明天要下雨"}
v.Train(docs)
@ -87,7 +87,7 @@ func TestTFIDFVectorizer(t *testing.T) {
}
func TestTFIDFVectorizerEmpty(t *testing.T) {
v := NewTFIDFVectorizer(2)
v := NewTFIDFVectorizer(NGramTokenizer(2))
v.Train(nil)
vec := v.Vectorize("test")
if len(vec) == 0 {
@ -127,7 +127,7 @@ func TestInvertedIndex(t *testing.T) {
func TestStoreInsertAndSearch(t *testing.T) {
s := NewStore()
v := NewTFIDFVectorizer(2)
v := NewTFIDFVectorizer(NGramTokenizer(2))
v.Train([]string{"hello world", "goodbye world"})
s.Insert("1", "hello world", v.Vectorize("hello world"), nil)
@ -148,7 +148,7 @@ func TestStoreInsertAndSearch(t *testing.T) {
func TestStoreRemove(t *testing.T) {
s := NewStore()
v := NewTFIDFVectorizer(1)
v := NewTFIDFVectorizer(NGramTokenizer(1))
v.Train([]string{"a"})
s.Insert("1", "a", v.Vectorize("a"), nil)
@ -175,7 +175,7 @@ func TestStoreEmpty(t *testing.T) {
func TestStoreAll(t *testing.T) {
s := NewStore()
v := NewTFIDFVectorizer(1)
v := NewTFIDFVectorizer(NGramTokenizer(1))
v.Train([]string{"a", "b"})
s.Insert("1", "a", v.Vectorize("a"), map[string]string{"k": "v"})