feat: 完整实现 NLP 三元组提取系统 + token budget 上下文分配

- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并
- parser.go: 分句循环 + TransE 向量验证(h+r≈t)
- fallback.go: jieba POS 降级解析器
- bridge.go: nlp.Triple ↔ memory.Triple 转换
- pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则
- distill.go: docToTriples 改用 NLP 提取器
- reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断
- Provider 接口加 MaxContextTokens() + 模型窗口映射表
- tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率)
- process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline

internal/memory/vector/store.go

@@ -121,21 +121,31 @@ func (s *Store) All() []DocVector {
 	return out
 }
 
-// TFIDFVectorizer 使用字符 bigram + TF-IDF
-type TFIDFVectorizer struct {
-	mu     sync.RWMutex
-	docFreq map[string]float64 // feature → 文档频率
-	totalDocs int
-	maxNGram int
+// Tokenizer 将文本拆分为词级 token
+type Tokenizer func(string) []string
+
+// NGramTokenizer 创建字符 n-gram tokenizer（降级方案）
+func NGramTokenizer(maxN int) Tokenizer {
+	return func(text string) []string {
+		return extractNGrams(text, maxN)
+	}
 }
 
-func NewTFIDFVectorizer(maxNGram int) *TFIDFVectorizer {
-	if maxNGram <= 0 {
-		maxNGram = 2
+// TFIDFVectorizer 使用 tokenizer + TF-IDF
+type TFIDFVectorizer struct {
+	mu        sync.RWMutex
+	tokenizer Tokenizer
+	docFreq   map[string]float64 // feature → 文档频率
+	totalDocs int
+}
+
+func NewTFIDFVectorizer(tokenizer Tokenizer) *TFIDFVectorizer {
+	if tokenizer == nil {
+		tokenizer = NGramTokenizer(2)
 	}
 	return &TFIDFVectorizer{
-		docFreq:  make(map[string]float64),
-		maxNGram: maxNGram,
+		tokenizer: tokenizer,
+		docFreq:   make(map[string]float64),
 	}
 }
 
@@ -148,7 +158,7 @@ func (v *TFIDFVectorizer) Train(docs []string) {
 
 	seen := make(map[string]map[string]bool)
 	for _, doc := range docs {
-		features := extractNGrams(doc, v.maxNGram)
+		features := v.tokenizer(doc)
 		key := doc
 		if seen[key] == nil {
 			seen[key] = make(map[string]bool)
@@ -166,7 +176,7 @@ func (v *TFIDFVectorizer) Vectorize(text string) Vector {
 	v.mu.RLock()
 	defer v.mu.RUnlock()
 
-	features := extractNGrams(text, v.maxNGram)
+	features := v.tokenizer(text)
 	tf := make(map[string]float64)
 	for _, f := range features {
 		tf[f]++

internal/memory/vector/store_test.go

@@ -65,7 +65,7 @@ func TestCosineSimilarity(t *testing.T) {
 }
 
 func TestTFIDFVectorizer(t *testing.T) {
-	v := NewTFIDFVectorizer(2)
+	v := NewTFIDFVectorizer(NGramTokenizer(2))
 	docs := []string{"今天天气很好", "今天心情不错", "明天要下雨"}
 	v.Train(docs)
 
@@ -87,7 +87,7 @@ func TestTFIDFVectorizer(t *testing.T) {
 }
 
 func TestTFIDFVectorizerEmpty(t *testing.T) {
-	v := NewTFIDFVectorizer(2)
+	v := NewTFIDFVectorizer(NGramTokenizer(2))
 	v.Train(nil)
 	vec := v.Vectorize("test")
 	if len(vec) == 0 {
@@ -127,7 +127,7 @@ func TestInvertedIndex(t *testing.T) {
 
 func TestStoreInsertAndSearch(t *testing.T) {
 	s := NewStore()
-	v := NewTFIDFVectorizer(2)
+	v := NewTFIDFVectorizer(NGramTokenizer(2))
 	v.Train([]string{"hello world", "goodbye world"})
 
 	s.Insert("1", "hello world", v.Vectorize("hello world"), nil)
@@ -148,7 +148,7 @@ func TestStoreInsertAndSearch(t *testing.T) {
 
 func TestStoreRemove(t *testing.T) {
 	s := NewStore()
-	v := NewTFIDFVectorizer(1)
+	v := NewTFIDFVectorizer(NGramTokenizer(1))
 	v.Train([]string{"a"})
 
 	s.Insert("1", "a", v.Vectorize("a"), nil)
@@ -175,7 +175,7 @@ func TestStoreEmpty(t *testing.T) {
 
 func TestStoreAll(t *testing.T) {
 	s := NewStore()
-	v := NewTFIDFVectorizer(1)
+	v := NewTFIDFVectorizer(NGramTokenizer(1))
 	v.Train([]string{"a", "b"})
 
 	s.Insert("1", "a", v.Vectorize("a"), map[string]string{"k": "v"})