mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-22 18:08:04 +00:00
feat: 完整实现 NLP 三元组提取系统 + token budget 上下文分配
- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并 - parser.go: 分句循环 + TransE 向量验证(h+r≈t) - fallback.go: jieba POS 降级解析器 - bridge.go: nlp.Triple ↔ memory.Triple 转换 - pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则 - distill.go: docToTriples 改用 NLP 提取器 - reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断 - Provider 接口加 MaxContextTokens() + 模型窗口映射表 - tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率) - process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline
This commit is contained in:
@ -65,7 +65,7 @@ func TestCosineSimilarity(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTFIDFVectorizer(t *testing.T) {
|
||||
v := NewTFIDFVectorizer(2)
|
||||
v := NewTFIDFVectorizer(NGramTokenizer(2))
|
||||
docs := []string{"今天天气很好", "今天心情不错", "明天要下雨"}
|
||||
v.Train(docs)
|
||||
|
||||
@ -87,7 +87,7 @@ func TestTFIDFVectorizer(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTFIDFVectorizerEmpty(t *testing.T) {
|
||||
v := NewTFIDFVectorizer(2)
|
||||
v := NewTFIDFVectorizer(NGramTokenizer(2))
|
||||
v.Train(nil)
|
||||
vec := v.Vectorize("test")
|
||||
if len(vec) == 0 {
|
||||
@ -127,7 +127,7 @@ func TestInvertedIndex(t *testing.T) {
|
||||
|
||||
func TestStoreInsertAndSearch(t *testing.T) {
|
||||
s := NewStore()
|
||||
v := NewTFIDFVectorizer(2)
|
||||
v := NewTFIDFVectorizer(NGramTokenizer(2))
|
||||
v.Train([]string{"hello world", "goodbye world"})
|
||||
|
||||
s.Insert("1", "hello world", v.Vectorize("hello world"), nil)
|
||||
@ -148,7 +148,7 @@ func TestStoreInsertAndSearch(t *testing.T) {
|
||||
|
||||
func TestStoreRemove(t *testing.T) {
|
||||
s := NewStore()
|
||||
v := NewTFIDFVectorizer(1)
|
||||
v := NewTFIDFVectorizer(NGramTokenizer(1))
|
||||
v.Train([]string{"a"})
|
||||
|
||||
s.Insert("1", "a", v.Vectorize("a"), nil)
|
||||
@ -175,7 +175,7 @@ func TestStoreEmpty(t *testing.T) {
|
||||
|
||||
func TestStoreAll(t *testing.T) {
|
||||
s := NewStore()
|
||||
v := NewTFIDFVectorizer(1)
|
||||
v := NewTFIDFVectorizer(NGramTokenizer(1))
|
||||
v.Train([]string{"a", "b"})
|
||||
|
||||
s.Insert("1", "a", v.Vectorize("a"), map[string]string{"k": "v"})
|
||||
|
||||
Reference in New Issue
Block a user