mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 01:18:08 +00:00
refactor: output channel interface (payload/meta/type) + memory fixes
- Redesign output_send__ tools: content JSON string -> structured payload/meta/type params for LLM reliability - executeOutputSendTool: route by type with capability check - executeOutputSendHelp: show meta format + type enum - Updated system prompt rules for new interface - docToTriples: use jieba exact mode adjacent co-occurrence - Unify vector space: Doc.Vector field, ContextToDoc vectorizer, ReindexWithVectorizer on startup
This commit is contained in:
@ -117,3 +117,26 @@ func ExtractKeywords(text string) []string {
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
// CutExact 精确模式分词:返回去停用词后的所有有义项(不限数量),用于 doc→graph 蒸馏
|
||||
func CutExact(text string) []string {
|
||||
text = CleanTemplateText(text)
|
||||
x := GetJieba()
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
words := x.Cut(text, false)
|
||||
var result []string
|
||||
seen := make(map[string]bool)
|
||||
for _, w := range words {
|
||||
if stopWords[w] || seen[w] {
|
||||
continue
|
||||
}
|
||||
if !validEntityName(w) {
|
||||
continue
|
||||
}
|
||||
seen[w] = true
|
||||
result = append(result, w)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
107
internal/memory/cut_test.go
Normal file
107
internal/memory/cut_test.go
Normal file
@ -0,0 +1,107 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func needJieba(t *testing.T) {
|
||||
t.Helper()
|
||||
if GetJieba() == nil {
|
||||
t.Skip("jieba dictionaries not found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutExact(t *testing.T) {
|
||||
needJieba(t)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
min int
|
||||
not []string
|
||||
}{
|
||||
{
|
||||
name: "chinese_sentence",
|
||||
text: "今天天气怎么样",
|
||||
min: 2,
|
||||
not: nil,
|
||||
},
|
||||
{
|
||||
name: "stop_words_removed",
|
||||
text: "和天气地",
|
||||
min: 1,
|
||||
not: []string{"的", "和"},
|
||||
},
|
||||
{
|
||||
name: "short_words_filtered",
|
||||
text: "今天好天气",
|
||||
min: 1,
|
||||
not: []string{"好"},
|
||||
},
|
||||
{
|
||||
name: "empty_text",
|
||||
text: "",
|
||||
min: 0,
|
||||
not: nil,
|
||||
},
|
||||
{
|
||||
name: "qq_conversation",
|
||||
text: "今天天气怎么样 → 今天天气很好",
|
||||
min: 2,
|
||||
not: nil,
|
||||
},
|
||||
{
|
||||
name: "all_stop_words",
|
||||
text: "的了呢吗",
|
||||
min: 0,
|
||||
not: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := CutExact(tt.text)
|
||||
if len(got) < tt.min {
|
||||
t.Errorf("CutExact(%q) = %v (len=%d), want at least %d terms", tt.text, got, len(got), tt.min)
|
||||
}
|
||||
for _, forbid := range tt.not {
|
||||
for _, g := range got {
|
||||
if g == forbid {
|
||||
t.Errorf("CutExact(%q) = %v, should not contain %q", tt.text, got, forbid)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutExactNoDuplicates(t *testing.T) {
|
||||
needJieba(t)
|
||||
|
||||
got := CutExact("天气天气天气")
|
||||
if len(got) > 1 {
|
||||
t.Errorf("expected deduplicated result, got %v (len=%d)", got, len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutExactValidEntityName(t *testing.T) {
|
||||
needJieba(t)
|
||||
|
||||
got := CutExact("a b c")
|
||||
for _, g := range got {
|
||||
if !validEntityName(g) {
|
||||
t.Errorf("CutExact returned invalid entity name %q", g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutExactRemoveTimestamp(t *testing.T) {
|
||||
needJieba(t)
|
||||
|
||||
got := CutExact("[15:04] 今天天气不错")
|
||||
for _, g := range got {
|
||||
if g == "15" || g == "04" || g == "15:04" {
|
||||
t.Errorf("timestamp should be removed by CleanTemplateText, got %q in %v", g, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -28,6 +28,7 @@ type Doc struct {
|
||||
Meta map[string]string `json:"meta,omitempty"`
|
||||
AccessCount int `json:"access_count"` // 访问次数
|
||||
LastAccess time.Time `json:"last_access"` // 最后访问时间
|
||||
Vector vector.Vector `json:"vector,omitempty"` // 预计算向量(与 context 同空间),nil 则用 TF-IDF 兜底
|
||||
}
|
||||
|
||||
// Store — 文档记忆存储,包含向量索引
|
||||
@ -37,12 +38,31 @@ type Store struct {
|
||||
veczer *vector.TFIDFVectorizer
|
||||
mu sync.RWMutex
|
||||
|
||||
docs map[string]*Doc
|
||||
summaries []string // 用于训练向量化器,最大 10000 条
|
||||
docs map[string]*Doc
|
||||
summaries []string // 用于训练向量化器,最大 10000 条
|
||||
vectorizer vector.Vectorizer // 可选:与 context 同空间的向量化器
|
||||
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func (s *Store) SetVectorizer(v vector.Vectorizer) {
|
||||
s.vectorizer = v
|
||||
}
|
||||
|
||||
// ReindexWithVectorizer 用给定的向量化器重建所有文档的向量索引
|
||||
func (s *Store) ReindexWithVectorizer(v vector.Vectorizer) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
log.Printf("[document memory] reindex with vectorizer (%d docs)", len(s.docs))
|
||||
s.vec = vector.NewStore()
|
||||
for _, doc := range s.docs {
|
||||
doc.Vector = v.Vectorize(doc.Summary + " " + doc.Content)
|
||||
s.vec.Insert(doc.ID, doc.Summary, doc.Vector, doc.Meta)
|
||||
}
|
||||
log.Printf("[document memory] reindex with vectorizer complete (%d vectors)", s.vec.Size())
|
||||
}
|
||||
|
||||
const maxSummaries = 10000
|
||||
|
||||
func NewStore(dir string) *Store {
|
||||
@ -88,7 +108,10 @@ func (s *Store) Insert(doc *Doc) error {
|
||||
|
||||
// 增量训练向量化器并加入向量索引
|
||||
s.addSummary(doc.Summary)
|
||||
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
vec := doc.Vector
|
||||
if vec == nil {
|
||||
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
}
|
||||
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
|
||||
|
||||
// 立即写盘
|
||||
@ -101,7 +124,7 @@ func (s *Store) Insert(doc *Doc) error {
|
||||
}
|
||||
|
||||
// ContextToDoc — 将一段上下文对话历史提炼为文档(带内容去重)
|
||||
func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error) {
|
||||
func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.Vectorizer) (*Doc, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@ -140,6 +163,12 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
|
||||
}
|
||||
|
||||
id := fmt.Sprintf("doc_%d", time.Now().UnixNano())
|
||||
var docVec vector.Vector
|
||||
if vec != nil {
|
||||
docVec = vec.Vectorize(summary + " " + content)
|
||||
} else {
|
||||
docVec = s.veczer.Vectorize(summary + " " + content)
|
||||
}
|
||||
doc := &Doc{
|
||||
ID: id,
|
||||
Summary: summary,
|
||||
@ -152,13 +181,13 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
|
||||
AccessCount: 1,
|
||||
Source: source,
|
||||
Meta: map[string]string{"content_hash": contentHash},
|
||||
Vector: docVec,
|
||||
}
|
||||
s.docs[id] = doc
|
||||
|
||||
// 增量训练向量化器并加入向量索引
|
||||
// 加入向量索引
|
||||
s.addSummary(summary)
|
||||
vec := s.veczer.Vectorize(summary + " " + content)
|
||||
s.vec.Insert(id, summary, vec, nil)
|
||||
s.vec.Insert(id, summary, doc.Vector, nil)
|
||||
|
||||
s.dirty = true
|
||||
s.mu.Unlock()
|
||||
@ -180,7 +209,7 @@ func (s *Store) Consume(text string, topK int) []*Doc {
|
||||
topK = 5
|
||||
}
|
||||
|
||||
vec := s.veczer.Vectorize(text)
|
||||
vec := s.vectorizeQuery(text)
|
||||
results := s.vec.Search(vec, topK)
|
||||
|
||||
var docs []*Doc
|
||||
@ -194,6 +223,14 @@ func (s *Store) Consume(text string, topK int) []*Doc {
|
||||
return docs
|
||||
}
|
||||
|
||||
// vectorizeQuery 用语义向量化器(首选)或 TF-IDF(兜底)处理查询文本
|
||||
func (s *Store) vectorizeQuery(text string) vector.Vector {
|
||||
if s.vectorizer != nil {
|
||||
return s.vectorizer.Vectorize(text)
|
||||
}
|
||||
return s.veczer.Vectorize(text)
|
||||
}
|
||||
|
||||
// Query — 向量相似度查询文档
|
||||
func (s *Store) Query(text string, topK int) []*Doc {
|
||||
s.mu.RLock()
|
||||
@ -203,7 +240,7 @@ func (s *Store) Query(text string, topK int) []*Doc {
|
||||
topK = 5
|
||||
}
|
||||
|
||||
vec := s.veczer.Vectorize(text)
|
||||
vec := s.vectorizeQuery(text)
|
||||
results := s.vec.Search(vec, topK)
|
||||
|
||||
var docs []*Doc
|
||||
@ -228,7 +265,10 @@ func (s *Store) Reindex() {
|
||||
|
||||
s.vec = vector.NewStore()
|
||||
for _, doc := range s.docs {
|
||||
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
vec := doc.Vector
|
||||
if vec == nil {
|
||||
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
}
|
||||
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
|
||||
}
|
||||
|
||||
@ -348,7 +388,10 @@ func (s *Store) loadAll() error {
|
||||
|
||||
// 重建向量索引
|
||||
for _, doc := range s.docs {
|
||||
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
vec := doc.Vector
|
||||
if vec == nil {
|
||||
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
||||
}
|
||||
s.vec.Insert(doc.ID, doc.Summary, vec, nil)
|
||||
}
|
||||
|
||||
|
||||
@ -82,7 +82,7 @@ func TestContextToDoc(t *testing.T) {
|
||||
{Timestamp: time.Now(), Source: "user", Content: "特别是Go语言", Response: "Go很棒"},
|
||||
}
|
||||
|
||||
doc, err := s.ContextToDoc("test", entries)
|
||||
doc, err := s.ContextToDoc("test", entries, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user