refactor: output channel interface (payload/meta/type) + memory fixes

- Redesign output_send__ tools: content JSON string -> structured
  payload/meta/type params for LLM reliability
- executeOutputSendTool: route by type with capability check
- executeOutputSendHelp: show meta format + type enum
- Updated system prompt rules for new interface
- docToTriples: use jieba exact mode adjacent co-occurrence
- Unify vector space: Doc.Vector field, ContextToDoc vectorizer,
  ReindexWithVectorizer on startup
This commit is contained in:
root
2026-07-19 10:29:21 +08:00
parent 9b3a751d98
commit 8ae1d19869
10 changed files with 453 additions and 109 deletions

View File

@ -117,3 +117,26 @@ func ExtractKeywords(text string) []string {
}
return keywords
}
// CutExact 精确模式分词:返回去停用词后的所有有义项(不限数量),用于 doc→graph 蒸馏
func CutExact(text string) []string {
text = CleanTemplateText(text)
x := GetJieba()
if x == nil {
return nil
}
words := x.Cut(text, false)
var result []string
seen := make(map[string]bool)
for _, w := range words {
if stopWords[w] || seen[w] {
continue
}
if !validEntityName(w) {
continue
}
seen[w] = true
result = append(result, w)
}
return result
}

107
internal/memory/cut_test.go Normal file
View File

@ -0,0 +1,107 @@
package memory
import (
"testing"
)
func needJieba(t *testing.T) {
t.Helper()
if GetJieba() == nil {
t.Skip("jieba dictionaries not found")
}
}
func TestCutExact(t *testing.T) {
needJieba(t)
tests := []struct {
name string
text string
min int
not []string
}{
{
name: "chinese_sentence",
text: "今天天气怎么样",
min: 2,
not: nil,
},
{
name: "stop_words_removed",
text: "和天气地",
min: 1,
not: []string{"的", "和"},
},
{
name: "short_words_filtered",
text: "今天好天气",
min: 1,
not: []string{"好"},
},
{
name: "empty_text",
text: "",
min: 0,
not: nil,
},
{
name: "qq_conversation",
text: "今天天气怎么样 → 今天天气很好",
min: 2,
not: nil,
},
{
name: "all_stop_words",
text: "的了呢吗",
min: 0,
not: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := CutExact(tt.text)
if len(got) < tt.min {
t.Errorf("CutExact(%q) = %v (len=%d), want at least %d terms", tt.text, got, len(got), tt.min)
}
for _, forbid := range tt.not {
for _, g := range got {
if g == forbid {
t.Errorf("CutExact(%q) = %v, should not contain %q", tt.text, got, forbid)
}
}
}
})
}
}
func TestCutExactNoDuplicates(t *testing.T) {
needJieba(t)
got := CutExact("天气天气天气")
if len(got) > 1 {
t.Errorf("expected deduplicated result, got %v (len=%d)", got, len(got))
}
}
func TestCutExactValidEntityName(t *testing.T) {
needJieba(t)
got := CutExact("a b c")
for _, g := range got {
if !validEntityName(g) {
t.Errorf("CutExact returned invalid entity name %q", g)
}
}
}
func TestCutExactRemoveTimestamp(t *testing.T) {
needJieba(t)
got := CutExact("[15:04] 今天天气不错")
for _, g := range got {
if g == "15" || g == "04" || g == "15:04" {
t.Errorf("timestamp should be removed by CleanTemplateText, got %q in %v", g, got)
}
}
}

View File

@ -28,6 +28,7 @@ type Doc struct {
Meta map[string]string `json:"meta,omitempty"`
AccessCount int `json:"access_count"` // 访问次数
LastAccess time.Time `json:"last_access"` // 最后访问时间
Vector vector.Vector `json:"vector,omitempty"` // 预计算向量(与 context 同空间nil 则用 TF-IDF 兜底
}
// Store — 文档记忆存储,包含向量索引
@ -37,12 +38,31 @@ type Store struct {
veczer *vector.TFIDFVectorizer
mu sync.RWMutex
docs map[string]*Doc
summaries []string // 用于训练向量化器,最大 10000 条
docs map[string]*Doc
summaries []string // 用于训练向量化器,最大 10000 条
vectorizer vector.Vectorizer // 可选:与 context 同空间的向量化器
dirty bool
}
func (s *Store) SetVectorizer(v vector.Vectorizer) {
s.vectorizer = v
}
// ReindexWithVectorizer 用给定的向量化器重建所有文档的向量索引
func (s *Store) ReindexWithVectorizer(v vector.Vectorizer) {
s.mu.Lock()
defer s.mu.Unlock()
log.Printf("[document memory] reindex with vectorizer (%d docs)", len(s.docs))
s.vec = vector.NewStore()
for _, doc := range s.docs {
doc.Vector = v.Vectorize(doc.Summary + " " + doc.Content)
s.vec.Insert(doc.ID, doc.Summary, doc.Vector, doc.Meta)
}
log.Printf("[document memory] reindex with vectorizer complete (%d vectors)", s.vec.Size())
}
const maxSummaries = 10000
func NewStore(dir string) *Store {
@ -88,7 +108,10 @@ func (s *Store) Insert(doc *Doc) error {
// 增量训练向量化器并加入向量索引
s.addSummary(doc.Summary)
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
vec := doc.Vector
if vec == nil {
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
}
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
// 立即写盘
@ -101,7 +124,7 @@ func (s *Store) Insert(doc *Doc) error {
}
// ContextToDoc — 将一段上下文对话历史提炼为文档(带内容去重)
func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error) {
func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.Vectorizer) (*Doc, error) {
if len(entries) == 0 {
return nil, nil
}
@ -140,6 +163,12 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
}
id := fmt.Sprintf("doc_%d", time.Now().UnixNano())
var docVec vector.Vector
if vec != nil {
docVec = vec.Vectorize(summary + " " + content)
} else {
docVec = s.veczer.Vectorize(summary + " " + content)
}
doc := &Doc{
ID: id,
Summary: summary,
@ -152,13 +181,13 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry) (*Doc, error
AccessCount: 1,
Source: source,
Meta: map[string]string{"content_hash": contentHash},
Vector: docVec,
}
s.docs[id] = doc
// 增量训练向量化器并加入向量索引
// 加入向量索引
s.addSummary(summary)
vec := s.veczer.Vectorize(summary + " " + content)
s.vec.Insert(id, summary, vec, nil)
s.vec.Insert(id, summary, doc.Vector, nil)
s.dirty = true
s.mu.Unlock()
@ -180,7 +209,7 @@ func (s *Store) Consume(text string, topK int) []*Doc {
topK = 5
}
vec := s.veczer.Vectorize(text)
vec := s.vectorizeQuery(text)
results := s.vec.Search(vec, topK)
var docs []*Doc
@ -194,6 +223,14 @@ func (s *Store) Consume(text string, topK int) []*Doc {
return docs
}
// vectorizeQuery 用语义向量化器(首选)或 TF-IDF兜底处理查询文本
func (s *Store) vectorizeQuery(text string) vector.Vector {
if s.vectorizer != nil {
return s.vectorizer.Vectorize(text)
}
return s.veczer.Vectorize(text)
}
// Query — 向量相似度查询文档
func (s *Store) Query(text string, topK int) []*Doc {
s.mu.RLock()
@ -203,7 +240,7 @@ func (s *Store) Query(text string, topK int) []*Doc {
topK = 5
}
vec := s.veczer.Vectorize(text)
vec := s.vectorizeQuery(text)
results := s.vec.Search(vec, topK)
var docs []*Doc
@ -228,7 +265,10 @@ func (s *Store) Reindex() {
s.vec = vector.NewStore()
for _, doc := range s.docs {
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
vec := doc.Vector
if vec == nil {
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
}
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
}
@ -348,7 +388,10 @@ func (s *Store) loadAll() error {
// 重建向量索引
for _, doc := range s.docs {
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
vec := doc.Vector
if vec == nil {
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
}
s.vec.Insert(doc.ID, doc.Summary, vec, nil)
}

View File

@ -82,7 +82,7 @@ func TestContextToDoc(t *testing.T) {
{Timestamp: time.Now(), Source: "user", Content: "特别是Go语言", Response: "Go很棒"},
}
doc, err := s.ContextToDoc("test", entries)
doc, err := s.ContextToDoc("test", entries, nil)
if err != nil {
t.Fatal(err)
}