mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-23 10:28:06 +00:00
feat: multi-language embedding with CleanTemplateText + three-branch vector strategy
- StaticEmbedder: pre-trained ConceptNet Numberbatch/fastText word embeddings, auto-download with TF-IDF fallback, comma-separated multi-model paths - CleanTemplateText: regex stripping of QQ tool call templates and noise - textForVector: per-source vector strategy (agent→Response, user→Input, cold_storage→both) - Indexer.BuildContext and ExtractKeywords now clean input before vectorization - Protect recent 10 events in Prune (regression fix: use local var not const)
This commit is contained in:
@ -15,32 +15,29 @@ import (
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||
)
|
||||
|
||||
// ContextEvent — 单条上下文事件
|
||||
type ContextEvent struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Source string `json:"source"`
|
||||
Input string `json:"input"`
|
||||
Response string `json:"response,omitempty"`
|
||||
ToolsUsed []string `json:"tools_used,omitempty"`
|
||||
Vector vector.Vector `json:"-"` // 缓存向量，避免重复计算
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Source string `json:"source"`
|
||||
Input string `json:"input"`
|
||||
Response string `json:"response,omitempty"`
|
||||
ToolsUsed []string `json:"tools_used,omitempty"`
|
||||
Vector vector.Vector `json:"-"`
|
||||
}
|
||||
|
||||
const contextFlushInterval = 5 * time.Second
|
||||
|
||||
// RelevanceContext — 基于相关性的上下文管理，非固定阈值
|
||||
type RelevanceContext struct {
|
||||
mu sync.Mutex
|
||||
events []*ContextEvent
|
||||
embedder *memory.LocalWordEmbedder
|
||||
trained bool
|
||||
embedder *memory.StaticEmbedder
|
||||
savePath string
|
||||
saveTimer *time.Timer
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func NewRelevanceContext(savePath string) *RelevanceContext {
|
||||
func NewRelevanceContext(savePath string, embedder *memory.StaticEmbedder) *RelevanceContext {
|
||||
rc := &RelevanceContext{
|
||||
embedder: memory.NewLocalWordEmbedder(),
|
||||
embedder: embedder,
|
||||
savePath: savePath,
|
||||
}
|
||||
if savePath != "" {
|
||||
@ -49,7 +46,6 @@ func NewRelevanceContext(savePath string) *RelevanceContext {
|
||||
return rc
|
||||
}
|
||||
|
||||
// load 从文件恢复上下文事件
|
||||
func (c *RelevanceContext) load() {
|
||||
data, err := os.ReadFile(c.savePath)
|
||||
if err != nil {
|
||||
@ -60,12 +56,27 @@ func (c *RelevanceContext) load() {
|
||||
return
|
||||
}
|
||||
for _, evt := range events {
|
||||
evt.Vector = c.embedder.Vectorize(evt.Input + " " + evt.Response)
|
||||
evt.Input = memory.CleanTemplateText(evt.Input)
|
||||
evt.Vector = c.computeVector(evt)
|
||||
}
|
||||
c.events = events
|
||||
}
|
||||
|
||||
// Save 持久化上下文事件到文件
|
||||
func textForVector(evt *ContextEvent) string {
|
||||
switch {
|
||||
case evt.Source == "agent" && evt.Response != "":
|
||||
return memory.CleanTemplateText(evt.Response)
|
||||
case evt.Source == "cold_storage":
|
||||
return memory.CleanTemplateText(evt.Input + " " + evt.Response)
|
||||
default:
|
||||
return memory.CleanTemplateText(evt.Input)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *RelevanceContext) computeVector(evt *ContextEvent) vector.Vector {
|
||||
return c.embedder.Vectorize(textForVector(evt))
|
||||
}
|
||||
|
||||
func (c *RelevanceContext) Save() error {
|
||||
if c.savePath == "" {
|
||||
return nil
|
||||
@ -84,15 +95,13 @@ func (c *RelevanceContext) Append(evt ContextEvent) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
evt.Vector = c.embedder.Vectorize(evt.Input + " " + evt.Response)
|
||||
evt.Input = memory.CleanTemplateText(evt.Input)
|
||||
evt.Vector = c.computeVector(&evt)
|
||||
c.events = append(c.events, &evt)
|
||||
|
||||
c.trained = false
|
||||
|
||||
c.save()
|
||||
}
|
||||
|
||||
// save 无锁版本，Append/Prune 内部持有锁时调用。带 debounce，每 5s 写一次盘。
|
||||
func (c *RelevanceContext) save() error {
|
||||
if c.savePath == "" {
|
||||
return nil
|
||||
@ -124,8 +133,6 @@ func (c *RelevanceContext) flush() {
|
||||
c.dirty = false
|
||||
}
|
||||
|
||||
// Prune — 基于当前输入计算每条上下文的相关性，归档最不相关的
|
||||
// 返回被归档的事件（转为文档），保留 topK 个最相关的在活跃上下文中
|
||||
func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *document.Store) int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
@ -134,23 +141,19 @@ func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *docume
|
||||
return 0
|
||||
}
|
||||
|
||||
// 保护最近 10 条记录不被淘汰，从更早的记录中选择淘汰对象
|
||||
protectCount := 10
|
||||
if protectCount > len(c.events) {
|
||||
protectCount = len(c.events)
|
||||
pCount := 10
|
||||
if pCount > len(c.events) {
|
||||
pCount = len(c.events)
|
||||
}
|
||||
protected := c.events[len(c.events)-protectCount:]
|
||||
candidates := c.events[:len(c.events)-protectCount]
|
||||
protected := c.events[len(c.events)-pCount:]
|
||||
candidates := c.events[:len(c.events)-pCount]
|
||||
|
||||
if len(candidates) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
c.ensureTrained()
|
||||
queryVec := c.embedder.VectorizeClean(currentInput)
|
||||
|
||||
queryVec := c.embedder.Vectorize(currentInput)
|
||||
|
||||
// 计算每条候选上下文与当前输入的相关性
|
||||
type scored struct {
|
||||
event *ContextEvent
|
||||
score float64
|
||||
@ -162,12 +165,10 @@ func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *docume
|
||||
scoredEvents[i] = scored{event: evt, score: score, idx: i}
|
||||
}
|
||||
|
||||
// 按相关性从高到低排序
|
||||
sort.Slice(scoredEvents, func(i, j int) bool {
|
||||
return scoredEvents[i].score > scoredEvents[j].score
|
||||
})
|
||||
|
||||
// 从候选中选 topK 最相关的保留，其余淘汰
|
||||
keepCount := topK - len(protected)
|
||||
if keepCount < 0 {
|
||||
keepCount = 0
|
||||
@ -178,19 +179,16 @@ func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *docume
|
||||
}
|
||||
archive := scoredEvents[keepCount:]
|
||||
|
||||
// 重建 events 为保留的候选 + 受保护的最新记录
|
||||
c.events = make([]*ContextEvent, 0, len(keep)+len(protected))
|
||||
for _, s := range keep {
|
||||
c.events = append(c.events, s.event)
|
||||
}
|
||||
c.events = append(c.events, protected...)
|
||||
|
||||
// 按时间重新排序
|
||||
sort.Slice(c.events, func(i, j int) bool {
|
||||
return c.events[i].Timestamp.Before(c.events[j].Timestamp)
|
||||
})
|
||||
|
||||
// 归档到文档记忆
|
||||
archived := 0
|
||||
if docStore != nil && len(archive) > 0 {
|
||||
entries := make([]document.ContextEntry, len(archive))
|
||||
@ -213,7 +211,6 @@ func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *docume
|
||||
return archived
|
||||
}
|
||||
|
||||
// Format — 输出活跃上下文的文本，用于注入 prompt
|
||||
func (c *RelevanceContext) Format() string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
@ -234,7 +231,6 @@ func (c *RelevanceContext) Format() string {
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// Recent — 返回最近 n 条
|
||||
func (c *RelevanceContext) Recent(n int) []ContextEvent {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
@ -249,23 +245,8 @@ func (c *RelevanceContext) Recent(n int) []ContextEvent {
|
||||
return result
|
||||
}
|
||||
|
||||
// Len — 当前上下文事件数
|
||||
func (c *RelevanceContext) Len() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return len(c.events)
|
||||
}
|
||||
|
||||
func (c *RelevanceContext) ensureTrained() {
|
||||
if !c.trained && len(c.events) > 0 {
|
||||
texts := make([]string, len(c.events))
|
||||
for i, evt := range c.events {
|
||||
texts[i] = evt.Input + " " + evt.Response
|
||||
}
|
||||
c.embedder.Train(texts)
|
||||
for _, evt := range c.events {
|
||||
evt.Vector = c.embedder.Vectorize(evt.Input + " " + evt.Response)
|
||||
}
|
||||
c.trained = true
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user