feat: multi-language embedding with CleanTemplateText + three-branch vector strategy

- StaticEmbedder: pre-trained ConceptNet Numberbatch/fastText word embeddings,
  auto-download with TF-IDF fallback, comma-separated multi-model paths
- CleanTemplateText: regex stripping of QQ tool call templates and noise
- textForVector: per-source vector strategy (agent→Response, user→Input,
  cold_storage→both)
- Indexer.BuildContext and ExtractKeywords now clean input before vectorization
- Protect recent 10 events in Prune (regression fix: use local var not const)
This commit is contained in:
2026-07-17 21:02:35 +08:00
parent d2aec1fd5f
commit 7892d7b0f2
11 changed files with 1734 additions and 69 deletions

View File

@ -0,0 +1,57 @@
package memory
import (
"regexp"
"strings"
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
)
var (
reQQGroupSuffix = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复该群聊content 设为 JSON 字符串:\{[^}]*\}`,
)
reQQPrivateSuffix = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复对方content 设为 JSON 字符串:\{[^}]*\}`,
)
reQQOldReply = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文。获取后必须使用[^。]+。`,
)
reQQOldForbid = regexp.MustCompile(
`你只能通过qq_get_message先看消息然后直接用%!s\(MISSING\)send_private_msg回复中间的思考过程禁止调用任何其他工具\s*→\s*`,
)
reQQGeneral = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文[。,][^。]*?(?:回复|发送消息)`,
)
reTimestamp = regexp.MustCompile(
`\[\d{2}:\d{2}\]\s*`,
)
reAgentPrefix = regexp.MustCompile(
`冷知识|注意|提示|核心要求|规则`,
)
reMultiSpace = regexp.MustCompile(`\s+`)
)
func CleanTemplateText(text string) string {
text = reQQGroupSuffix.ReplaceAllString(text, "")
text = reQQPrivateSuffix.ReplaceAllString(text, "")
text = reQQOldReply.ReplaceAllString(text, "")
text = reQQOldForbid.ReplaceAllString(text, "")
text = reQQGeneral.ReplaceAllString(text, "")
text = reTimestamp.ReplaceAllString(text, "")
text = reMultiSpace.ReplaceAllString(text, " ")
text = strings.TrimSpace(text)
if text == "" {
return ""
}
text = strings.TrimPrefix(text, "")
text = strings.TrimPrefix(text, "")
text = strings.TrimSpace(text)
return text
}
func (e *StaticEmbedder) VectorizeClean(text string) vector.Vector {
return e.Vectorize(CleanTemplateText(text))
}