refactor: pluginize text cleaning and tool NoMemory control

- SDK: ToolDef.NoMemory field, PluginSDK.RegisterTextCleaner/TextCleaners
- Registry: aggregate text cleaners from plugins, expose CleanText()
- Memory: replace hardcoded QQ regex CleanTemplateText with dynamic CleanText/SetTextCleaner
- StageHost: add ToolDef(name) lookup
- eventloop: check ToolDef.NoMemory before emitMemoryCandidate
- context/Prune: replace hardcoded agentcli/terminal source filter with ToolsUsed NoMemory check
- agentcli/cmd: mark tools with NoMemory: true
- main.go: wire memory.SetTextCleaner(pluginReg.CleanText)
This commit is contained in:
root
2026-07-24 14:49:08 +08:00
parent 097c8e80b7
commit 3fc2151588
32 changed files with 2368 additions and 69 deletions

View File

@ -180,7 +180,7 @@ func TestBilingualVectorizeClean(t *testing.T) {
cleanVec := e.VectorizeClean(inp)
sim := cosineSim(rawVec, cleanVec)
rawTokens := len(e.tokenize(inp))
cleanTokens := len(e.tokenize(CleanTemplateText(inp)))
cleanTokens := len(e.tokenize(CleanText(inp)))
t.Logf("[%d] sim(raw,clean)=%.4f tokens: raw=%d clean=%d", i, sim, rawTokens, cleanTokens)
}
}
@ -236,8 +236,8 @@ func genBilingualEvents() []bilingualEvent {
func textForBilingual(ev bilingualEvent, modelPaths []string) string {
switch {
case ev.source == "agent" && ev.text != "":
return CleanTemplateText(ev.text)
return CleanText(ev.text)
default:
return CleanTemplateText(ev.text)
return CleanText(ev.text)
}
}

View File

@ -2,10 +2,31 @@ package memory
import (
"fmt"
"regexp"
"sort"
"testing"
)
func init() {
globalTextCleaner = func(text string) string {
reQQGroupSuffix := regexp.MustCompile(`通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复该群聊content 设为 JSON 字符串:\{[^}]*\}`)
reQQPrivateSuffix := regexp.MustCompile(`通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复对方content 设为 JSON 字符串:\{[^}]*\}`)
reQQOldReply := regexp.MustCompile(`通过id\d+使用qq_get_message工具获取消息正文。获取后必须使用[^。]+。`)
reQQOldForbid := regexp.MustCompile(`你只能通过qq_get_message先看消息然后直接用%!s\(MISSING\)send_private_msg回复中间的思考过程禁止调用任何其他工具\s*→\s*`)
reQQGeneral := regexp.MustCompile(`通过id\d+使用qq_get_message工具获取消息正文[。,][^。]*?(?:回复|发送消息)`)
reTimestamp := regexp.MustCompile(`\[\d{2}:\d{2}\]\s*`)
reMultiSpace := regexp.MustCompile(`\s+`)
text = reQQGroupSuffix.ReplaceAllString(text, "")
text = reQQPrivateSuffix.ReplaceAllString(text, "")
text = reQQOldReply.ReplaceAllString(text, "")
text = reQQOldForbid.ReplaceAllString(text, "")
text = reQQGeneral.ReplaceAllString(text, "")
text = reTimestamp.ReplaceAllString(text, "")
text = reMultiSpace.ReplaceAllString(text, " ")
return text
}
}
type cleanTestEvent struct {
idx int
source string
@ -173,7 +194,7 @@ func TestCleanVectorConsistency(t *testing.T) {
}
for _, tmpl := range templates {
cleaned := CleanTemplateText(tmpl)
cleaned := CleanText(tmpl)
t.Logf("template {%q} → {%q} (%d chars)", trimLen(tmpl, 60), cleaned, len(cleaned))
}
@ -286,11 +307,11 @@ func genStressEvents(n int) []cleanTestEvent {
func cleanEventText(source, input, response string) string {
switch {
case source == "agent" && response != "":
return CleanTemplateText(response)
return CleanText(response)
case source == "cold_storage":
return CleanTemplateText(input + " " + response)
return CleanText(input + " " + response)
default:
return CleanTemplateText(input)
return CleanText(input)
}
}

View File

@ -1,45 +1,22 @@
package memory
import (
"regexp"
"strings"
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
)
var (
reQQGroupSuffix = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复该群聊content 设为 JSON 字符串:\{[^}]*\}`,
)
reQQPrivateSuffix = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复对方content 设为 JSON 字符串:\{[^}]*\}`,
)
reQQOldReply = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文。获取后必须使用[^。]+。`,
)
reQQOldForbid = regexp.MustCompile(
`你只能通过qq_get_message先看消息然后直接用%!s\(MISSING\)send_private_msg回复中间的思考过程禁止调用任何其他工具\s*→\s*`,
)
reQQGeneral = regexp.MustCompile(
`通过id\d+使用qq_get_message工具获取消息正文[。,][^。]*?(?:回复|发送消息)`,
)
reTimestamp = regexp.MustCompile(
`\[\d{2}:\d{2}\]\s*`,
)
reAgentPrefix = regexp.MustCompile(
`冷知识|注意|提示|核心要求|规则`,
)
reMultiSpace = regexp.MustCompile(`\s+`)
)
var globalTextCleaner func(string) string
func SetTextCleaner(fn func(string) string) {
globalTextCleaner = fn
}
func CleanText(text string) string {
if globalTextCleaner != nil {
text = globalTextCleaner(text)
}
func CleanTemplateText(text string) string {
text = reQQGroupSuffix.ReplaceAllString(text, "")
text = reQQPrivateSuffix.ReplaceAllString(text, "")
text = reQQOldReply.ReplaceAllString(text, "")
text = reQQOldForbid.ReplaceAllString(text, "")
text = reQQGeneral.ReplaceAllString(text, "")
text = reTimestamp.ReplaceAllString(text, "")
text = reMultiSpace.ReplaceAllString(text, " ")
text = strings.TrimSpace(text)
if text == "" {
@ -53,5 +30,5 @@ func CleanTemplateText(text string) string {
}
func (e *StaticEmbedder) VectorizeClean(text string) vector.Vector {
return e.Vectorize(CleanTemplateText(text))
return e.Vectorize(CleanText(text))
}

View File

@ -0,0 +1,83 @@
package memory
import (
"testing"
)
func TestCleanTextTrim(t *testing.T) {
prev := globalTextCleaner
globalTextCleaner = nil
defer func() { globalTextCleaner = prev }()
tests := []struct {
input string
expected string
}{
{" hello ", "hello"},
{"hello", "hello"},
{"hello", "hello"},
{" hello ", "hello"},
{"", ""},
{" ", ""},
{"", ""},
{"x", "x"},
}
for _, tt := range tests {
got := CleanText(tt.input)
if got != tt.expected {
t.Errorf("CleanText(%q) = %q, want %q", tt.input, got, tt.expected)
}
}
}
func TestCleanTextWithRegisteredCleaner(t *testing.T) {
prev := globalTextCleaner
globalTextCleaner = func(text string) string {
return "prefix_" + text
}
defer func() { globalTextCleaner = prev }()
got := CleanText(" hello ")
if got != "prefix_ hello" {
t.Errorf("CleanText with cleaner = %q, want %q", got, "prefix_ hello")
}
}
func TestCleanTextCleanerChain(t *testing.T) {
prev := globalTextCleaner
globalTextCleaner = func(text string) string {
text = text + "_step1"
text = text + "_step2"
return text
}
defer func() { globalTextCleaner = prev }()
got := CleanText("test")
if got != "test_step1_step2" {
t.Errorf("CleanText chain = %q, want %q", got, "test_step1_step2")
}
}
func TestSetTextCleanerReplace(t *testing.T) {
prev := globalTextCleaner
globalTextCleaner = func(text string) string { return "old_" + text }
SetTextCleaner(func(text string) string { return "new_" + text })
defer func() { globalTextCleaner = prev }()
got := CleanText("x")
if got != "new_x" {
t.Errorf("after SetTextCleaner = %q, want %q", got, "new_x")
}
}
func TestCleanTextEmptyAfterCleaner(t *testing.T) {
prev := globalTextCleaner
globalTextCleaner = func(text string) string { return "" }
defer func() { globalTextCleaner = prev }()
got := CleanText("something")
if got != "" {
t.Errorf("expected empty, got %q", got)
}
}

View File

@ -93,7 +93,7 @@ var stopWords = map[string]bool{
}
func ExtractKeywords(text string) []string {
text = CleanTemplateText(text)
text = CleanText(text)
x := GetJieba()
if x == nil {
return nil
@ -120,7 +120,7 @@ func ExtractKeywords(text string) []string {
// CutExact 精确模式分词:返回去停用词后的所有有义项(不限数量),用于 doc→graph 蒸馏
func CutExact(text string) []string {
text = CleanTemplateText(text)
text = CleanText(text)
x := GetJieba()
if x == nil {
return nil

View File

@ -101,7 +101,7 @@ func TestCutExactRemoveTimestamp(t *testing.T) {
got := CutExact("[15:04] 今天天气不错")
for _, g := range got {
if g == "15" || g == "04" || g == "15:04" {
t.Errorf("timestamp should be removed by CleanTemplateText, got %q in %v", g, got)
t.Errorf("timestamp should be removed by CleanText, got %q in %v", g, got)
}
}
}

View File

@ -90,7 +90,7 @@ func (idx *Indexer) BuildContext(userInput string) *InjectedContext {
return &InjectedContext{Summary: ""}
}
input := CleanTemplateText(userInput)
input := CleanText(userInput)
// 1. 向量搜索:从实体名向量索引中找到相关实体
vectorEntities := idx.vectorSearchEntities(input)

View File

@ -16,7 +16,7 @@ type realEvent struct {
Response string `json:"response"`
}
func TestCleanTemplateText(t *testing.T) {
func TestCleanText(t *testing.T) {
cases := []struct {
input string
expected string
@ -53,7 +53,7 @@ func TestCleanTemplateText(t *testing.T) {
}
for i, c := range cases {
got := CleanTemplateText(c.input)
got := CleanText(c.input)
if c.expected != "" && got != c.expected {
t.Errorf("case %d:\n input: %q\n expected: %q\n got: %q", i, trimLen(c.input, 60), c.expected, got)
}
@ -91,11 +91,11 @@ func TestRealContextPerSourceVector(t *testing.T) {
clean := func(ev realEvent) string {
switch {
case ev.Source == "agent" && ev.Response != "":
return CleanTemplateText(ev.Response)
return CleanText(ev.Response)
case ev.Source == "cold_storage":
return CleanTemplateText(ev.Input + " " + ev.Response)
return CleanText(ev.Input + " " + ev.Response)
default:
return CleanTemplateText(ev.Input)
return CleanText(ev.Input)
}
}
@ -275,11 +275,11 @@ func TestRealContextEmbedderStats(t *testing.T) {
var text string
switch {
case ev.Source == "agent" && ev.Response != "":
text = CleanTemplateText(ev.Response)
text = CleanText(ev.Response)
case ev.Source == "cold_storage":
text = CleanTemplateText(ev.Input + " " + ev.Response)
text = CleanText(ev.Input + " " + ev.Response)
default:
text = CleanTemplateText(ev.Input)
text = CleanText(ev.Input)
}
vec := e.Vectorize(text)
origLen := len(ev.Input + ev.Response)