mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 01:18:08 +00:00
refactor: pluginize text cleaning and tool NoMemory control
- SDK: ToolDef.NoMemory field, PluginSDK.RegisterTextCleaner/TextCleaners - Registry: aggregate text cleaners from plugins, expose CleanText() - Memory: replace hardcoded QQ regex CleanTemplateText with dynamic CleanText/SetTextCleaner - StageHost: add ToolDef(name) lookup - eventloop: check ToolDef.NoMemory before emitMemoryCandidate - context/Prune: replace hardcoded agentcli/terminal source filter with ToolsUsed NoMemory check - agentcli/cmd: mark tools with NoMemory: true - main.go: wire memory.SetTextCleaner(pluginReg.CleanText)
This commit is contained in:
@ -180,7 +180,7 @@ func TestBilingualVectorizeClean(t *testing.T) {
|
||||
cleanVec := e.VectorizeClean(inp)
|
||||
sim := cosineSim(rawVec, cleanVec)
|
||||
rawTokens := len(e.tokenize(inp))
|
||||
cleanTokens := len(e.tokenize(CleanTemplateText(inp)))
|
||||
cleanTokens := len(e.tokenize(CleanText(inp)))
|
||||
t.Logf("[%d] sim(raw,clean)=%.4f tokens: raw=%d clean=%d", i, sim, rawTokens, cleanTokens)
|
||||
}
|
||||
}
|
||||
@ -236,8 +236,8 @@ func genBilingualEvents() []bilingualEvent {
|
||||
func textForBilingual(ev bilingualEvent, modelPaths []string) string {
|
||||
switch {
|
||||
case ev.source == "agent" && ev.text != "":
|
||||
return CleanTemplateText(ev.text)
|
||||
return CleanText(ev.text)
|
||||
default:
|
||||
return CleanTemplateText(ev.text)
|
||||
return CleanText(ev.text)
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,10 +2,31 @@ package memory
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func init() {
|
||||
globalTextCleaner = func(text string) string {
|
||||
reQQGroupSuffix := regexp.MustCompile(`,通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复该群聊,content 设为 JSON 字符串:\{[^}]*\}`)
|
||||
reQQPrivateSuffix := regexp.MustCompile(`,通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复对方,content 设为 JSON 字符串:\{[^}]*\}`)
|
||||
reQQOldReply := regexp.MustCompile(`通过id\d+使用qq_get_message工具获取消息正文。获取后必须使用[^。]+。`)
|
||||
reQQOldForbid := regexp.MustCompile(`你只能通过qq_get_message先看消息,然后直接用%!s\(MISSING\)send_private_msg回复,中间的思考过程禁止调用任何其他工具\s*→\s*`)
|
||||
reQQGeneral := regexp.MustCompile(`通过id\d+使用qq_get_message工具获取消息正文[。,][^。]*?(?:回复|发送消息)`)
|
||||
reTimestamp := regexp.MustCompile(`\[\d{2}:\d{2}\]\s*`)
|
||||
reMultiSpace := regexp.MustCompile(`\s+`)
|
||||
text = reQQGroupSuffix.ReplaceAllString(text, "")
|
||||
text = reQQPrivateSuffix.ReplaceAllString(text, "")
|
||||
text = reQQOldReply.ReplaceAllString(text, "")
|
||||
text = reQQOldForbid.ReplaceAllString(text, "")
|
||||
text = reQQGeneral.ReplaceAllString(text, "")
|
||||
text = reTimestamp.ReplaceAllString(text, "")
|
||||
text = reMultiSpace.ReplaceAllString(text, " ")
|
||||
return text
|
||||
}
|
||||
}
|
||||
|
||||
type cleanTestEvent struct {
|
||||
idx int
|
||||
source string
|
||||
@ -173,7 +194,7 @@ func TestCleanVectorConsistency(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, tmpl := range templates {
|
||||
cleaned := CleanTemplateText(tmpl)
|
||||
cleaned := CleanText(tmpl)
|
||||
t.Logf("template {%q} → {%q} (%d chars)", trimLen(tmpl, 60), cleaned, len(cleaned))
|
||||
}
|
||||
|
||||
@ -286,11 +307,11 @@ func genStressEvents(n int) []cleanTestEvent {
|
||||
func cleanEventText(source, input, response string) string {
|
||||
switch {
|
||||
case source == "agent" && response != "":
|
||||
return CleanTemplateText(response)
|
||||
return CleanText(response)
|
||||
case source == "cold_storage":
|
||||
return CleanTemplateText(input + " " + response)
|
||||
return CleanText(input + " " + response)
|
||||
default:
|
||||
return CleanTemplateText(input)
|
||||
return CleanText(input)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,45 +1,22 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||
)
|
||||
|
||||
var (
|
||||
reQQGroupSuffix = regexp.MustCompile(
|
||||
`,通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复该群聊,content 设为 JSON 字符串:\{[^}]*\}`,
|
||||
)
|
||||
reQQPrivateSuffix = regexp.MustCompile(
|
||||
`,通过id\d+使用qq_get_message工具获取消息正文。获取内容后使用 output_send\(channel="qq"\) 回复对方,content 设为 JSON 字符串:\{[^}]*\}`,
|
||||
)
|
||||
reQQOldReply = regexp.MustCompile(
|
||||
`通过id\d+使用qq_get_message工具获取消息正文。获取后必须使用[^。]+。`,
|
||||
)
|
||||
reQQOldForbid = regexp.MustCompile(
|
||||
`你只能通过qq_get_message先看消息,然后直接用%!s\(MISSING\)send_private_msg回复,中间的思考过程禁止调用任何其他工具\s*→\s*`,
|
||||
)
|
||||
reQQGeneral = regexp.MustCompile(
|
||||
`通过id\d+使用qq_get_message工具获取消息正文[。,][^。]*?(?:回复|发送消息)`,
|
||||
)
|
||||
reTimestamp = regexp.MustCompile(
|
||||
`\[\d{2}:\d{2}\]\s*`,
|
||||
)
|
||||
reAgentPrefix = regexp.MustCompile(
|
||||
`冷知识|注意|提示|核心要求|规则`,
|
||||
)
|
||||
reMultiSpace = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
var globalTextCleaner func(string) string
|
||||
|
||||
func SetTextCleaner(fn func(string) string) {
|
||||
globalTextCleaner = fn
|
||||
}
|
||||
|
||||
func CleanText(text string) string {
|
||||
if globalTextCleaner != nil {
|
||||
text = globalTextCleaner(text)
|
||||
}
|
||||
|
||||
func CleanTemplateText(text string) string {
|
||||
text = reQQGroupSuffix.ReplaceAllString(text, "")
|
||||
text = reQQPrivateSuffix.ReplaceAllString(text, "")
|
||||
text = reQQOldReply.ReplaceAllString(text, "")
|
||||
text = reQQOldForbid.ReplaceAllString(text, "")
|
||||
text = reQQGeneral.ReplaceAllString(text, "")
|
||||
text = reTimestamp.ReplaceAllString(text, "")
|
||||
text = reMultiSpace.ReplaceAllString(text, " ")
|
||||
text = strings.TrimSpace(text)
|
||||
|
||||
if text == "" {
|
||||
@ -53,5 +30,5 @@ func CleanTemplateText(text string) string {
|
||||
}
|
||||
|
||||
func (e *StaticEmbedder) VectorizeClean(text string) vector.Vector {
|
||||
return e.Vectorize(CleanTemplateText(text))
|
||||
return e.Vectorize(CleanText(text))
|
||||
}
|
||||
|
||||
83
internal/memory/clean_text_test.go
Normal file
83
internal/memory/clean_text_test.go
Normal file
@ -0,0 +1,83 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCleanTextTrim(t *testing.T) {
|
||||
prev := globalTextCleaner
|
||||
globalTextCleaner = nil
|
||||
defer func() { globalTextCleaner = prev }()
|
||||
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{" hello ", "hello"},
|
||||
{",hello", "hello"},
|
||||
{",,hello", "hello"},
|
||||
{" ,,hello ", "hello"},
|
||||
{"", ""},
|
||||
{" ", ""},
|
||||
{",", ""},
|
||||
{",x", "x"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := CleanText(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("CleanText(%q) = %q, want %q", tt.input, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanTextWithRegisteredCleaner(t *testing.T) {
|
||||
prev := globalTextCleaner
|
||||
globalTextCleaner = func(text string) string {
|
||||
return "prefix_" + text
|
||||
}
|
||||
defer func() { globalTextCleaner = prev }()
|
||||
|
||||
got := CleanText(" hello ")
|
||||
if got != "prefix_ hello" {
|
||||
t.Errorf("CleanText with cleaner = %q, want %q", got, "prefix_ hello")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanTextCleanerChain(t *testing.T) {
|
||||
prev := globalTextCleaner
|
||||
globalTextCleaner = func(text string) string {
|
||||
text = text + "_step1"
|
||||
text = text + "_step2"
|
||||
return text
|
||||
}
|
||||
defer func() { globalTextCleaner = prev }()
|
||||
|
||||
got := CleanText("test")
|
||||
if got != "test_step1_step2" {
|
||||
t.Errorf("CleanText chain = %q, want %q", got, "test_step1_step2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetTextCleanerReplace(t *testing.T) {
|
||||
prev := globalTextCleaner
|
||||
globalTextCleaner = func(text string) string { return "old_" + text }
|
||||
|
||||
SetTextCleaner(func(text string) string { return "new_" + text })
|
||||
defer func() { globalTextCleaner = prev }()
|
||||
|
||||
got := CleanText("x")
|
||||
if got != "new_x" {
|
||||
t.Errorf("after SetTextCleaner = %q, want %q", got, "new_x")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanTextEmptyAfterCleaner(t *testing.T) {
|
||||
prev := globalTextCleaner
|
||||
globalTextCleaner = func(text string) string { return "" }
|
||||
defer func() { globalTextCleaner = prev }()
|
||||
|
||||
got := CleanText("something")
|
||||
if got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
@ -93,7 +93,7 @@ var stopWords = map[string]bool{
|
||||
}
|
||||
|
||||
func ExtractKeywords(text string) []string {
|
||||
text = CleanTemplateText(text)
|
||||
text = CleanText(text)
|
||||
x := GetJieba()
|
||||
if x == nil {
|
||||
return nil
|
||||
@ -120,7 +120,7 @@ func ExtractKeywords(text string) []string {
|
||||
|
||||
// CutExact 精确模式分词:返回去停用词后的所有有义项(不限数量),用于 doc→graph 蒸馏
|
||||
func CutExact(text string) []string {
|
||||
text = CleanTemplateText(text)
|
||||
text = CleanText(text)
|
||||
x := GetJieba()
|
||||
if x == nil {
|
||||
return nil
|
||||
|
||||
@ -101,7 +101,7 @@ func TestCutExactRemoveTimestamp(t *testing.T) {
|
||||
got := CutExact("[15:04] 今天天气不错")
|
||||
for _, g := range got {
|
||||
if g == "15" || g == "04" || g == "15:04" {
|
||||
t.Errorf("timestamp should be removed by CleanTemplateText, got %q in %v", g, got)
|
||||
t.Errorf("timestamp should be removed by CleanText, got %q in %v", g, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -90,7 +90,7 @@ func (idx *Indexer) BuildContext(userInput string) *InjectedContext {
|
||||
return &InjectedContext{Summary: ""}
|
||||
}
|
||||
|
||||
input := CleanTemplateText(userInput)
|
||||
input := CleanText(userInput)
|
||||
|
||||
// 1. 向量搜索:从实体名向量索引中找到相关实体
|
||||
vectorEntities := idx.vectorSearchEntities(input)
|
||||
|
||||
@ -16,7 +16,7 @@ type realEvent struct {
|
||||
Response string `json:"response"`
|
||||
}
|
||||
|
||||
func TestCleanTemplateText(t *testing.T) {
|
||||
func TestCleanText(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
expected string
|
||||
@ -53,7 +53,7 @@ func TestCleanTemplateText(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
got := CleanTemplateText(c.input)
|
||||
got := CleanText(c.input)
|
||||
if c.expected != "" && got != c.expected {
|
||||
t.Errorf("case %d:\n input: %q\n expected: %q\n got: %q", i, trimLen(c.input, 60), c.expected, got)
|
||||
}
|
||||
@ -91,11 +91,11 @@ func TestRealContextPerSourceVector(t *testing.T) {
|
||||
clean := func(ev realEvent) string {
|
||||
switch {
|
||||
case ev.Source == "agent" && ev.Response != "":
|
||||
return CleanTemplateText(ev.Response)
|
||||
return CleanText(ev.Response)
|
||||
case ev.Source == "cold_storage":
|
||||
return CleanTemplateText(ev.Input + " " + ev.Response)
|
||||
return CleanText(ev.Input + " " + ev.Response)
|
||||
default:
|
||||
return CleanTemplateText(ev.Input)
|
||||
return CleanText(ev.Input)
|
||||
}
|
||||
}
|
||||
|
||||
@ -275,11 +275,11 @@ func TestRealContextEmbedderStats(t *testing.T) {
|
||||
var text string
|
||||
switch {
|
||||
case ev.Source == "agent" && ev.Response != "":
|
||||
text = CleanTemplateText(ev.Response)
|
||||
text = CleanText(ev.Response)
|
||||
case ev.Source == "cold_storage":
|
||||
text = CleanTemplateText(ev.Input + " " + ev.Response)
|
||||
text = CleanText(ev.Input + " " + ev.Response)
|
||||
default:
|
||||
text = CleanTemplateText(ev.Input)
|
||||
text = CleanText(ev.Input)
|
||||
}
|
||||
vec := e.Vectorize(text)
|
||||
origLen := len(ev.Input + ev.Response)
|
||||
|
||||
Reference in New Issue
Block a user