mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 17:38:10 +00:00
sdk: embed non-toolchain SDK in third_party, add NoMemory/Cleaner support
- Embed sdk/, example/, meta/, go.mod from homeagent-sdk (no .git)
- Core .gitignore excludes SDK toolchain: bin/, tools/, package/
- RegisterInputChannel + ChannelDef(NoMemory, Cleaner) in SDK
- IOManager input channel registry with GetInputChannelDef
- eventloop: apply channel Cleaner/NoMemory to interrupt text
- context engine: channelDefLookup applied in textForVector
- document store: ChannelCleaner param for archive functions
- All callers/adapters updated with ChannelDef{} default
This commit is contained in:
root
@ -178,6 +178,9 @@ func New(cfg AgentConfig) *Agent {
|
||||
if cfg.StageHost != nil {
|
||||
rc.SetToolDefLookup(cfg.StageHost.ToolDef)
|
||||
}
|
||||
if cfg.IO != nil {
|
||||
rc.SetChannelDefLookup(cfg.IO.GetInputChannelDef)
|
||||
}
|
||||
|
||||
return &Agent{
|
||||
id: cfg.ID,
|
||||
|
||||
@ -22,6 +22,7 @@ func (d *mockOutputDevice) Tools() []agentIO.ToolDef { return d.too
|
||||
func (d *mockOutputDevice) Start() error { return nil }
|
||||
func (d *mockOutputDevice) Stop() error { return nil }
|
||||
func (d *mockOutputDevice) OutputCapabilities() agentIO.OutputCapability { return d.caps }
|
||||
func (d *mockOutputDevice) ChannelDef() agentIO.ChannelDef { return agentIO.ChannelDef{} }
|
||||
func (d *mockOutputDevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
|
||||
if d.toolFn != nil {
|
||||
return d.toolFn(tool, args)
|
||||
|
||||
@ -34,13 +34,14 @@ type ContextEvent struct {
|
||||
const contextFlushInterval = 5 * time.Second
|
||||
|
||||
type RelevanceContext struct {
|
||||
mu sync.Mutex
|
||||
events []*ContextEvent
|
||||
embedder *memory.StaticEmbedder
|
||||
savePath string
|
||||
saveTimer *time.Timer
|
||||
dirty bool
|
||||
toolDefLookup func(name string) *sdk.ToolDef
|
||||
mu sync.Mutex
|
||||
events []*ContextEvent
|
||||
embedder *memory.StaticEmbedder
|
||||
savePath string
|
||||
saveTimer *time.Timer
|
||||
dirty bool
|
||||
toolDefLookup func(name string) *sdk.ToolDef
|
||||
channelDefLookup func(name string) (sdk.ChannelDef, bool)
|
||||
}
|
||||
|
||||
func NewRelevanceContext(savePath string, embedder *memory.StaticEmbedder) *RelevanceContext {
|
||||
@ -60,6 +61,12 @@ func (c *RelevanceContext) SetToolDefLookup(fn func(name string) *sdk.ToolDef) {
|
||||
c.toolDefLookup = fn
|
||||
}
|
||||
|
||||
func (c *RelevanceContext) SetChannelDefLookup(fn func(name string) (sdk.ChannelDef, bool)) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.channelDefLookup = fn
|
||||
}
|
||||
|
||||
func (c *RelevanceContext) load() {
|
||||
data, err := os.ReadFile(c.savePath)
|
||||
if err != nil {
|
||||
@ -75,7 +82,7 @@ func (c *RelevanceContext) load() {
|
||||
c.events = events
|
||||
}
|
||||
|
||||
func textForVector(evt *ContextEvent, toolDefLookup func(name string) *sdk.ToolDef) string {
|
||||
func textForVector(evt *ContextEvent, toolDefLookup func(name string) *sdk.ToolDef, channelDefLookup func(name string) (sdk.ChannelDef, bool)) string {
|
||||
var text string
|
||||
switch {
|
||||
case evt.Source == "agent" && evt.Response != "":
|
||||
@ -86,6 +93,16 @@ func textForVector(evt *ContextEvent, toolDefLookup func(name string) *sdk.ToolD
|
||||
text = evt.Input
|
||||
}
|
||||
|
||||
// 计算层：应用输入通道的 Cleaner（不改原文，仅在计算层清洗）
|
||||
if channelDefLookup != nil {
|
||||
if chDef, ok := channelDefLookup(evt.Source); ok && chDef.Cleaner != nil {
|
||||
text = chDef.Cleaner(text)
|
||||
}
|
||||
if chDef, ok := channelDefLookup(evt.Source); ok && chDef.NoMemory {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// 计算层：附加工具输出，NoMemory 跳过，其余经 Cleaner 过滤
|
||||
if toolDefLookup != nil {
|
||||
noMemory := make(map[string]bool)
|
||||
@ -130,8 +147,41 @@ func (c *RelevanceContext) toolOutputClean(name, output string) string {
|
||||
return output
|
||||
}
|
||||
|
||||
// inputChannelClean 根据输入通道的 Def 清洗输入文本，用于计算层。
|
||||
func (c *RelevanceContext) inputChannelClean(source, input string) string {
|
||||
if c.channelDefLookup == nil {
|
||||
return input
|
||||
}
|
||||
chDef, ok := c.channelDefLookup(source)
|
||||
if !ok {
|
||||
return input
|
||||
}
|
||||
if chDef.Cleaner != nil {
|
||||
return chDef.Cleaner(input)
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
// channelCleanerForDoc 返回 ChannelCleaner，使 document 包在存档时能按来源查找 Cleaner。
|
||||
func (c *RelevanceContext) channelCleanerForDoc() document.ChannelCleaner {
|
||||
if c.channelDefLookup == nil {
|
||||
return nil
|
||||
}
|
||||
return func(source string) func(string) string {
|
||||
chDef, ok := c.channelDefLookup(source)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return chDef.Cleaner
|
||||
}
|
||||
}
|
||||
|
||||
func (c *RelevanceContext) computeVector(evt *ContextEvent) vector.Vector {
|
||||
return c.embedder.Vectorize(textForVector(evt, c.toolDefLookup))
|
||||
text := textForVector(evt, c.toolDefLookup, c.channelDefLookup)
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
return c.embedder.Vectorize(text)
|
||||
}
|
||||
|
||||
func (c *RelevanceContext) Save() error {
|
||||
@ -274,7 +324,7 @@ func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *docume
|
||||
ToolResults: convertToolResults(s.event.ToolResults),
|
||||
}
|
||||
}
|
||||
doc, err := docStore.ContextToDoc("context_archived", entries, c.embedder, nil, c.toolOutputClean)
|
||||
doc, err := docStore.ContextToDoc("context_archived", entries, c.embedder, nil, c.toolOutputClean, c.channelCleanerForDoc())
|
||||
if err == nil && doc != nil {
|
||||
archived = len(entries)
|
||||
}
|
||||
|
||||
@ -281,6 +281,11 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
||||
if v, ok := evt.Payload["no_memory"].(bool); ok {
|
||||
noMemory = v
|
||||
}
|
||||
if !noMemory && a.io != nil {
|
||||
if chDef, ok := a.io.GetInputChannelDef(evt.Source); ok && chDef.NoMemory {
|
||||
noMemory = true
|
||||
}
|
||||
}
|
||||
|
||||
stageCtx := a.stageCtxFromInput(input, evt.Source, "")
|
||||
stageCtx.Extra["input_source"] = evt.Source
|
||||
@ -297,12 +302,20 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
||||
|
||||
input = stageCtx.RawMessage
|
||||
|
||||
// 计算层用的清洗文本（不改原文）：通道 Cleaner 提取语义内容后用于向量化/提关键词
|
||||
cleanInput := input
|
||||
if a.io != nil {
|
||||
if chDef, ok := a.io.GetInputChannelDef(evt.Source); ok && chDef.Cleaner != nil {
|
||||
cleanInput = chDef.Cleaner(input)
|
||||
}
|
||||
}
|
||||
|
||||
a.publishEvent(events.EventRawInput, map[string]interface{}{
|
||||
"content": input,
|
||||
"source": evt.Source,
|
||||
})
|
||||
|
||||
archived := a.context.Prune(input, a.maxContextSize-1, a.docStore)
|
||||
archived := a.context.Prune(cleanInput, a.maxContextSize-1, a.docStore)
|
||||
if archived > 0 {
|
||||
log.Printf("[agent] pruned %d low-relevance events to document memory", archived)
|
||||
}
|
||||
@ -328,7 +341,7 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
||||
a.context.Append(ContextEvent{
|
||||
Timestamp: time.Now(),
|
||||
Source: "agent",
|
||||
Input: input,
|
||||
Input: cleanInput,
|
||||
Response: response,
|
||||
ToolsUsed: toolsUsed,
|
||||
ToolResults: toolResults,
|
||||
@ -337,7 +350,7 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
||||
a.emitResponse(evt, response)
|
||||
|
||||
if !stageCtx.NoMemory {
|
||||
a.emitMemoryCandidate(evt.Source, input, response, toolResults, toolsUsed)
|
||||
a.emitMemoryCandidate(evt.Source, cleanInput, response, toolResults, toolsUsed)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -6,8 +6,13 @@ import (
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||
)
|
||||
|
||||
// ChannelDef 描述通道在记忆计算层的行为，与 ToolDef.NoMemory/Cleaner 语义一致。
|
||||
type ChannelDef = pubsdk.ChannelDef
|
||||
|
||||
type DeviceType int
|
||||
|
||||
const (
|
||||
@ -60,6 +65,7 @@ type Device interface {
|
||||
Start() error
|
||||
Stop() error
|
||||
OutputCapabilities() OutputCapability
|
||||
ChannelDef() ChannelDef
|
||||
}
|
||||
|
||||
type ToolHandler func(args map[string]interface{}) (interface{}, error)
|
||||
@ -90,20 +96,22 @@ type OutputEvent struct {
|
||||
}
|
||||
|
||||
type IOManager struct {
|
||||
mu sync.RWMutex
|
||||
devices map[string]Device
|
||||
inputCh chan *InputEvent
|
||||
interruptCh chan *InputEvent
|
||||
outputCh chan *OutputEvent
|
||||
nextReqID int64
|
||||
mu sync.RWMutex
|
||||
devices map[string]Device
|
||||
inputCh chan *InputEvent
|
||||
interruptCh chan *InputEvent
|
||||
outputCh chan *OutputEvent
|
||||
nextReqID int64
|
||||
inputChannels map[string]ChannelDef
|
||||
}
|
||||
|
||||
func NewIOManager() *IOManager {
|
||||
return &IOManager{
|
||||
devices: make(map[string]Device),
|
||||
inputCh: make(chan *InputEvent, 256),
|
||||
interruptCh: make(chan *InputEvent, 64),
|
||||
outputCh: make(chan *OutputEvent, 256),
|
||||
devices: make(map[string]Device),
|
||||
inputCh: make(chan *InputEvent, 256),
|
||||
interruptCh: make(chan *InputEvent, 64),
|
||||
outputCh: make(chan *OutputEvent, 256),
|
||||
inputChannels: make(map[string]ChannelDef),
|
||||
}
|
||||
}
|
||||
|
||||
@ -333,6 +341,28 @@ func (m *IOManager) EmitTextTo(target, outputChannel, text string) {
|
||||
func (m *IOManager) InputChan() <-chan *InputEvent { return m.inputCh }
|
||||
func (m *IOManager) OutputChan() <-chan *OutputEvent { return m.outputCh }
|
||||
|
||||
// RegisterInputChannel 注册输入通道的记忆行为
|
||||
func (m *IOManager) RegisterInputChannel(name string, def ChannelDef) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.inputChannels[name] = def
|
||||
}
|
||||
|
||||
// UnregisterInputChannel 注销输入通道
|
||||
func (m *IOManager) UnregisterInputChannel(name string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.inputChannels, name)
|
||||
}
|
||||
|
||||
// GetInputChannelDef 查询输入通道的记忆行为定义
|
||||
func (m *IOManager) GetInputChannelDef(name string) (ChannelDef, bool) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
def, ok := m.inputChannels[name]
|
||||
return def, ok
|
||||
}
|
||||
|
||||
func (m *IOManager) GetAllTools() []ToolDef {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
@ -434,6 +464,7 @@ func (d *Microphone) OutputCapabilities() OutputCapability { return 0 } // 纯
|
||||
func (d *Microphone) Description() string { return fmt.Sprintf("麦克风 (%s, %dHz)", d.name, d.sampleRate) }
|
||||
func (d *Microphone) Start() error { return nil }
|
||||
func (d *Microphone) Stop() error { return nil }
|
||||
func (d *Microphone) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||
|
||||
func (d *Microphone) Tools() []ToolDef {
|
||||
return []ToolDef{{
|
||||
@ -468,6 +499,7 @@ func (d *Speaker) OutputCapabilities() OutputCapability { return CapText | CapAu
|
||||
func (d *Speaker) Description() string { return fmt.Sprintf("扬声器 (%s)", d.name) }
|
||||
func (d *Speaker) Start() error { return nil }
|
||||
func (d *Speaker) Stop() error { return nil }
|
||||
func (d *Speaker) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||
|
||||
func (d *Speaker) Tools() []ToolDef {
|
||||
return []ToolDef{{
|
||||
@ -504,6 +536,7 @@ func (d *Camera) OutputCapabilities() OutputCapability { return CapImage } //
|
||||
func (d *Camera) Description() string { return fmt.Sprintf("摄像头 (%s)", d.name) }
|
||||
func (d *Camera) Start() error { return nil }
|
||||
func (d *Camera) Stop() error { return nil }
|
||||
func (d *Camera) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||
|
||||
func (d *Camera) Tools() []ToolDef {
|
||||
return []ToolDef{
|
||||
@ -551,6 +584,7 @@ func (d *RobotArm) OutputCapabilities() OutputCapability { return CapStructured
|
||||
func (d *RobotArm) Description() string { return fmt.Sprintf("机械臂 (%s)", d.name) }
|
||||
func (d *RobotArm) Start() error { return nil }
|
||||
func (d *RobotArm) Stop() error { return nil }
|
||||
func (d *RobotArm) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||
|
||||
func (d *RobotArm) Tools() []ToolDef {
|
||||
return []ToolDef{
|
||||
@ -602,6 +636,7 @@ func (d *GPIODevice) OutputCapabilities() OutputCapability { return CapStructure
|
||||
func (d *GPIODevice) Description() string { return "GPIO 通用引脚" }
|
||||
func (d *GPIODevice) Start() error { return nil }
|
||||
func (d *GPIODevice) Stop() error { return nil }
|
||||
func (d *GPIODevice) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||
|
||||
func (d *GPIODevice) Tools() []ToolDef {
|
||||
return []ToolDef{
|
||||
|
||||
@ -43,6 +43,7 @@ func (d *mockDevice) Stop() error {
|
||||
return nil
|
||||
}
|
||||
func (d *mockDevice) OutputCapabilities() OutputCapability { return d.caps }
|
||||
func (d *mockDevice) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||
func (d *mockDevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
|
||||
if d.executeFn != nil {
|
||||
return d.executeFn(tool, args)
|
||||
|
||||
@ -6,10 +6,39 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/yanyiwu/gojieba"
|
||||
)
|
||||
|
||||
// contentPOS 有实义的词性标签：只保留名词/动词/形容词/专名等
|
||||
var contentPOS = map[string]bool{
|
||||
"n": true, // 普通名词
|
||||
"nr": true, // 人名
|
||||
"ns": true, // 地名
|
||||
"nt": true, // 机构名
|
||||
"nw": true, // 作品名/URL
|
||||
"nz": true, // 其他专名
|
||||
"v": true, // 动词
|
||||
"vd": true, // 副动词
|
||||
"vn": true, // 名动词
|
||||
"a": true, // 形容词
|
||||
"ad": true, // 副形词
|
||||
"an": true, // 名形词
|
||||
"i": true, // 成语
|
||||
"l": true, // 习用语
|
||||
"j": true, // 简称
|
||||
"s": true, // 处所词
|
||||
"f": true, // 方位词
|
||||
"b": true, // 区别词
|
||||
"z": true, // 状态词
|
||||
"t": true, // 时间词
|
||||
"eng": true, // 英文
|
||||
"x": true, // 非语素字
|
||||
"zg": true, // 其他
|
||||
"un": true, // 未知词性——保守保留
|
||||
}
|
||||
|
||||
var (
|
||||
jiebaOnce sync.Once
|
||||
jiebaInst *gojieba.Jieba
|
||||
@ -69,32 +98,109 @@ func jiebaDictDir() string {
|
||||
}
|
||||
|
||||
var stopWords = map[string]bool{
|
||||
"的": true, "了": true, "是": true, "在": true, "有": true,
|
||||
"和": true, "就": true, "不": true, "人": true, "都": true,
|
||||
"一": true, "一个": true, "上": true, "也": true, "很": true,
|
||||
"到": true, "说": true, "要": true, "去": true, "你": true,
|
||||
"会": true, "着": true, "没有": true, "看": true, "好": true,
|
||||
"自己": true, "这": true, "他": true, "她": true, "它": true,
|
||||
"我": true, "我们": true, "你们": true, "他们": true,
|
||||
"吗": true, "吧": true, "啊": true,
|
||||
"嗯": true, "哦": true, "哈": true, "呀": true, "嘛": true,
|
||||
"然后": true, "因为": true, "所以": true, "如果": true, "但是": true,
|
||||
"可能": true, "还是": true, "已经": true,
|
||||
"就是": true, "不是": true, "是的": true,
|
||||
"非常": true, "比较": true, "应该": true, "需要": true,
|
||||
"能够": true, "目前": true, "现在": true, "今天": true, "昨天": true,
|
||||
"明天": true, "知道": true, "觉得": true, "认为": true,
|
||||
"能": true, "没": true, "对": true,
|
||||
// ── 代词 ──
|
||||
"我": true, "你": true, "他": true, "她": true, "它": true,
|
||||
"我们": true, "你们": true, "他们": true, "她们": true, "它们": true,
|
||||
"自己": true, "别人": true, "大家": true,
|
||||
"这": true, "那": true, "哪": true,
|
||||
"这个": true, "那个": true, "哪个": true,
|
||||
"这里": true, "那里": true, "哪里": true,
|
||||
"这些": true, "那些": true, "哪些": true,
|
||||
"什么": true, "怎么": true, "怎样": true, "怎么样": true,
|
||||
"谁": true, "为什么": true,
|
||||
// ── 量词 ──
|
||||
"一个": true, "每个": true,
|
||||
"种": true, "个": true, "些": true, "点": true,
|
||||
// ── 介词 ──
|
||||
"在": true, "到": true, "对": true, "从": true, "把": true,
|
||||
"被": true, "让": true, "给": true, "向": true, "往": true,
|
||||
"跟": true, "和": true, "与": true, "同": true, "比": true,
|
||||
"关于": true, "对于": true, "按照": true, "根据": true, "通过": true,
|
||||
"因为": true, "由于": true, "为了": true, "为": true,
|
||||
// ── 连词 ──
|
||||
"或": true, "或者": true,
|
||||
"但是": true, "但": true, "可是": true, "不过": true,
|
||||
"然而": true, "虽然": true, "尽管": true, "即使": true,
|
||||
"如果": true, "假如": true, "只要": true, "除非": true,
|
||||
"而且": true, "并且": true, "同时": true, "此外": true,
|
||||
"所以": true, "因此": true, "于是": true,
|
||||
"然后": true, "接着": true, "从而": true,
|
||||
"不是": true, "就是": true, "而是": true,
|
||||
// ── 助词 ──
|
||||
"的": true, "地": true, "得": true,
|
||||
"了": true, "着": true, "过": true,
|
||||
"所": true,
|
||||
"吗": true, "吧": true, "啊": true, "呢": true, "啦": true,
|
||||
"嗯": true, "哦": true, "哈": true, "呀": true, "嘛": true, "哟": true,
|
||||
"噢": true, "喔": true, "呵": true, "嘿": true, "喂": true,
|
||||
"呐": true, "呗": true, "咚": true, "噗": true,
|
||||
// ── 副词 ──
|
||||
"不": true, "没": true, "没有": true, "别": true, "不要": true,
|
||||
"很": true, "太": true, "非常": true, "十分": true, "特别": true,
|
||||
"比较": true, "相当": true, "更": true, "最": true,
|
||||
"都": true, "也": true, "还": true, "又": true, "再": true,
|
||||
"就": true, "才": true, "便": true,
|
||||
"已经": true, "曾经": true, "刚": true, "刚刚": true,
|
||||
"正在": true, "正": true,
|
||||
"将要": true, "将": true,
|
||||
"一直": true, "总是": true, "从来": true,
|
||||
"经常": true, "通常": true, "往往": true,
|
||||
"可能": true, "也许": true, "大概": true, "大约": true,
|
||||
"一定": true, "肯定": true, "必须": true,
|
||||
"当然": true, "其实": true, "确实": true,
|
||||
"一起": true, "一块": true,
|
||||
"仍然": true, "依然": true, "还是": true,
|
||||
"互相": true, "分别": true,
|
||||
"是否": true, "能否": true, "可否": true,
|
||||
"越": true, "挺": true,
|
||||
// ── 判断动词 ──
|
||||
"是": true, "有": true,
|
||||
"是的": true, "有的": true,
|
||||
// ── 能愿动词 ──
|
||||
"能": true, "能够": true, "可以": true, "会": true,
|
||||
"要": true, "需要": true, "想要": true,
|
||||
"应该": true, "应当": true, "该": true,
|
||||
"愿意": true, "肯": true, "敢": true,
|
||||
// ── 常用弱义动词 ──
|
||||
"说": true, "看": true, "做": true, "来": true, "去": true,
|
||||
"用": true, "想": true, "知道": true,
|
||||
"觉得": true, "认为": true, "发现": true, "看到": true,
|
||||
"表示": true, "告诉": true, "问": true, "回答": true,
|
||||
"成为": true, "作为": true, "进行": true, "使用": true,
|
||||
// ── 时间词（泛化） ──
|
||||
"今天": true, "昨天": true, "明天": true, "前天": true, "后天": true,
|
||||
"早上": true, "上午": true, "中午": true, "下午": true, "晚上": true,
|
||||
"现在": true, "目前": true, "之前": true, "之后": true, "以后": true,
|
||||
"最近": true, "刚才": true, "以前": true, "原来": true, "将来": true,
|
||||
// ── 人称/泛指 ──
|
||||
"人": true, "人们": true, "东西": true, "事情": true,
|
||||
"问题": true, "情况": true, "时候": true, "地方": true,
|
||||
"方式": true, "方法": true, "原因": true, "结果": true,
|
||||
// ── 高频语气组合 ──
|
||||
"好的": true, "好吧": true, "好": true,
|
||||
"好了": true, "对了": true, "行了": true,
|
||||
"没事": true, "没关系": true, "算了": true,
|
||||
"看起来": true, "看上去": true, "听起来": true,
|
||||
"可以说": true, "也就是说": true, "的话": true,
|
||||
"来着": true, "罢了": true, "便是": true,
|
||||
// ── 对话/消息类弱义词 ──
|
||||
"消息": true, "回复": true, "发送": true,
|
||||
"查看": true, "显示": true, "输出": true, "输入": true,
|
||||
"文件": true, "内容": true, "信息": true,
|
||||
"来自": true, "收到": true,
|
||||
// ── 英文停用词 ──
|
||||
"the": true, "a": true, "an": true, "is": true, "are": true,
|
||||
"was": true, "were": true, "be": true, "been": true, "being": true,
|
||||
"have": true, "has": true, "had": true, "do": true, "does": true,
|
||||
"did": true, "will": true, "would": true, "could": true, "should": true,
|
||||
"may": true, "might": true, "can": true, "shall": true, "this": true,
|
||||
"that": true, "these": true, "those": true, "it": true, "its": true,
|
||||
"may": true, "might": true, "can": true, "shall": true,
|
||||
"this": true, "that": true, "these": true, "those": true,
|
||||
"it": true, "its": true,
|
||||
"and": true, "or": true, "but": true, "in": true, "on": true,
|
||||
"at": true, "to": true, "for": true, "of": true, "with": true,
|
||||
"what": true, "how": true, "why": true, "which": true, "where": true,
|
||||
"when": true, "who": true, "whom": true,
|
||||
"please": true, "yes": true, "no": true, "not": true,
|
||||
}
|
||||
|
||||
// TokenizeWords 使用 jieba 精确模式分词，返回去重后的所有词 token（不过滤停用词）
|
||||
@ -118,30 +224,45 @@ func TokenizeWords(text string) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
func ExtractKeywords(text string) []string {
|
||||
// TokenizeContentWords 使用 jieba 精确模式分词 + 词性过滤，只保留名词/动词/形容词/专名等有实义的词
|
||||
// 过滤停用词 + 短词（<2 字符），返回去重结果。适用于向量化、关键词提取等需要语义质量的任务。
|
||||
func TokenizeContentWords(text string) []string {
|
||||
text = CleanText(text)
|
||||
x := GetJieba()
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
words := x.Cut(text, false)
|
||||
var keywords []string
|
||||
tagged := x.Tag(text)
|
||||
var result []string
|
||||
seen := make(map[string]bool)
|
||||
for _, w := range words {
|
||||
if stopWords[w] || seen[w] {
|
||||
for _, t := range tagged {
|
||||
idx := strings.LastIndex(t, "/")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
r := []rune(w)
|
||||
if len(r) < 2 {
|
||||
word := t[:idx]
|
||||
tag := t[idx+1:]
|
||||
word = strings.TrimSpace(word)
|
||||
if word == "" || seen[word] {
|
||||
continue
|
||||
}
|
||||
seen[w] = true
|
||||
keywords = append(keywords, w)
|
||||
if stopWords[word] {
|
||||
continue
|
||||
}
|
||||
if utf8.RuneCountInString(word) < 2 {
|
||||
continue
|
||||
}
|
||||
if !contentPOS[tag] {
|
||||
continue
|
||||
}
|
||||
seen[word] = true
|
||||
result = append(result, word)
|
||||
}
|
||||
if len(keywords) > 5 {
|
||||
keywords = keywords[:5]
|
||||
}
|
||||
return keywords
|
||||
return result
|
||||
}
|
||||
|
||||
func ExtractKeywords(text string) []string {
|
||||
return TokenizeContentWords(text)
|
||||
}
|
||||
|
||||
// CutExact 精确模式分词：返回去停用词后的所有有义项（不限数量），用于 doc→graph 蒸馏
|
||||
|
||||
@ -23,8 +23,8 @@ func TestCutExact(t *testing.T) {
|
||||
{
|
||||
name: "chinese_sentence",
|
||||
text: "今天天气怎么样",
|
||||
min: 2,
|
||||
not: nil,
|
||||
min: 1, // "怎么样" 已加入停用词表
|
||||
not: []string{"怎么样"},
|
||||
},
|
||||
{
|
||||
name: "stop_words_removed",
|
||||
@ -47,8 +47,8 @@ func TestCutExact(t *testing.T) {
|
||||
{
|
||||
name: "qq_conversation",
|
||||
text: "今天天气怎么样 → 今天天气很好",
|
||||
min: 2,
|
||||
not: nil,
|
||||
min: 1, // "怎么样" "很" 已加入停用词表
|
||||
not: []string{"怎么样", "很好", "很"},
|
||||
},
|
||||
{
|
||||
name: "all_stop_words",
|
||||
|
||||
@ -15,6 +15,10 @@ import (
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||
)
|
||||
|
||||
// ChannelCleaner 按事件来源查找输入通道的 Cleaner 函数。
|
||||
// 返回 nil 表示不使用额外清洗。
|
||||
type ChannelCleaner func(source string) func(string) string
|
||||
|
||||
// Doc — 记忆文档：由上下文提炼而来
|
||||
type Doc struct {
|
||||
ID string `json:"id"`
|
||||
@ -128,7 +132,7 @@ func (s *Store) Insert(doc *Doc) error {
|
||||
// toolCleanFn 可选，func(name, output string) string，按工具名对输出进行过滤/清洗：
|
||||
// - 返回 "" → 跳过该工具输出（NoMemory）
|
||||
// - 返回清洗后文本 → 用于计算层（Cleaner），原文不受影响
|
||||
func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.Vectorizer, cleanFn func(string) string, toolCleanFn func(name, output string) string) (*Doc, error) {
|
||||
func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.Vectorizer, cleanFn func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) (*Doc, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@ -151,9 +155,9 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.V
|
||||
content := strings.Join(parts, "\n")
|
||||
contentHash := simpleHash(content)
|
||||
|
||||
summary := summarizeEntries(entries, cleanFn, toolCleanFn)
|
||||
tags := extractTags(entries, cleanFn, toolCleanFn)
|
||||
entities := extractEntities(entries, cleanFn, toolCleanFn)
|
||||
summary := summarizeEntries(entries, cleanFn, toolCleanFn, channelCleaner)
|
||||
tags := extractTags(entries, cleanFn, toolCleanFn, channelCleaner)
|
||||
entities := extractEntities(entries, cleanFn, toolCleanFn, channelCleaner)
|
||||
|
||||
s.mu.Lock()
|
||||
|
||||
@ -441,7 +445,7 @@ type ContextEntry struct {
|
||||
ToolResults []ToolResultItem
|
||||
}
|
||||
|
||||
func summarizeEntries(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string) string {
|
||||
func summarizeEntries(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) string {
|
||||
if len(entries) == 0 {
|
||||
return ""
|
||||
}
|
||||
@ -449,7 +453,13 @@ func summarizeEntries(entries []ContextEntry, cleanText func(string) string, too
|
||||
var topics []string
|
||||
for _, e := range entries {
|
||||
sources[e.Source]++
|
||||
words := memory.ExtractKeywords(cleanText(e.Content))
|
||||
content := e.Content
|
||||
if channelCleaner != nil {
|
||||
if c := channelCleaner(e.Source); c != nil {
|
||||
content = c(content)
|
||||
}
|
||||
}
|
||||
words := memory.ExtractKeywords(cleanText(content))
|
||||
topics = append(topics, words...)
|
||||
for _, tr := range e.ToolResults {
|
||||
out := tr.Output
|
||||
@ -490,10 +500,16 @@ func summarizeEntries(entries []ContextEntry, cleanText func(string) string, too
|
||||
return summary
|
||||
}
|
||||
|
||||
func extractTags(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string) []string {
|
||||
func extractTags(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) []string {
|
||||
tagSet := make(map[string]bool)
|
||||
for _, e := range entries {
|
||||
for _, kw := range memory.ExtractKeywords(cleanText(e.Content)) {
|
||||
content := e.Content
|
||||
if channelCleaner != nil {
|
||||
if c := channelCleaner(e.Source); c != nil {
|
||||
content = c(content)
|
||||
}
|
||||
}
|
||||
for _, kw := range memory.ExtractKeywords(cleanText(content)) {
|
||||
tagSet[kw] = true
|
||||
}
|
||||
for _, tr := range e.ToolResults {
|
||||
@ -520,11 +536,17 @@ func extractTags(entries []ContextEntry, cleanText func(string) string, toolClea
|
||||
return tags
|
||||
}
|
||||
|
||||
func extractEntities(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string) []string {
|
||||
func extractEntities(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) []string {
|
||||
var entities []string
|
||||
seen := make(map[string]bool)
|
||||
for _, e := range entries {
|
||||
for _, kw := range memory.ExtractKeywords(cleanText(e.Content)) {
|
||||
content := e.Content
|
||||
if channelCleaner != nil {
|
||||
if c := channelCleaner(e.Source); c != nil {
|
||||
content = c(content)
|
||||
}
|
||||
}
|
||||
for _, kw := range memory.ExtractKeywords(cleanText(content)) {
|
||||
if len(kw) >= 2 && !seen[kw] {
|
||||
seen[kw] = true
|
||||
entities = append(entities, kw)
|
||||
|
||||
@ -82,7 +82,7 @@ func TestContextToDoc(t *testing.T) {
|
||||
{Timestamp: time.Now(), Source: "user", Content: "特别是Go语言", Response: "Go很棒"},
|
||||
}
|
||||
|
||||
doc, err := s.ContextToDoc("test", entries, nil, nil, nil)
|
||||
doc, err := s.ContextToDoc("test", entries, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -178,7 +178,7 @@ func TestSummarizeEntries(t *testing.T) {
|
||||
{Source: "user", Content: "今天天气如何"},
|
||||
{Source: "user", Content: "明天会下雨吗"},
|
||||
}
|
||||
summary := summarizeEntries(entries, func(s string) string { return s }, nil)
|
||||
summary := summarizeEntries(entries, func(s string) string { return s }, nil, nil)
|
||||
if summary == "" {
|
||||
t.Error("summary should not be empty")
|
||||
}
|
||||
@ -198,7 +198,7 @@ func TestExtractTags(t *testing.T) {
|
||||
entries := []ContextEntry{
|
||||
{Content: "我喜欢喝咖啡和编程"},
|
||||
}
|
||||
tags := extractTags(entries, func(s string) string { return s }, nil)
|
||||
tags := extractTags(entries, func(s string) string { return s }, nil, nil)
|
||||
if len(tags) == 0 {
|
||||
t.Error("should extract tags")
|
||||
}
|
||||
@ -384,7 +384,7 @@ func TestSummarizeEntriesWithToolCleanFn(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
summary := summarizeEntries([]ContextEntry{entry}, func(s string) string { return s }, tc.toolCleanFn)
|
||||
summary := summarizeEntries([]ContextEntry{entry}, func(s string) string { return s }, tc.toolCleanFn, nil)
|
||||
for _, w := range tc.wantTopics {
|
||||
if !contains(summary, w) {
|
||||
t.Errorf("summary should contain %q, got: %s", w, summary)
|
||||
@ -410,7 +410,7 @@ func TestExtractTagsWithToolCleanFn(t *testing.T) {
|
||||
}
|
||||
|
||||
// toolCleanFn 返回 "" → NoMemory，工具输出被跳过
|
||||
tagsSkip := extractTags(entries, func(s string) string { return s }, func(name, output string) string { return "" })
|
||||
tagsSkip := extractTags(entries, func(s string) string { return s }, func(name, output string) string { return "" }, nil)
|
||||
for _, tag := range tagsSkip {
|
||||
if tag == "编程" || tag == "咖啡" {
|
||||
t.Errorf("NoMemory tool should not contribute keywords, got tag: %s", tag)
|
||||
@ -418,7 +418,7 @@ func TestExtractTagsWithToolCleanFn(t *testing.T) {
|
||||
}
|
||||
|
||||
// toolCleanFn 返回清洗文本 → 用清洗后内容提取关键词
|
||||
tagsClean := extractTags(entries, func(s string) string { return s }, func(name, output string) string { return "咖啡 编程" })
|
||||
tagsClean := extractTags(entries, func(s string) string { return s }, func(name, output string) string { return "咖啡 编程" }, nil)
|
||||
found := false
|
||||
for _, tag := range tagsClean {
|
||||
if tag == "编程" {
|
||||
@ -456,7 +456,7 @@ func TestContextToDocContentPreservesRawToolOutput(t *testing.T) {
|
||||
cleaner := func(name, output string) string {
|
||||
return "天气 温度"
|
||||
}
|
||||
doc, err := s.ContextToDoc("test", entries, nil, nil, cleaner)
|
||||
doc, err := s.ContextToDoc("test", entries, nil, nil, cleaner, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@ -271,19 +271,31 @@ func (e *StaticEmbedder) tokenize(text string) []string {
|
||||
if e.jieba == nil {
|
||||
return nil
|
||||
}
|
||||
words := e.jieba.Cut(text, false)
|
||||
tagged := e.jieba.Tag(text)
|
||||
var result []string
|
||||
seen := make(map[string]bool)
|
||||
for _, w := range words {
|
||||
w = strings.TrimSpace(w)
|
||||
if w == "" || e.stopWords[w] || seen[w] {
|
||||
for _, t := range tagged {
|
||||
idx := strings.LastIndex(t, "/")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
if utf8.RuneCountInString(w) < 2 {
|
||||
word := t[:idx]
|
||||
tag := t[idx+1:]
|
||||
word = strings.TrimSpace(word)
|
||||
if word == "" || seen[word] {
|
||||
continue
|
||||
}
|
||||
seen[w] = true
|
||||
result = append(result, w)
|
||||
if e.stopWords[word] {
|
||||
continue
|
||||
}
|
||||
if utf8.RuneCountInString(word) < 2 {
|
||||
continue
|
||||
}
|
||||
if !contentPOS[tag] {
|
||||
continue
|
||||
}
|
||||
seen[word] = true
|
||||
result = append(result, word)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@ -191,12 +191,18 @@ func (v *TFIDFVectorizer) Vectorize(text string) Vector {
|
||||
vec := make(Vector)
|
||||
for f, count := range tf {
|
||||
tfNorm := count / maxTF
|
||||
idf := 1.0
|
||||
if v.totalDocs > 0 {
|
||||
df := v.docFreq[f]
|
||||
if df > 0 {
|
||||
idf = math.Log(float64(v.totalDocs+1)/df+1) + 1
|
||||
}
|
||||
if v.totalDocs < 3 {
|
||||
vec[f] = tfNorm
|
||||
continue
|
||||
}
|
||||
df := v.docFreq[f]
|
||||
if df <= 0 {
|
||||
continue
|
||||
}
|
||||
// 平滑 IDF，高频词趋近 0，低频词趋近 log(N)
|
||||
idf := math.Log(float64(v.totalDocs+1) / (df + 1))
|
||||
if idf < 0.1 {
|
||||
continue
|
||||
}
|
||||
vec[f] = tfNorm * idf
|
||||
}
|
||||
|
||||
@ -149,7 +149,7 @@ func TestStoreInsertAndSearch(t *testing.T) {
|
||||
func TestStoreRemove(t *testing.T) {
|
||||
s := NewStore()
|
||||
v := NewTFIDFVectorizer(NGramTokenizer(1))
|
||||
v.Train([]string{"a"})
|
||||
v.Train([]string{"hello world", "hello a", "foo bar", "baz qux", "test doc"})
|
||||
|
||||
s.Insert("1", "a", v.Vectorize("a"), nil)
|
||||
s.Insert("2", "a", v.Vectorize("a"), nil)
|
||||
|
||||
@ -320,7 +320,7 @@ func go_core_dispatch(methodID C.int, ctx unsafe.Pointer, s1, s2, s3 *C.char, i1
|
||||
case 3: // CORE_REGISTER_OUTPUT_CH
|
||||
pid := pluginID
|
||||
chName := a1
|
||||
s.RegisterOutputChannel(chName, n1, a2, func(args map[string]interface{}) (interface{}, error) {
|
||||
s.RegisterOutputChannel(chName, n1, a2, sdk.ChannelDef{}, func(args map[string]interface{}) (interface{}, error) {
|
||||
// Output is async: return immediately, send in background
|
||||
// to avoid nested cgo calls (cgo within cgo can crash)
|
||||
go func() {
|
||||
|
||||
@ -121,6 +121,7 @@ type channelDevice struct {
|
||||
desc string
|
||||
caps agentIO.OutputCapability
|
||||
handler sdk.ToolHandler
|
||||
chDef agentIO.ChannelDef
|
||||
}
|
||||
|
||||
func (d *channelDevice) Name() string { return d.name }
|
||||
@ -133,6 +134,7 @@ func (d *channelDevice) Tools() []agentIO.ToolDef { return nil }
|
||||
func (d *channelDevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
|
||||
return d.handler(args)
|
||||
}
|
||||
func (d *channelDevice) ChannelDef() agentIO.ChannelDef { return d.chDef }
|
||||
|
||||
func (r *Registry) buildSDK(name string) *sdk.PluginSDK {
|
||||
sett := sdk.NewSettings(name, r.cfgReg)
|
||||
@ -152,7 +154,7 @@ func (r *Registry) buildSDK(name string) *sdk.PluginSDK {
|
||||
regAPI = func(name string) error { return nil }
|
||||
}
|
||||
|
||||
regOutput := func(chName string, caps int, desc string, handler sdk.ToolHandler) error {
|
||||
regOutput := func(chName string, caps int, desc string, def sdk.ChannelDef, handler sdk.ToolHandler) error {
|
||||
if r.iom == nil {
|
||||
return nil
|
||||
}
|
||||
@ -161,9 +163,18 @@ func (r *Registry) buildSDK(name string) *sdk.PluginSDK {
|
||||
caps: agentIO.OutputCapability(caps),
|
||||
desc: desc,
|
||||
handler: handler,
|
||||
chDef: agentIO.ChannelDef(def),
|
||||
})
|
||||
}
|
||||
|
||||
regInput := func(name string, def sdk.ChannelDef) error {
|
||||
if r.iom == nil {
|
||||
return nil
|
||||
}
|
||||
r.iom.RegisterInputChannel(name, agentIO.ChannelDef(def))
|
||||
return nil
|
||||
}
|
||||
|
||||
return sdk.New(name, sdk.SDKConfig{
|
||||
IOManager: r.iom,
|
||||
EventBus: r.evBus,
|
||||
@ -177,6 +188,7 @@ func (r *Registry) buildSDK(name string) *sdk.PluginSDK {
|
||||
RegStage: regStage,
|
||||
RegAPI: regAPI,
|
||||
RegOutput: regOutput,
|
||||
RegInput: regInput,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -190,7 +190,7 @@ func (r *ChannelRegistry) Dispatch(data json.RawMessage, pluginName string, sp *
|
||||
caps = 1
|
||||
}
|
||||
desc := fmt.Sprintf("OC channel %s (from %s)", chName, pn)
|
||||
s.RegisterOutputChannel(chName, caps, desc, func(args map[string]interface{}) (interface{}, error) {
|
||||
s.RegisterOutputChannel(chName, caps, desc, sdk.ChannelDef{}, func(args map[string]interface{}) (interface{}, error) {
|
||||
return sp.CallTool(chName, args)
|
||||
})
|
||||
|
||||
|
||||
@ -72,7 +72,7 @@ func (p *Plugin) Name() string { return p.name }
|
||||
func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||||
s.SetAutoRestart(true)
|
||||
|
||||
s.RegisterOutputChannel("cli", 1, "CLI 终端", func(args map[string]interface{}) (interface{}, error) {
|
||||
s.RegisterOutputChannel("cli", 1, "CLI 终端", sdk.ChannelDef{}, func(args map[string]interface{}) (interface{}, error) {
|
||||
payload, _ := args["payload"].(string)
|
||||
if payload != "" {
|
||||
fmt.Println(payload)
|
||||
|
||||
@ -157,7 +157,7 @@ func (p *Plugin) Name() string { return p.name }
|
||||
func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||||
s.SetAutoRestart(true)
|
||||
|
||||
s.RegisterOutputChannel("webui", 1, "Web 控制台", func(args map[string]interface{}) (interface{}, error) {
|
||||
s.RegisterOutputChannel("webui", 1, "Web 控制台", sdk.ChannelDef{}, func(args map[string]interface{}) (interface{}, error) {
|
||||
payload, _ := args["payload"].(string)
|
||||
if payload != "" {
|
||||
p.evBus.Publish(&events.Event{
|
||||
|
||||
@ -48,6 +48,8 @@ const (
|
||||
)
|
||||
type APIRegistrar = pubsdk.APIRegistrar
|
||||
type OutputChannelRegistrar = pubsdk.OutputChannelRegistrar
|
||||
type InputChannelRegistrar = pubsdk.InputChannelRegistrar
|
||||
type ChannelDef = pubsdk.ChannelDef
|
||||
|
||||
type PluginSDK struct {
|
||||
*pubsdk.PluginSDK
|
||||
@ -92,6 +94,7 @@ type SDKConfig struct {
|
||||
RegStage StageRegistrar
|
||||
RegAPI APIRegistrar
|
||||
RegOutput OutputChannelRegistrar
|
||||
RegInput InputChannelRegistrar
|
||||
}
|
||||
|
||||
func New(name string, cfg SDKConfig) *PluginSDK {
|
||||
@ -99,6 +102,9 @@ func New(name string, cfg SDKConfig) *PluginSDK {
|
||||
if cfg.IOManager != nil {
|
||||
base.SetIOInjector(ioAdapter{iom: cfg.IOManager})
|
||||
}
|
||||
if cfg.RegInput != nil {
|
||||
base.SetInputChannelRegistrar(cfg.RegInput)
|
||||
}
|
||||
base.SetMemoryAPI(cfg.Memory)
|
||||
base.SetTextMemoryAPI(cfg.TextMemory)
|
||||
base.SetDocMemoryAPI(cfg.DocMemory)
|
||||
|
||||
Reference in New Issue
Block a user