sdk: embed non-toolchain SDK in third_party, add NoMemory/Cleaner support

- Embed sdk/, example/, meta/, go.mod from homeagent-sdk (no .git)
- Core .gitignore excludes SDK toolchain: bin/, tools/, package/
- RegisterInputChannel + ChannelDef(NoMemory, Cleaner) in SDK
- IOManager input channel registry with GetInputChannelDef
- eventloop: apply channel Cleaner/NoMemory to interrupt text
- context engine: channelDefLookup applied in textForVector
- document store: ChannelCleaner param for archive functions
- All callers/adapters updated with ChannelDef{} default
This commit is contained in:
root
2026-07-29 14:48:14 +08:00
parent 4218890aed
commit a899d777c3
29 changed files with 1265 additions and 94 deletions

View File

@ -271,19 +271,31 @@ func (e *StaticEmbedder) tokenize(text string) []string {
if e.jieba == nil {
return nil
}
words := e.jieba.Cut(text, false)
tagged := e.jieba.Tag(text)
var result []string
seen := make(map[string]bool)
for _, w := range words {
w = strings.TrimSpace(w)
if w == "" || e.stopWords[w] || seen[w] {
for _, t := range tagged {
idx := strings.LastIndex(t, "/")
if idx < 0 {
continue
}
if utf8.RuneCountInString(w) < 2 {
word := t[:idx]
tag := t[idx+1:]
word = strings.TrimSpace(word)
if word == "" || seen[word] {
continue
}
seen[w] = true
result = append(result, w)
if e.stopWords[word] {
continue
}
if utf8.RuneCountInString(word) < 2 {
continue
}
if !contentPOS[tag] {
continue
}
seen[word] = true
result = append(result, word)
}
return result
}