mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-03 15:53:56 +00:00
sdk: embed non-toolchain SDK in third_party, add NoMemory/Cleaner support
- Embed sdk/, example/, meta/, go.mod from homeagent-sdk (no .git)
- Core .gitignore excludes SDK toolchain: bin/, tools/, package/
- RegisterInputChannel + ChannelDef(NoMemory, Cleaner) in SDK
- IOManager input channel registry with GetInputChannelDef
- eventloop: apply channel Cleaner/NoMemory to interrupt text
- context engine: channelDefLookup applied in textForVector
- document store: ChannelCleaner param for archive functions
- All callers/adapters updated with ChannelDef{} default
This commit is contained in:
@ -271,19 +271,31 @@ func (e *StaticEmbedder) tokenize(text string) []string {
|
||||
if e.jieba == nil {
|
||||
return nil
|
||||
}
|
||||
words := e.jieba.Cut(text, false)
|
||||
tagged := e.jieba.Tag(text)
|
||||
var result []string
|
||||
seen := make(map[string]bool)
|
||||
for _, w := range words {
|
||||
w = strings.TrimSpace(w)
|
||||
if w == "" || e.stopWords[w] || seen[w] {
|
||||
for _, t := range tagged {
|
||||
idx := strings.LastIndex(t, "/")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
if utf8.RuneCountInString(w) < 2 {
|
||||
word := t[:idx]
|
||||
tag := t[idx+1:]
|
||||
word = strings.TrimSpace(word)
|
||||
if word == "" || seen[word] {
|
||||
continue
|
||||
}
|
||||
seen[w] = true
|
||||
result = append(result, w)
|
||||
if e.stopWords[word] {
|
||||
continue
|
||||
}
|
||||
if utf8.RuneCountInString(word) < 2 {
|
||||
continue
|
||||
}
|
||||
if !contentPOS[tag] {
|
||||
continue
|
||||
}
|
||||
seen[word] = true
|
||||
result = append(result, word)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user