sdk: embed non-toolchain SDK in third_party, add NoMemory/Cleaner support

- Embed sdk/, example/, meta/, go.mod from homeagent-sdk (no .git)
- Core .gitignore excludes SDK toolchain: bin/, tools/, package/
- RegisterInputChannel + ChannelDef(NoMemory, Cleaner) in SDK
- IOManager input channel registry with GetInputChannelDef
- eventloop: apply channel Cleaner/NoMemory to interrupt text
- context engine: channelDefLookup applied in textForVector
- document store: ChannelCleaner param for archive functions
- All callers/adapters updated with ChannelDef{} default
This commit is contained in:
root
2026-07-29 14:48:14 +08:00
parent 4218890aed
commit a899d777c3
29 changed files with 1265 additions and 94 deletions

View File

@ -191,12 +191,18 @@ func (v *TFIDFVectorizer) Vectorize(text string) Vector {
vec := make(Vector)
for f, count := range tf {
tfNorm := count / maxTF
idf := 1.0
if v.totalDocs > 0 {
df := v.docFreq[f]
if df > 0 {
idf = math.Log(float64(v.totalDocs+1)/df+1) + 1
}
if v.totalDocs < 3 {
vec[f] = tfNorm
continue
}
df := v.docFreq[f]
if df <= 0 {
continue
}
// 平滑 IDF高频词趋近 0低频词趋近 log(N)
idf := math.Log(float64(v.totalDocs+1) / (df + 1))
if idf < 0.1 {
continue
}
vec[f] = tfNorm * idf
}

View File

@ -149,7 +149,7 @@ func TestStoreInsertAndSearch(t *testing.T) {
func TestStoreRemove(t *testing.T) {
s := NewStore()
v := NewTFIDFVectorizer(NGramTokenizer(1))
v.Train([]string{"a"})
v.Train([]string{"hello world", "hello a", "foo bar", "baz qux", "test doc"})
s.Insert("1", "a", v.Vectorize("a"), nil)
s.Insert("2", "a", v.Vectorize("a"), nil)