mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 17:38:10 +00:00
sdk: embed non-toolchain SDK in third_party, add NoMemory/Cleaner support
- Embed sdk/, example/, meta/, go.mod from homeagent-sdk (no .git)
- Core .gitignore excludes SDK toolchain: bin/, tools/, package/
- RegisterInputChannel + ChannelDef(NoMemory, Cleaner) in SDK
- IOManager input channel registry with GetInputChannelDef
- eventloop: apply channel Cleaner/NoMemory to interrupt text
- context engine: channelDefLookup applied in textForVector
- document store: ChannelCleaner param for archive functions
- All callers/adapters updated with ChannelDef{} default
This commit is contained in:
@ -191,12 +191,18 @@ func (v *TFIDFVectorizer) Vectorize(text string) Vector {
|
||||
vec := make(Vector)
|
||||
for f, count := range tf {
|
||||
tfNorm := count / maxTF
|
||||
idf := 1.0
|
||||
if v.totalDocs > 0 {
|
||||
df := v.docFreq[f]
|
||||
if df > 0 {
|
||||
idf = math.Log(float64(v.totalDocs+1)/df+1) + 1
|
||||
}
|
||||
if v.totalDocs < 3 {
|
||||
vec[f] = tfNorm
|
||||
continue
|
||||
}
|
||||
df := v.docFreq[f]
|
||||
if df <= 0 {
|
||||
continue
|
||||
}
|
||||
// 平滑 IDF,高频词趋近 0,低频词趋近 log(N)
|
||||
idf := math.Log(float64(v.totalDocs+1) / (df + 1))
|
||||
if idf < 0.1 {
|
||||
continue
|
||||
}
|
||||
vec[f] = tfNorm * idf
|
||||
}
|
||||
|
||||
@ -149,7 +149,7 @@ func TestStoreInsertAndSearch(t *testing.T) {
|
||||
func TestStoreRemove(t *testing.T) {
|
||||
s := NewStore()
|
||||
v := NewTFIDFVectorizer(NGramTokenizer(1))
|
||||
v.Train([]string{"a"})
|
||||
v.Train([]string{"hello world", "hello a", "foo bar", "baz qux", "test doc"})
|
||||
|
||||
s.Insert("1", "a", v.Vectorize("a"), nil)
|
||||
s.Insert("2", "a", v.Vectorize("a"), nil)
|
||||
|
||||
Reference in New Issue
Block a user