mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-03 15:53:56 +00:00
feat: complete HomeAgent architecture v2
- IO abstraction layer with OutputChannel routing and capability validation - Three-layer memory (Context-Document-Graph) with TF-IDF relevance pruning - OneBot V11 QQ protocol plugin with Reverse WebSocket client - Plugin system with hot-reload (SKILL.md + native factories) - Knowledge system with TF-IDF vector indexing - Personality system (personal.md) - Text memory (JSONL with rotation) - Change tracker (overlayfs) with rollback - Lua adapter VM - Design document (DESIGN.md) Module: gitcode.com/JianFeeeee/HomeAgent
This commit is contained in:
81
internal/tokenizer/jieba.go
Normal file
81
internal/tokenizer/jieba.go
Normal file
@ -0,0 +1,81 @@
|
||||
package tokenizer
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
jieba "github.com/yanyiwu/gojieba"
|
||||
)
|
||||
|
||||
type Jieba struct {
|
||||
mu sync.Mutex
|
||||
handle *jieba.Jieba
|
||||
}
|
||||
|
||||
var (
|
||||
global *Jieba
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
func Global() *Jieba {
|
||||
once.Do(func() {
|
||||
global = &Jieba{
|
||||
handle: jieba.NewJieba(),
|
||||
}
|
||||
})
|
||||
return global
|
||||
}
|
||||
|
||||
func (j *Jieba) Close() {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if j.handle != nil {
|
||||
j.handle.Free()
|
||||
j.handle = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (j *Jieba) ExtractKeywords(text string, topK int) []string {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
|
||||
words := j.handle.ExtractWithWeight(text, topK)
|
||||
result := make([]string, 0, len(words))
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, w := range words {
|
||||
if seen[w.Word] {
|
||||
continue
|
||||
}
|
||||
if len([]rune(w.Word)) < 2 {
|
||||
continue
|
||||
}
|
||||
seen[w.Word] = true
|
||||
result = append(result, w.Word)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (j *Jieba) Cut(text string) []string {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
|
||||
return j.handle.Cut(text, true)
|
||||
}
|
||||
|
||||
func (j *Jieba) Tag(text string) map[string]string {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
|
||||
words := j.handle.Tag(text)
|
||||
result := make(map[string]string, len(words))
|
||||
for _, pair := range words {
|
||||
if idx := strings.Index(pair, "/"); idx > 0 {
|
||||
result[pair[:idx]] = pair[idx+1:]
|
||||
} else {
|
||||
result[pair] = ""
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user