mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 09:28:14 +00:00
feat: 完整实现 NLP 三元组提取系统 + token budget 上下文分配
- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并 - parser.go: 分句循环 + TransE 向量验证(h+r≈t) - fallback.go: jieba POS 降级解析器 - bridge.go: nlp.Triple ↔ memory.Triple 转换 - pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则 - distill.go: docToTriples 改用 NLP 提取器 - reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断 - Provider 接口加 MaxContextTokens() + 模型窗口映射表 - tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率) - process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline
This commit is contained in:
@ -148,6 +148,47 @@ type Provider interface {
|
||||
Name() string
|
||||
Chat(ctx context.Context, req *CompletionRequest) (*CompletionResponse, error)
|
||||
ChatStream(ctx context.Context, req *CompletionRequest) (<-chan StreamChunk, error)
|
||||
MaxContextTokens() int
|
||||
}
|
||||
|
||||
// ModelContextWindow 返回模型的最大上下文窗口(token 数)
|
||||
// 标称窗口 ≠ 有效窗口:接近满时注意力涣散,调用方应取 70-80% 为目标利用率
|
||||
func ModelContextWindow(model string) int {
|
||||
model = strings.ToLower(model)
|
||||
switch {
|
||||
case strings.Contains(model, "deepseek-r1") || strings.Contains(model, "deepseek-chat"):
|
||||
return 65536
|
||||
case strings.Contains(model, "gpt-4") && (strings.Contains(model, "turbo") || strings.Contains(model, "mini") || strings.Contains(model, "omni")):
|
||||
return 128000
|
||||
case strings.Contains(model, "gpt-4"):
|
||||
return 8192
|
||||
case strings.Contains(model, "gpt-3.5"):
|
||||
return 16384
|
||||
case strings.Contains(model, "claude-3.5") || strings.Contains(model, "claude-3"):
|
||||
return 200000
|
||||
case strings.Contains(model, "claude"):
|
||||
return 100000
|
||||
case strings.Contains(model, "gemini-1.5") || strings.Contains(model, "gemini-2"):
|
||||
return 1048576
|
||||
case strings.Contains(model, "gemini"):
|
||||
return 32768
|
||||
case strings.Contains(model, "qwen"):
|
||||
return 131072
|
||||
case strings.Contains(model, "glm") || strings.Contains(model, "chatglm"):
|
||||
return 131072
|
||||
case strings.Contains(model, "llama-3"):
|
||||
return 8192
|
||||
case strings.Contains(model, "llama-2"):
|
||||
return 4096
|
||||
case strings.Contains(model, "mistral") || strings.Contains(model, "mixtral"):
|
||||
return 32768
|
||||
case strings.Contains(model, "yi-") || strings.Contains(model, "零一"):
|
||||
return 200000
|
||||
case strings.Contains(model, "moonshot") || strings.Contains(model, "kimi"):
|
||||
return 131072
|
||||
default:
|
||||
return 32768
|
||||
}
|
||||
}
|
||||
|
||||
type BaseConfig struct {
|
||||
@ -409,6 +450,18 @@ func NewLuaAdaptedProvider(cfg BaseConfig, vm *luaVM.VM, adapter string) *LuaAda
|
||||
}
|
||||
}
|
||||
|
||||
func (p *OpenAIProvider) MaxContextTokens() int {
|
||||
return ModelContextWindow(p.cfg.Model)
|
||||
}
|
||||
|
||||
func (p *OllamaProvider) MaxContextTokens() int {
|
||||
return ModelContextWindow(p.cfg.Model)
|
||||
}
|
||||
|
||||
func (p *LuaAdaptedProvider) MaxContextTokens() int {
|
||||
return ModelContextWindow(p.cfg.Model)
|
||||
}
|
||||
|
||||
func (p *LuaAdaptedProvider) Name() string { return p.name }
|
||||
|
||||
func (p *LuaAdaptedProvider) Chat(ctx context.Context, req *CompletionRequest) (*CompletionResponse, error) {
|
||||
|
||||
Reference in New Issue
Block a user