mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-15 12:15:16 +00:00
feat: 完整实现 NLP 三元组提取系统 + token budget 上下文分配
- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并 - parser.go: 分句循环 + TransE 向量验证(h+r≈t) - fallback.go: jieba POS 降级解析器 - bridge.go: nlp.Triple ↔ memory.Triple 转换 - pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则 - distill.go: docToTriples 改用 NLP 提取器 - reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断 - Provider 接口加 MaxContextTokens() + 模型窗口映射表 - tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率) - process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline
This commit is contained in:
103
internal/nlp/download.go
Normal file
103
internal/nlp/download.go
Normal file
@ -0,0 +1,103 @@
|
||||
package nlp
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// ModelSource 模型来源:本地路径或远程 URL
|
||||
type ModelSource struct {
|
||||
Path string // 本地路径(优先)
|
||||
URL string // 远程下载地址
|
||||
}
|
||||
|
||||
// EnsureModel 确保模型文件存在,返回最终路径
|
||||
func EnsureModel(dstDir string, src ModelSource, filename string) (string, error) {
|
||||
if err := os.MkdirAll(dstDir, 0755); err != nil {
|
||||
return "", fmt.Errorf("create dir %s: %w", dstDir, err)
|
||||
}
|
||||
|
||||
dst := filepath.Join(dstDir, filename)
|
||||
|
||||
// 1. 本地路径优先
|
||||
if src.Path != "" {
|
||||
if _, err := os.Stat(src.Path); err == nil {
|
||||
if err := copyFile(src.Path, dst); err != nil {
|
||||
return "", fmt.Errorf("copy from %s: %w", src.Path, err)
|
||||
}
|
||||
log.Printf("[nlp] model ready (local): %s", dst)
|
||||
return dst, nil
|
||||
}
|
||||
log.Printf("[nlp] local path %s not found, trying remote...", src.Path)
|
||||
}
|
||||
|
||||
// 2. 远程下载
|
||||
if src.URL != "" {
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
return dst, nil // 已存在
|
||||
}
|
||||
log.Printf("[nlp] downloading model from %s ...", src.URL)
|
||||
if err := downloadFile(dst, src.URL); err != nil {
|
||||
return "", fmt.Errorf("download from %s: %w", src.URL, err)
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("model not found: no local path or remote URL")
|
||||
}
|
||||
|
||||
func downloadFile(dst, url string) error {
|
||||
tmp := dst + ".download." + fmt.Sprintf("%x", md5.Sum([]byte(url)))
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http get %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("http status %s", resp.Status)
|
||||
}
|
||||
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temp %s: %w", tmp, err)
|
||||
}
|
||||
|
||||
written, err := io.Copy(f, resp.Body)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("write: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("rename: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("[nlp] downloaded %d bytes to %s", written, dst)
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user