mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-21 09:28:14 +00:00
1cb3e87dde
- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并 - parser.go: 分句循环 + TransE 向量验证(h+r≈t) - fallback.go: jieba POS 降级解析器 - bridge.go: nlp.Triple ↔ memory.Triple 转换 - pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则 - distill.go: docToTriples 改用 NLP 提取器 - reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断 - Provider 接口加 MaxContextTokens() + 模型窗口映射表 - tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率) - process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline
104 lines
2.2 KiB
Go
104 lines
2.2 KiB
Go
package nlp
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
)
|
|
|
|
// ModelSource 模型来源：本地路径或远程 URL
|
|
type ModelSource struct {
|
|
Path string // 本地路径（优先）
|
|
URL string // 远程下载地址
|
|
}
|
|
|
|
// EnsureModel 确保模型文件存在，返回最终路径
|
|
func EnsureModel(dstDir string, src ModelSource, filename string) (string, error) {
|
|
if err := os.MkdirAll(dstDir, 0755); err != nil {
|
|
return "", fmt.Errorf("create dir %s: %w", dstDir, err)
|
|
}
|
|
|
|
dst := filepath.Join(dstDir, filename)
|
|
|
|
// 1. 本地路径优先
|
|
if src.Path != "" {
|
|
if _, err := os.Stat(src.Path); err == nil {
|
|
if err := copyFile(src.Path, dst); err != nil {
|
|
return "", fmt.Errorf("copy from %s: %w", src.Path, err)
|
|
}
|
|
log.Printf("[nlp] model ready (local): %s", dst)
|
|
return dst, nil
|
|
}
|
|
log.Printf("[nlp] local path %s not found, trying remote...", src.Path)
|
|
}
|
|
|
|
// 2. 远程下载
|
|
if src.URL != "" {
|
|
if _, err := os.Stat(dst); err == nil {
|
|
return dst, nil // 已存在
|
|
}
|
|
log.Printf("[nlp] downloading model from %s ...", src.URL)
|
|
if err := downloadFile(dst, src.URL); err != nil {
|
|
return "", fmt.Errorf("download from %s: %w", src.URL, err)
|
|
}
|
|
return dst, nil
|
|
}
|
|
|
|
return "", fmt.Errorf("model not found: no local path or remote URL")
|
|
}
|
|
|
|
func downloadFile(dst, url string) error {
|
|
tmp := dst + ".download." + fmt.Sprintf("%x", md5.Sum([]byte(url)))
|
|
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return fmt.Errorf("http get %s: %w", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("http status %s", resp.Status)
|
|
}
|
|
|
|
f, err := os.Create(tmp)
|
|
if err != nil {
|
|
return fmt.Errorf("create temp %s: %w", tmp, err)
|
|
}
|
|
|
|
written, err := io.Copy(f, resp.Body)
|
|
f.Close()
|
|
if err != nil {
|
|
os.Remove(tmp)
|
|
return fmt.Errorf("write: %w", err)
|
|
}
|
|
|
|
if err := os.Rename(tmp, dst); err != nil {
|
|
os.Remove(tmp)
|
|
return fmt.Errorf("rename: %w", err)
|
|
}
|
|
|
|
log.Printf("[nlp] downloaded %d bytes to %s", written, dst)
|
|
return nil
|
|
}
|
|
|
|
func copyFile(src, dst string) error {
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
|
|
_, err = io.Copy(out, in)
|
|
return err
|
|
}
|