mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-28 21:33:05 +00:00
1cb3e87dde
- 重写 extractor.go: 分句、17条 POS 模板、依存模板 + COO 链、ATT合并 - parser.go: 分句循环 + TransE 向量验证(h+r≈t) - fallback.go: jieba POS 降级解析器 - bridge.go: nlp.Triple ↔ memory.Triple 转换 - pipeline.go: extractKeyTriples 改用 NLP 提取器, 删除5条旧前缀规则 - distill.go: docToTriples 改用 NLP 提取器 - reorgGraph: 语义相似度增强检测, 保持纯 LLM 决断 - Provider 接口加 MaxContextTokens() + 模型窗口映射表 - tokenbudget.go: 中文 token 估算器 + budget 分配(80%利用率) - process.go/buildSystemPrompt: 按 token 预算截断 memory+timeline
384 lines
10 KiB
Go
384 lines
10 KiB
Go
package nlp
|
||
|
||
import "strings"
|
||
|
||
// ——— 分句 ———
|
||
|
||
func splitSentences(text string) []string {
|
||
var sentences []string
|
||
buf := strings.Builder{}
|
||
for _, r := range text {
|
||
buf.WriteRune(r)
|
||
if r == '。' || r == '！' || r == '？' || r == '；' || r == '\n' {
|
||
s := strings.TrimSpace(buf.String())
|
||
if s != "" {
|
||
sentences = append(sentences, s)
|
||
}
|
||
buf.Reset()
|
||
}
|
||
}
|
||
if tail := strings.TrimSpace(buf.String()); tail != "" {
|
||
sentences = append(sentences, tail)
|
||
}
|
||
return sentences
|
||
}
|
||
|
||
// ——— 依存句法模板 ———
|
||
|
||
type depTemplate struct {
|
||
subjRel string
|
||
objRel string
|
||
score float64
|
||
}
|
||
|
||
var depTemplates = []depTemplate{
|
||
{subjRel: "SBV", objRel: "VOB", score: 0.9},
|
||
{subjRel: "SBV", objRel: "IOB", score: 0.85},
|
||
{subjRel: "SBV", objRel: "FOB", score: 0.8},
|
||
{subjRel: "SBV", objRel: "POB", score: 0.75},
|
||
}
|
||
|
||
// extractFromDep 基于依存句法树提取三元组
|
||
func extractFromDep(result *ParseResult) []Triple {
|
||
if len(result.Tokens) < 2 {
|
||
return nil
|
||
}
|
||
var triples []Triple
|
||
|
||
verbIndices := findPredicates(result.POS, result.Tokens)
|
||
for _, vi := range verbIndices {
|
||
var subj, obj string
|
||
var objIdx int
|
||
|
||
for i, head := range result.Heads {
|
||
if head == 0 {
|
||
continue
|
||
}
|
||
parentIdx := head - 1
|
||
if parentIdx != vi {
|
||
continue
|
||
}
|
||
rel := result.DepRels[i]
|
||
|
||
if isSubjRel(rel) && subj == "" {
|
||
subj = result.Tokens[i]
|
||
} else if isObjRel(rel) && obj == "" {
|
||
obj = result.Tokens[i]
|
||
objIdx = i
|
||
}
|
||
}
|
||
|
||
if subj == "" {
|
||
for j := vi - 1; j >= 0; j-- {
|
||
if isNounLike(result.POS[j]) {
|
||
subj = result.Tokens[j]
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
if subj != "" && obj != "" {
|
||
relLabel := result.Tokens[vi]
|
||
score := 0.8
|
||
if objIdx < len(result.Heads) && result.Heads[objIdx] == vi+1 {
|
||
for _, t := range depTemplates {
|
||
if t.objRel == result.DepRels[objIdx] {
|
||
score = t.score
|
||
break
|
||
}
|
||
}
|
||
}
|
||
triples = append(triples, Triple{
|
||
Subject: subj,
|
||
Relation: relLabel,
|
||
Object: obj,
|
||
Score: score,
|
||
Src: "dep",
|
||
})
|
||
}
|
||
|
||
// COO 链扩展：如果宾语有并列结构，为每个并列项生成三元组
|
||
if obj != "" {
|
||
cooExpanded := expandCOO(result, objIdx, vi)
|
||
for _, cooObj := range cooExpanded {
|
||
if cooObj == obj {
|
||
continue
|
||
}
|
||
relLabel := result.Tokens[vi]
|
||
triples = append(triples, Triple{
|
||
Subject: subj,
|
||
Relation: relLabel,
|
||
Object: cooObj,
|
||
Score: 0.7,
|
||
Src: "dep_coo",
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
triples = mergeAttTriples(result, triples)
|
||
return triples
|
||
}
|
||
|
||
// expandCOO 从宾语开始沿 COO 链展开所有并列项
|
||
func expandCOO(result *ParseResult, startIdx, excludeParent int) []string {
|
||
var expanded []string
|
||
seen := make(map[int]bool)
|
||
|
||
var walk func(idx int)
|
||
walk = func(idx int) {
|
||
if idx < 0 || idx >= len(result.Tokens) || seen[idx] {
|
||
return
|
||
}
|
||
seen[idx] = true
|
||
expanded = append(expanded, result.Tokens[idx])
|
||
for i, head := range result.Heads {
|
||
if head == 0 {
|
||
continue
|
||
}
|
||
if result.DepRels[i] == "COO" && head-1 == idx && i != excludeParent {
|
||
walk(i)
|
||
}
|
||
}
|
||
}
|
||
walk(startIdx)
|
||
return expanded
|
||
}
|
||
|
||
// ——— POS 序列模板（降级） ———
|
||
|
||
type posTemplate struct {
|
||
pattern []string
|
||
subj int // 主语在 pattern 中的绝对索引
|
||
verb int // 谓语在 pattern 中的绝对索引
|
||
obj int // 宾语在 pattern 中的绝对索引
|
||
score float64
|
||
}
|
||
|
||
var posTemplates = []posTemplate{
|
||
// 我/r 吃/v 苹果/n
|
||
{pattern: []string{"r", "v", "n"}, subj: 0, verb: 1, obj: 2, score: 0.7},
|
||
// 我/r 吃/v 苹果/n
|
||
{pattern: []string{"r", "v", "nr"}, subj: 0, verb: 1, obj: 2, score: 0.7},
|
||
// 我/r 是/v 学生/n
|
||
{pattern: []string{"r", "v", "n"}, subj: 0, verb: 1, obj: 2, score: 0.7},
|
||
// 小明/nr 喜欢/v 篮球/n
|
||
{pattern: []string{"nr", "v", "n"}, subj: 0, verb: 1, obj: 2, score: 0.7},
|
||
// 小明/nr 打/v 篮球/n
|
||
{pattern: []string{"nr", "v", "nr"}, subj: 0, verb: 1, obj: 2, score: 0.65},
|
||
// 我/r 在/p 杭州/ns 读书/v
|
||
{pattern: []string{"r", "p", "ns", "v"}, subj: 0, verb: 3, obj: 2, score: 0.65},
|
||
// 我/r 在/p 杭州/ns 读书/n（读书被标为 n）
|
||
{pattern: []string{"r", "p", "ns", "n"}, subj: 0, verb: 3, obj: 2, score: 0.55},
|
||
// 我/r 在/p 杭州/ns 工作/vn
|
||
{pattern: []string{"r", "p", "ns", "vn"}, subj: 0, verb: 3, obj: 2, score: 0.6},
|
||
// 小明/nr 在/p 杭州/ns 读书/v
|
||
{pattern: []string{"nr", "p", "ns", "v"}, subj: 0, verb: 3, obj: 2, score: 0.65},
|
||
// 我/r 住在/p 杭州/ns （"住"被标为 v，"在"是 p）
|
||
{pattern: []string{"r", "v", "p", "ns"}, subj: 0, verb: 1, obj: 3, score: 0.6},
|
||
// 小明/nr 住在/p 北京/ns
|
||
{pattern: []string{"nr", "v", "p", "ns"}, subj: 0, verb: 1, obj: 3, score: 0.6},
|
||
// 天气/n 很/d 好/a
|
||
{pattern: []string{"n", "d", "a"}, subj: 0, verb: 2, obj: 2, score: 0.5},
|
||
// 天气/n 很/zg 好/a（很 被标为 zg 而非 d）
|
||
{pattern: []string{"n", "zg", "a"}, subj: 0, verb: 2, obj: 2, score: 0.45},
|
||
// 今天/t 天气/n 好/a
|
||
{pattern: []string{"t", "n", "a"}, subj: 1, verb: 2, obj: 2, score: 0.5},
|
||
// 我/r 喜欢/v 跑步/vn
|
||
{pattern: []string{"r", "v", "vn"}, subj: 0, verb: 1, obj: 2, score: 0.6},
|
||
// 我/r 喜欢/v 游泳/vn
|
||
{pattern: []string{"r", "v", "v"}, subj: 0, verb: 1, obj: 2, score: 0.65},
|
||
// 我/r 叫/v 小明/nr
|
||
{pattern: []string{"r", "v", "nr"}, subj: 0, verb: 1, obj: 2, score: 0.7},
|
||
// 通用：代词/名词 + 动词 + 名词
|
||
{pattern: []string{"r", "v", "ns"}, subj: 0, verb: 1, obj: 2, score: 0.6},
|
||
{pattern: []string{"n", "v", "n"}, subj: 0, verb: 1, obj: 2, score: 0.65},
|
||
// 我/r 吃/v 了/u 苹果/n
|
||
{pattern: []string{"r", "v", "u", "n"}, subj: 0, verb: 1, obj: 3, score: 0.6},
|
||
// 名词跟在代词后作为谓语（打球/n 在 我/r 后）
|
||
{pattern: []string{"r", "n"}, subj: 0, verb: 1, obj: 1, score: 0.5},
|
||
// 小明/x 喜欢/v 吃/v 苹果/n（x 为人名，连动结构）
|
||
{pattern: []string{"x", "v", "v", "n"}, subj: 0, verb: 1, obj: 3, score: 0.55},
|
||
// 小明/x 喜欢/v 苹果/n
|
||
{pattern: []string{"x", "v", "n"}, subj: 0, verb: 1, obj: 2, score: 0.55},
|
||
// 我/r 喜欢/v 吃/v 苹果/n
|
||
{pattern: []string{"r", "v", "v", "n"}, subj: 0, verb: 1, obj: 3, score: 0.6},
|
||
// 通用：x 标签代词 + 动词 + vn
|
||
{pattern: []string{"x", "v", "vn"}, subj: 0, verb: 1, obj: 2, score: 0.5},
|
||
// 小明/x 在/p 北京/ns 工作/v
|
||
{pattern: []string{"x", "p", "ns", "v"}, subj: 0, verb: 3, obj: 2, score: 0.55},
|
||
// 小明/x 在/p 北京/ns 上班/vn
|
||
{pattern: []string{"x", "p", "ns", "vn"}, subj: 0, verb: 3, obj: 2, score: 0.5},
|
||
// 名词/n + 动词/v + 动词/v + 名词/n（连动）
|
||
{pattern: []string{"n", "v", "v", "n"}, subj: 0, verb: 1, obj: 3, score: 0.55},
|
||
}
|
||
|
||
// extractFromPOS 基于 POS 序列匹配模板提取三元组
|
||
func extractFromPOS(result *ParseResult) []Triple {
|
||
if len(result.Tokens) < 2 {
|
||
return nil
|
||
}
|
||
|
||
var triples []Triple
|
||
pos := result.POS
|
||
tokens := result.Tokens
|
||
|
||
for _, tpl := range posTemplates {
|
||
pat := tpl.pattern
|
||
if len(pat) > len(pos) {
|
||
continue
|
||
}
|
||
for i := 0; i <= len(pos)-len(pat); i++ {
|
||
if !matchPOS(pos[i:i+len(pat)], pat) {
|
||
continue
|
||
}
|
||
|
||
subj := tokens[i+tpl.subj]
|
||
verb := tokens[i+tpl.verb]
|
||
obj := tokens[i+tpl.obj]
|
||
if subj == "" || verb == "" || obj == "" {
|
||
continue
|
||
}
|
||
// 跳过自指谓语/无宾语谓语
|
||
if subj == obj {
|
||
continue
|
||
}
|
||
// 跳过谓语等于宾语（形容词谓语等无实际宾语的情况）
|
||
if verb == obj {
|
||
continue
|
||
}
|
||
triples = append(triples, Triple{
|
||
Subject: subj,
|
||
Relation: verb,
|
||
Object: obj,
|
||
Score: tpl.score,
|
||
Src: "pos",
|
||
})
|
||
}
|
||
}
|
||
|
||
// 去重（相同 subj/rel/obj 只保留一个）
|
||
triples = dedupTriples(triples)
|
||
return triples
|
||
}
|
||
|
||
func matchPOS(got, want []string) bool {
|
||
if len(got) != len(want) {
|
||
return false
|
||
}
|
||
for i := range got {
|
||
if got[i] != want[i] {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
func dedupTriples(triples []Triple) []Triple {
|
||
seen := make(map[string]bool)
|
||
var out []Triple
|
||
for _, t := range triples {
|
||
key := t.Subject + "\x00" + t.Relation + "\x00" + t.Object
|
||
if seen[key] {
|
||
continue
|
||
}
|
||
seen[key] = true
|
||
out = append(out, t)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ——— ATT 链合并 ———
|
||
|
||
// mergeAttTriples ATT 链合并：将定语合并到被修饰词
|
||
func mergeAttTriples(result *ParseResult, triples []Triple) []Triple {
|
||
attMap := make(map[int][]int)
|
||
for i, head := range result.Heads {
|
||
if head == 0 {
|
||
continue
|
||
}
|
||
if i >= len(result.DepRels) {
|
||
continue
|
||
}
|
||
if result.DepRels[i] == "ATT" {
|
||
parentIdx := head - 1
|
||
attMap[parentIdx] = append(attMap[parentIdx], i)
|
||
}
|
||
}
|
||
if len(attMap) == 0 {
|
||
return triples
|
||
}
|
||
for i := range triples {
|
||
for headIdx, attIds := range attMap {
|
||
if headIdx >= len(result.Tokens) {
|
||
continue
|
||
}
|
||
headWord := result.Tokens[headIdx]
|
||
var attWords []string
|
||
for _, aid := range attIds {
|
||
if aid < len(result.Tokens) {
|
||
attWords = append(attWords, result.Tokens[aid])
|
||
}
|
||
}
|
||
if len(attWords) == 0 {
|
||
continue
|
||
}
|
||
expanded := strings.Join(attWords, "") + headWord
|
||
if triples[i].Subject == headWord {
|
||
triples[i].Subject = expanded
|
||
}
|
||
if triples[i].Object == headWord {
|
||
triples[i].Object = expanded
|
||
}
|
||
}
|
||
}
|
||
return triples
|
||
}
|
||
|
||
// ——— helper ———
|
||
|
||
func findPredicates(pos []string, tokens []string) []int {
|
||
var indices []int
|
||
for i, p := range pos {
|
||
if isVerb(p) || isAdj(p) {
|
||
indices = append(indices, i)
|
||
continue
|
||
}
|
||
if isNounLike(p) && i > 0 && isPronoun(pos[i-1]) {
|
||
indices = append(indices, i)
|
||
continue
|
||
}
|
||
if isNounLike(p) && i > 0 && isNounLike(pos[i-1]) {
|
||
indices = append(indices, i)
|
||
continue
|
||
}
|
||
}
|
||
return indices
|
||
}
|
||
|
||
func isVerb(p string) bool {
|
||
return p == "v" || p == "vd" || strings.HasPrefix(p, "v")
|
||
}
|
||
|
||
func isNounLike(p string) bool {
|
||
return p == "n" || p == "nr" || p == "ns" || p == "nt" || p == "nz" ||
|
||
p == "an" || p == "vn" || p == "x" ||
|
||
strings.HasPrefix(p, "n")
|
||
}
|
||
|
||
func isPronoun(p string) bool {
|
||
return p == "r"
|
||
}
|
||
|
||
func isAdj(p string) bool {
|
||
return p == "a"
|
||
}
|
||
|
||
func isSubjRel(rel string) bool {
|
||
return rel == "SBV"
|
||
}
|
||
|
||
func isObjRel(rel string) bool {
|
||
return rel == "VOB" || rel == "IOB" || rel == "FOB" || rel == "POB"
|
||
}
|