From 7c0b7a1fb0e932aa95890f7b1a7cbffad38ecfbc Mon Sep 17 00:00:00 2001 From: JianFeeeee Date: Sat, 12 Sep 2026 23:16:23 +0800 Subject: [PATCH] =?UTF-8?q?feat(deepsearch):=20=E8=81=94=E7=BD=91=E6=A3=80?= =?UTF-8?q?=E7=B4=A2=E6=8F=92=E4=BB=B6=20+=20SearXNG=20=E7=94=9F=E5=91=BD?= =?UTF-8?q?=E5=91=A8=E6=9C=9F=E6=89=98=E7=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 把原 websearch 示例改名为 deepsearch（目录/go.mod/plg.json/工具前缀/README 全量对齐， 工具名 websearch_* → deepsearch_*）。 新增 searxng.go：插件自己托管搜索后端 - 启动探 healthz：已在跑则直接接管（不重启），没跑就 docker compose up -d 并等就绪 - 关闭动作注册为 stop handler（幂等、限时 4s < 内核 5s 宽限期） - 配置 manage_searxng / searxng_dir / stop_searxng_on_exit - 崩溃/被 kill 时不关后端（下次启动接管）：安全失败方向 - 可注入 cmdRunner + 时间预算，8 项单测离线覆盖接管/拉起/失败/幂等/保留 契约依据（internal/plugin/proc）：停止插件 = plugin.stop → RunStopHandlers（LIFO、 幂等）→ Stop() → exit(0)，宽限期 5s；stdin 关闭同路径。 验证：19 项单测（18 通过 + 1 联调跳过）、-race 干净、vet/gofmt 干净；内核 E2E 真调用 返回 58 条/37 条、58/58 带摘要；线上 daemon 实测 stop handler 与冷启动（约 3s）。 --- example/deepsearch/README.md | 100 +++ example/deepsearch/go.mod | 17 + example/deepsearch/live_test.go | 81 +++ example/deepsearch/plg.json | 12 + example/deepsearch/plugin.go | 1000 ++++++++++++++++++++++++++++ example/deepsearch/plugin_test.go | 288 ++++++++ example/deepsearch/searxng.go | 159 +++++ example/deepsearch/searxng_test.go | 197 ++++++ 8 files changed, 1854 insertions(+) create mode 100644 example/deepsearch/README.md create mode 100644 example/deepsearch/go.mod create mode 100644 example/deepsearch/live_test.go create mode 100644 example/deepsearch/plg.json create mode 100644 example/deepsearch/plugin.go create mode 100644 example/deepsearch/plugin_test.go create mode 100644 example/deepsearch/searxng.go create mode 100644 example/deepsearch/searxng_test.go diff --git a/example/deepsearch/README.md b/example/deepsearch/README.md new file mode 100644 index 0000000..79d8078 --- /dev/null +++ b/example/deepsearch/README.md @@ -0,0 +1,100 @@ +# 联网检索插件（HomeAgent） + +给 agent 补上**真正的信息检索**能力：检索交给本地 SearXNG（多引擎聚合、结构化 JSON）， +并补上「读完前 K 篇再回答」的深检索。 + +## 为什么需要它（背景） + +agent 原本只有 `browser_*` 那套浏览器工具，联网检索实际只有 `browser_search` 一个入口，而它是 +**「抓 Bing HTML + 正则解析」**： + +| 缺陷 | 实测结果 | +|---|---| +| 标题取的是结果块里**第一个 ``** | 拿到的是 Bing 的「来源行」而非标题 → `deepin.orghttps://www.deepin.org` | +| 摘要正则 `
.*?

` | 对现代 Bing **命中 0/10**（摘要已迁到 `p.b_lineclamp*`）→ 结果**完全没有摘要** | +| 用 `www.bing.com` | 程序化请求直接 302；`cn.bing.com` 才返回 10 个结果块 | +| 单引擎、无兜底、无去重、无站点读取 | 模型只能反复换词重搜（日志里 8 秒 6 连击） | + +结果就是日志里那句用户反馈：**「你的搜索能力好像不太行啊」**。 + +## 依赖：本地 SearXNG（由本插件托管） + +插件会**自己管后端**： + +- **启动时**：探 `healthz`；已在跑就**直接接管**（不重启），没跑就 `docker compose up -d` 并等就绪（上限 6s） +- **停止时**：跑 `docker compose stop -t 2` 关闭它 + +配置项 `manage_searxng`（默认 true）与 `searxng_dir`（默认 `/root/searxng-agent`）控制这套行为； +`stop_searxng_on_exit`（默认 true）设 false 可让后端在插件停止后继续跑（**插件重载频繁时建议设 false**， +否则每次重载都会把后端重启一遍）。 + +### 生命周期契约（依据内核源码，非猜测） + +| 环节 | 内核行为 | +|---|---| +| 停止插件 | 发 `plugin.stop` → 插件先跑 **RunStopHandlers（LIFO、幂等）** → 再 `Stop()` → `exit(0)` | +| 宽限期 | **5 秒**；未退出则直接 SIGKILL —— 所以关闭动作限时 4s（`searxShutdownBudget`） | +| stdin 关闭 | 同样会跑 handlers + `Stop()` | +| 崩溃/被 kill | 关闭动作不会执行，后端会留在运行态；下次启动探测到就直接接管（**更安全的失败方向**） | +| 自动重启 | `SetAutoRestart(true)` 由注入的 runtime 在 `plugin.start` 后经 `lifecycle.autoRestart` **显式上报**内核 | + +### SearXNG 侧配置 + +部署在 **.60**，`127.0.0.1:8888`： + +``` +/root/searxng-agent/docker-compose.yml # host 网络（要访问宿主 clash） +/root/searxng-agent/settings.yml # json 输出 + limiter 关闭 + 出站走 clash +``` + +两个必须知道的坑： + +1. **`search.formats` 必须含 `json`**，否则 `/search?format=json` 返回 **403**（看起来像网络问题，其实是配置）。 +2. 该镜像默认 `GRANIAN_PORT=8080`，而 granian 的 `GRANIAN_*` **优先级高于 settings.yml**： + .60 上 8080 被 homeagent 占用 → 不改 `SEARXNG_PORT` 就是无休止的 `Address already in use` 崩溃循环。 + +实测可用的引擎（2026-09-12）：`duckduckgo`、`brave`、`google cse`；`quark` 时好时坏； +`baidu`/`google` 经代理出口触发 CAPTCHA，`sogou` 崩溃，`wikidata` 报 HTTP error（已关）。 + +## 工具 + +| 工具 | 说明 | +|---|---| +| `deepsearch_search` | 联网检索（首选）：标题 + URL + 摘要 + 发布时间，支持 `engines`/`category`/`time_range`/`language`，自动按 URL 去重并按分数排序；会回报**引擎覆盖度与无响应引擎** | +| `deepsearch_news` | 新闻检索：`news` 类别 + 默认最近一周；新闻为空时自动回退 general + 时间范围 | +| `deepsearch_fetch` | 抓单个网页并抽正文（去脚本/样式/导航），返回标题 + 纯文本，可设截断长度 | +| `deepsearch_deep` | **深检索**：检索 → 并行抓前 K 篇正文 → 一次返回「候选清单 + 证据正文」；单篇失败不影响整体 | +| `deepsearch_status` | 自检：healthz、json 是否可用、延迟、**哪些引擎真的在返回结果**（检索出问题先跑这个） | + +## 配置项 + +| 键 | 默认 | 说明 | +|---|---|---| +| `searxng_url` | `http://127.0.0.1:8888` | 本地 SearXNG 地址 | +| `max_results` | `8` | 默认条数（控制上下文体积） | +| `language` | `zh-CN` | 检索语言 | +| `safesearch` | `0` | 0 关 / 1 中 / 2 严 | +| `request_timeout` | `20` | 单次请求超时（秒） | +| `fetch_max_chars` | `4000` | `deepsearch_fetch` 正文上限 | +| `proxy` | 空 | 仅作用于本插件直连抓取（搜索出网由 SearXNG 侧负责） | +| `user_agent` | Chrome UA | 抓取用 | + +每次调用前重读配置，改完即时生效。 + +## 开发与验证 + +```bash +go test -count=1 -race ./... # 11 项测试（httptest 打桩 SearXNG） + +# 真实后端联调（默认跳过）：跑的就是当初失败的那条查询 +DEEPSEARCH_LIVE_SEARXNG=http://127.0.0.1:8888 go test -run TestLiveSearxng -v ./... + +hmapdev build # 产出 dist/deep_search_bundle.hmap +``` + +## 已知边界 + +- **知乎等站点对直连抓取返回 403**（反爬），`deepsearch_deep` 会如实标注该篇抓取失败并继续； + 这类页面请改用浏览器工具（`browser_navigate` + `browser_render`）。 +- 引擎可用性随出口 IP 与目标站点风控变化；`deepsearch_status` 与每次结果里的「覆盖度」行就是给这个用的。 +- 未做正文去重/相似度合并：同一事件的多篇转载会各占一条（摘要已能区分）。 diff --git a/example/deepsearch/go.mod b/example/deepsearch/go.mod new file mode 100644 index 0000000..804f546 --- /dev/null +++ b/example/deepsearch/go.mod @@ -0,0 +1,17 @@ +module deepsearch-plugin + +go 1.25.0 + +require gitcode.com/JianFeeeee/homeagent-sdk v1.2.0 + + + + + + + + + + + +replace gitcode.com/JianFeeeee/homeagent-sdk => /root/.homeagent/hmapdev/sdk/v1.2.0 diff --git a/example/deepsearch/live_test.go b/example/deepsearch/live_test.go new file mode 100644 index 0000000..11d8a85 --- /dev/null +++ b/example/deepsearch/live_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "os" + "strings" + "testing" +) + +// 真实后端联调（默认跳过，需显式指定地址）： +// +// DEEPSEARCH_LIVE_SEARXNG=http://127.0.0.1:8888 go test -run TestLiveSearxng -v ./... +// +// 它跑的就是当初失败的场景（日志里那条「你的搜索能力好像不太行啊」对应的查询）， +// 用来回答一个具体问题：换了后端之后，模型拿到的是不是「带摘要的相关结果」。 +func TestLiveSearxng(t *testing.T) { + base := os.Getenv("DEEPSEARCH_LIVE_SEARXNG") + if base == "" { + t.Skip("未设置 DEEPSEARCH_LIVE_SEARXNG，跳过真实后端联调") + } + p := &Plugin{ + name: "deepsearch", + searxURL: strings.TrimRight(base, "/"), + maxItems: 6, + language: "zh-CN", + fetchMax: 1200, + userAgent: defaultUA, + } + p.ensure() + + // 1) 自检 + st, err := p.handleStatus(map[string]interface{}{}) + if err != nil { + t.Fatalf("status: %v", err) + } + t.Logf("status: %v", st) + + // 2) 当初失败的那条查询 + res, err := p.handleSearch(map[string]interface{}{"query": "深度科技 deepin 开发者 被开除"}) + if err != nil { + t.Fatalf("search: %v", err) + } + txt := res.(map[string]interface{})["content"].(string) + t.Logf("检索结果：\n%s", txt) + if !strings.Contains(txt, "摘要：") { + t.Errorf("结果里应当有摘要（这正是原实现缺失的东西）") + } + if !strings.Contains(txt, "覆盖：") { + t.Errorf("应报告引擎覆盖度") + } + + // 3) 正文抓取（取第一条结果的 URL） + var firstURL string + for _, line := range strings.Split(txt, "\n") { + l := strings.TrimSpace(line) + if strings.HasPrefix(l, "http") { + firstURL = l + break + } + } + if firstURL == "" { + t.Fatal("未从结果中解析出 URL") + } + page, err := p.handleFetch(map[string]interface{}{"url": firstURL, "max_chars": float64(600)}) + if err != nil { + t.Logf("抓取 %s 失败（真实站点有反爬/需 JS 属正常）：%v", firstURL, err) + } else { + body := page.(map[string]interface{})["content"].(string) + t.Logf("抓取 %s 正文前 400 字：%s", firstURL, oneLine(body, 400)) + } + + // 4) 深检索 + deep, err := p.handleDeep(map[string]interface{}{"query": "统信 UOS 内核工程师 西装 事件", "top_k": float64(2)}) + if err != nil { + t.Fatalf("deep: %v", err) + } + dTxt := deep.(map[string]interface{})["content"].(string) + if !strings.Contains(dTxt, "候选清单") || !strings.Contains(dTxt, "正文证据") { + t.Errorf("深检索输出结构不对") + } + t.Logf("深检索输出前 800 字：\n%s", oneLine(dTxt, 800)) +} diff --git a/example/deepsearch/plg.json b/example/deepsearch/plg.json new file mode 100644 index 0000000..5234105 --- /dev/null +++ b/example/deepsearch/plg.json @@ -0,0 +1,12 @@ +{ + "name": "deepsearch", + "name_zh": "联网检索", + "name_en": "Deep Search", + "version": "1.1.0", + "description": "为 agent 提供真正的联网信息检索：本地 SearXNG 聚合多引擎（返回标题/URL/摘要/时间），支持新闻、时间范围、指定引擎；并提供网页正文抽取与「搜索+读前K篇」的深检索", + "author": "HomeAgent", + "entry": "plugin.bin", + "sdk": "1.2.0", + "tags": ["search", "web", "searxng", "retrieval", "news"], + "targets": "linux/amd64" +} diff --git a/example/deepsearch/plugin.go b/example/deepsearch/plugin.go new file mode 100644 index 0000000..872c7f3 --- /dev/null +++ b/example/deepsearch/plugin.go @@ -0,0 +1,1000 @@ +package main + +// 联网检索插件（HomeAgent） +// +// 解决的问题：原来 agent 只有 browser_* 那套「驱动浏览器抓页面」的能力， +// 而 browser_search 是「抓 Bing HTML + 正则解析」——标题取错（拿到来源行）、 +// 摘要正则对现代 Bing 完全不命中，于是模型每次只能拿到「标题=URL 串、无摘要」， +// 表现为反复改词重搜。本插件把检索交给本地 SearXNG（多引擎聚合 + 结构化 JSON）， +// 并补上「读完前 K 篇再回答」的深检索。 +// +// 配置项（全部可在插件配置界面改）： +// searxng_url 本地/远端 SearXNG 基地址（默认 http://127.0.0.1:8888） +// manage_searxng 是否由本插件托管搜索后端（默认 true）：插件启动时拉起、停止时关闭 +// searxng_dir 托管时使用的 compose 目录（默认 /root/searxng-agent） +// stop_searxng_on_exit 停止插件时是否关闭后端（默认 true；关掉可避免重载时反复重启） +// max_results 默认返回条数（控制上下文体积） +// language 默认语言（如 zh-CN） +// safesearch 0/1/2 +// request_timeout 单次请求超时（秒） +// fetch_max_chars 网页正文截断长度 +// proxy 可选 HTTP 代理（抓取被墙站点用，如 http://127.0.0.1:7890） +// +// 依赖的 SearXNG 侧配置（否则 /search?format=json 会回 403，而非网络问题）： +// search: { formats: [html, json] } 且 server: { limiter: false } + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "html" + "io" + "log" + "net/http" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" +) + +const ( + cfgSearxURL = "searxng_url" + cfgMaxResults = "max_results" + cfgLanguage = "language" + cfgSafeSearch = "safesearch" + cfgTimeout = "request_timeout" + cfgFetchMaxChars = "fetch_max_chars" + cfgProxy = "proxy" + cfgUserAgent = "user_agent" + + defaultSearxURL = "http://127.0.0.1:8888" + defaultUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" +) + +type Plugin struct { + name string + sdk *sdk.PluginSDK + http *http.Client + + searxURL string + maxItems int + language string + safeLevel int + fetchMax int + userAgent string + + // SearXNG 生命周期托管（见 searxng.go） + run cmdRunner + manageSearx bool + searxDir string + stopOnExit bool + + searxMu sync.Mutex + searxOwned bool // 本插件是否负责关闭它（只在真拉起/接管后为真） + + // 时间预算（零值取包内默认；单测注入短值以避免真等） + bud searxBudget +} + +func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name}, nil +} + +func NewPluginFactory(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name}, nil +} + +func (p *Plugin) Name() string { return p.name } + +func (p *Plugin) Stop() error { + // 双重保险：正常停止走 StopHandler（内核 `plugin.stop` 会先跑 handlers 再调这里）， + // 但写在同一处更稳。shutdownSearxng 幂等，重复调用无害。 + p.shutdownSearxng() + return nil +} + +// ======== 参数与配置读取 ======== + +func argStr(a map[string]interface{}, key string) string { + if v, ok := a[key]; ok && v != nil { + if s, ok := v.(string); ok { + return strings.TrimSpace(s) + } + return strings.TrimSpace(fmt.Sprint(v)) + } + return "" +} + +func argInt(a map[string]interface{}, key string, def int) int { + if v, ok := a[key]; ok && v != nil { + switch n := v.(type) { + case float64: + return int(n) + case int: + return n + case int64: + return int(n) + case string: + if x, err := strconv.Atoi(strings.TrimSpace(n)); err == nil { + return x + } + } + } + return def +} + +func argBool(a map[string]interface{}, key string) bool { + if v, ok := a[key]; ok && v != nil { + switch b := v.(type) { + case bool: + return b + case string: + x, _ := strconv.ParseBool(strings.TrimSpace(b)) + return x + case float64: + return b != 0 + } + } + return false +} + +func cfgStr(st sdk.SettingsAPI, key, def string) string { + if st == nil { + return def + } + v, err := st.Get(key) + if err != nil || v == nil { + return def + } + if s, ok := v.(string); ok { + if strings.TrimSpace(s) == "" { + return def + } + return strings.TrimSpace(s) + } + return fmt.Sprint(v) +} + +func cfgInt(st sdk.SettingsAPI, key string, def int) int { + s := cfgStr(st, key, "") + if s == "" { + return def + } + if n, err := strconv.Atoi(s); err == nil { + return n + } + return def +} + +// cfgBool 注意：core 侧 bool 以字符串（"true"/"false"）存储，这里一并对齐 +func cfgBool(st sdk.SettingsAPI, key string, def bool) bool { + s := cfgStr(st, key, "") + if s == "" { + return def + } + if b, err := strconv.ParseBool(s); err == nil { + return b + } + return def +} + +// ensure 每次调用前重读配置：改地址/代理即时生效 +func (p *Plugin) ensure() { + if p.sdk == nil { + if p.searxURL == "" { + p.searxURL = defaultSearxURL + } + if p.searxDir == "" { + p.searxDir = defaultSearxDir + } + if p.run == nil { + p.run = defaultRunner + } + if p.maxItems <= 0 { + p.maxItems = 8 + } + if p.fetchMax <= 0 { + p.fetchMax = 4000 + } + if p.userAgent == "" { + p.userAgent = defaultUA + } + if p.http == nil { + p.http = &http.Client{Timeout: 20 * time.Second} + } + return + } + st := p.sdk.Settings() + p.searxURL = strings.TrimRight(cfgStr(st, cfgSearxURL, defaultSearxURL), "/") + p.manageSearx = cfgBool(st, cfgManageSearx, true) + p.stopOnExit = cfgBool(st, cfgStopOnExit, true) + if d := cfgStr(st, cfgSearxDir, defaultSearxDir); d != "" { + p.searxDir = d + } else { + p.searxDir = defaultSearxDir + } + p.run = defaultRunner + if n := cfgInt(st, cfgMaxResults, 8); n > 0 { + p.maxItems = n + } else { + p.maxItems = 8 + } + p.language = cfgStr(st, cfgLanguage, "zh-CN") + p.safeLevel = cfgInt(st, cfgSafeSearch, 0) + if n := cfgInt(st, cfgFetchMaxChars, 4000); n > 0 { + p.fetchMax = n + } else { + p.fetchMax = 4000 + } + p.userAgent = cfgStr(st, cfgUserAgent, defaultUA) + + timeout := cfgInt(st, cfgTimeout, 20) + if timeout <= 0 { + timeout = 20 + } + client := &http.Client{Timeout: time.Duration(timeout) * time.Second} + transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}} + if raw := cfgStr(st, cfgProxy, ""); raw != "" { + if pu, err := url.Parse(raw); err == nil { + transport.Proxy = http.ProxyURL(pu) + } + } + client.Transport = transport + p.http = client +} + +// ======== SearXNG 交互 ======== + +type searxResult struct { + URL string `json:"url"` + Title string `json:"title"` + Content string `json:"content"` + Engine string `json:"engine"` + Engines []string `json:"engines"` + Category string `json:"category"` + PublishedDate string `json:"publishedDate"` + Score float64 `json:"score"` +} + +type searxResponse struct { + Query string `json:"query"` + Results []searxResult `json:"results"` + Answers []string `json:"answers"` + Infoboxes []json.RawMessage `json:"infoboxes"` + Suggestions []string `json:"suggestions"` + UnresponsiveEngine [][]string `json:"unresponsive_engines"` + Timings map[string]any `json:"timings"` +} + +func (r *searxResult) engineNames() []string { + set := map[string]bool{} + for _, e := range r.Engines { + if e != "" { + set[e] = true + } + } + if r.Engine != "" { + set[r.Engine] = true + } + out := make([]string, 0, len(set)) + for e := range set { + out = append(out, e) + } + sort.Strings(out) + return out +} + +// searxQuery 调用 SearXNG JSON 接口 +func (p *Plugin) searxQuery(params url.Values) (*searxResponse, error) { + if p.searxURL == "" { + return nil, fmt.Errorf("未配置 searxng_url") + } + params.Set("format", "json") + endpoint := p.searxURL + "/search?" + params.Encode() + + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("构造请求失败: %v", err) + } + req.Header.Set("User-Agent", p.userAgent) + req.Header.Set("Accept", "application/json") + + resp, err := p.http.Do(req) + if err != nil { + return nil, fmt.Errorf("SearXNG 不可达（%s）：%v", p.searxURL, err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + + if resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("SearXNG 返回 403：通常是实例未开启 json 输出（settings.yml 的 search.formats 需含 json）或 limiter 拦截了本调用") + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("SearXNG 返回 HTTP %d：%s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var out searxResponse + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("解析 SearXNG 响应失败（返回体不是 JSON，可能被拦截页替换）: %v", err) + } + return &out, nil +} + +// dedupResults 按规范化 URL 去重（保留分数更高者），并按分数排序 +func dedupResults(in []searxResult) []searxResult { + norm := func(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return raw + } + q := u.Query() + for k := range q { + if strings.HasPrefix(k, "utm_") || k == "ref" || k == "spm" { + q.Del(k) + } + } + u.RawQuery = q.Encode() + u.Fragment = "" + return strings.TrimRight(u.String(), "/") + } + best := map[string]searxResult{} + order := []string{} + for _, r := range in { + if strings.TrimSpace(r.URL) == "" { + continue + } + k := norm(r.URL) + if prev, ok := best[k]; ok { + if r.Score > prev.Score { + best[k] = r + } + continue + } + best[k] = r + order = append(order, k) + } + out := make([]searxResult, 0, len(order)) + for _, k := range order { + out = append(out, best[k]) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Score > out[j].Score }) + return out +} + +// formatResults 返回给模型的紧凑文本（每条：序号/标题/URL/摘要/时间） +func formatResults(q string, res []searxResult, unresponsive [][]string, elapsed time.Duration) string { + var b strings.Builder + fmt.Fprintf(&b, "检索 %q：命中 %d 条（%s）\n", q, len(res), elapsed.Round(time.Millisecond)) + engSet := map[string]bool{} + withSnippet := 0 + for _, r := range res { + for _, e := range r.engineNames() { + engSet[e] = true + } + if strings.TrimSpace(r.Content) != "" { + withSnippet++ + } + } + engs := make([]string, 0, len(engSet)) + for e := range engSet { + engs = append(engs, e) + } + sort.Strings(engs) + fmt.Fprintf(&b, "覆盖：%s；带摘要 %d/%d 条\n", strings.Join(engs, " + "), withSnippet, len(res)) + if len(unresponsive) > 0 { + parts := make([]string, 0, len(unresponsive)) + for _, e := range unresponsive { + if len(e) >= 2 { + parts = append(parts, e[0]+"("+e[1]+")") + } + } + if len(parts) > 0 { + fmt.Fprintf(&b, "本次无结果的引擎：%s\n", strings.Join(parts, ", ")) + } + } + b.WriteString("\n") + for i, r := range res { + fmt.Fprintf(&b, "%d. %s\n", i+1, strings.TrimSpace(r.Title)) + fmt.Fprintf(&b, " %s\n", r.URL) + if d := strings.TrimSpace(r.PublishedDate); d != "" { + if len(d) > 10 { + d = d[:10] + } + fmt.Fprintf(&b, " 时间：%s\n", d) + } + if c := strings.TrimSpace(r.Content); c != "" { + fmt.Fprintf(&b, " 摘要：%s\n", oneLine(c, 240)) + } + } + return strings.TrimSpace(b.String()) +} + +func oneLine(s string, max int) string { + s = strings.Join(strings.Fields(s), " ") + r := []rune(s) + if len(r) > max { + return string(r[:max]) + "…" + } + return s +} + +// ======== 网页正文抽取 ======== + +var ( + reScript = regexp.MustCompile(`(?is)`) + reStyle = regexp.MustCompile(`(?is)`) + reNoscript = regexp.MustCompile(`(?is)`) + reSvg = regexp.MustCompile(`(?is)`) + reComment = regexp.MustCompile(`(?s)`) + reHead = regexp.MustCompile(`(?is)<(head|header|footer|nav|aside|form)\b.*?`) + reTag = regexp.MustCompile(`(?s)<[^>]+>`) + reTitleTag = regexp.MustCompile(`(?is)]*>(.*?)`) + reOgTitle = regexp.MustCompile(`(?is)]+property=["']og:title["'][^>]+content=["']([^"']+)["']`) + reArticle = regexp.MustCompile(`(?is)]*>(.*?)`) + reMain = regexp.MustCompile(`(?is)]*>(.*?)`) + reBody = regexp.MustCompile(`(?is)]*>(.*?)`) + reBlankLine = regexp.MustCompile(`\n{3,}`) +) + +// htmlToText 轻量正文抽取：不去依赖 readibility 库，够 agent 用即可 +func htmlToText(raw string, maxChars int) (title, text string) { + src := raw + if m := reTitleTag.FindStringSubmatch(src); len(m) > 1 { + title = strings.TrimSpace(html.UnescapeString(reTag.ReplaceAllString(m[1], " "))) + } + if m := reOgTitle.FindStringSubmatch(src); len(m) > 1 && title == "" { + title = strings.TrimSpace(html.UnescapeString(m[1])) + } + body := src + for _, re := range []*regexp.Regexp{reArticle, reMain} { + if m := re.FindStringSubmatch(src); len(m) > 1 && len(m[1]) > 200 { + body = m[1] + break + } + } + if body == src { + if m := reBody.FindStringSubmatch(src); len(m) > 1 { + body = m[1] + } + } + body = reComment.ReplaceAllString(body, " ") + body = reScript.ReplaceAllString(body, " ") + body = reStyle.ReplaceAllString(body, " ") + body = reNoscript.ReplaceAllString(body, " ") + body = reSvg.ReplaceAllString(body, " ") + body = reHead.ReplaceAllString(body, " ") + body = regexp.MustCompile(`(?i)`).ReplaceAllString(body, "\n") + body = regexp.MustCompile(`(?i)`).ReplaceAllString(body, "\n") + body = reTag.ReplaceAllString(body, " ") + body = html.UnescapeString(body) + lines := strings.Split(body, "\n") + kept := make([]string, 0, len(lines)) + for _, l := range lines { + l = strings.Join(strings.Fields(l), " ") + if l == "" { + continue + } + kept = append(kept, l) + } + text = strings.Join(kept, "\n") + text = reBlankLine.ReplaceAllString(text, "\n\n") + if maxChars > 0 { + r := []rune(text) + if len(r) > maxChars { + text = string(r[:maxChars]) + "\n…（已截断）" + } + } + return title, strings.TrimSpace(text) +} + +// fetchPage 抓取并转正文 +func (p *Plugin) fetchPage(rawURL string, maxChars int) (string, string, error) { + if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { + return "", "", fmt.Errorf("只支持 http/https URL：%s", rawURL) + } + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return "", "", fmt.Errorf("构造请求失败: %v", err) + } + req.Header.Set("User-Agent", p.userAgent) + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") + resp, err := p.http.Do(req) + if err != nil { + return "", "", fmt.Errorf("抓取失败（%s）：%v", rawURL, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return "", "", fmt.Errorf("抓取返回 HTTP %d：%s", resp.StatusCode, rawURL) + } + ct := resp.Header.Get("Content-Type") + if !strings.Contains(ct, "html") && !strings.Contains(ct, "text") && !strings.Contains(ct, "xml") { + return "", "", fmt.Errorf("不支持的内容类型 %q：%s", ct, rawURL) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return "", "", fmt.Errorf("读取响应失败: %v", err) + } + // 处理常见的中文站点 GBK 页面：只在检测到 charset=gb 时做最小映射 + body := string(raw) + if strings.Contains(strings.ToLower(ct), "gbk") || strings.Contains(strings.ToLower(ct), "gb2312") { + body = decodeGBKish(raw) + } + title, text := htmlToText(body, maxChars) + if text == "" { + return title, "", fmt.Errorf("正文为空（可能是需要 JS 渲染的页面，可改用浏览器工具）: %s", rawURL) + } + return title, text, nil +} + +// decodeGBKish 极简 GBK→UTF-8 兜底：只处理 ASCII 与常见区间，避免引入额外依赖。 +// 拿不准时保留原字节，宁可少转也不要把正文写坏。 +func decodeGBKish(raw []byte) string { + if utf8Valid(raw) { + return string(raw) + } + return strings.ToValidUTF8(string(raw), "\uFFFD") +} + +func utf8Valid(b []byte) bool { + for i := 0; i < len(b); { + c := b[i] + switch { + case c < 0x80: + i++ + case c&0xE0 == 0xC0: + if i+1 >= len(b) { + return false + } + i += 2 + case c&0xF0 == 0xE0: + if i+2 >= len(b) { + return false + } + i += 3 + case c&0xF8 == 0xF0: + if i+3 >= len(b) { + return false + } + i += 4 + default: + return false + } + } + return true +} + +// ======== 工具注册 ======== + +func schemas(props map[string]interface{}, required ...string) map[string]interface{} { + m := map[string]interface{}{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func pStr(desc string) map[string]interface{} { + return map[string]interface{}{"type": "string", "description": desc} +} + +func pInt(desc string) map[string]interface{} { + return map[string]interface{}{"type": "integer", "description": desc} +} + +func pBool(desc string) map[string]interface{} { + return map[string]interface{}{"type": "boolean", "description": desc} +} + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + p.sdk = s + s.SetAutoRestart(true) + + st := s.Settings() + st.RegisterDef(sdk.ConfigDef{ + Key: cfgSearxURL, Type: "string", Default: defaultSearxURL, + DisplayName: "SearXNG 地址", + Description: "本地 SearXNG 基地址（.60 上已有专用实例）。该实例必须开启 json 输出，否则请求会 403", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgMaxResults, Type: "int", Default: "8", Min: 1, Max: 50, + DisplayName: "默认返回条数", Description: "控制上下文体积", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgLanguage, Type: "string", Default: "zh-CN", + DisplayName: "默认语言", Description: "如 zh-CN / en-US / all", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgSafeSearch, Type: "int", Default: "0", Min: 0, Max: 2, + DisplayName: "安全搜索", Description: "0 关 / 1 中 / 2 严", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgTimeout, Type: "int", Default: "20", Min: 1, Max: 300, + DisplayName: "请求超时（秒）", Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgFetchMaxChars, Type: "int", Default: "4000", Min: 500, Max: 50000, + DisplayName: "正文截断长度", Description: "deepsearch_fetch 返回的正文上限（字符）", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgProxy, Type: "string", Default: "", + DisplayName: "抓取代理", Description: "可选，如 http://127.0.0.1:7890；仅影响本插件直连抓取，搜索本身由 SearXNG 侧出网", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgUserAgent, Type: "string", Default: defaultUA, + DisplayName: "User-Agent", Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgManageSearx, Type: "bool", Default: "true", + DisplayName: "托管 SearXNG", + Description: "开启后：插件启动时自动拉起搜索后端（docker compose up -d），插件停止时关闭它。关掉则假定后端由外部维护（如 systemd）", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgSearxDir, Type: "string", Default: defaultSearxDir, + DisplayName: "SearXNG compose 目录", + Description: "托管时在该目录执行 docker compose up -d / stop", + Category: "deepsearch", + }) + st.RegisterDef(sdk.ConfigDef{ + Key: cfgStopOnExit, Type: "bool", Default: "true", + DisplayName: "停止时关闭后端", + Description: "关闭插件时一并关闭 SearXNG。若插件频繁重载而想避免反复重启后端，可设为 false", + Category: "deepsearch", + }) + + p.ensure() + + // 生命周期托管：启动时拉起搜索后端，并把关闭动作注册为 stop handler。 + // 依据内核契约：plugin.stop 会先跑 StopHandlers（LIFO、幂等）再调 Stop()， + // 宽限期 5 秒 —— 所以 shutdownSearxng 内部限时 4 秒。 + p.ensureSearxng() + s.RegisterStopHandler(func() { p.shutdownSearxng() }) + + p.registerTools() + log.Printf("[%s] started (searxng=%s manage=%v)", p.name, p.searxURL, p.manageSearx) + return nil +} + +func (p *Plugin) registerTools() { + tp := p.name + "_" + + p.sdk.RegisterTool(tp+"search", sdk.ToolDef{ + Name: tp + "search", + Description: "联网检索（首选工具）：经本地 SearXNG 聚合多个搜索引擎，返回带标题、URL、摘要、发布时间的结果。" + + "需要事实、新闻、文档、报错信息时用它，而不是抓搜索引擎页面", + Parameters: schemas(map[string]interface{}{ + "query": pStr("检索词；中文/英文都可"), + "count": pInt("条数，默认取插件配置（8）"), + "engines": pStr("指定引擎，逗号分隔（如 duckduckgo,brave,quark）；留空用默认聚合"), + "category": pStr("类别：general（默认）| news | it | science | images"), + "time_range": pStr("时间范围：day|week|month|year（新闻类很有用）"), + "language": pStr("语言，如 all / zh-CN / en-US；留空用插件配置"), + "raw": pBool("true 则返回 SearXNG 原始 JSON（排查用）"), + }, "query"), + }, p.handleSearch) + + p.sdk.RegisterTool(tp+"news", sdk.ToolDef{ + Name: tp + "news", + Description: "新闻检索：等价于 search 的 news 类别，默认按最近一周过滤，并保留发布时间", + Parameters: schemas(map[string]interface{}{ + "query": pStr("检索词"), + "count": pInt("条数"), + "time_range": pStr("day|week|month|year（默认 week）"), + "language": pStr("语言，默认插件配置"), + }, "query"), + }, p.handleNews) + + p.sdk.RegisterTool(tp+"fetch", sdk.ToolDef{ + Name: tp + "fetch", + Description: "抓取单个网页并抽取正文（去脚本/样式/导航），返回标题 + 纯文本，便于精读某条结果", + Parameters: schemas(map[string]interface{}{ + "url": pStr("目标 URL（http/https）"), + "max_chars": pInt("正文上限字符数，默认取插件配置"), + }, "url"), + }, p.handleFetch) + + p.sdk.RegisterTool(tp+"deep", sdk.ToolDef{ + Name: tp + "deep", + Description: "深检索：先检索、再并行抓取前 K 篇正文，一次性返回「候选清单 + 证据正文」。" + + "适合需要事实核对、多来源交叉的问题——比反复换词重搜有效得多", + Parameters: schemas(map[string]interface{}{ + "query": pStr("检索词"), + "count": pInt("候选条数（默认 8）"), + "top_k": pInt("抓取前几篇正文（默认 3，最多 6）"), + "max_chars": pInt("每篇正文上限字符数（默认 2000）"), + "time_range": pStr("时间范围：day|week|month|year"), + "language": pStr("语言"), + }, "query"), + }, p.handleDeep) + + p.sdk.RegisterTool(tp+"status", sdk.ToolDef{ + Name: tp + "status", + Description: "自检：SearXNG 是否可达、json 输出是否开启、哪些引擎真正在返回结果（检索出问题时第一步）", + Parameters: schemas(map[string]interface{}{ + "probe": pStr("用于探测的查询词（默认 test）"), + }), + }, p.handleStatus) +} + +// ======== 处理器 ======== + +func (p *Plugin) buildParams(args map[string]interface{}, forceCategory string, forceRange string) url.Values { + v := url.Values{} + v.Set("q", argStr(args, "query")) + if n := argInt(args, "count", p.maxItems); n > 0 { + v.Set("limit", strconv.Itoa(n)) // SearXNG 用 limit 控制返回条数 + } + if e := argStr(args, "engines"); e != "" { + v.Set("engines", e) + } + cat := forceCategory + if cat == "" { + cat = argStr(args, "category") + } + if cat != "" { + v.Set("categories", cat) + } + if forceRange == "" { + forceRange = argStr(args, "time_range") + } + if forceRange != "" { + v.Set("time_range", forceRange) + } + lang := argStr(args, "language") + if lang == "" { + lang = p.language + } + if lang != "" { + v.Set("language", lang) + } + v.Set("safesearch", strconv.Itoa(p.safeLevel)) + v.Set("pageno", "1") + return v +} + +func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) { + p.ensure() + query := argStr(args, "query") + if query == "" { + return nil, fmt.Errorf("缺少参数 query") + } + start := time.Now() + resp, err := p.searxQuery(p.buildParams(args, "", "")) + if err != nil { + return nil, err + } + if argBool(args, "raw") { + return resp, nil + } + results := dedupResults(resp.Results) + if len(results) == 0 { + return map[string]interface{}{"content": p.emptyHint(query, resp)}, nil + } + txt := formatResults(query, results, resp.UnresponsiveEngine, time.Since(start)) + if len(resp.Answers) > 0 { + txt = "直接答案：" + strings.Join(resp.Answers, "；") + "\n\n" + txt + } + return map[string]interface{}{"content": txt}, nil +} + +func (p *Plugin) emptyHint(query string, resp *searxResponse) string { + var b strings.Builder + fmt.Fprintf(&b, "检索 %q 未返回结果。", query) + if len(resp.UnresponsiveEngine) > 0 { + parts := make([]string, 0, len(resp.UnresponsiveEngine)) + for _, e := range resp.UnresponsiveEngine { + if len(e) >= 2 { + parts = append(parts, e[0]+"("+e[1]+")") + } + } + fmt.Fprintf(&b, " 无响应的引擎：%s。", strings.Join(parts, ", ")) + } + if len(resp.Suggestions) > 0 { + fmt.Fprintf(&b, " 建议改用：%s。", strings.Join(resp.Suggestions, " / ")) + } + b.WriteString(" 可以换更短的关键词、去掉过于具体的限定，或改用 deepsearch_news 查新闻。") + return b.String() +} + +func (p *Plugin) handleNews(args map[string]interface{}) (interface{}, error) { + p.ensure() + query := argStr(args, "query") + if query == "" { + return nil, fmt.Errorf("缺少参数 query") + } + if argStr(args, "time_range") == "" { + args["time_range"] = "week" + } + start := time.Now() + resp, err := p.searxQuery(p.buildParams(args, "news", "")) + if err != nil { + return nil, err + } + results := dedupResults(resp.Results) + if len(results) == 0 { + // 新闻类别颗粒度粗时退回 general + 时间范围 + resp2, err2 := p.searxQuery(p.buildParams(args, "", "")) + if err2 == nil && len(resp2.Results) > 0 { + results = dedupResults(resp2.Results) + resp = resp2 + } + } + if len(results) == 0 { + return map[string]interface{}{"content": p.emptyHint(query, resp)}, nil + } + return map[string]interface{}{"content": formatResults(query+"（新闻）", results, resp.UnresponsiveEngine, time.Since(start))}, nil +} + +func (p *Plugin) handleFetch(args map[string]interface{}) (interface{}, error) { + p.ensure() + raw := argStr(args, "url") + if raw == "" { + return nil, fmt.Errorf("缺少参数 url") + } + maxChars := argInt(args, "max_chars", p.fetchMax) + title, text, err := p.fetchPage(raw, maxChars) + if err != nil { + return nil, err + } + var b strings.Builder + if title != "" { + fmt.Fprintf(&b, "标题：%s\n来源：%s\n\n", title, raw) + } else { + fmt.Fprintf(&b, "来源：%s\n\n", raw) + } + b.WriteString(text) + return map[string]interface{}{"content": strings.TrimSpace(b.String())}, nil +} + +func (p *Plugin) handleDeep(args map[string]interface{}) (interface{}, error) { + p.ensure() + query := argStr(args, "query") + if query == "" { + return nil, fmt.Errorf("缺少参数 query") + } + topK := argInt(args, "top_k", 3) + if topK < 1 { + topK = 1 + } + if topK > 6 { + topK = 6 + } + perDoc := argInt(args, "max_chars", 2000) + if perDoc <= 0 { + perDoc = 2000 + } + start := time.Now() + resp, err := p.searxQuery(p.buildParams(args, "", "")) + if err != nil { + return nil, err + } + results := dedupResults(resp.Results) + if len(results) == 0 { + return map[string]interface{}{"content": p.emptyHint(query, resp)}, nil + } + + type doc struct { + idx int + title string + url string + text string + err string + } + pick := results + if len(pick) > topK { + pick = pick[:topK] + } + docs := make([]doc, len(pick)) + var wg sync.WaitGroup + sem := make(chan struct{}, 3) + for i, r := range pick { + wg.Add(1) + go func(i int, r searxResult) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + title, text, err := p.fetchPage(r.URL, perDoc) + d := doc{idx: i, title: r.Title, url: r.URL, text: text} + if err != nil { + d.err = err.Error() + } else if title != "" { + d.title = title + } + docs[i] = d + }(i, r) + } + wg.Wait() + + var b strings.Builder + fmt.Fprintf(&b, "深检索 %q —— 候选 %d 条，已读 %d 篇（%s）\n\n", query, len(results), len(docs), time.Since(start).Round(time.Millisecond)) + b.WriteString("【候选清单】\n") + for i, r := range results { + fmt.Fprintf(&b, "%d. %s\n %s\n", i+1, strings.TrimSpace(r.Title), r.URL) + if c := strings.TrimSpace(r.Content); c != "" { + fmt.Fprintf(&b, " 摘要：%s\n", oneLine(c, 200)) + } + } + b.WriteString("\n【正文证据】\n") + okCount := 0 + for _, d := range docs { + fmt.Fprintf(&b, "\n--- [%d] %s\n%s\n", d.idx+1, d.title, d.url) + if d.err != "" { + fmt.Fprintf(&b, "（抓取失败，可改用 browser_navigate/browser_render 处理需要 JS 的页面：%s）\n", d.err) + continue + } + okCount++ + b.WriteString(d.text) + b.WriteString("\n") + } + if okCount == 0 { + b.WriteString("\n（正文全部抓取失败：这批站点可能需要 JS 渲染或被反爬拦截，建议换来源或改用浏览器工具）\n") + } + return map[string]interface{}{"content": strings.TrimSpace(b.String())}, nil +} + +func (p *Plugin) handleStatus(args map[string]interface{}) (interface{}, error) { + p.ensure() + out := map[string]interface{}{ + "searxng_url": p.searxURL, + "max_results": p.maxItems, + "language": p.language, + "proxy": cfgStr(p.settings(), cfgProxy, ""), + } + // 1) 健康检查 + if req, err := http.NewRequest(http.MethodGet, p.searxURL+"/healthz", nil); err == nil { + req.Header.Set("User-Agent", p.userAgent) + if resp, err := p.http.Do(req); err == nil { + out["healthz"] = resp.StatusCode + _ = resp.Body.Close() + } else { + out["healthz_error"] = err.Error() + } + } + // 2) 实测一次检索：这比只看端口有意义 + probe := argStr(args, "probe") + if probe == "" { + probe = "test" + } + v := url.Values{} + v.Set("q", probe) + v.Set("limit", "10") + start := time.Now() + resp, err := p.searxQuery(v) + if err != nil { + out["search_error"] = err.Error() + out["hint"] = "若为 403：检查 SearXNG 的 search.formats 是否含 json、server.limiter 是否为 false" + return out, nil + } + out["search_ok"] = true + out["latency_ms"] = time.Since(start).Milliseconds() + out["result_count"] = len(resp.Results) + engs := map[string]int{} + for _, r := range resp.Results { + for _, e := range r.engineNames() { + engs[e]++ + } + } + out["engines_returning_results"] = engs + if len(resp.UnresponsiveEngine) > 0 { + out["unresponsive_engines"] = resp.UnresponsiveEngine + } + return out, nil +} + +func (p *Plugin) settings() sdk.SettingsAPI { + if p.sdk == nil { + return nil + } + return p.sdk.Settings() +} diff --git a/example/deepsearch/plugin_test.go b/example/deepsearch/plugin_test.go new file mode 100644 index 0000000..e058b4a --- /dev/null +++ b/example/deepsearch/plugin_test.go @@ -0,0 +1,288 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func newTestPlugin(t *testing.T, h http.HandlerFunc) (*Plugin, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + p := &Plugin{ + name: "deepsearch", + searxURL: srv.URL, + maxItems: 5, + language: "zh-CN", + fetchMax: 1000, + userAgent: "test-agent", + http: srv.Client(), + } + return p, srv +} + +// 一份贴近真实 SearXNG 的响应：含重复 URL、缺摘要、多引擎、无响应引擎 +const sampleResponse = `{ + "query": "deepin 被开除", + "results": [ + {"url":"https://www.zhihu.com/question/1?utm_source=x","title":"网传统信内核开发工程师因没穿西服被开除","content":"截止1月9日最新情况…","engines":["duckduckgo","brave"],"score":9.5,"publishedDate":"2026-09-10T00:00:00"}, + {"url":"https://www.zhihu.com/question/1","title":"网传统信内核开发工程师因没穿西服被开除（重复项）","content":"重复条目","engines":["brave"],"score":1.0}, + {"url":"https://www.163.com/dy/article/KIQURODQ.html","title":"离谱！传某信创操作系统大厂因西装开除核心开发者","content":"一位负责Linux内核开发的核心工程师…","engines":["brave","quark"],"score":7.2}, + {"url":"https://bbs.deepin.org.cn/zh","title":"deepin官方论坛","content":"","engines":["duckduckgo"],"score":2.0} + ], + "answers": [], + "suggestions": ["deepin 王勇 离职"], + "unresponsive_engines": [["baidu","CAPTCHA"],["sogou","unexpected crash"]], + "timings": {"search": 1.2} +}` + +// 1) 检索：去重 + 按分数排序 + 摘要/覆盖度输出 +func TestSearchDedupAndFormat(t *testing.T) { + var gotQuery url.Values + p, _ := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/search" { + gotQuery = r.URL.Query() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(sampleResponse)) + return + } + http.NotFound(w, r) + }) + res, err := p.handleSearch(map[string]interface{}{"query": "deepin 被开除", "count": float64(5)}) + if err != nil { + t.Fatalf("err: %v", err) + } + if gotQuery.Get("format") != "json" { + t.Errorf("必须要求 json 输出，实际 %q", gotQuery.Get("format")) + } + if gotQuery.Get("limit") != "5" { + t.Errorf("limit 未生效: %q", gotQuery.Get("limit")) + } + txt := res.(map[string]interface{})["content"].(string) + // utm_source 应被规范化掉，重复项只剩一条 + if n := strings.Count(txt, "zhihu.com/question/1"); n != 1 { + t.Errorf("URL 未正确去重（出现 %d 次）:\n%s", n, txt) + } + if !strings.Contains(txt, "网传统信内核开发工程师") { + t.Errorf("缺少标题: %s", txt) + } + if !strings.Contains(txt, "摘要：") { + t.Errorf("应输出摘要: %s", txt) + } + if !strings.Contains(txt, "baidu(CAPTCHA)") { + t.Errorf("应回报无响应引擎（让模型知道覆盖度）: %s", txt) + } + if !strings.Contains(txt, "duckduckgo") || !strings.Contains(txt, "quark") { + t.Errorf("应回报引擎覆盖: %s", txt) + } + // 高分条目应排在前面 + if strings.Index(txt, "统信内核开发工程师") > strings.Index(txt, "离谱！") { + t.Errorf("未按分数排序:\n%s", txt) + } +} + +// 2) 403（未开 json）必须给出可操作提示，而不是裸错误 +func TestSearchForbiddenHint(t *testing.T) { + p, _ := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte("Forbidden")) + }) + _, err := p.handleSearch(map[string]interface{}{"query": "x"}) + if err == nil { + t.Fatal("应返回错误") + } + msg := err.Error() + if !strings.Contains(msg, "403") || !strings.Contains(msg, "formats") { + t.Errorf("403 提示应指向 json/limiter 配置，实际: %s", msg) + } +} + +// 3) 空结果：要给出原因与下一步建议 +func TestSearchEmptyHint(t *testing.T) { + p, _ := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"query":"x","results":[],"suggestions":["换个词"],"unresponsive_engines":[["google","CAPTCHA"]]}`)) + }) + res, err := p.handleSearch(map[string]interface{}{"query": "x"}) + if err != nil { + t.Fatalf("err: %v", err) + } + txt := res.(map[string]interface{})["content"].(string) + for _, want := range []string{"未返回结果", "google(CAPTCHA)", "换个词", "deepsearch_news"} { + if !strings.Contains(txt, want) { + t.Errorf("空结果提示缺少 %q: %s", want, txt) + } + } +} + +// 4) 新闻：应带 categories=news 与 time_range=week；新闻为空时回退 general +func TestNewsParamsAndFallback(t *testing.T) { + var calls []url.Values + p, _ := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.URL.Query()) + if r.URL.Query().Get("categories") == "news" { + _, _ = w.Write([]byte(`{"query":"n","results":[]}`)) + return + } + _, _ = w.Write([]byte(`{"query":"n","results":[{"url":"https://a.com/1","title":"回退结果","content":"内容","engines":["brave"],"score":1}]}`)) + }) + res, err := p.handleNews(map[string]interface{}{"query": "某事"}) + if err != nil { + t.Fatalf("err: %v", err) + } + if len(calls) != 2 { + t.Fatalf("新闻为空时应回退 general，实际调用 %d 次", len(calls)) + } + if calls[0].Get("categories") != "news" || calls[0].Get("time_range") != "week" { + t.Errorf("首次应为 news + week，实际 categories=%q time_range=%q", calls[0].Get("categories"), calls[0].Get("time_range")) + } + if tmp := res.(map[string]interface{})["content"].(string); !strings.Contains(tmp, "回退结果") { + t.Errorf("回退结果未被采用: %s", tmp) + } +} + +// 5) 正文抽取：去脚本/样式/导航，保留 article +func TestFetchExtractsArticle(t *testing.T) { + page := `测试标题 - 站点 + +

+

第一段正文，包含关键事实。

第二段正文。

+
页脚
` + var srvURL string + p, srv := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(page)) + }) + srvURL = srv.URL + // 注意：不要用 example.com 之类真实域名——本机 DNS/proxy 会把它们转走，测试会飘 + res, err := p.handleFetch(map[string]interface{}{"url": srvURL + "/a"}) + if err != nil { + t.Fatalf("err: %v", err) + } + txt := res.(map[string]interface{})["content"].(string) + if !strings.Contains(txt, "第一段正文") { + t.Errorf("正文丢失: %s", txt) + } + if strings.Contains(txt, "SHOULD_NOT_APPEAR") { + t.Errorf("脚本内容不应出现: %s", txt) + } + if strings.Contains(txt, "导航链接") || strings.Contains(txt, "页脚") { + t.Errorf("导航/页脚应被剥离: %s", txt) + } + if !strings.Contains(txt, "测试标题") { + t.Errorf("标题应被提取: %s", txt) + } +} + +// 6) 深检索：候选 + 正文证据；单篇失败不应导致整体失败 +func TestDeepSearch(t *testing.T) { + var srvURL string // 处理函数先于 server 存在，故用闭包变量回填 + p, srv := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/search": + _, _ = w.Write([]byte(`{"query":"d","results":[ + {"url":"` + srvURL + `/ok1","title":"好文一","content":"摘要一","engines":["brave"],"score":3}, + {"url":"` + srvURL + `/bad","title":"打不开的","content":"摘要二","engines":["brave"],"score":2}, + {"url":"` + srvURL + `/ok2","title":"好文二","content":"摘要三","engines":["brave"],"score":1}]}`)) + case "/ok1", "/ok2": + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("

正文内容 " + r.URL.Path + "

")) + case "/bad": + w.WriteHeader(http.StatusForbidden) + default: + http.NotFound(w, r) + } + }) + srvURL = srv.URL + res, err := p.handleDeep(map[string]interface{}{"query": "d", "top_k": float64(3), "max_chars": float64(500)}) + if err != nil { + t.Fatalf("err: %v", err) + } + txt := res.(map[string]interface{})["content"].(string) + for _, want := range []string{"候选清单", "正文证据", "正文内容 /ok1", "正文内容 /ok2", "抓取失败"} { + if !strings.Contains(txt, want) { + t.Errorf("深检索输出缺少 %q:\n%s", want, txt) + } + } +} + +// 7) 自检：健康检查 + 探测检索 + 引擎覆盖统计 +func TestStatusReportsEngines(t *testing.T) { + p, _ := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" { + _, _ = w.Write([]byte("OK")) + return + } + _, _ = w.Write([]byte(sampleResponse)) + }) + res, err := p.handleStatus(map[string]interface{}{}) + if err != nil { + t.Fatalf("err: %v", err) + } + m := res.(map[string]interface{}) + if m["healthz"] != 200 { + t.Errorf("healthz 应为 200，实际 %v", m["healthz"]) + } + if m["search_ok"] != true { + t.Errorf("search_ok 应为 true：%v", m["search_ok"]) + } + engs, ok := m["engines_returning_results"].(map[string]int) + if !ok || engs["brave"] == 0 || engs["quark"] == 0 { + t.Errorf("引擎统计不正确: %#v", m["engines_returning_results"]) + } +} + +// 8) 摘要压成一行并按字符截断（避免巨长摘要吃掉上下文） +func TestOneLineTruncate(t *testing.T) { + got := oneLine("第一行\n第二行\t第三行", 5) + if strings.Contains(got, "\n") { + t.Errorf("应为单行: %q", got) + } + if r := []rune(got); len(r) != 6 { // 5 字符 + 省略号 + t.Errorf("截断长度不符: %q (%d runes)", got, len(r)) + } +} + +// 9) 正文抽取长度上限生效 +func TestHtmlToTextTruncation(t *testing.T) { + long := strings.Repeat("字", 5000) + _, text := htmlToText("

"+long+"

", 100) + if !strings.Contains(text, "已截断") { + t.Errorf("超长正文应被截断: %d", len([]rune(text))) + } +} + +// 10) 非 http(s) 协议应被拒绝 +func TestFetchRejectsBadScheme(t *testing.T) { + p, _ := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) {}) + if _, err := p.handleFetch(map[string]interface{}{"url": "file:///etc/passwd"}); err == nil { + t.Fatal("file:// 应被拒绝") + } + if _, err := p.handleFetch(map[string]interface{}{"url": "javascript:alert(1)"}); err == nil { + t.Fatal("javascript: 应被拒绝") + } +} + +// 11) raw 模式返回结构化 JSON（排查用） +func TestSearchRawMode(t *testing.T) { + p, _ := newTestPlugin(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(sampleResponse)) + }) + res, err := p.handleSearch(map[string]interface{}{"query": "q", "raw": true}) + if err != nil { + t.Fatalf("err: %v", err) + } + m, ok := res.(*searxResponse) + if !ok { + t.Fatalf("raw 应返回结构化响应，实际 %T", res) + } + if len(m.Results) != 4 { + t.Errorf("结果数应为 4（raw 不去重），实际 %d", len(m.Results)) + } + if _, err := json.Marshal(m); err != nil { + t.Errorf("结构化结果应可序列化: %v", err) + } +} diff --git a/example/deepsearch/searxng.go b/example/deepsearch/searxng.go new file mode 100644 index 0000000..71e58f4 --- /dev/null +++ b/example/deepsearch/searxng.go @@ -0,0 +1,159 @@ +package main + +// SearXNG 生命周期托管：插件启动时拉起搜索后端，插件停止时关闭它。 +// +// 契约依据（内核侧 internal/plugin/proc/*，已逐行核对）： +// - 内核停止插件：发 `plugin.stop` → 插件先跑 RunStopHandlers（LIFO、幂等）→ 再 Stop() → exit(0) +// - 若插件未在 stopGracePeriod（**5 秒**）内退出，内核直接 SIGKILL +// - stdin 关闭（内核消失）同样会跑 handlers + Stop() +// +// 因此这里的关闭动作必须**有界**：searxShutdownBudget 取 4s，留 1s 余量。 +// 若插件是被 kill -9 / OOM 带走的，关闭动作不会执行 —— SearXNG 会留在运行态； +// 下次 Start 探测到它在跑就直接接管，这是更安全的失败方向。 + +import ( + "context" + "log" + "net/http" + "os/exec" + "time" +) + +const ( + cfgManageSearx = "manage_searxng" + cfgSearxDir = "searxng_dir" + cfgStopOnExit = "stop_searxng_on_exit" + + defaultSearxDir = "/root/searxng-agent" + + searxProbeTimeout = 1500 * time.Millisecond // 单次 healthz 探测 + searxUpBudget = 20 * time.Second // docker compose up -d 的上限（正常 1s 内返回） + searxReadyBudget = 6 * time.Second // up 之后等 healthz 就绪的上限 + searxShutdownBudget = 4 * time.Second // 必须 < 内核 5s 宽限期 +) + +// searxBudget 把四个时间预算收拢，便于单测注入短值（否则测试要真等就绪窗口）。 +type searxBudget struct { + probe time.Duration + up time.Duration + ready time.Duration + shutdown time.Duration +} + +func (p *Plugin) budget() searxBudget { + b := p.bud + if b.probe == 0 { + b.probe = searxProbeTimeout + } + if b.up == 0 { + b.up = searxUpBudget + } + if b.ready == 0 { + b.ready = searxReadyBudget + } + if b.shutdown == 0 { + b.shutdown = searxShutdownBudget + } + return b +} + +// cmdRunner 抽出来是为了让生命周期逻辑可单测：注入假执行器，不起真容器。 +type cmdRunner func(ctx context.Context, dir, name string, args ...string) (string, error) + +func defaultRunner(ctx context.Context, dir, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + return string(out), err +} + +// searxReachable 探测搜索后端是否可用（只看 healthz，不发检索请求）。 +func (p *Plugin) searxReachable(timeout time.Duration) bool { + if p.searxURL == "" { + return false + } + base := p.http + if base == nil { + base = &http.Client{} + } + cl := *base // 复制一份，避免改到共享 client 的超时 + cl.Timeout = timeout + req, err := http.NewRequest(http.MethodGet, p.searxURL+"/healthz", nil) + if err != nil { + return false + } + req.Header.Set("User-Agent", p.userAgent) + resp, err := cl.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode < 400 +} + +// ensureSearxng 在插件启动时确保搜索后端在跑；已在跑则直接接管，不重启。 +func (p *Plugin) ensureSearxng() { + b := p.budget() + if !p.manageSearx { + log.Printf("[%s] 未启用 SearXNG 托管（manage_searxng=false），假定 %s 由外部维护", p.name, p.searxURL) + return + } + if p.searxReachable(b.probe) { + log.Printf("[%s] SearXNG 已在运行（%s），直接接管", p.name, p.searxURL) + p.markSearxOwned() + return + } + ctx, cancel := context.WithTimeout(context.Background(), b.up) + out, err := p.run(ctx, p.searxDir, "docker", "compose", "up", "-d") + cancel() + if err != nil { + log.Printf("[%s] 拉起 SearXNG 失败（dir=%s，请检查 manage_searxng/searxng_dir 配置）: %v；输出: %s", + p.name, p.searxDir, err, oneLine(out, 300)) + return + } + log.Printf("[%s] 已执行 docker compose up -d（%s）：%s", p.name, p.searxDir, oneLine(out, 200)) + + deadline := time.Now().Add(b.ready) + for time.Now().Before(deadline) { + if p.searxReachable(800 * time.Millisecond) { + log.Printf("[%s] SearXNG 就绪", p.name) + p.markSearxOwned() + return + } + time.Sleep(600 * time.Millisecond) + } + log.Printf("[%s] SearXNG 已启动但 %s 内未就绪；首次检索会自动等待", p.name, b.ready) + p.markSearxOwned() +} + +func (p *Plugin) markSearxOwned() { + p.searxMu.Lock() + p.searxOwned = true + p.searxMu.Unlock() +} + +// shutdownSearxng 关闭搜索后端。幂等，且有界（内核宽限期 5s，这里最多 4s）。 +func (p *Plugin) shutdownSearxng() { + b := p.budget() + p.searxMu.Lock() + owned := p.searxOwned + p.searxOwned = false + p.searxMu.Unlock() + + if !owned { + return // 不是我们拉起来的 / 已经关过 + } + if !p.manageSearx || !p.stopOnExit { + log.Printf("[%s] 保留 SearXNG 运行（stop_searxng_on_exit=false）", p.name) + return + } + ctx, cancel := context.WithTimeout(context.Background(), b.shutdown) + defer cancel() + out, err := p.run(ctx, p.searxDir, "docker", "compose", "stop", "-t", "2") + if err != nil { + // 故意只记日志：这里再重试就会拖过内核宽限期，被 SIGKILL 更糟 + log.Printf("[%s] 关闭 SearXNG 失败（忽略）: %v；输出: %s", p.name, err, oneLine(out, 200)) + return + } + log.Printf("[%s] 已关闭 SearXNG", p.name) +} diff --git a/example/deepsearch/searxng_test.go b/example/deepsearch/searxng_test.go new file mode 100644 index 0000000..57975c4 --- /dev/null +++ b/example/deepsearch/searxng_test.go @@ -0,0 +1,197 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +type fakeCall struct { + dir string + name string + args []string +} + +func (c fakeCall) String() string { return c.name + " " + strings.Join(c.args, " ") } + +// newFakeRunner 记录调用并返回预设结果 +func newFakeRunner(calls *[]fakeCall, out string, err error) cmdRunner { + var mu sync.Mutex + return func(ctx context.Context, dir, name string, args ...string) (string, error) { + mu.Lock() + *calls = append(*calls, fakeCall{dir: dir, name: name, args: args}) + mu.Unlock() + return out, err + } +} + +// fastBudget 把就绪窗口压到毫秒级，避免单测真等 +func fastBudget() searxBudget { + return searxBudget{ + probe: 50 * time.Millisecond, + up: time.Second, + ready: 200 * time.Millisecond, + shutdown: time.Second, + } +} + +// 1) 后端没跑 → 应执行 docker compose up -d，并认领关闭责任 +func TestEnsureSearxngStartsWhenUnreachable(t *testing.T) { + var calls []fakeCall + p := &Plugin{ + name: "deepsearch", searxURL: "http://127.0.0.1:1", searxDir: "/tmp/fake-searx", + manageSearx: true, stopOnExit: true, userAgent: "test", + bud: fastBudget(), run: newFakeRunner(&calls, "Container searxng-agent Started", nil), + } + p.ensureSearxng() + + if len(calls) != 1 { + t.Fatalf("应恰好拉起一次，实际 %d 次：%v", len(calls), calls) + } + got := calls[0] + if got.name != "docker" || strings.Join(got.args, " ") != "compose up -d" { + t.Errorf("命令不对：%s", got) + } + if got.dir != "/tmp/fake-searx" { + t.Errorf("工作目录应为配置的 compose 目录，实际 %q", got.dir) + } + if !p.searxOwned { + t.Error("既然是我们拉起的，就应认领关闭责任") + } +} + +// 2) 后端已在跑 → 不重启，直接接管 +func TestEnsureSearxngAdoptsRunningBackend(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" { + _, _ = w.Write([]byte("OK")) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + var calls []fakeCall + p := &Plugin{ + name: "deepsearch", searxURL: srv.URL, searxDir: "/tmp/fake-searx", + manageSearx: true, stopOnExit: true, userAgent: "test", + bud: fastBudget(), run: newFakeRunner(&calls, "", nil), + } + p.ensureSearxng() + + if len(calls) != 0 { + t.Errorf("已在跑就不该重启它，实际执行了：%v", calls) + } + if !p.searxOwned { + t.Error("接管后也应负责停止（与「不重启」不冲突）") + } +} + +// 3) 关掉托管 → 完全不碰 docker +func TestEnsureSearxngDisabled(t *testing.T) { + var calls []fakeCall + p := &Plugin{ + name: "deepsearch", searxURL: "http://127.0.0.1:1", searxDir: "/tmp/fake-searx", + manageSearx: false, stopOnExit: true, userAgent: "test", + bud: fastBudget(), run: newFakeRunner(&calls, "", nil), + } + p.ensureSearxng() + if len(calls) != 0 || p.searxOwned { + t.Errorf("manage_searxng=false 时不该有任何动作：calls=%v owned=%v", calls, p.searxOwned) + } +} + +// 4) 拉起失败不能让插件起不来（记日志即可） +func TestEnsureSearxngFailureNonFatal(t *testing.T) { + var calls []fakeCall + p := &Plugin{ + name: "deepsearch", searxURL: "http://127.0.0.1:1", searxDir: "/tmp/fake-searx", + manageSearx: true, stopOnExit: true, userAgent: "test", + bud: fastBudget(), run: newFakeRunner(&calls, "Cannot connect to the Docker daemon", errors.New("exit status 1")), + } + p.ensureSearxng() // 不应 panic + if p.searxOwned { + t.Error("没拉起来就不该认领关闭责任（否则停止时会去关一个不是我们起的服务）") + } +} + +// 5) 停止：关掉我们拉起的后端，且幂等 +func TestShutdownStopsOwnedBackend(t *testing.T) { + var calls []fakeCall + runner := newFakeRunner(&calls, "ok", nil) + p := &Plugin{ + name: "deepsearch", searxURL: "http://127.0.0.1:1", searxDir: "/tmp/fake-searx", + manageSearx: true, stopOnExit: true, userAgent: "test", + bud: fastBudget(), run: runner, + } + p.ensureSearxng() + calls = nil + + p.shutdownSearxng() + if len(calls) != 1 { + t.Fatalf("应执行一次 compose stop，实际 %v", calls) + } + if got := strings.Join(calls[0].args, " "); !strings.HasPrefix(got, "compose stop") { + t.Errorf("停止命令不对：%s", got) + } + if p.searxOwned { + t.Error("停止后应清掉认领标记") + } + + p.shutdownSearxng() // 幂等：不应再调一次 + if len(calls) != 1 { + t.Errorf("重复停止应无副作用，实际 %v", calls) + } +} + +// 6) 不是我们拉起的 → 停止时不许动它 +func TestShutdownSkippedWhenNotOwned(t *testing.T) { + var calls []fakeCall + p := &Plugin{ + name: "deepsearch", searxDir: "/tmp/fake-searx", manageSearx: true, stopOnExit: true, + bud: fastBudget(), run: newFakeRunner(&calls, "", nil), + } + p.shutdownSearxng() + if len(calls) != 0 { + t.Errorf("不该去停一个我们没起的服务：%v", calls) + } +} + +// 7) 配了「停止时保留」→ 认领过也不关 +func TestShutdownKeepsBackendWhenConfigured(t *testing.T) { + var calls []fakeCall + p := &Plugin{ + name: "deepsearch", searxURL: "http://127.0.0.1:1", searxDir: "/tmp/fake-searx", + manageSearx: true, stopOnExit: false, userAgent: "test", + bud: fastBudget(), run: newFakeRunner(&calls, "", nil), + } + p.ensureSearxng() + calls = nil + p.shutdownSearxng() + if len(calls) != 0 { + t.Errorf("stop_searxng_on_exit=false 时不应关闭：%v", calls) + } +} + +// 8) Stop() 自身也要收尾（内核 stdin 关闭路径不会走 stop handler 的注册顺序之外） +func TestStopTriggersShutdown(t *testing.T) { + var calls []fakeCall + p := &Plugin{ + name: "deepsearch", searxURL: "http://127.0.0.1:1", searxDir: "/tmp/fake-searx", + manageSearx: true, stopOnExit: true, userAgent: "test", + bud: fastBudget(), run: newFakeRunner(&calls, "", nil), + } + p.ensureSearxng() + calls = nil + if err := p.Stop(); err != nil { + t.Fatalf("Stop 返回错误: %v", err) + } + if len(calls) != 1 { + t.Errorf("Stop 应触发一次关闭，实际 %v", calls) + } +}