mirror of
https://gitcode.com/JianFeeeee/homeagent-sdk.git
synced 2026-09-20 00:48:12 +00:00
fix(browser): browser_search 解析现代 Bing 版式,并把解析失败显式报错
旧实现三处叠加,模型只拿到「标题=来源行 URL 串、无摘要」,表现为反复换词重搜: - www.bing.com 对程序化请求常回 302,拿不到结果块 → 改 cn.bing.com - 块内第一个 <a> 当标题 → 抓到来源行 `deepin.orghttps://www.deepin.org`;改取 h2 > a, 并解开 /ck/a?...&u=a1<base64url> 跳转包装 - 摘要正则 <div class="b_caption">.*?<p> 对现代版式 0 命中(已迁到 p.b_lineclamp*) - 分块不再用 (.*?)</li>:块内可能嵌套 <li>(deep links)会在错误位置截断 - 解析不出结果时明确报错,不再伪装成 "No results found." 夹具 testdata/bing_cn.html 为真实 cn.bing.com 响应裁剪;新增 5 项单测。 版本 2.4.0 → 2.4.1(2.4.0 = 交互式 timeout 改必填,此前已提交)。 验证:go test -race 全过、vet/gofmt 干净;线上实测返回真实标题+摘要+规整链接。
This commit is contained in:
@ -2,7 +2,7 @@
|
|||||||
"name": "browser",
|
"name": "browser",
|
||||||
"name_zh": "浏览器",
|
"name_zh": "浏览器",
|
||||||
"name_en": "Browser",
|
"name_en": "Browser",
|
||||||
"version": "2.4.0",
|
"version": "2.4.1",
|
||||||
"description": "统一浏览器插件:搜索、HTTP抓取(quick)、无头渲染(normal)、交互式浏览器(interactive/CDP)",
|
"description": "统一浏览器插件:搜索、HTTP抓取(quick)、无头渲染(normal)、交互式浏览器(interactive/CDP)",
|
||||||
"author": "HomeAgent",
|
"author": "HomeAgent",
|
||||||
"entry": "plugin.so",
|
"entry": "plugin.so",
|
||||||
|
|||||||
@ -6,6 +6,7 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"html"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net"
|
"net"
|
||||||
@ -484,7 +485,9 @@ type searchResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *Plugin) bingSearch(query string, count int) ([]searchResult, error) {
|
func (p *Plugin) bingSearch(query string, count int) ([]searchResult, error) {
|
||||||
u := fmt.Sprintf("https://www.bing.com/search?q=%s&count=%d", url.QueryEscape(query), count)
|
// 用 cn.bing.com:www.bing.com 对程序化请求常回 302(同意/重定向页),拿不到结果块。
|
||||||
|
// 另:Bing 忽略 count 参数,翻页靠 first=,这里保留 count 只为兼容旧调用语义。
|
||||||
|
u := fmt.Sprintf("https://cn.bing.com/search?q=%s&first=1&count=%d&setlang=zh-CN", url.QueryEscape(query), count)
|
||||||
req, _ := http.NewRequest("GET", u, nil)
|
req, _ := http.NewRequest("GET", u, nil)
|
||||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
||||||
@ -494,38 +497,112 @@ func (p *Plugin) bingSearch(query string, count int) ([]searchResult, error) {
|
|||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
body, _ := io.ReadAll(resp.Body)
|
body, _ := io.ReadAll(resp.Body)
|
||||||
return parseBingResults(string(body), count), nil
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("Bing 返回 HTTP %d(%d 字节)", resp.StatusCode, len(body))
|
||||||
|
}
|
||||||
|
results := parseBingResults(string(body), count)
|
||||||
|
if len(results) == 0 {
|
||||||
|
// 关键:把「解析不出来」与「真的没结果」区分开。
|
||||||
|
// 以前两者都变成 "No results found.",版式一变就静默退化成「搜不到」。
|
||||||
|
return nil, fmt.Errorf("Bing 返回 %d 字节但未解析出结果(可能被反爬或版式变更,可改用 deepsearch 插件)", len(body))
|
||||||
|
}
|
||||||
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseBingResults(html string, count int) []searchResult {
|
var (
|
||||||
|
bingBlockRe = regexp.MustCompile(`<li class="b_algo"`)
|
||||||
|
bingTitleRe = regexp.MustCompile(`(?s)<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>`)
|
||||||
|
bingAnyLinkRe = regexp.MustCompile(`(?s)<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>`)
|
||||||
|
bingSnipRe = regexp.MustCompile(`(?s)<p class="b_lineclamp[^"]*"[^>]*>(.*?)</p>`)
|
||||||
|
bingCaptionRe = regexp.MustCompile(`(?s)<div class="b_caption"[^>]*>(.*?)</div>`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// splitBingBlocks 按块标记切分,每块内容延伸到下一个块标记为止。
|
||||||
|
//
|
||||||
|
// 不用 `<li class="b_algo"(?s)(.*?)</li>`:结果块内部可能嵌套 <li>(deep links),
|
||||||
|
// 非贪婪匹配会在错误位置截断;而且块内第一个 <a> 往往是 Bing 的「来源行」,
|
||||||
|
// 取到的是 `deepin.orghttps://www.deepin.org` 这种垃圾标题。
|
||||||
|
func splitBingBlocks(pageHTML string) []string {
|
||||||
|
locs := bingBlockRe.FindAllStringIndex(pageHTML, -1)
|
||||||
|
if len(locs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
blocks := make([]string, 0, len(locs))
|
||||||
|
for i, loc := range locs {
|
||||||
|
end := len(pageHTML)
|
||||||
|
if i+1 < len(locs) {
|
||||||
|
end = locs[i+1][0]
|
||||||
|
}
|
||||||
|
blocks = append(blocks, pageHTML[loc[1]:end])
|
||||||
|
}
|
||||||
|
return blocks
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseBingResults(pageHTML string, count int) []searchResult {
|
||||||
|
if count <= 0 {
|
||||||
|
count = 5
|
||||||
|
}
|
||||||
var results []searchResult
|
var results []searchResult
|
||||||
re := regexp.MustCompile(`<li class="b_algo"(?s)(.*?)</li>`)
|
for _, block := range splitBingBlocks(pageHTML) {
|
||||||
matches := re.FindAllStringSubmatch(html, -1)
|
|
||||||
for _, m := range matches {
|
|
||||||
if len(results) >= count {
|
if len(results) >= count {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
block := m[1]
|
// 标题:现代 Bing 是 <h2><a href=...>标题</a></h2>;没有 h2 时才退回到块内第一个链接。
|
||||||
var r searchResult
|
var href, title string
|
||||||
hrefRe := regexp.MustCompile(`<a[^>]+href="([^"]+)"[^>]*>`)
|
if m := bingTitleRe.FindStringSubmatch(block); m != nil {
|
||||||
if hm := hrefRe.FindStringSubmatch(block); len(hm) > 1 {
|
href, title = m[1], html.UnescapeString(stripTags(m[2]))
|
||||||
r.URL = hm[1]
|
} else if m := bingAnyLinkRe.FindStringSubmatch(block); m != nil {
|
||||||
|
href, title = m[1], html.UnescapeString(stripTags(m[2]))
|
||||||
}
|
}
|
||||||
titleRe := regexp.MustCompile(`<a[^>]+href="[^"]+"[^>]*>(.*?)</a>`)
|
href = bingRealURL(html.UnescapeString(href))
|
||||||
if tm := titleRe.FindStringSubmatch(block); len(tm) > 1 {
|
|
||||||
r.Title = stripTags(tm[1])
|
// 摘要:新版在 p.b_lineclamp*,旧版在 div.b_caption > p
|
||||||
|
var snippet string
|
||||||
|
if m := bingSnipRe.FindStringSubmatch(block); m != nil {
|
||||||
|
snippet = html.UnescapeString(stripTags(m[1]))
|
||||||
|
} else if m := bingCaptionRe.FindStringSubmatch(block); m != nil {
|
||||||
|
snippet = html.UnescapeString(stripTags(m[1]))
|
||||||
}
|
}
|
||||||
snipRe := regexp.MustCompile(`<div class="b_caption">.*?<p>(.*?)</p>`)
|
|
||||||
if sm := snipRe.FindStringSubmatch(block); len(sm) > 1 {
|
title, snippet = strings.TrimSpace(title), strings.TrimSpace(snippet)
|
||||||
r.Snippet = stripTags(sm[1])
|
if href == "" || title == "" || !strings.HasPrefix(href, "http") {
|
||||||
}
|
continue
|
||||||
if r.URL != "" && r.Title != "" {
|
|
||||||
results = append(results, r)
|
|
||||||
}
|
}
|
||||||
|
results = append(results, searchResult{Title: title, URL: href, Snippet: snippet})
|
||||||
}
|
}
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bingRealURL 解开 Bing 的跳转包装:/ck/a?...&u=a1<base64url>&... → 真实 URL。
|
||||||
|
// 不解的话模型拿到的是 `https://cn.bing.com/ck/a?...` 这种不可读地址。
|
||||||
|
func bingRealURL(href string) string {
|
||||||
|
href = strings.TrimSpace(href)
|
||||||
|
if href == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if !strings.Contains(href, "/ck/a") && !strings.Contains(href, "u=a1") {
|
||||||
|
return href
|
||||||
|
}
|
||||||
|
u, err := url.Parse(href)
|
||||||
|
if err != nil {
|
||||||
|
return href
|
||||||
|
}
|
||||||
|
raw := u.Query().Get("u")
|
||||||
|
if !strings.HasPrefix(raw, "a1") {
|
||||||
|
return href
|
||||||
|
}
|
||||||
|
b64 := raw[2:]
|
||||||
|
for _, enc := range []*base64.Encoding{base64.RawURLEncoding, base64.URLEncoding, base64.RawStdEncoding} {
|
||||||
|
if dec, err := enc.DecodeString(b64); err == nil {
|
||||||
|
s := string(dec)
|
||||||
|
if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return href
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) {
|
func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) {
|
||||||
query := readArg(args, "query", "")
|
query := readArg(args, "query", "")
|
||||||
if query == "" {
|
if query == "" {
|
||||||
|
|||||||
@ -1,6 +1,10 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -60,3 +64,150 @@ func TestBrowserStartReuseResetsExplicitCloseTime(t *testing.T) {
|
|||||||
t.Fatalf("deadline not reset: createdAt=%v timeout=%v", s.createdAt, s.timeout)
|
t.Fatalf("deadline not reset: createdAt=%v timeout=%v", s.createdAt, s.timeout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Bing 解析器(2026-09 版式)─────────────────────────────
|
||||||
|
//
|
||||||
|
// 背景:旧实现把块内**第一个 <a>** 当标题 —— 拿到的是 Bing 的「来源行」
|
||||||
|
// `deepin.orghttps://www.deepin.org`;摘要正则 `<div class="b_caption">.*?<p>`
|
||||||
|
// 对现代 Bing 命中 0/N(摘要已迁到 p.b_lineclamp*),于是结果「有标题没摘要」,
|
||||||
|
// 模型只好反复换词重搜。夹具 testdata/bing_cn.html 是真实 cn.bing.com 响应裁剪。
|
||||||
|
|
||||||
|
func TestParseBingResultsRealBingHTML(t *testing.T) {
|
||||||
|
page, err := os.ReadFile("testdata/bing_cn.html")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("读取夹具失败: %v", err)
|
||||||
|
}
|
||||||
|
results := parseBingResults(string(page), 3)
|
||||||
|
if len(results) != 3 {
|
||||||
|
t.Fatalf("应解析出 3 条,实际 %d 条: %+v", len(results), results)
|
||||||
|
}
|
||||||
|
for i, r := range results {
|
||||||
|
if !strings.HasPrefix(r.URL, "http") {
|
||||||
|
t.Errorf("第 %d 条 URL 不是真实地址: %q", i+1, r.URL)
|
||||||
|
}
|
||||||
|
if strings.Contains(r.Title, "http") || strings.Contains(r.Title, "://") {
|
||||||
|
t.Errorf("第 %d 条标题混入了 URL(旧 bug 的典型症状): %q", i+1, r.Title)
|
||||||
|
}
|
||||||
|
if r.Snippet == "" {
|
||||||
|
t.Errorf("第 %d 条没有摘要(旧 bug 的典型症状): %+v", i+1, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 第一条必须与样本里的真实结果一致
|
||||||
|
if results[0].URL != "https://www.deepin.org/" {
|
||||||
|
t.Errorf("第一条 URL 应为 https://www.deepin.org/,实际 %q", results[0].URL)
|
||||||
|
}
|
||||||
|
if !strings.Contains(results[0].Title, "deepin") {
|
||||||
|
t.Errorf("第一条标题不对: %q", results[0].Title)
|
||||||
|
}
|
||||||
|
if len(results[0].Snippet) < 10 || strings.Contains(results[0].Snippet, "://") {
|
||||||
|
t.Errorf("第一条摘要不对(应是有内容的文本): %q", results[0].Snippet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 块内嵌套 <li>(deep links)时不能截断 —— 旧的 `<li class="b_algo"(?s)(.*?)</li>` 会在此翻车
|
||||||
|
func TestParseBingResultsNestedLiKeepsResult(t *testing.T) {
|
||||||
|
page := `<ol id="b_results"><li class="b_algo" data-id iid=SERP.1>` +
|
||||||
|
`<h2><a href="https://a.example/x" h="ID=SERP,1">真标题</a></h2>` +
|
||||||
|
`<div class="b_caption"><p class="b_lineclamp2">真摘要</p></div>` +
|
||||||
|
`<div><ul><li><a href="https://sub.example/deeplink">子链接</a></li></ul></div>` +
|
||||||
|
`</li><li class="b_algo"><h2><a href="https://b.example/y">第二条</a></h2>` +
|
||||||
|
`<p class="b_lineclamp3">摘要二</p></li></ol>`
|
||||||
|
rs := parseBingResults(page, 5)
|
||||||
|
if len(rs) != 2 {
|
||||||
|
t.Fatalf("应解析 2 条,实际 %d 条: %+v", len(rs), rs)
|
||||||
|
}
|
||||||
|
if rs[0].URL != "https://a.example/x" || rs[0].Title != "真标题" || rs[0].Snippet != "真摘要" {
|
||||||
|
t.Errorf("第一条解析错误: %+v", rs[0])
|
||||||
|
}
|
||||||
|
if rs[1].Title != "第二条" || rs[1].Snippet != "摘要二" {
|
||||||
|
t.Errorf("第二条(无 b_caption,摘要走 b_lineclamp3)解析错误: %+v", rs[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBingRealURLDecodesRedirectWrapper(t *testing.T) {
|
||||||
|
// Bing 跳转包装:/ck/a?...&u=a1<base64url>
|
||||||
|
wrapped := "/ck/a?!&&p=abc&u=a1aHR0cHM6Ly93d3cuZGVlcGluLm9yZy96aC9EZWVwaW4v&ntb=1"
|
||||||
|
if got := bingRealURL(wrapped); got != "https://www.deepin.org/zh/Deepin/" {
|
||||||
|
t.Errorf("未解开跳转包装: %q", got)
|
||||||
|
}
|
||||||
|
if got := bingRealURL("https://direct.example/p"); got != "https://direct.example/p" {
|
||||||
|
t.Errorf("直链不应被改动: %q", got)
|
||||||
|
}
|
||||||
|
// 解不开时保守返回原值,不能返回空
|
||||||
|
bad := "/ck/a?u=a1!!!!"
|
||||||
|
if got := bingRealURL(bad); got == "" {
|
||||||
|
t.Errorf("解不开时应保留原值,实际返回空")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// roundTripFunc 把任意请求转给本地测试服务器,从而离线测 bingSearch 的完整路径
|
||||||
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestBingSearchReportsParseFailureInsteadOfEmptyResult(t *testing.T) {
|
||||||
|
var seenURL string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte("<html><body>no result blocks here</body></html>"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := &Plugin{name: "browser", client: &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||||
|
seenURL = r.URL.String()
|
||||||
|
return srv.Client().Transport.RoundTrip(&http.Request{
|
||||||
|
Method: r.Method, URL: mustParseURL(t, srv.URL), Header: r.Header, Body: r.Body,
|
||||||
|
})
|
||||||
|
})}}
|
||||||
|
|
||||||
|
if _, err := p.bingSearch("任意查询", 5); err == nil {
|
||||||
|
t.Fatal("解析不出结果时必须报错,而不是伪装成「没有结果」")
|
||||||
|
} else if !strings.Contains(err.Error(), "未解析出结果") {
|
||||||
|
t.Errorf("错误信息应说明是解析失败: %v", err)
|
||||||
|
}
|
||||||
|
// 数据源必须是 cn.bing.com(www.bing.com 对程序化请求回 302,拿不到结果块)
|
||||||
|
if !strings.Contains(seenURL, "cn.bing.com") {
|
||||||
|
t.Errorf("应请求 cn.bing.com,实际 %q", seenURL)
|
||||||
|
}
|
||||||
|
if strings.Contains(seenURL, "www.bing.com") {
|
||||||
|
t.Errorf("不应再请求 www.bing.com: %q", seenURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 正常路径:能解析出结果时返回结果且不报错
|
||||||
|
func TestBingSearchParsesFixtureThroughClient(t *testing.T) {
|
||||||
|
page, err := os.ReadFile("testdata/bing_cn.html")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("读取夹具失败: %v", err)
|
||||||
|
}
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
_, _ = w.Write(page)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := &Plugin{name: "browser", client: &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||||
|
return srv.Client().Transport.RoundTrip(&http.Request{
|
||||||
|
Method: r.Method, URL: mustParseURL(t, srv.URL), Header: r.Header, Body: r.Body,
|
||||||
|
})
|
||||||
|
})}}
|
||||||
|
|
||||||
|
results, err := p.bingSearch("deepin", 2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("应成功,实际 %v", err)
|
||||||
|
}
|
||||||
|
if len(results) != 2 {
|
||||||
|
t.Fatalf("应返回 2 条(count 生效),实际 %d", len(results))
|
||||||
|
}
|
||||||
|
if results[0].Snippet == "" {
|
||||||
|
t.Errorf("摘要不应为空: %+v", results[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustParseURL(t *testing.T, raw string) *url.URL {
|
||||||
|
t.Helper()
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("解析测试 URL 失败: %v", err)
|
||||||
|
}
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
|||||||
1
example/browser/testdata/bing_cn.html
vendored
Normal file
1
example/browser/testdata/bing_cn.html
vendored
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user