diff --git a/Dockerfile b/Dockerfile index 5fb7da0..a6927dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -# syntax=docker/dockerfile:1 +# 不用 `# syntax=docker/dockerfile:1`:该指令会让 BuildKit 去 pull +# docker/dockerfile 前端镜像,在无外网/仅代理的环境里会直接构建失败。 +# 本文件未使用任何 BuildKit 专有语法,去掉后走内置前端即可。 # ── Stage 1: Go binaries ── FROM golang:1.21-bookworm AS gobuild diff --git a/README.md b/README.md index 431394e..af121f2 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,20 @@ icon: fas fa-star # Font Awesome - 每 30 秒检查本地队列,**仅在有新数据时 POST** - 搜索引擎 URL 在本地过滤,不上传 - 「常用服务」和「历史访问」按总访问频次自动排序 +- 历史条目的图标由服务端自动抓取并缓存(见下) + +### 历史条目的站点图标 + +history 里的站点不可能像手工导航项那样预先配好图标,所以服务端会**后台抓取每个 +host 的 favicon** 并缓存到 `data/item-icons/favicon-.png`,缓存命中后 +conf.yml 里写的是本地文件名,与手工图标走同一套解析。 + +- 抓取顺序:`/favicon.ico` → 首页 `` +- `.ico` 会先尝试抽出内嵌 PNG;若是传统 BMP/DIB 编码,则服务端解码并转成 PNG + (Dashy 的 `isImage()` 只认 `svg/png/jpg`,直接写 `.ico` 是渲染不出来的) +- 内网地址(含端口,如 `192.168.1.10:3000`)只走直连;外网站点先直连, + 失败时自动复用 SearXNG `outgoing.proxies` 的代理(也可用 `NAVI_FAVICON_PROXY` 单独指定) +- 抓不到就保持 `fas fa-globe`,6 小时后再试;缓存文件持久化在 data 卷 ## 管理后台 diff --git a/history-server/favicon.go b/history-server/favicon.go new file mode 100644 index 0000000..1e0c0ba --- /dev/null +++ b/history-server/favicon.go @@ -0,0 +1,658 @@ +package main + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "image" + "image/color" + "image/png" + "io" + "log" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "gopkg.in/yaml.v3" +) + +// ── 历史条目的站点图标 ── +// +// history 里的站点不可能像手工导航项那样预先配好图标,若统一写死 fa-globe +// 会很难看。这里在后台抓取每个 host 的 favicon,缓存到 data 卷的 item-icons/; +// 缓存命中后 conf.yml 里写的是本地文件名,与手工图标走同一套解析。 +// +// 关键约束:Dashy 的 isImage() 只把 svg/png/jpg 当图片渲染 +// (源码里的白名单就是 ["svg","png","jpg"]),其它扩展名写进 conf.yml +// 会渲染不出任何东西 —— 所以只缓存这三种;.ico 先尝试抽出内嵌 PNG。 +// +// 抓取顺序:/favicon.ico → 首页 指向的图标。 +// 每个 host 先试直连;仅当目标不是内网地址时才走代理。 +const ( + faviconMaxBytes = 512 << 10 + faviconTimeout = 6 * time.Second + faviconRetryGap = 6 * time.Hour + faviconUA = "Mozilla/5.0 (compatible; HomeDashFavicon/1.0)" +) + +// dashyIconExts 是 Dashy 能渲染的图标扩展名(顺序即查找优先级)。 +var dashyIconExts = []string{".png", ".svg", ".jpg"} + +var faviconProxy string + +var ( + faviconClients []*http.Client + faviconMu sync.Mutex + faviconQueued = map[string]bool{} + faviconAttempted = map[string]time.Time{} + faviconQueue = make(chan string, 128) + faviconSignal = make(chan struct{}, 1) + rebuildMu sync.Mutex +) + +func faviconDir() string { return filepath.Join(dataDir, "item-icons") } + +func newFaviconClient(proxy string) *http.Client { + tr := &http.Transport{ + MaxIdleConns: 8, + IdleConnTimeout: 30 * time.Second, + DisableKeepAlives: true, + TLSHandshakeTimeout: 5 * time.Second, + } + if proxy != "" { + if pu, err := url.Parse(proxy); err == nil { + tr.Proxy = http.ProxyURL(pu) + } + } + return &http.Client{ + Timeout: faviconTimeout, + Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 4 { + return errors.New("too many redirects") + } + return nil + }, + } +} + +func initFavicon() { + faviconProxy = detectFaviconProxy() + faviconClients = []*http.Client{newFaviconClient("")} + if faviconProxy != "" { + faviconClients = append(faviconClients, newFaviconClient(faviconProxy)) + log.Printf("favicon: proxy fallback enabled") + } + os.MkdirAll(faviconDir(), 0755) + for i := 0; i < 3; i++ { + go faviconWorker() + } + go faviconRebuildLoop() +} + +// detectFaviconProxy 优先用 NAVI_FAVICON_PROXY;未设置时直接复用 +// SearXNG 的 outgoing.proxies —— 需要代理才能出网的环境里, +// 不必为了图标再多配一个变量(也就不用重建容器)。 +func detectFaviconProxy() string { + if v := strings.TrimSpace(os.Getenv("NAVI_FAVICON_PROXY")); v != "" { + return v + } + paths := []string{os.Getenv("NAVI_SEARXNG_SETTINGS"), "/etc/searxng/settings.yml"} + for _, p := range paths { + if p == "" { + continue + } + data, err := os.ReadFile(p) + if err != nil { + continue + } + var cfg struct { + Outgoing struct { + Proxies map[string]string `yaml:"proxies"` + } `yaml:"outgoing"` + } + if err := yaml.Unmarshal(data, &cfg); err != nil { + continue + } + if v := strings.TrimSpace(cfg.Outgoing.Proxies["all://"]); v != "" { + return v + } + } + return "" +} + +// clientsFor 返回该 host 可用的客户端。内网/回环地址只直连, +// 避免每个内网服务都白等一次代理超时。 +func clientsFor(hostport string) []*http.Client { + host := hostport + if h, _, err := net.SplitHostPort(hostport); err == nil { + host = h + } + host = strings.Trim(host, "[]") + if ip := net.ParseIP(host); ip != nil && (ip.IsPrivate() || ip.IsLoopback()) { + return faviconClients[:1] + } + if !strings.Contains(host, ".") { + return faviconClients[:1] + } + return faviconClients +} + +// sanitizeHostKey 把 host:port 变成安全文件名片段。 +func sanitizeHostKey(hostport string) string { + var b strings.Builder + for _, r := range strings.ToLower(hostport) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + +// cachedFavicon 返回已缓存的图标文件名(相对 item-icons/),未命中返回空串。 +// 只认 Dashy 能渲染的格式,历史遗留的 .ico 缓存会被忽略并重新抓取。 +func cachedFavicon(hostport string) string { + if hostport == "" { + return "" + } + key := sanitizeHostKey(hostport) + for _, ext := range dashyIconExts { + name := "favicon-" + key + ext + if st, err := os.Stat(filepath.Join(faviconDir(), name)); err == nil && st.Size() > 0 { + return name + } + } + return "" +} + +// queueFavicon 把 host 排进后台抓取队列(幂等、非阻塞)。 +func queueFavicon(hostport string) { + if hostport == "" || cachedFavicon(hostport) != "" { + return + } + faviconMu.Lock() + if faviconQueued[hostport] { + faviconMu.Unlock() + return + } + if t, ok := faviconAttempted[hostport]; ok && time.Since(t) < faviconRetryGap { + faviconMu.Unlock() + return + } + faviconQueued[hostport] = true + faviconMu.Unlock() + + select { + case faviconQueue <- hostport: + default: + faviconMu.Lock() + delete(faviconQueued, hostport) + faviconMu.Unlock() + } +} + +func faviconWorker() { + for hostport := range faviconQueue { + err := fetchFavicon(hostport) + faviconMu.Lock() + delete(faviconQueued, hostport) + faviconAttempted[hostport] = time.Now() + faviconMu.Unlock() + if err != nil { + log.Printf("favicon: %s: %v", hostport, err) + continue + } + select { + case faviconSignal <- struct{}{}: + default: + } + } +} + +// faviconRebuildLoop 把短时间内完成的多个抓取合并成一次 conf.yml 重建。 +func faviconRebuildLoop() { + for range faviconSignal { + time.Sleep(1500 * time.Millisecond) + drain: + for { + select { + case <-faviconSignal: + default: + break drain + } + } + rebuildDashySections() + } +} + +func httpGet(rawURL string, max int, clients []*http.Client) ([]byte, string, error) { + var lastErr error + for _, c := range clients { + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return nil, "", err + } + req.Header.Set("User-Agent", faviconUA) + req.Header.Set("Accept", "*/*") + resp, err := c.Do(req) + if err != nil { + lastErr = err + continue + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + lastErr = fmt.Errorf("http %d", resp.StatusCode) + continue + } + data, err := io.ReadAll(io.LimitReader(resp.Body, int64(max)+1)) + ct := resp.Header.Get("Content-Type") + resp.Body.Close() + if err != nil { + lastErr = err + continue + } + if len(data) > max { + lastErr = errors.New("body too large") + continue + } + return data, ct, nil + } + if lastErr == nil { + lastErr = errors.New("no http client") + } + return nil, "", lastErr +} + +// httpGetTruncated 用于抓 HTML:只需要前若干字节里的 , +// 页面比上限大不应该算失败(GitHub 首页就超 512KB)。 +func httpGetTruncated(rawURL string, max int, clients []*http.Client) ([]byte, string, error) { + var lastErr error + for _, c := range clients { + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return nil, "", err + } + req.Header.Set("User-Agent", faviconUA) + req.Header.Set("Accept", "text/html,*/*;q=0.8") + resp, err := c.Do(req) + if err != nil { + lastErr = err + continue + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + lastErr = fmt.Errorf("http %d", resp.StatusCode) + continue + } + data, _ := io.ReadAll(io.LimitReader(resp.Body, int64(max))) + ct := resp.Header.Get("Content-Type") + resp.Body.Close() + return data, ct, nil + } + if lastErr == nil { + lastErr = errors.New("no http client") + } + return nil, "", lastErr +} + +var ( + iconLinkRe = regexp.MustCompile(`(?is)]*rel\s*=\s*["']?[^"'>]*icon[^"'>]*["']?[^>]*>`) + hrefRe = regexp.MustCompile(`(?is)href\s*=\s*["']([^"']+)["']`) +) + +// findIconHref 从首页 HTML 里挑出最合适的图标地址。 +func findIconHref(pageURL string, clients []*http.Client) (string, error) { + data, ct, err := httpGetTruncated(pageURL, 384<<10, clients) + if err != nil { + return "", err + } + lower := strings.ToLower(ct) + if ct != "" && !strings.Contains(lower, "html") { + return "", fmt.Errorf("not html: %s", ct) + } + best, bestScore := "", 0 + for _, tag := range iconLinkRe.FindAllString(string(data), -1) { + m := hrefRe.FindStringSubmatch(tag) + if len(m) < 2 { + continue + } + href := strings.TrimSpace(m[1]) + if href == "" || strings.HasPrefix(href, "data:") { + continue + } + low := strings.ToLower(tag) + score := 0 + switch { + case strings.Contains(low, "apple-touch-icon"): + score = 1 + case strings.Contains(low, "icon"): + score = 2 + } + // 同分时优先 Dashy 能渲染的格式 + if ext := lowerExtOf(href); ext != "" { + if _, ok := renderableByDashy(ext); ok { + score += 2 + } + } + if score > bestScore { + best, bestScore = href, score + } + } + if best == "" { + return "", errors.New("no icon link") + } + return best, nil +} + +func lowerExtOf(raw string) string { + if u, err := url.Parse(raw); err == nil { + return strings.ToLower(filepath.Ext(u.Path)) + } + return "" +} + +func renderableByDashy(ext string) (string, bool) { + switch ext { + case ".png", ".jpg", ".jpeg": + if ext == ".jpeg" { + return ".jpg", true + } + return ext, true + case ".svg": + return ".svg", true + } + return "", false +} + +func resolveURL(base, href string) string { + bu, err := url.Parse(base) + if err != nil { + return "" + } + hu, err := url.Parse(href) + if err != nil { + return "" + } + return bu.ResolveReference(hu).String() +} + +// sniffExt 只返回 Dashy 可渲染的扩展名,以及需要二次处理的 .ico。 +// 返回空串表示这份内容不能当图标用(例如 SPA 用 index.html 兜底的 200)。 +func sniffExt(data []byte, contentType, rawURL string) string { + ct := strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) + // 很多 SPA 对 /favicon.ico 也回 200 + text/html(把 index.html 吐回来), + // 这种必须先排掉,否则会存下一份 HTML 当图标。 + switch ct { + case "text/html", "application/xhtml+xml", "text/plain": + return "" + case "image/x-icon", "image/vnd.microsoft.icon": + return ".ico" + } + switch ct { + case "image/png": + return ".png" + case "image/jpeg": + return ".jpg" + case "image/svg+xml": + return ".svg" + } + + switch { + case bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")): + return ".png" + case bytes.HasPrefix(data, []byte("\xff\xd8\xff")): + return ".jpg" + case bytes.HasPrefix(data, []byte("\x00\x00\x01\x00")): + return ".ico" + } + + trimmed := bytes.TrimSpace(data) + if len(trimmed) > 0 && trimmed[0] == '<' { + head := bytes.ToLower(trimmed) + if len(head) > 512 { + head = head[:512] + } + // 必须根元素就是 = len(data) || int(start)+int(size) > len(data) { + continue + } + out = append(out, data[start:int(start)+int(size)]) + } + return out +} + +// pngFromICO 抽出 ICO 里内嵌的 PNG(现代 ICO 常常直接内嵌 PNG 数据)。 +func pngFromICO(data []byte) ([]byte, bool) { + var best []byte + for _, e := range icoEntries(data) { + if bytes.HasPrefix(e, []byte("\x89PNG\r\n\x1a\n")) && len(e) > len(best) { + best = e + } + } + if best == nil { + return nil, false + } + return best, true +} + +// dibToPNG 把 ICO 里的传统 DIB(BMP)位图转成 PNG。 +// 这样 Google 这类只给老式 .ico 的站点也能有图标 —— +// Dashy 不认 .ico,只能自己转。 +func dibToPNG(dib []byte) ([]byte, error) { + if len(dib) < 40 { + return nil, errors.New("short dib header") + } + hdrSize := int(binary.LittleEndian.Uint32(dib[0:4])) + if hdrSize < 40 || hdrSize > len(dib) { + return nil, errors.New("bad dib header size") + } + width := int(int32(binary.LittleEndian.Uint32(dib[4:8]))) + dblHeight := int(int32(binary.LittleEndian.Uint32(dib[8:12]))) + bitCount := int(binary.LittleEndian.Uint16(dib[14:16])) + compression := binary.LittleEndian.Uint32(dib[16:20]) + height := dblHeight / 2 // ICO 里的高度是实际高度的两倍(含 AND 掩码) + if width <= 0 || height <= 0 || width > 4096 || height > 4096 { + return nil, errors.New("bad dib dimensions") + } + if compression != 0 { + return nil, errors.New("compressed dib unsupported") + } + if bitCount != 32 && bitCount != 24 { + return nil, fmt.Errorf("unsupported dib bitcount %d", bitCount) + } + + bpp := bitCount / 8 + stride := ((width*bpp + 3) / 4) * 4 + pixels := dib[hdrSize:] + if len(pixels) < stride*height { + return nil, errors.New("short dib pixel data") + } + + // 有些老图标 alpha 全为 0;那样整张图会变全透明,此时按不透明处理。 + alphaUsed := false + if bpp == 4 { + for y := 0; y < height && !alphaUsed; y++ { + row := pixels[(height-1-y)*stride:] + for x := 0; x < width; x++ { + if row[x*4+3] != 0 { + alphaUsed = true + break + } + } + } + } + + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + row := pixels[(height-1-y)*stride:] // DIB 自下而上 + for x := 0; x < width; x++ { + i := x * bpp + b, g, r := row[i], row[i+1], row[i+2] + a := uint8(255) + if bpp == 4 && alphaUsed { + a = row[i+3] + } + img.SetNRGBA(x, y, color.NRGBA{R: r, G: g, B: b, A: a}) + } + } + + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// dibToPNGFromICO 逐个条目尝试解码,优先取像素数据最多的(分辨率最高)。 +func dibToPNGFromICO(data []byte) ([]byte, error) { + entries := icoEntries(data) + if len(entries) == 0 { + return nil, errors.New("no ico entries") + } + sort.Slice(entries, func(i, j int) bool { return len(entries[i]) > len(entries[j]) }) + var lastErr error + for _, e := range entries { + out, err := dibToPNG(e) + if err != nil { + lastErr = err + continue + } + return out, nil + } + if lastErr == nil { + lastErr = errors.New("no decodable ico entry") + } + return nil, lastErr +} + +func saveFavicon(hostport string, data []byte, ext string) error { + name := "favicon-" + sanitizeHostKey(hostport) + ext + dir := faviconDir() + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + dst := filepath.Join(dir, name) + tmp := dst + ".tmp" + if err := os.WriteFile(tmp, data, 0644); err != nil { + return err + } + return os.Rename(tmp, dst) +} + +// storeIcon 落盘;ICO 会先尝试抽内嵌 PNG,抽不出就放弃(避免存下 Dashy 渲染不了的格式)。 +func storeIcon(hostport string, data []byte, ext string) error { + if _, ok := renderableByDashy(ext); ok { + target, _ := renderableByDashy(ext) + return saveFavicon(hostport, data, target) + } + if ext == ".ico" { + if png, ok := pngFromICO(data); ok { + return saveFavicon(hostport, png, ".png") + } + pngData, err := dibToPNGFromICO(data) + if err != nil { + return fmt.Errorf("ico not usable: %w", err) + } + return saveFavicon(hostport, pngData, ".png") + } + return fmt.Errorf("unsupported icon type %q", ext) +} + +func fetchFavicon(hostport string) error { + if hostport == "" || strings.ContainsAny(hostport, "/\\ \t") { + return errors.New("invalid host") + } + clients := clientsFor(hostport) + var lastErr error + + for _, scheme := range []string{"https", "http"} { + base := scheme + "://" + hostport + icoURL := base + "/favicon.ico" + + if data, ct, err := httpGet(icoURL, faviconMaxBytes, clients); err == nil { + if ext := sniffExt(data, ct, icoURL); ext != "" { + if err := storeIcon(hostport, data, ext); err == nil { + return nil + } else { + lastErr = err + } + } else { + lastErr = errors.New("favicon.ico is not an image") + } + } else { + lastErr = err + } + + href, err := findIconHref(base+"/", clients) + if err != nil { + lastErr = err + continue + } + target := resolveURL(base+"/", href) + if target == "" { + continue + } + data, ct, err := httpGet(target, faviconMaxBytes, clients) + if err != nil { + lastErr = err + continue + } + ext := sniffExt(data, ct, target) + if ext == "" { + lastErr = errors.New("declared icon is not a usable image") + continue + } + if err := storeIcon(hostport, data, ext); err != nil { + lastErr = err + continue + } + return nil + } + return lastErr +} + +// hostPortOf 返回 host[:port](内网不同端口的服务要区分开)。 +func hostPortOf(rawURL string) string { + if u, err := url.Parse(rawURL); err == nil && u.Host != "" { + return strings.ToLower(u.Host) + } + return "" +} diff --git a/history-server/main.go b/history-server/main.go index 8c3168c..b69db7b 100644 --- a/history-server/main.go +++ b/history-server/main.go @@ -291,6 +291,10 @@ func extensionInstallItem() map[string]any { // 3) 「常用服务」= 访问最高的 4 个导航(Gitea/代码托管恒置顶) // 4) 「历史访问」= 非导航 host 的 top 20 func rebuildDashySections() { + // 序列化重建:同步请求、启动、以及 favicon 抓完后的重试都会调到这里。 + rebuildMu.Lock() + defer rebuildMu.Unlock() + confFile := filepath.Join(dataDir, "conf.yml") data, err := os.ReadFile(confFile) if err != nil { @@ -375,6 +379,7 @@ func rebuildDashySections() { }) // 导航项若均无访问记录(首次/纯外站访问),保持常用服务原手动配置不动 + updateFavorites := true { anyNav := false for _, c := range candidates { @@ -383,23 +388,9 @@ func rebuildDashySections() { break } } - if !anyNav { - // 尚无导航访问数据:保持「常用服务」的手工配置不动, - // 「历史访问」只放插件安装入口(装好后由 navi.js 隐藏)。 - for _, si := range secs { - if isHistSection(si.name) { - si.raw["items"] = []any{extensionInstallItem()} - } - } - out, err := yaml.Marshal(cfg) - if err == nil { - tmp := confFile + ".sort-tmp" - if err2 := os.WriteFile(tmp, out, 0644); err2 == nil { - os.Rename(tmp, confFile) - } - } - return - } + // 导航项都没有访问数据时不重排「常用服务」,保留手工配置; + // 但「历史访问」仍要正常生成 —— 只浏览站外站点时,那些历史恰恰是最该展示的。 + updateFavorites = anyNav } // host 去重:同 host 只保留访问最高的一条导航项 @@ -448,7 +439,7 @@ func rebuildDashySections() { // 更新各 section for _, si := range secs { - if isFavSection(si.name) { + if updateFavorites && isFavSection(si.name) { newItems := make([]any, 0, len(finalNavs)) for _, n := range finalNavs { it := map[string]any{ @@ -498,10 +489,18 @@ func rebuildDashySections() { if title == "" { title = e.URL } + icon := "fas fa-globe" + if hp := hostPortOf(e.URL); hp != "" { + if name := cachedFavicon(hp); name != "" { + icon = name // 与手工导航项同一套本地图标机制 + } else { + queueFavicon(hp) // 后台抓取,完成后会重建 conf.yml + } + } histItems = append(histItems, map[string]any{ "title": title, "description": fmt.Sprintf("%d\u6b21\u8bbf\u95ee", e.TotalVisits), - "icon": "fas fa-globe", "url": e.URL, + "icon": icon, "url": e.URL, }) if len(histItems) >= 21 { break @@ -552,6 +551,7 @@ func visitCount(rawURL string) int { func main() { os.MkdirAll(dataDir, 0755) load() + initFavicon() // 启动时重建一次:让「历史访问」首条的插件安装入口立即生效, // 而不必等到下一次浏览器插件同步。 rebuildDashySections()