diff --git a/admin-server/main.go b/admin-server/main.go index e7391c6..465286d 100644 --- a/admin-server/main.go +++ b/admin-server/main.go @@ -13,10 +13,23 @@ import ( ) var ( - confFile = envOr("DASHY_CONF", "/app/data/conf.yml") - mu sync.Mutex + confFile = envOr("DASHY_CONF", "/app/data/conf.yml") + adminToken = os.Getenv("NAVI_ADMIN_TOKEN") + mu sync.Mutex ) +func requireToken(w http.ResponseWriter, r *http.Request, expected string) bool { + if expected == "" { + return true // 未配置 token 则不校验 + } + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, "Bearer ") && strings.TrimPrefix(auth, "Bearer ") == expected { + return true + } + jsonErr(w, http.StatusUnauthorized, "invalid or missing token") + return false +} + type Config struct { AppConfig map[string]any `yaml:"appConfig"` PageInfo map[string]any `yaml:"pageInfo"` @@ -110,6 +123,9 @@ func handleAddItem(w http.ResponseWriter, r *http.Request) { jsonErr(w, 405, "method not allowed") return } + if !requireToken(w, r, adminToken) { + return + } mu.Lock() defer mu.Unlock() @@ -153,6 +169,9 @@ func handleDeleteItem(w http.ResponseWriter, r *http.Request) { jsonErr(w, 405, "method not allowed") return } + if !requireToken(w, r, adminToken) { + return + } mu.Lock() defer mu.Unlock() diff --git a/extension/README.md b/extension/README.md index 3a1694c..7c3593a 100644 --- a/extension/README.md +++ b/extension/README.md @@ -1,48 +1,48 @@ # JianF Nav Sync — 浏览器扩展 -将浏览历史同步到 JianF 统一导航门户，按总访问频次自动排序显示。 +将浏览历史同步到自建导航门户，按总访问频次自动排序「常用服务」与「历史访问」。 ## 安装 ### Chrome / Edge（MV3） 1. 打开 `chrome://extensions/` -2. 开启「开发者模式」 -3. 点击「加载已解压的扩展程序」，选择 `chrome/` 目录 +2. 开启右上角「开发者模式」 +3. 点「加载已解压的扩展程序」，选择本目录的 `chrome/` 文件夹 4. Edge 同理：`edge://extensions/` ### Firefox（MV2） 1. 打开 `about:debugging#/runtime/this-firefox` -2. 点击「临时加载附加组件」，选择 `firefox/manifest.json` -3. 如需永久安装：打包后通过 `about:addons` 安装 +2. 点「临时加载附加组件」，选择本目录 `firefox/` 下的 `manifest.json` +3. 长期使用：`web-ext build` 打包后到 `about:addons` 安装 -### iOS +### iOS / 手机浏览器 -Safari 不支持自定义扩展直接注入 `chrome.history` API。 -推荐方案：使用 Safari Web Extension（需 Xcode）或通过快捷指令定时拉取。 +系统浏览器扩展 API 不适用于 iOS Safari 普通网页。可用备选： +- iOS Safari Web Extension（需 Xcode 工程封装） +- Android 上使用 Kiwi / Yandex 等支持 MV3 的浏览器加载 `chrome/` -## 配置 - -点击扩展图标 → 设置： +## 配置（点击扩展图标） | 字段 | 说明 | |------|------| -| 服务地址 | `https://nav.example.com`（你自己的导航服务地址） | -| 同步令牌 | 服务端环境变量 `NAVI_SYNC_TOKEN` 的值，留空则不鉴权 | -| 设备名称 | 用于区分多端，如 `desktop-chrome`、`android-chrome` | +| 导航服务地址 | 你的门户根地址，如 `https://nav.example.com`（不要带 `/search` 等路径） | +| 同步令牌 | 服务端环境变量 `NAVI_SYNC_TOKEN` 的值（**服务端开启鉴权时必填**） | +| 设备名称 | 可选，留空自动检测（desktop-chrome / desktop-firefox / mobile-*） | ## 行为 -- 每 30 秒检查一次本地队列 -- **仅在有待同步数据时发送 POST 请求**（无空轮询） -- 搜索引擎 URL 在本地过滤，不上传 -- 同步失败时数据保留，下次重试 -- 导航栏「历史访问」分组按 `total_visits` 降序自动填充 +- **每 30 秒**用 `chrome.history.search` 拉取最近 2 小时新记录（不依赖 onVisited 事件，MV3 休眠也可靠） +- 本地过滤：搜索引擎域名、`/search?q=` 结果页、`chrome://`、`about:` 等一律**不上传** +- 同一网址 10 分钟内去重、同批 URL 去重 +- **仅在有待同步数据时发 POST**（无空轮询） +- 同步失败自动保留队列，下轮重试 +- 上报带 `Authorization: Bearer `；门户「常用服务/历史访问」按 `total_visits` 自动更新 ## 技术细节 -- Chrome/Edge: Manifest V3 (Service Worker) -- Firefox: Manifest V2 (Event Page) -- 存储: `chrome.storage.local` -- API: `chrome.history.onVisited` + `chrome.alarms` +- Chrome/Edge: Manifest V3，service worker + `chrome.alarms`(最小周期 30s) +- Firefox: Manifest V2，event page + `browser.alarms` +- 存储: `storage.local`（pending 队列 + 配置） +- API: `history.search` + `alarms` + `runtime.onMessage` diff --git a/extension/chrome/background.js b/extension/chrome/background.js index 3f789b3..bcee467 100644 --- a/extension/chrome/background.js +++ b/extension/chrome/background.js @@ -1,132 +1,198 @@ // JianF Nav Sync — Chrome/Edge MV3 service worker +// 定期用 chrome.history.search 拉取最近访问，本地过滤搜索引擎后批量上报。 'use strict'; const ALARM_NAME = 'navi-sync'; -const SYNC_INTERVAL_MINUTES = 0.5; // 30 seconds -const SEARCH_DOMAINS = new Set([ - 'google.com', 'www.google.com', 'baidu.com', 'www.baidu.com', - 'bing.com', 'www.bing.com', 'duckduckgo.com', 'www.duckduckgo.com', - 'sogou.com', 'www.sogou.com', 'so.com', 'www.so.com', - 'yahoo.com', 'www.yahoo.com', 'ask.com', 'www.ask.com', - 'ecosia.org', 'www.ecosia.org', 'qwant.com', 'www.qwant.com', +const SYNC_MINUTES = 0.5; // 30 秒一轮 +const FETCH_WINDOW = 2 * 60 * 60 * 1000; // 每次拉取过去 2 小时 +const MAX_BATCH = 200; // 单批最多上报条数 +const DEDUP_SECONDS = 600; // 10 分钟内同一 URL 只记一次 + +const SEARCH_HOSTS = new Set([ + 'google.com', 'www.google.com', 'googleusercontent.com', + 'bing.com', 'www.bing.com', 'cn.bing.com', + 'baidu.com', 'www.baidu.com', 'm.baidu.com', + 'duckduckgo.com', 'www.duckduckgo.com', 'lite.duckduckgo.com', + 'sogou.com', 'www.sogou.com', + 'so.com', 'www.so.com', + 'yahoo.com', 'www.yahoo.com', 'search.yahoo.com', + 'ecosia.org', 'www.ecosia.org', + 'qwant.com', 'www.qwant.com', + 'startpage.com', 'www.startpage.com', + 'brave.com', 'search.brave.com', + 'yandex.com', 'yandex.ru', + '360.cn', 'www.360.cn', + 'sm.cn', 'quark.sm.cn', + 'searx.be', 'searxng.site', 'paulgo.io', ]); +let lastSyncTime = 0; let pendingItems = []; -// Load pending from storage on startup -chrome.storage.local.get(['pending', 'serverUrl', 'syncToken'], (data) => { - pendingItems = data.pending || []; +// ── 初始化 ── +chrome.runtime.onInstalled.addListener(() => { + ensureAlarm(); + chrome.storage.local.get(['pending', 'lastSyncTime'], (data) => { + pendingItems = data.pending || []; + lastSyncTime = data.lastSyncTime || 0; + }); }); -// Listen to history visits -chrome.history.onVisited.addListener((historyItem) => { +function ensureAlarm() { + chrome.alarms.get(ALARM_NAME, (alarm) => { + if (!alarm) chrome.alarms.create(ALARM_NAME, { periodInMinutes: SYNC_MINUTES }); + }); +} + +// ── 本地过滤：搜索引擎与无意义页面 ── +function isJunk(url) { try { - const url = new URL(historyItem.url); - // Skip non-http(s) - if (url.protocol !== 'http:' && url.protocol !== 'https:') return; - // Skip search engine URLs - const domain = url.hostname.replace(/^www\./, ''); - if (SEARCH_DOMAINS.has(domain)) return; - - pendingItems.push({ - url: historyItem.url, - title: historyItem.title || '', - timestamp: historyItem.lastVisitTime / 1000, // ms -> seconds - visit_count: historyItem.visitCount || 1, - }); - chrome.storage.local.set({ pending: pendingItems }); + const u = new URL(url); + if (u.protocol !== 'http:' && u.protocol !== 'https:') return true; + const host = u.hostname.toLowerCase().replace(/^www\./, ''); + if (SEARCH_HOSTS.has(host)) return true; + if (host.endsWith('.search.yahoo.com')) return true; + // 常见搜索引擎结果特征：路径 /search、/s、/results 且带 q 参数 + const q = u.searchParams.get('q'); + const path = u.pathname.toLowerCase(); + if (q && (path === '/search' || path === '/s' || path === '/results' || path === '/web')) return true; + // 浏览器内部页 / 空白页 / 下载页 + if (u.hostname === 'newtab' || u.hostname === 'chrome-search' || + url.startsWith('chrome://') || url.startsWith('edge://') || + url.startsWith('about:')) return true; + return false; } catch (e) { - // invalid URL, skip + return true; } -}); +} -// Periodic sync alarm -chrome.alarms.create(ALARM_NAME, { periodInMinutes: SYNC_INTERVAL_MINUTES }); +function hostFrom(url) { + try { return new URL(url).hostname.replace(/^www\./, ''); } catch { return ''; } +} -chrome.alarms.onAlarm.addListener((alarm) => { - if (alarm.name !== ALARM_NAME) return; - syncNow(); -}); +// ── 拉取最近浏览历史（增量）── +async function collectRecent() { + const since = Math.max(0, Date.now() - FETCH_WINDOW); + const items = await new Promise((resolve) => { + chrome.history.search({ text: '', startTime: since, maxResults: 10000 }, resolve); + }); -// Manual sync from options page -chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => { - if (msg.action === 'sync') { - syncNow().then((result) => sendResponse(result)).catch((e) => sendResponse({ error: e.message })); - return true; // async response - } - if (msg.action === 'getConfig') { - chrome.storage.local.get(['serverUrl', 'syncToken', 'deviceName'], (data) => { - sendResponse(data); + const cutoff = Date.now() - DEDUP_SECONDS * 1000; + const seen = new Set(); + const collected = []; + + // 按时间从新到旧去重：同一 host+path 最近 10 分钟内只保留一次 + for (const item of items) { + if (isJunk(item.url)) continue; + const key = hostFrom(item.url) + '|' + (item.url.split('?')[0] || ''); + if (seen.has(key)) continue; + seen.add(key); + const ts = item.lastVisitTime || Date.now(); + if (ts < cutoff) continue; // 只收集最近窗口 + collected.push({ + url: item.url, + title: item.title || '', + timestamp: Math.floor(ts / 1000), + visit_count: item.visitCount || 1, }); + if (collected.length >= MAX_BATCH) break; + } + return collected; +} + +// ── 定时同步 ── +chrome.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === ALARM_NAME) syncNow(); +}); + +chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => { + if (msg && msg.action === 'sync') { + syncNow().then((r) => sendResponse(r)).catch((e) => sendResponse({ ok: false, error: e.message })); return true; } - if (msg.action === 'setConfig') { - chrome.storage.local.set(msg.data, () => sendResponse({ ok: true })); + if (msg && msg.action === 'getConfig') { + chrome.storage.local.get(['serverUrl', 'syncToken', 'deviceName'], (d) => sendResponse(d)); return true; } - if (msg.action === 'clearPending') { + if (msg && msg.action === 'setConfig') { + chrome.storage.local.set(msg.data || {}, () => sendResponse({ ok: true })); + return true; + } + if (msg && msg.action === 'clearPending') { pendingItems = []; chrome.storage.local.set({ pending: [] }, () => sendResponse({ ok: true })); return true; } + if (msg && msg.action === 'getStatus') { + chrome.storage.local.get(['serverUrl', 'syncToken'], (d) => { + sendResponse({ + serverConfigured: !!(d.serverUrl), + pendingCount: pendingItems.length, + device: msg.deviceName || '', + }); + }); + return true; + } }); -function detectDevice() { - const ua = navigator.userAgent.toLowerCase(); - if (/mobile|android|iphone|ipad/.test(ua)) { - if (/firefox/.test(ua)) return 'mobile-firefox'; - if (/edg/.test(ua)) return 'mobile-edge'; - return 'mobile-chrome'; - } - if (/firefox/.test(ua)) return 'desktop-firefox'; - if (/edg/.test(ua)) return 'desktop-edge'; - return 'desktop-chrome'; -} - async function syncNow() { - if (pendingItems.length === 0) return { ok: true, skipped: true, reason: 'no pending' }; - + ensureAlarm(); const { serverUrl, syncToken, deviceName } = await new Promise((resolve) => { chrome.storage.local.get(['serverUrl', 'syncToken', 'deviceName'], resolve); }); + if (!serverUrl) return { ok: false, reason: 'not configured' }; - if (!serverUrl) return { ok: false, reason: 'server not configured' }; - - // Snapshot and clear pending - const toSend = [...pendingItems]; - pendingItems = []; - chrome.storage.local.set({ pending: [] }); + // 增量拉取最近历史，合并到待上报队列 + let recent; + try { recent = await collectRecent(); } catch (e) { recent = []; } + const merged = dedup(recent.concat(pendingItems)); + if (merged.length === 0) return { ok: true, skipped: true, reason: 'nothing new' }; const body = JSON.stringify({ - items: toSend.map((item) => ({ - ...item, - device: deviceName || detectDevice(), + items: merged.map((it) => ({ + url: it.url, + title: it.title || '', + device: deviceName || defaultDevice(), + timestamp: it.timestamp, + visit_count: it.visit_count || 1, })), }); + const headers = { 'Content-Type': 'application/json' }; + if (syncToken) headers['Authorization'] = 'Bearer ' + syncToken; + try { - const headers = { 'Content-Type': 'application/json' }; - if (syncToken) headers['Authorization'] = `Bearer ${syncToken}`; - - const resp = await fetch(`${serverUrl}/api/history/sync`, { - method: 'POST', - headers, - body, + const resp = await fetch(serverUrl.replace(/\/$/, '') + '/api/history/sync', { + method: 'POST', headers, body, }); - if (!resp.ok) { - const text = await resp.text(); - // On failure, put items back - pendingItems = [...toSend, ...pendingItems]; - chrome.storage.local.set({ pending: pendingItems }); - return { ok: false, status: resp.status, error: text }; + const text = await resp.text().catch(() => ''); + return { ok: false, status: resp.status, error: text.slice(0, 200) }; } - - return await resp.json(); + const data = await resp.json().catch(() => ({})); + pendingItems = []; + lastSyncTime = Date.now(); + await new Promise((res) => chrome.storage.local.set({ pending: [], lastSyncTime }, res)); + return { ok: true, count: merged.length, total: data.total || 0 }; } catch (e) { - // Network error — put items back - pendingItems = [...toSend, ...pendingItems]; - chrome.storage.local.set({ pending: pendingItems }); + // 网络失败：保留队列下次重试 + await new Promise((res) => chrome.storage.local.set({ pending: merged }, res)); return { ok: false, error: e.message }; } } + +function dedup(list) { + const seen = new Set(); + const out = []; + for (const it of list) { + const key = it.url; + if (seen.has(key)) continue; + seen.add(key); + out.push(it); + } + return out; +} + +function defaultDevice() { + return 'desktop-browser'; +} diff --git a/extension/chrome/icons/icon128.png b/extension/chrome/icons/icon128.png new file mode 100644 index 0000000..76eaa6b Binary files /dev/null and b/extension/chrome/icons/icon128.png differ diff --git a/extension/chrome/icons/icon16.png b/extension/chrome/icons/icon16.png new file mode 100644 index 0000000..57e350d Binary files /dev/null and b/extension/chrome/icons/icon16.png differ diff --git a/extension/chrome/icons/icon48.png b/extension/chrome/icons/icon48.png new file mode 100644 index 0000000..f78ca3d Binary files /dev/null and b/extension/chrome/icons/icon48.png differ diff --git a/extension/chrome/manifest.json b/extension/chrome/manifest.json index a02a76c..14e38e0 100644 --- a/extension/chrome/manifest.json +++ b/extension/chrome/manifest.json @@ -1,7 +1,8 @@ { + "manifest_version": 3, "name": "JianF Nav Sync", "version": "1.0.0", - "description": "Sync browsing history to JianF unified navigation portal", + "description": "同步浏览历史到自建导航门户，按访问频次自动排序显示常用服务", "permissions": [ "history", "storage", @@ -13,6 +14,14 @@ "background": { "service_worker": "background.js" }, + "action": { + "default_title": "JianF Nav Sync 设置", + "default_popup": "options.html", + "default_icon": { + "16": "icons/icon16.png", + "48": "icons/icon48.png" + } + }, "icons": { "16": "icons/icon16.png", "48": "icons/icon48.png", diff --git a/extension/chrome/options.html b/extension/chrome/options.html index 9c178d1..49355ae 100644 --- a/extension/chrome/options.html +++ b/extension/chrome/options.html @@ -5,7 +5,7 @@ JianF Nav Sync 设置 +

JianF Nav Sync

diff --git a/history-server/main.go b/history-server/main.go index a2b4472..0b7aa1b 100644 --- a/history-server/main.go +++ b/history-server/main.go @@ -90,11 +90,29 @@ func cors(next http.HandlerFunc) http.HandlerFunc { } } +func requireToken(w http.ResponseWriter, r *http.Request, expected string) bool { + if expected == "" { + return true // 未配置 token 则不校验 + } + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, "Bearer ") && strings.TrimPrefix(auth, "Bearer ") == expected { + return true + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{"error": "invalid or missing token"}) + return false +} + func handleSync(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "method not allowed", 405) return } + // 浏览器插件回传：必须携带 NAVI_SYNC_TOKEN（免 portal 登录，改用自定义 token） + if !requireToken(w, r, syncToken) { + return + } var req SyncRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, err.Error(), 400) @@ -106,7 +124,6 @@ func handleSync(w http.ResponseWriter, r *http.Request) { } mu.Lock() - defer mu.Unlock() for _, item := range req.Items { normalized, err := normalizeURL(item.URL) @@ -145,10 +162,16 @@ func handleSync(w http.ResponseWriter, r *http.Request) { } } save() + mu.Unlock() // 写锁释放后再排序，避免 sortDashyFavorites 内 RLock 死锁 + sortDashyFavorites() + mu.RLock() + total := len(history) + mu.RUnlock() + w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]any{"status": "ok", "count": len(req.Items), "total": len(history)}) + json.NewEncoder(w).Encode(map[string]any{"status": "ok", "count": len(req.Items), "total": total}) } func handleGet(w http.ResponseWriter, r *http.Request) { @@ -176,9 +199,12 @@ func handleClear(w http.ResponseWriter, r *http.Request) { return } mu.Lock() - defer mu.Unlock() history = map[string]*Entry{} save() + mu.Unlock() + + sortDashyFavorites() // 清空后重置 conf.yml 历史 section 为占位 + w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]string{"status": "cleared"}) } @@ -345,11 +371,18 @@ func main() { http.HandleFunc("/sync", cors(handleSync)) http.HandleFunc("/", cors(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodGet { + switch r.Method { + case http.MethodGet: + if !requireToken(w, r, syncToken) { + return + } handleGet(w, r) - } else if r.Method == http.MethodDelete { + case http.MethodDelete: + if !requireToken(w, r, adminToken) { + return + } handleClear(w, r) - } else { + default: http.Error(w, "method not allowed", 405) } }))