diff --git a/Makefile b/Makefile index 9940bed..835e51b 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # ModelRouter - single-binary gateway + optional Electron desktop GUI (embedded core) export BINARY -BINARY=llmsproxy +BINARY=modelrouter export BUILD_DIR BUILD_DIR=build export GUI_DIR @@ -20,7 +20,7 @@ all: build # ---- core binary (server users / embedded core) ---- build: @mkdir -p $(BUILD_DIR) - CGO_ENABLED=1 go build -tags luajit -trimpath -o $(BUILD_DIR)/$(BINARY) ./cmd/llmsproxy + CGO_ENABLED=1 go build -tags luajit -trimpath -o $(BUILD_DIR)/$(BINARY) ./cmd/modelrouter @echo "Built: $(BUILD_DIR)/$(BINARY) ($(VERSION))" # ---- core distribution packages (deb + rpm + tar.gz) ----- diff --git a/README.md b/README.md index b001c6a..eb28285 100644 --- a/README.md +++ b/README.md @@ -4,103 +4,46 @@ > > **AI-assisted**: 本项目使用 AI 辅助编程（代码与文档由 AI 协作完成，经人工复核）。 -面向多 llms 订阅者，部署在内网，实现一次配置多个服务共同使用的效果。支持指定模型或 auto 模式，按照配置的优先级选择可用模型提供服务。 +面向多 LLM 订阅者的内网网关：一次配置，多个服务共用一组上游额度。支持指定模型或 +`AUTO` 模式，按配置的优先级选择可用模型提供服务。 -> **轻量且增源无需重编译**：网关本体是单 Go 二进制（约 10MB，零运行时依赖）。新增/切换上游 -> 只需在 `config.yaml`（或 WebUI）加一个 `sources` 条目或挂一个 `.lua` 适配器——**不改 Go、 -> 不重编译**。源与适配器的改动经 WebUI 提交时即时生效（热更新）；直接编辑 -> `config.yaml` 或 `adapter_dir` 下的 `.lua` 文件则需要重启进程生效。 +网关本体是单个 Go 二进制（约 10 MB，零运行时依赖）。新增或切换上游只需在 +`config.yaml`（或 WebUI）加一个 `sources` 条目、或挂一个 `.lua` 适配器——**不改 Go、 +不重编译**。源与适配器的改动经 WebUI 提交时即时生效（热更新）；直接编辑 +`config.yaml` 或 `adapter_dir` 下的 `.lua` 文件需要重启进程生效。 -## 核心优势 +## 核心能力 -### 极致轻量 - -- **单二进制**：编译后 8~12MB（`-s -w` strip 后约 8MB），零运行时依赖（仅依赖系统 libc），部署即用 -- **低内存占用**：与源数量相关，非与运行时长相关——单源 ~10MB、16 源生产实例 ~32–35MB（[实测分解与调优](#内存占用实测与调优)） -- **日志按需加载**：统计数字来自**全量**审计日志（启动时流式扫一遍即释放，29MB/22 万行约 260ms），但原始记录不常驻内存——默认只加载首屏，下滚自动分页，CSV 导出流式写出（O(1) 内存），离页即释放 -- **零运行时依赖**：纯 Go + LuaJIT 静态链接，无需安装 Python/Node/Java 等运行时 -- **启动极快**：冷启动 < 200ms，热重载配置 < 10ms - -### 强大的多租户调度能力 - -- **多密钥多租户**：支持无限密钥，每个密钥独立角色、模型范围、Token 配额、重置周期 -- **按模型配额**：每把 key 的每个模型单独配 token + 请求数配额与重置周期（小时/周/月/自定义 N 小时）；一个模型用满只影响该模型，同 key 其它模型照常；耗尽返 429 + `Retry-After` 并点名模型，admin key 永不受限 -- **AUTO 智能调度**：基于优先级档位的分级调度，同优先级源自动轮询负载均衡，故障自动毫秒级故障转移 -- **自愈冷却**：冷却上限 5 分钟，过半后放行 1 个探测请求，上游/额度恢复即刻回归轮询，无需等满冷却窗口 -- **Token 配额管理**：精确到模型级别的 Token 配额控制，支持小时/周/月/自定义小时周期自动重置 -- **同优先级源负载均衡**：同一优先级档位的多个源，请求自动 Round-Robin 均匀分发，故障毫秒级故障转移 -- **Source-Model 前缀路由**：支持 `source-model`/`source:model`/`source/model` 精确指定上游源 - -### 生产级可靠性 - -- **热加载配置**：WebUI 修改源/密钥/适配器/AUTO链即时生效，无需重启 -- **密钥加密落盘**：AES-256-GCM 加密存储 `api_key`、网关密钥、请求头，master.key 0600 权限保护 -- **实时源探测**：`GET {base}/models` 定期探测，毫秒级感知上游状态，不污染调度退避状态 -- **全链路审计**：HTTP 访问日志、登录/配置变更审计、请求记录 CSV 导出、密钥用量统计 CSV 导出 - -### 灵活的协议适配 - -- **LuaJIT VM**：每个适配器独立 VM + **弹性 worker 池**（按需扩缩容，空闲时自动回收），安全并发，Lua 脚本热加载无需重启 -- **Lua 适配器协议**：`transform_request` / `transform_response` / `transform_stream_chunk` 双向转换 -- **动态请求头钩子**：`build_headers(meta)` 支持 HMAC 签名、动态 Header 注入 -- **多模态透传**：`image_url` 等多模态内容在多源间无损透传，Anthropic/Gemini/Ollama 自动转换 - -## 实现 - -统一 OpenAI 兼容网关：把多个上游 LLM 源（DeepSeek、Qijiar、OpenAI、Anthropic、Gemini、 -Groq、Mistral、Ollama、KimiCode…）通过 **Lua 适配器** 做协议转换，对内网暴露 -一个标准 `OpenAI Chat Completions` 接口（`/v1/chat/completions` + `/v1/models`）， -支持单次结算、SSE 流式、**AUTO 模型路由**与**多模态**透传。 - -从 [HomeAgent](https://gitcode.com/JianFeeeee/HomeAgent) 的多源 LLM 适配层 -（`internal/agent/api/provider.go` + `internal/lua/adapters/*`）抽离而来并独立演进。 - -## 特性 - -- **多源**：一个进程内配置任意多个上游源，按请求的 `model` 自动路由。 -- **AUTO 模式**：`default_model: AUTO` 时按优先级页保存的 AUTO 链档位逐档调度并发请求。 -- **统一输出**：所有源对外都是 OpenAI 格式（含 `reasoning_content`、`tool_calls`、`usage`）。 -- **生图**：`POST /v1/images/generations`，`kind: image` 的模型独立路由。 -- **多模态**：`content` 数组（`image_url` 等）在多源间无损透传，Anthropic/Gemini/Ollama 自动转换。 -- **LuaJIT VM**：基于 golua 绑定的 LuaJIT，每个适配器独立 VM + worker 池，安全并发。 - 池是 **弹性** 的：`Σ max_concurrent` 只是上限而非预分配，状态按需创建、空闲时自动回收， - 因此空载网关几乎不持有 Lua 状态（启动 0 个）。 - - 扩容步长由适配器最大并发决定：`clamp(ceil(max/8), 1, 8)`，且仅在**真实并发争用** - （所有现有状态均已占用）时批量预热，顺序流量始终只用 1 个状态。 - - 缩容步长由当前连接数决定：`clamp(ceil(冗余/(1+占用)), 1, 冗余)`——无连接时一轮收到 - 常驻下限，忙时每轮只释放 1 个，保护热路径。WebUI「适配器」页展示实时池指标。 -- **thinking 预算**：请求里的推理开关/预算归一为中立的 `thinking:{enabled,budget_tokens}`， - 适配器映射到各自协议（Anthropic `thinking.budget_tokens`、Gemini `thinkingConfig`、 - DeepSeek `thinking.type="disabled"`）。 -- **Lua 适配协议**：每个源挂一个 `.lua` 适配器，完成 `transform_request` / - `transform_response` / `transform_stream_chunk` 双向转换，协议差异全在 Lua 层。 -- **签名 / 请求头钩子**：适配器可定义 `build_headers(meta)`，在 Go 发 HTTP 前 - 动态注入/签名请求头——用于云端 API 校验调用方 app（如 KimiCode 只放行特定 - agent）。提供 `hmac_sha256_hex` / `sha256_hex` / `base64_encode` 等签名辅助。 -- **WebUI**：内置管理界面，可在线查看/新增/编辑上游源与模型、配置 AUTO 优先级链、 - 管理密钥模型范围，写入运行时文件持久化。 -- **密钥加密存储**：运行时文件中的上游 `api_key`、自定义请求头值、网关 key 均以 - AES-256-GCM 加密落盘（`master.key` 独立 0600，或 `LLMS_PROXY_MASTER_KEY`）。 -- **实时源探测**：状态页探测各源可达性（`GET {base}/models`，失败回退最小请求）， - 不污染正常调度的退避状态，错误信息在 UI 可悬停查看。 -- **鉴权**：网关自身用 `gateway_keys` 校验客户端 Bearer key；与上游各自的 key 相互独立。 -- **流式**：SSE `chat.completion.chunk`，含角色首包与 `[DONE]` 收尾。 -- **全链路审计**：HTTP 访问日志、登录/配置变更审计、请求记录 CSV 导出、密钥用量统计 CSV 导出 -- **AES-256-GCM 加密存储**：运行时文件敏感字段加密落盘，master.key 0600 权限，支持环境变量注入主密钥 +- **统一接口**：对内暴露 OpenAI 兼容的 `/v1/chat/completions`、`/v1/models`、 + `/v1/images/generations`；所有源对外都是同一套格式（含 `reasoning_content`、 + `tool_calls`、`usage`）。 +- **多源 + AUTO 链**：一个进程内任意多个上游源；`default_model: AUTO` 时按优先级页 + 保存的档位逐档调度，源故障自动降级并冷却，恢复后半冷却探测自愈。 +- **协议适配在 Lua 层**：每个源挂一个 `.lua` 适配器，完成请求/响应/流式分片的双向 + 转换，并可用 `build_headers(meta)` 注入签名头。协议差异不进 Go 内核。 +- **插件独立**：业务扩展（渠道接入、计费、额度巡检）是外置 Lua 插件，由插件目录加载， + 不编进二进制、不随发行包分发；插件只通过 `modelrouter` SDK 与内核交互。 +- **多租户密钥**：网关 key 与上游 key 相互独立，按 key 限定模型范围、AUTO 链与配额。 +- **加密存储**：上游 key、自定义请求头、网关 key 均以 AES-256-GCM 落盘， + `master.key` 单独 0600 权限（或由 `MODELROUTER_MASTER_KEY` 注入）。 +- **可观测**：状态页实时探测源可达性；HTTP 访问日志、配置变更审计、请求与用量 CSV 导出； + 写操作不落在请求热路径上。 ## 快速开始 ```bash -GOMODCACHE=... GOPROXY=off go build -tags luajit -o llmsproxy ./cmd/llmsproxy -./llmsproxy -config config.yaml # 首次运行自动生成默认配置并打印随机 admin key +# 需要 LuaJIT 开发库；-tags luajit 是必须的 +GOMODCACHE=... GOPROXY=off go build -tags luajit -o modelrouter ./cmd/modelrouter + +# 首次运行会生成默认配置，并在日志里打印随机 admin key +./modelrouter -config config.yaml ``` -> 依赖 [golua](https://github.com/aarzilli/golua)（LuaJIT 绑定）。**必须**带 `-tags luajit` -> 构建，否则默认走内置超集 gopher-lua 路径（行为略有差异）。 +> 依赖 [golua](https://github.com/aarzilli/golua)（LuaJIT 绑定）。不带 `-tags luajit` +> 会走 gopher-lua 路径，行为略有差异。 -> **配置不入库**：仓库不携带任何 config 文件（配置文件含密钥）。二进制首次运行时 -> 会在 `-config` 指定的位置生成默认配置：随机 admin key（打印在启动日志中）、 -> 仅绑定 `127.0.0.1:8080`。首次登录后请在 WebUI「密钥」页更换管理员密钥。 +> **配置不入库**：仓库不携带任何 config 文件（其中含密钥）。默认配置只绑定 +> `127.0.0.1:8080`，首次登录后请在 WebUI「密钥」页更换管理员密钥。 ```bash # 无 key -> 401 @@ -117,526 +60,73 @@ curl -N -H "Authorization: Bearer $KEY" \ http://127.0.0.1:8080/v1/chat/completions ``` -任何 OpenAI SDK 把 `base_url` 指到网关地址、`api_key` 用任意一个已创建并授权的网关 key 即可。 +任何 OpenAI SDK 把 `base_url` 指向网关、`api_key` 换成任一已授权的网关 key 即可。 -## 配置 +## 文档 -首次运行在 `-config` 路径自动生成默认配置（默认 `./config.yaml`），核心字段： +| 主题 | 文档 | +|---|---| +| 实现与完整特性清单 | [docs/features.md](docs/features.md) | +| 配置：`config.yaml`、自动探测模型、多密钥 | [docs/configuration.md](docs/configuration.md) | +| 路由：模型路由、AUTO 链、冷却与半冷却、源与适配器、源模板 | [docs/routing.md](docs/routing.md) | +| 适配器：Lua 协议、内置适配器清单 | [docs/adapters.md](docs/adapters.md) · [docs/lua-adapters.md](docs/lua-adapters.md)（[EN](docs/lua-adapters-en.md)） | +| 插件：SDK 契约、钩子、UI 扩展、HTTP 端点 | [docs/plugins.md](docs/plugins.md) | +| 桌面端：Headless / Desktop 两种产物、构建与运行 | [docs/gui.md](docs/gui.md) | +| 性能：实测内存占用与调优、thinking 预算 | [docs/performance.md](docs/performance.md) | +| WebUI 使用 | [docs/webui.md](docs/webui.md) | +| 开发：目录职责、测试、分支与发布纪律 | [docs/development.md](docs/development.md) · [docs/git-workflow.md](docs/git-workflow.md)（[EN](docs/git-workflow-en.md)） | +| 部署与迁移 | [docs/deployment.md](docs/deployment.md) | -```yaml -listen: 127.0.0.1:8080 # 网关监听地址（建议绑内网/回环） -gateway_keys: [sk-gw-<随机>] # 初始 admin 密钥（每次安装随机生成，仅首启写入运行时存储用） -default_model: AUTO # model 无法路由时自动按优先级链选源 -adapter_dir: adapters # Lua 适配目录；目录不存在时首启创建并 seed 内置，存在则只读 -runtime_file: runtime.json # WebUI 编辑的源/密钥/AUTO 链持久化到此文件 +## 从 llmsproxy 改名 -sources: - - name: deepseek - base_url: https://api.deepseek.com - api_key: sk-... # 也可用 api_key_env: SOME_ENV 引用环境变量（不落盘明文） - adapter: deepseek - max_concurrent: 8 - models: - - id: deepseek-v4-flash - priority: 100 # YAML 源首次启动会 seed 进 AUTO 链的初值 - kind: chat - - id: deepseek-v4-pro - priority: 60 - kind: chat - # 静态请求头（比适配器默认优先） - headers: { X-Tenant: prod } - # 传给 Lua build_headers 的透传元数据 - meta: { app_id: x, app_secret: y } - temperature: 0.7 - max_tokens: 4096 - timeout: 120s # 请求超时，默认 120s -``` +产品名早已是 ModelRouter，二进制、配置目录、服务名与环境变量现已统一为 +`modelrouter`： +| 旧 | 新 | +|---|---| +| `llmsproxy`（二进制） | `modelrouter` | +| `/etc/llmsproxy/` | `/etc/modelrouter/` | +| `llmsproxy.service` | `modelrouter.service` | +| `LLMSPROXY_*` | `MODELROUTER_*` | -### 自动探测模型列表 - -不想手抄模型 id 时，只填 URL 和 key，让网关去扫： +已部署的机器用迁移脚本升级（**默认 dry-run**，复制不移动，可回滚）： ```bash -# 只探测，不写任何东西 -curl -X POST http://127.0.0.1:8080/api/sources/discover \ - -H "Authorization: Bearer sk-gw-…" -H "Content-Type: application/json" \ - -d '{"base_url":"https://api.vendor.com/v1","api_key":"sk-…"}' -# → {"ok":true,"base_url":"…","endpoint":"","models":["gpt-x",…], -# "image_models":["gpt-image-2"],"models_path":"/models","latency_ms":1387,…} - -# 探测后直接落库 -curl -X POST http://127.0.0.1:8080/api/sources/discover/apply \ - -H "Authorization: Bearer sk-gw-…" -H "Content-Type: application/json" \ - -d '{"base_url":"https://api.vendor.com/v1","api_key":"sk-…","name":"vendor"}' +sudo ./packaging/migrate-to-modelrouter.sh # 预演 +sudo ./packaging/migrate-to-modelrouter.sh --apply # 执行 +sudo ./packaging/migrate-to-modelrouter.sh --rollback --apply ``` -WebUI 源编辑框里也有「自动探测模型」按钮：填好 URL 和 key 点一下，列表自动 -填好，仍需点保存才生效。 - -几点值得知道： - -- **URL 形态不用猜。** 厂商对「`/v1` 放 base 还是放 path」没有共识，探测会 - 依次尝试若干候选并返回答对的那个，同时给出归一化后的 `base_url` + - `endpoint`（`endpoint` 平时留空，因为 openai adapter 自带 `/chat/completions`）。 -- **`image_endpoint` 是另一套规则。** 网关的图像默认路径是**绝对**的 - `/v1/images/generations`，而 chat 默认是相对的。所以 `base_url` 带 `/v1` 时 - 必须显式给 `image_endpoint: /images/generations`，否则图像请求 404。 - 探测会替你填好。 -- **图像模型只看上游自己声明的 modality**，绝不从模型名猜。`gpt-image-2` - 这类 id 也出现在同时提供对话服务的目录上，猜错会让 AUTO 链选中一个只支持 - 图像生成的模型，其后每一层都被它拖垮。 -- **不会逐模型探活。** 34 个模型的目录跑一轮要花钱且耗时数分钟，`/models` - 是足够便宜的信号；模型是否真能应答留给正常请求去发现。 -- **对已存在的源，必须显式 `replace_models: true`** 才会覆盖模型列表。重新 - 探测顺手抹掉为 AUTO 链手工调过的 priority，是最难排查的一类数据丢失。 -- **`api_key` 留空表示沿用已存值**，不会把凭据清空。 - - -`config.yaml`、`runtime.json`、`master.key`、`adapters/` 均不提交版本库（见 -`.gitignore`），请随备份带走——`master.key` 丢失后运行时密文无法解密。 - -运行文件生成在 `-config` 所在目录：`adapter_dir` / `runtime_file` 自动指向 -配置文件旁边的 `adapters/` 与 `runtime.json`，从任意工作目录启动都可用。 - -运行时文件（`runtime_file`）中的敏感字段自动加密： - -- 加密算法 AES-256-GCM，格式 `enc:v1:`。 -- 主密钥来源：环境变量 `LLMS_PROXY_MASTER_KEY`（64 位 hex）；否则读取 - `runtime_file` 同目录的 `master.key`；都不存在时首次启动自动生成 `master.key`（0600）。 -- 升级时旧明文文件自动兼容：首次运行正常读取，任何 UI 保存操作触发全文件加密迁移。 -- 注意：**master.key 丢失后密文无法解密**，请随配置一起备份；切勿提交到版本库。 - -### 网关密钥（多密钥） - -网关鉴权采用「多密钥 + 角色 + 模型范围」架构，密钥持久化在 `runtime_file` 的 -`keys` 字段（加密存储）： - -- `gateway_keys` 配置只是**初始 admin 密钥种子**：首次启动迁移为运行时 admin - key，之后不再参与鉴权管理。 -- **重要：首次启动后请在 WebUI「密钥」页更换管理员密钥**——初始密钥明文写在 - `config.yaml` 里（每次安装随机生成），继续使用存在被盗风险；用新密钥登录后删除初始密钥。 -- WebUI **密钥页**可创建/删除密钥；每个密钥可指定 `admin`（管理全部）或 - `user`（仅看自己的 key）角色，并配置**模型范围**（模型 + 源 + token 配额 + - 重置周期）。 -- 客户端用任意一个已授权的密钥明文作为 Bearer（`Authorization: Bearer `）。 -- 删除密钥即从运行时存储移除，立即失效。 - -#### 密钥用量配额 - -**配额按模型单独设置**：每个密钥的 `models[]` 里，每一条模型范围各自带一份 -token 配额与请求数配额。一个模型用满只影响该模型，同一密钥的其它模型照常工作。 - -```yaml -keys: - - key: sk-gw- - role: user - name: agent-alice - models: - - model: deepseek-v4-flash - token_quota: 1000000 # 本周期内该 key 用这个模型的 token 预算 - req_quota: 20000 # 本周期内的请求次数 - period: nhour # "" | hour | week | month | nhour - hours: 6 # 仅 nhour - - model: AUTO - token_quota: 5000000 # AUTO 也是一条独立配额 - period: day # ← 这种写法会被拒绝（词表只有 hour/week/month/nhour） - - model: kimi-k3 # 未列配额 = 无限 -``` - -- **没有「整钥总配额」**：这是刻意的设计。整钥总额会让「把 A 模型的额度挪给 - B 模型」变成一次全局重分配；按模型独立计费则每个模型各自可控，运维可以 - 看出哪个模型吃掉了预算。 -- `period` 词表：空 = 永不过期（累计总量），`hour` / `week` / `month` = 固定窗口， - `nhour` + `hours` = 自定义小时数。**拼错的周期在写入时就被拒**，不会静默变成 - 永不过期。 -- 配额严格按密钥隔离，且**同一密钥内按模型隔离**：A 密钥用满 `m1` 不会消耗 - B 密钥的额度，同一密钥的 `m2` 也不受影响。 -- 配额统计含聊天、流式、生图，跨重启从审计日志回放（保留 40 天，覆盖最长的 - month 窗口）。 -- 配额耗尽返回 **429 + `Retry-After`**（`rate_limit_exceeded`），消息里点名是哪个 - 模型用满了，客户端可等窗口重置后自动恢复；模型越权才是 403。 - **admin 密钥永不受配额限制**（其 scope 上的配额也不强制），避免把管理员 - 锁在门外。 -- 窗口用量按整点小时分桶统计，实际释放比配置窗口最多晚 1 小时（配额宁可晚释放 - 也不超发）。 -- `PUT /api/keys/{key}` 提交 `models` 即同时提交它们的配额（配额就是 scope 的一部分， - 不存在会与模型列表脱节的第二份预算）。显式 `0` = 解除该模型的限制。 - -##### 配额拒绝 vs 容量拒绝：两种「拒绝」含义不同 - -配额在**入口**检查，不占用任何上游槽位，因此是廉价拒绝；容量不足则要走 -`busyWait` 有界等待后才返回 503。实测（容量 8、0.6s/请求、100 并发）： - -| 情况 | 状态码 | 延迟 | 是否打到上游 | -|---|---|---|---| -| 配额用尽（`req_quota` 50 → 100 并发） | `429 rate_limit_exceeded` + `Retry-After` | **19–21ms** | 否 | -| 容量打满（同一拓扑，未限配额） | `503 upstream_error` | ~2.6s（`busyWait`） | 超出部分否 | - -客户端因此可以区分「等窗口重置」与「等上游腾出容量」：前者只需等待，后者 -通常该降并发或换源。 - -##### 配额的开销 - -配额桶按 (密钥, 模型, 整点小时) 分桶保留 40 天，实测（AMD 7840HS）： - -- 窗口查询按窗口长度而非保留总量扫描：1h 窗口 49ns、24h 窗口 55ns、 - 30d 窗口 3.9μs（此前全扫保留总量，960 桶时 5.9μs）。 -- 记录一条请求：283ns、3 分配（与引入配额前相同，分配来自 ring buffer）。 -- 内存：生产形态（7 密钥 × 8 模型 × 2 源 × 满 40 天 retention）约 **3.7MB**。 - 按源 pin 的 `source::model` 桶**惰性创建**——只有当某条配额真的 pin 了 - 某个源时才维护，否则每条记录多写一份桶，在 20 密钥 × 8 模型 × 3 源下会 - 多占 18MB。 - 代价：配置 pinned 配额之前的历史用量无法事后按源拆分，所以 **pinned 配额的 - 首个窗口可能少算**。 - -```bash -# 配额耗尽时客户端看到（点名了具体模型） -HTTP/1.1 429 Too Many Requests -Retry-After: 2100 -{"error":{"type":"rate_limit_exceeded","message":"token quota exceeded for \"deepseek-v4-flash\" (5000000/5000000)"}} -``` - -### 模型路由 - -`/v1/chat/completions` 的 `model` 解析顺序： - -1. `source/model` 或 `source:model` 前缀 → 指定源； -2. 精确匹配某个源的 `model`； -3. 配置 `default_model: AUTO` 时 → 走优先级页保存的 AUTO 链（见下）； -4. 否则回落（返回错误）。 - -任何 OpenAI 客户端，只要 `model` 设为某个源的 `name/任意名`，即可锁定走该源； -设为 `AUTO`（或网关配了 `default_model: AUTO`）即自动按优先级链选源。 - -### AUTO 链（优先级页） - -AUTO 调度**只**由优先级页保存的规则决定，源配置里的 `models[].priority` 数字不再 -参与调度、也不再显示。**聊天与生图是两套独立的链**，在优先级页面通过「聊天 / 生图」 -按钮切换编辑： - -- **聊天链**（`runtime_file` 的 `auto` 字段）：服务 `POST /v1/chat/completions` - 的 `model: AUTO` 请求；只接受 `kind: chat` 模型，配置在链里的生图槽位会被忽略。 -- **生图链**（`runtime_file` 的 `auto_image` 字段）：服务 `POST /v1/images/generations` - 的 `model: AUTO` 请求；只接受 `kind: image` 模型。未配置生图链时，AUTO 生图回落 - 到「所有含生图模型的源、按注册顺序」的旧行为，兼容旧配置。 - -每条规则 = 一个「槽位」：`{ model, source, tier, token_quota, period, hours }`。 - -- `tier` 表示优先级档位：同一档的模型并排、共享该优先级；档位从上到下递减。 -- 同一模型可配置多个槽位（如 A 源低配 → B 源低配 → A 源高配 → B 源高配），按档位顺延。 -- `token_quota` > 0 时该槽位在重置周期内用满即顺延到下一槽位；`period` 支持 - `hour` / `week` / `month` / `nhour`（配合 `hours`），空 = 不限。 -- 同 tier 内按「偏好分」排序，失败槽位冷却后自动跳过；冷却 / 配额耗尽 / hard - 错误会顺延到下一 tier，全部失败时返回 503 并附上每档失败摘要。 - -### 冷却与半冷却探测（自愈调度） - -槽位失败后进入指数退避冷却（5s → 10s → 20s …，上限 **5 分钟**）。冷却不是全时段 -硬闸： - -- **前半段完全静默**：刚失败的槽位不接任何流量，避免对故障上游持续施压。 -- **后半段放行 1 个探测**：窗口过半后，同一 `(源, 模型)` 最多允许 **一个** 请求作为 - 探测通过。探测槽数 = `clamp(max_concurrent / 10, 1, 2)`——`max_concurrent: 10` - 的源恰好放行 1 个探测请求。 -- **探测排在最后**：同 tier 内探测候选永远排在健康候选之后，只有没有正常槽可用时 - 才承接真实流量，因此探测不会抢走可用容量。 -- **探测即真实请求**：成功则立即清零失败计数与冷却，槽位当场回到正常轮询—— - 上游恢复/额度恢复后不必等满整个冷却窗口。失败则重开一个新窗口，下次探测顺延到 - 新窗口的中点，不会连续重试。 - -分类冷却策略： - -| 情况 | 冷却 | 说明 | -|---|---|---| -| 传输错误 / 5xx | 5s→10s→20s…，上限 5min | 指数退避 | -| 401 / 403 凭据错误 | 10min | 换 key 后由下一次探测自动接回 | -| 429 限流 | 30s 固定 | 「太快」不等于「坏了」，不进指数阶梯 | -| 配额耗尽（`insufficient_quota` / `余额不足` 等） | 对齐配额窗口，上限 30min | **不进指数阶梯、不涨失败计数**，额度恢复即可用 | - -WebUI「优先级」页的槽位健康标签会区分 `冷却` / `待探测` / `探测中`，悬停可看窗口 -起点、探测放行时刻与完全恢复时刻。 - -### 源（Source）与适配器（Adapter）的关系 - -- **源** 是到某个上游的连接描述：`name`、`base_url`、`api_key`、模型列表与优先级。 -- **适配器** 是协议转换逻辑（Lua 脚本）：把统一 OpenAI 格式请求转成上游原生格式， - 再把上游响应/流式分块转回统一格式。 -- 一个适配器可被多个源复用（如 `openai.lua` 同时服务多个 OpenAI 兼容站点）； - 同一个源也可以切换不同适配器（改 `adapter` 字段即可）。 -- 源决定“连谁、暴露哪些模型”，适配器决定“怎么对话”——二者在 `sources[]` 条目中 - 通过 `adapter` 字段关联。 - -加载流程（`internal/core` 的 `Core` 负责装配）： - -1. 启动时 `lua.NewVM(adapter_dir)` 加载全部适配器：内置适配器（编译期 embed）+ - `adapter_dir` 下的同名覆盖文件；内置文件写在代码内，覆盖文件要求更高优先级。 -2. `config.Load` 读取 `config.yaml`，`config.NewStore(runtime_file)` 读 WebUI 改动的 - 运行时源，二者按名称合并成完整源列表。 -3. `rebuildRegistry` 为每个源创建 `provider.Provider`（持有目标适配器），并按源的 - 并发上限配置适配器 worker 池**上限**（不预分配，见下）。 -4. 请求进来时 `Registry` 按 `model` 路由到 Provider，Provider 调适配器 - `transform_request` → HTTP 发送 → `transform_response` / `transform_stream_chunk`。 - -WebUI 上的"新增/编辑源"、"上传 Lua 适配器"、"改 AUTO 优先级链"与"密钥模型范围" -都即时生效（写入运行时文件或 `adapter_dir` 后重新装配，无需重启）；直接编辑 -`config.yaml` / `adapter_dir` 下的文件则需要重启进程才会重新加载。 - -在 WebUI 删除源/适配器时：运行时（WebUI 创建）的源与 `.lua` 文件会真正移除； -`config.yaml` 中定义的基源无法改写配置文件，采用删除标记隐藏（重启后仍隐藏）， -在 UI 里重新添加同名源即可恢复。 - -### 源模板（多 key 负载均衡） - -多个上游 key 共享同一套配置（URL / 适配器 / 模型列表 / 并发 / rpm）时，逐一复制 -整段 source 太冗余。**源模板**保存除 `name` 和 `api_key` 以外的全部源字段， -从模板一键创建多个仅 key 不同的源，天然支持多 key 负载均衡（每个展开源独立调度、 -独立冷却、独立健康状态）。 - -- WebUI「源」页右上角「**模板管理**」按钮：列出全部模板，可编辑 / 删除 / 新建。 -- 添加源弹窗右上角两个按钮： - - **从模板创建** → 选模板 → 表单自动填充（name + key 仍需你填）； - - **存为模板** → 把当前表单非 key/name 字段存为新模板。 -- 模板持久化在 `runtime_file` 的 `source_templates` 字段，与运行时源一样热管理， - 改完即时生效。模板本身不含密钥，可以安全共享 / 纳管。 - -模板不会自己变成源——它只是「配方」。真正承担流量的是从模板创建出来的、带 key -的具体源。 - -### 内存占用（实测与调优） - -内存不是一个定数：它随**配置的源数量**增长（每个源一个 `http.Transport` 与连接池、 -一组健康状态），与运行时长无关。本机实测（Linux x86_64，12 核）： - -| 部署形态 | 启动 RSS | 稳态 RSS | -|---|---|---| -| 1 源 / 1 适配器（最小配置） | ~4 MB | ~10 MB | -| 1 源 + 29 MB 历史审计日志 | ~19 MB | ~20 MB | -| **16 源 / 13 适配器 / 59 模型（本机生产）** | ~28 MB | **~32–35 MB** | - -> 历史参考：本项优化前同一生产配置为 **~105 MB**。降幅来自三处：审计日志的**原始记录** -> 不再常驻内存（约 25 MB；统计聚合仍扫全量，但扫完即释放）、Lua 状态池不再单调增长、 -> 以及下面两个运行时开关。 - -内存构成（生产实例分段测量，`/proc//smaps`）： - -| 区域 | RSS | 说明 | -|---|---|---| -| Go 堆 | ~14 MB | provider/registry/scheduler 结构 + 连接池缓冲 | -| 其他匹名（线程栈 / LuaJIT chunk / runtime） | ~12 MB | 与线程数、已加载适配器数相关 | -| 二进制 text+rodata | ~8 MB | 映射的可执行文件页（只读、可被内核回收） | -| Go runtime 预留 | ~4 MB | VSZ 上看到的 2 GB+ 是地址空间预留，不占物理内存 | -| 共享库 | ~3 MB | libc / libluajit / libm | - -**两个推荐的部署开关**（只适用于环境变量，无需改代码）： - -```ini -# /etc/systemd/system/llmsproxy.service -Environment=GOGC=50 -Environment=MALLOC_ARENA_MAX=2 -``` - -- `MALLOC_ARENA_MAX=2`：LuaJIT 的分配走 cgo → glibc malloc，glibc 默认允许 `8×nproc` - 个 per-thread arena，每个碰到 malloc 的 OS 线程会占用一个（各约 1 MB，且**不归还给系统**）。 - 实测从 8–15 个 arena（约 7–12 MB）降到 **0**。 -- `GOGC=50`：把 Go 堆增长目标减半。**单独使用无效**（省下的堆会立即被更多 glibc - arena 吃掉，实测 20.3 → 21.5 MB 反而变大），必须与 `MALLOC_ARENA_MAX` 配合， - 两者同时开启才降 **~19%**。网关是 I/O 密集型（本机 9 小时仅消耗 1min10s CPU）， - 多出的 GC 周期与它的空闲 CPU 相比可忽略。 - -> 桌面版（Electron）已在 `cmd/gui/main.js` 里默认为内嵌核心注入这两个开关；手动 export 同名 -> 环境变量可覆盖。如果你用自己的 systemd unit / 容器，建议照上面加上。 - -可选：`GOMEMLIMIT=48MiB` 作软上限，实测再省 ~1 MB；代价是逐近上限时 GC 转激进， -温和场景不必开。 - -### thinking 预算 - -请求体带 `"thinking": {"enabled": false}` 时关闭推理；`"thinking": {"enabled": true, -"budget_tokens": 4096}` 指定预算。内核只定义这个**中立**语义字段，映射到具体协议由 -适配器负责（Anthropic `thinking.budget_tokens`、Gemini `thinkingConfig.thinkingBudget`、 -DeepSeek `thinking.type = "disabled"`）。内核不再向适配器泄漏 `disable_thinking` / -`extra_body` 这类私有字段。 - -### WebUI - -内置管理界面（`GET /`），登录后可在浏览器完成： - -- **状态页**：源在线状态（实时探测 + 悬停看错误）、模型/源/key 用量统计、请求记录， - 支持按时间范围导出 CSV；点击模型可生成 pin 到该模型的连接配置。 -- **对话页**：流式/非流式调试。 -- **密钥页**：创建/编辑网关 key，为每个 key 配模型范围（模型 + 源 + token 配额 + 周期）， - 管理员管理全部 key，用户只看到自己的 key。每个模型砖块显示自己的配额徽标 - （如 `1.0K 77×·1h`，未设配额显示 `∞`），点开可编辑该模型的 token 配额、 - 请求数配额与重置周期 —— 配额按模型独立生效，一个用满不影响同一 key 的其它模型。 - 「我的密钥」页对用户展示本 key 的模型范围与各自配额。 -- **优先级页**：拖拽积木配置 AUTO 链档位。 -- **源页**：在线增删改上游源（API key 等敏感字段加密落盘）。 -- **适配器页**：上传 / 删除 Lua 适配器脚本。 - -改动写入 `runtime_file`（重启仍生效）。 - -## Lua 适配器协议 - -完整 API 见 **[Lua 适配器 API 文档](docs/lua-adapters.md)**（[English](docs/lua-adapters-en.md)）。 - -每个适配器是一个返回 table 的 Lua 脚本（`internal/lua/adapters/.lua`）， -加载时可被 `adapter_dir` 下的同名脚本覆盖——**改 Lua 脚本无需重编译**， -重启即生效；通过 WebUI 上传的适配器与在线编辑的源配置则即时生效。 - -```lua -return { - name = "mysrc", - version = "1.0.0", - endpoint = "/chat/completions", -- 上游路径（可被 source.endpoint 覆盖） - headers = { ["X-Static"] = "v" }, -- 静态默认请求头（build_headers 缺省时使用） - - -- 请求转换：把 OpenAI 格式 req 转成上游原生格式，返回字符串 - transform_request = function(raw_json) ... end, - - -- 响应转换：把上游原生响应转成统一格式字符串 - -- { content, reasoning_content, finish_reason, token_usage{...}, tool_calls[{...}] } - transform_response = function(raw_json) ... end, - - -- 流式分块转换：把上游 SSE data 转成 { content, done, ... }，返回 "" 则跳过 - transform_stream_chunk = function(raw_chunk) ... end, - - -- [可选] 动态请求头/签名钩子 - -- meta = { url, method, body, api_key, timestamp, source={ name, meta={...} } } - build_headers = function(meta) return { ["X-App-Sign"] = sign } end, - - -- [可选] 出向序列化（契约 v2）：上游不是 OpenAI 协议时接管写给客户端的字节。 - -- 不声明则沿用内核的 OpenAI 兜底；声明即完全取代，内核不再混合字段。 - serialize_response = function(result) return body_str end, - serialize_open_preamble = function(result) return bytes end, - serialize_chunk = function(delta) return bytes end, - serialize_close = function(result) return bytes end, -} -``` - -内置信号辅助：`hmac_sha256_hex(key, data)`、`sha256_hex(data)`、`base64_encode(s)`、 -`tohex(s)`、`json.encode/decode`、`log(level, msg)`。 - -### 内置适配器 - -`openai` `deepseek` `anthropic` `gemini` `github` `groq` `mistral` `ollama` `kimicode` -`qwen38` `sensenova` `trae`。 - -`anthropic`/`gemini`/`ollama` 适配器内置多模态转换（`image_url` → 各自上游格式）； -推理开关由中立的 `thinking` 字段表达，`deepseek` 适配器把它映射为 -`thinking.type = "disabled"`。 - -**kimicode** 是展示 `build_headers` 的样例：云端校验调用方 app，需要按 -`meta.app_secret` 对时间戳+URL+请求体哈希做 HMAC 签名并附 `X-App-Sign` 等头。 -配好 `sources[].meta.{app_id, app_secret, app_agent}` 即可。 - -## 桌面 GUI（Electron） - -可选独立产物，面向内网桌面端用户：**内嵌 ModelRouter 内核的托盘式桌面客户端**， -一键启停、系统托盘常驻、开机自启、静默启动，内容区一比一内嵌完整 WebUI -（状态/对话/密钥/优先级/源/适配器 六个页面，免登录）。服务器用户继续使用纯 Go 二进制。 - -### 两种安装包：Headless 与 Desktop - -发布到 Release 的安装包分为两类，面向不同用户： - -- **Headless（服务器版）**：纯核心二进制（无 Electron），配置驱动，适合服务器、 - 容器、systemd 等无人值守场景。发布包含 `deb` + `rpm`（含 systemd unit、内存调优 - 与示例配置）与 `tar.gz`（二进制 + `config.example.yaml` + `adapters/` + README）， - Windows 为 `.zip`。 -- **Desktop（桌面版）**：Electron GUI 安装包，内嵌核心，适合个人桌面日常使用。 - -### 选择 GUI 还是纯后端 - -| 场景 | 推荐 | 理由 | -| ------ | ------ | ------ | -| 服务器 / 内网网关 / 无人值守常驻 | **Headless**（单二进制） | 极轻量（二进制 ~8MB；RSS 随源数量，单源 ~10MB、16 源 ~32–35MB，见[内存占用](#内存占用实测与调优)），零依赖单进程，直接跑在 systemd/任意容器里，远程管理 | -| 个人桌面日常使用 / 多设备内网共享 | **Desktop**（GUI） | 免登录内嵌 WebUI、系统托盘一键启停、开机自启、静默后台，适合不懂命令行的使用者 | -| Windows 桌面 | **Desktop** | 纯后端在 Windows 上需自行注册服务，GUI 提供原生托盘/自启体验 | -| CI 一键出三平台安装包 | **Desktop 打包脚本** | `make gui-*` 系列出 deb/AppImage/NSIS，可进发行流水线 | - -两者完全同源：GUI 内嵌的就是纯后端的同一份 `llmsproxy` 核心（LuaJIT 版）， -配置/适配器格式完全一致，可随时互换。 - -### 构建与运行 - -```bash -cd cmd/gui -npm install -make gui # 或 npm run dev —— 开发运行（需先 make build 生成 bin/llmsproxy） -make gui-deb # 本机构建 deb（分享给其它 Linux 用户） -make gui-dist # 构建 deb + rpm + AppImage（linux，rpm 需系统 rpmbuild） -make core-dist # 构建核心 deb + rpm + tar.gz（服务器版，需 nfpm） -make gui-win # 构建 Windows nsis 安装包（本机 mingw + wine） -make gui-win-docker # 构建 Windows nsis 安装包（全 docker 自足，含 wine） -make gui-dist-dir # 仅解包产物（调试用，不产安装包） -``` - -产物在 `cmd/build/gui-dist/`：`ModelRouter-.AppImage`、 -`modelrouter-gui__amd64.deb`、`ModelRouter Setup .exe`（win）； -核心包在 `cmd/build/dist/`：`llmsproxy__amd64.deb`、`llmsproxy-.x86_64.rpm`。 -rpm 需系统 `rpmbuild`。每次打包后都会跑 `packaging/verify-dist.sh` 产物体检 -（大小下限门禁——1.3.0 曾因宿主 wine 损坏产出 264KB 的残缺 exe 却没有被拦住）。 -发布时会把产物重命名为 `ModelRouter-Desktop-*`（GUI）与 `ModelRouter-Headless-*`（纯后端） -两类上传到 GitCode Release。 - -#### Windows 打包（docker 固化方案，推荐） - -`make gui-win-docker` 一步完成：交叉编译核心 + NSIS 打包**全在 docker 内**，宿主机 -**无需 mingw、无需 wine、无需 node**。镜像内置 wine（含 i386 运行时），修复了 -此前宿主 wine 损坏（缺 syswow64 → `c0000135`）导致产出 264KB 残缺 exe 的问题。 - -- 构建镜像：`cmd/gui/docker/win-builder/Dockerfile`（`golang:1.25` + mingw-w64 + - node + wine32/wine64 + 预编译 LuaJIT windows 版 `lua51.dll` 与 import lib）。 - LuaJIT 源码取自 gitcode 镜像（国内可达），github 兜底；Go 依赖走 `https://goproxy.cn`。 -- 一键发布：`cmd/gui/scripts/dist-win-docker.sh` —— 核心编译 + electron-builder - NSIS 全部在容器内执行（`USE_SYSTEM_WINE=true` 走镜像内的 wine）；electron-builder - 的 ~400MB 工具链缓存挂到 docker volume，二次构建增量复用。 -- 首次运行自动构建 `modelrouter/win-builder` 镜像（约 5–10 分钟）；已存在则直接复用。 -- 打包完成后自动跑 `packaging/verify-dist.sh` 门禁：exe 低于 5MB / deb·rpm·7z 低于 - 10MB 即失败——残缺安装包不可能再溜进 Release。 - -```bash -# 全 Windows 发布（核心重编 + NSIS）： -make gui-win-docker -``` - -### 特性 - -- **内嵌核心**：自动拉起捆绑的 `llmsproxy`（LuaJIT 版），配置/密钥/adapter 存于 - 用户数据目录 `/profile/`，首次运行自动生成随机 admin key 写入 `keys` - （非 seed）并自注入，WebUI 免登录、不会提示"更换初始密钥"。端口可在设置中更改（默认 8787）。 -- **系统托盘**：内核状态点、开机自启/静默启动开关、启停/重启内核、退出。 -- **静默启动**：`--silent` 参数或设置项，启动仅驻托盘不弹窗；首次运行始终显示窗口。 -- **开机自启**：Windows/macOS 用 `setLoginItemSettings`；Linux 写 - `~/.config/autostart/modelrouter-gui.desktop`，静默开启时自动附加 `--silent`。 - -## 目录 - -``` -cmd/llmsproxy # 入口（纯 Go 服务端） -cmd/gui # Electron 桌面 GUI（内嵌核心 + 托盘 + 打包） -internal/config # YAML 配置加载/校验 -internal/lua # LuaJIT VM + worker 池 + AdapterCache + 内置适配器 (embed) +旧路径与旧环境变量在读取侧仍被接受，因此升级不会因为顺序问题立刻失效。 +`deploy.sh` 在检测到旧 `/etc/llmsproxy` 而新配置不存在时会**中止**，避免部署出 +一个「能启动但没有 key / 没有源」的空实例。 + +插件 SDK 同名调整：唯一公开入口是 `modelrouter`（`llmsproxy` 仅作兼容别名）， +详见 [docs/plugins.md](docs/plugins.md)。 + +## 目录结构 + +```text +cmd/modelrouter # 入口（纯 Go 服务端） +cmd/gui # Electron 桌面壳（内嵌同一份核心） +internal/config # 配置解析、加密存储、运行时文件 +internal/core # 调度、内核动作、状态与审计 +internal/lua # LuaJIT VM + 弹性 worker 池 + 内置适配器 (embed) +internal/lua/plugins # 可选插件示例（不随二进制/发行包分发） internal/provider # Provider(HTTP) + Registry(路由) -internal/gateway # OpenAI 兼容 HTTP 服务 + 鉴权 + SDK/流式 + WebUI +internal/gateway # OpenAI 兼容 HTTP 服务 + 鉴权 + 流式 + WebUI internal/types # 统一格式 & OpenAI wire 类型 +packaging/ # deb/rpm/tar 打包、systemd unit、迁移脚本 +docs/ # 主题文档 ``` -## 开发协作 - -- **Git 分支工作流**：主分支 / 特性分支 / 发布分支的规范见 - **[docs/git-workflow.md](docs/git-workflow.md)**（[English](docs/git-workflow-en.md)）。 - 主线：`main` 永可部署 → 特性合入 `main` → 切 `release/vX.Y.Z` 打 tag 发布 → - hotfix 提交发布分支并 cherry-pick 回 `main`。 -- **Lua 适配器协议**：见 **[docs/lua-adapters.md](docs/lua-adapters.md)** - （[English](docs/lua-adapters-en.md)）。 - -## 测试 +## 开发与测试 ```bash -go test -tags luajit ./... +go test -tags luajit ./... # 全量 +go test -tags luajit ./internal/lua # 单包（LuaJIT 相关改动必跑） ``` -覆盖：配置校验、适配器加载/变换、签名钩子、Gateway 鉴权、SDK 结算、SSE 流式、 -模型路由、多模态透传与 thinking 预算。 +- 分支与发布纪律见 [docs/git-workflow.md](docs/git-workflow.md)。 +- 插件放置在 `plugin_dir`，网关启动时**不会**创建、覆盖或删除任何插件文件； + 升级桌面端也不会动它们。 diff --git a/README_EN.md b/README_EN.md index 746ff26..cb39f46 100644 --- a/README_EN.md +++ b/README_EN.md @@ -2,625 +2,148 @@ > **中文**: [README.md](./README.md) > -> **AI-assisted**: This project is developed with AI-assisted coding (code and docs are produced in collaboration with AI, reviewed by humans). +> **AI-assisted**: code and documentation were produced with AI assistance and +> reviewed by a human. -A lightweight, unified OpenAI-compatible LLM gateway for internal networks — -configure once, and let multiple services (agents, SDKs, bots) share many -upstreams (DeepSeek, Qijiar, OpenAI, Anthropic, Gemini, Groq, Mistral, Ollama, -KimiCode…) behind a single endpoint. Pick a specific model or use **AUTO** mode, -which routes to the best healthy upstream by configured priority. +An intranet gateway for people with several LLM subscriptions: configure once and +let multiple services share the upstream capacity. Pick a model explicitly, or let +`AUTO` choose by priority among the sources that are currently healthy. -> **Lightweight, no recompile to add sources**: the gateway is a single Go -> binary (8-12 MB, ~8 MB stripped; zero runtime dependencies). Adding or switching an upstream is -> just a `sources` entry in `config.yaml` (or via the WebUI) or a `.lua` -> adapter — **no Go changes, no recompile**. Changes submitted through the -> WebUI take effect immediately (hot reload); editing `config.yaml` or a `.lua` -> file under `adapter_dir` by hand requires a process restart. +The gateway itself is a single Go binary (~10 MB, no runtime dependencies). Adding +or switching an upstream is a `sources` entry in `config.yaml` (or the WebUI), or a +dropped-in `.lua` adapter — **no Go changes, no rebuild**. Changes submitted through +the WebUI take effect immediately; editing `config.yaml` or a `.lua` adapter file +directly requires a restart. -## Overview +## What it does -Exposes a standard `OpenAI Chat Completions` API (`/v1/chat/completions` + -`/v1/models`) to your internal network, translating between multiple upstream -protocols via **Lua adapters**. Supports one-shot calls, SSE streaming, -**AUTO model routing**, multimodal passthrough, and tool calls. - -Extracted and independently evolved from the multi-source LLM adapter layer of -[HomeAgent](https://gitcode.com/JianFeeeee/HomeAgent) -(`internal/agent/api/provider.go` + `internal/lua/adapters/*`). - -## Features - -- **Multi-source**: any number of upstream sources in one process, routed by the - request's `model`. -- **AUTO mode**: with `default_model: AUTO`, scheduling follows the tiered AUTO - chain saved on the Priority page (see below). -- **Unified output**: every source speaks OpenAI format (incl. - `reasoning_content`, `tool_calls`, `usage`). -- **Image generation**: `POST /v1/images/generations`, routed to models with - `kind: image`. -- **Per-model quota**: each key gives every model its own token and request caps - plus a reset period (hour/week/month/custom N hours). One model running out - affects only that model — the key's other models keep working. Exhaustion - answers 429 + `Retry-After` naming the model, so a client resumes when the - window rolls over; admin keys are never capped. -- **Multimodal**: `content` arrays (`image_url` etc.) pass through losslessly; - Anthropic/Gemini/Ollama are translated automatically. -- **LuaJIT VM**: golua-binding LuaJIT; each adapter has its own VM + worker - pool for safe concurrency. Pools are **elastic**: the sum of `max_concurrent` - over an adapter's sources is a ceiling, not a preallocation — states are booted - on demand and reclaimed when demand drops, so an idle gateway holds close to - zero Lua states. The grow step follows the adapter's max concurrency - (`clamp(ceil(max/8), 1, 8)`, and never exceeds the number of queued callers); - the shrink step follows the live connection count - (`clamp(ceil(slack/(1+in_use)), 1, slack)`), so an idle pool collapses in one - round while a busy one gives up a single state at a time. -- **On-demand request logs**: dashboard totals are computed from the **full** - audit history (streamed once at startup and released — 29 MB / 221k lines in - ~260 ms), while the raw records are never held in memory: the dashboard loads - one screen, scrolling pages the rest straight off disk, CSV export streams in - O(1) memory, and leaving the page releases everything. -- **Self-healing cooldown**: cooldown is capped at 5 minutes and, past the - window's midpoint, exactly one probe request is allowed through; a recovered - upstream (or a reset quota) returns to full rotation on that probe instead of - waiting out the window. -- **disable_thinking**: `disable_thinking: true` toggles reasoning per-request. -- **Lua adapter protocol**: each source mounts a `.lua` adapter with - `transform_request` / `transform_response` / `transform_stream_chunk` — all - protocol differences live in Lua, Go only schedules and proxies. -- **Signature / header hooks**: adapters may define `build_headers(meta)` to - inject/sign request headers before the HTTP call (e.g. KimiCode style - app-validation), with helpers like `hmac_sha256_hex`, `sha256_hex`, - `base64_encode`. -- **WebUI**: built-in management page to view/add/edit sources & models, edit - the AUTO chain, and manage per-key model scopes, persisted to a runtime file. -- **Encrypted secrets at rest**: upstream `api_key`, custom header values and - gateway keys are stored AES-256-GCM encrypted (`master.key` 0600 next to the - runtime file, or `LLMS_PROXY_MASTER_KEY`). -- **Live source probing**: the status page probes each source - (`GET {base}/models`, falling back to a minimal chat call) without touching - the scheduler's backoff state; errors are shown on hover. -- **Auth**: the gateway validates client keys via `gateway_keys`; independent - from each upstream's own key. -- **Streaming**: SSE `chat.completion.chunk` with a role-first chunk and - `[DONE]` terminator. +- **One OpenAI-compatible API**: `/v1/chat/completions`, `/v1/models`, + `/v1/images/generations`. Every source speaks the same shape outwards, including + `reasoning_content`, `tool_calls` and `usage`. +- **Many sources, one AUTO chain**: configure any number of upstreams; `AUTO` + follows the priority tiers saved in the WebUI, fails over between them, cools + down broken sources and probes them back to health at half-cool. +- **Protocol differences live in Lua**: each source binds a `.lua` adapter that + transforms requests, responses and stream chunks both ways, and can sign headers + via `build_headers(meta)`. Vendor knowledge stays out of the Go kernel. +- **Plugins are independent**: business extensions (channel login, billing, quota + probing) are external Lua plugins loaded from `plugin_dir`. They are not compiled + into the binary and not shipped in release archives, and they talk to the kernel + only through the `modelrouter` SDK. +- **Multi-tenant keys**: gateway keys are separate from upstream keys, and can be + scoped to a model set, an AUTO chain and a quota. +- **Encrypted at rest**: upstream keys, custom header values and gateway keys are + sealed with AES-256-GCM; `master.key` is 0600, or injected via + `MODELROUTER_MASTER_KEY`. +- **Observability**: live source probing, HTTP access log, config-change audit, + request and usage CSV export. Writes are kept off the request hot path. ## Quick start ```bash -GOMODCACHE=... GOPROXY=off go build -tags luajit -o llmsproxy ./cmd/llmsproxy -./llmsproxy -config config.yaml # first run generates a default config and prints a random admin key +# LuaJIT development libraries are required; -tags luajit is mandatory +GOMODCACHE=... GOPROXY=off go build -tags luajit -o modelrouter ./cmd/modelrouter + +# First run generates a default config and prints a random admin key +./modelrouter -config config.yaml ``` -> Depends on [golua](https://github.com/aarzilli/golua) (LuaJIT bindings). -> You **must** build with `-tags luajit`; otherwise the built-in superset -> gopher-lua path is used (behavior differs slightly). +> Depends on [golua](https://github.com/aarzilli/golua) (LuaJIT bindings). Without +> `-tags luajit` the build falls back to gopher-lua, which behaves slightly +> differently. -> **No config is shipped in the repo** (config files carry real keys). On -> first run the binary generates a default config at the `-config` path: a -> random admin key (printed to the startup log), loopback-only `127.0.0.1:8080`, -> and a keyless source ready to chat. Rotate the admin key in the -> WebUI after first login. +> **No config in the repo**: the repository ships no config file (they contain +> keys). The generated default binds `127.0.0.1:8080` only; rotate the admin key in +> the WebUI after the first login. ```bash -# no key -> 401 +# No key -> 401 curl http://127.0.0.1:8080/v1/models -# one-shot ($KEY = the admin key printed at first startup) +# Non-streaming ($KEY = the admin key printed at first start) curl -H "Authorization: Bearer $KEY" \ -d '{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}' \ http://127.0.0.1:8080/v1/chat/completions -# streaming +# Streaming curl -N -H "Authorization: Bearer $KEY" \ -d '{"model":"AUTO","stream":true,"messages":[{"role":"user","content":"hi"}]}' \ http://127.0.0.1:8080/v1/chat/completions ``` -Any OpenAI SDK works: point `base_url` at the gateway, use any authorized -gateway key as the API key. - -## Configuration - -Generated on first run at the `-config` path (default `./config.yaml`). Core fields: - -```yaml -listen: 127.0.0.1:8080 # bind address (keep internal/loopback) -gateway_keys: [sk-gw-] # initial admin key seed (random per install), migrated to the store on first start -default_model: AUTO # when model is unroutable, follow the AUTO chain -adapter_dir: adapters # Lua adapter dir; created+seeded if missing, read-only otherwise -runtime_file: runtime.json # WebUI-edited sources/keys/AUTO chain persist here - -sources: - - name: deepseek - base_url: https://api.deepseek.com - api_key: sk-... # or api_key_env: SOME_ENV to read from an env var - adapter: deepseek - max_concurrent: 8 - models: - - id: deepseek-v4-flash - priority: 100 # YAML sources seed the AUTO chain on first start - kind: chat - - id: deepseek-v4-pro - priority: 60 - kind: chat - # static headers (take precedence over adapter defaults) - headers: { X-Tenant: prod } - # passthrough metadata for the Lua build_headers hook - meta: { app_id: x, app_secret: y } - temperature: 0.7 - max_tokens: 4096 - timeout: 120s # request timeout, default 120s -``` - -Sensitive fields in the runtime file (`runtime_file`) are encrypted at rest: - -- AES-256-GCM, stored as `enc:v1:`. -- Master key source: env var `LLMS_PROXY_MASTER_KEY` (64 hex chars), else a - `master.key` file next to the runtime file; if neither exists it is generated - on first start (mode 0600). -- Old plaintext files still load; the first UI save migrates the whole file to - ciphertext. -- Back up `master.key` with your config — losing it makes the secrets - undecryptable. Do not commit it. - -### Gateway keys (multi-key) - -Gateway auth uses a "multi-key + role + model scope" model. Keys are persisted -under the `keys` field of the runtime file (encrypted at rest): - -- The `gateway_keys` config is only an **initial admin key seed** — it is - migrated into the runtime store on first start and no longer drives auth. -- **Important: after first start, replace the admin key via the WebUI Keys - page.** The seed key is written in plaintext in `config.yaml`, so keeping it - active is a security risk; create a new admin key, log in with it, then - delete the seed key. -- The WebUI **Keys page** creates/deletes keys. Each key has a role (`admin` - manages everything, `user` sees only its own key) and an optional **model - scope** (model + source + token quota + reset period). Each model brick shows - its own budget badge (e.g. `1.0K 77×·1h`, `∞` when uncapped); clicking it - edits that model's token quota, request quota and reset period. Quotas apply - per model, so one model running out never blocks the key's others. The - "My key" view shows a user its model scopes and their budgets. -- Clients authenticate with any authorized key's plaintext as - `Authorization: Bearer `. -- Deleting a key removes it from the store immediately. - -#### Per-key usage quota - -**Quotas are per model.** Each key's `models[]` list gives every model its own -token budget and request budget. One model running out affects only that model -— the key's other models keep working. - -```yaml -keys: - - key: sk-gw- - role: user - name: agent-alice - models: - - model: deepseek-v4-flash - token_quota: 1000000 # this key's token budget for this model - req_quota: 20000 # requests within the window - period: nhour # "" | hour | week | month | nhour - hours: 6 # n-hour only - - model: AUTO - token_quota: 5000000 # AUTO is a quota entry like any other - period: hour - - model: kimi-k3 # no quota listed = unlimited -``` - -- **There is deliberately no key-wide total.** A key-wide cap would make - "move A's budget to B" a global reallocation; per-model budgets keep each - model independently controllable, so it stays visible which model is - actually consuming the spend. -- `period`: empty = never resets (lifetime total); `hour` / `week` / `month` = - fixed windows; `nhour` + `hours` = a custom hour count. **A misspelled - period is rejected at write time** rather than silently becoming a - never-resetting quota. -- Quotas are isolated per key *and* per model within a key: one key exhausting - `m1` never draws on another key's budget, and never blocks the same key's - `m2`. -- Usage counts chat, streaming and image requests, and survives a restart by - replaying the audit log (40 days retained, covering the longest `month` - window). -- An exhausted quota returns **429 + `Retry-After`** - (`rate_limit_exceeded`) and the message names the model that ran out, so a - client resumes when the window rolls over; a model the key may not use stays - 403. **Admin keys are never capped** (quotas on their scopes are not - enforced either), so a cap can never lock the operator out. -- Buckets are whole unix hours, so a window frees up at most an hour late - (deliberately freeing late rather than overspending). -- `PUT /api/keys/{key}` submits quotas by submitting `models` — the caps are - part of the scope, so there is no second budget that can drift out of sync - with the model list. An explicit `0` lifts that model's cap. - -##### Quota rejection vs capacity rejection - -A quota is checked at the entry point and occupies no upstream slot, so it is a -cheap rejection. Running out of capacity instead waits out the bounded -`busyWait` before answering 503. Measured (capacity 8, 0.6 s/request, 100 -concurrent clients): - -| Case | Status | Latency | Reaches upstream | -|---|---|---|---| -| Quota spent (`req_quota` 50, 100 concurrent) | `429 rate_limit_exceeded` + `Retry-After` | **19–21 ms** | no | -| Capacity exhausted (same topology, no quota) | `503 upstream_error` | ~2.6 s (`busyWait`) | overflow: no | - -So a client can tell "wait for the window to roll over" apart from "wait for -upstream capacity" — the first only needs patience, the second usually means -back off concurrency or switch sources. - -##### What the quota costs - -Buckets are kept per (key, model, whole unix hour) for 40 days. Measured on an -AMD 7840HS: - -- A window query scans the window, not the whole retention: 49 ns for 1 h, - 55 ns for 24 h, 3.9 µs for 30 d. -- Recording one request: 283 ns, 3 allocations (unchanged from before this - feature; the allocations come from the record ring buffer). -- Memory: the production shape (7 keys x 8 models x 2 sources at full 40-day - retention) costs about **3.7 MB**. The `source::model` bucket used by a - source-pinned quota is created **lazily** — it is maintained only once some - quota actually pins that source, because writing it on every record would - cost an extra 18 MB at 20 keys x 8 models x 3 sources. - The trade-off: usage recorded before a pinned quota was configured cannot be - split by source afterwards, so **a pinned quota may under-count its first - window**. - -``` -HTTP/1.1 429 Too Many Requests -Retry-After: 2100 -{"error":{"type":"rate_limit_exceeded","message":"token quota exceeded for \"deepseek-v4-flash\" (5000000/5000000)"}} -``` - -### Model routing - -`/v1/chat/completions` `model` resolution order: - -1. `source/model` or `source:model` prefix → pinned source; -2. exact match of a source's `model`; -3. with `default_model: AUTO` → follow the AUTO chain from the Priority page - (see below); -4. otherwise the request errors. - -Any OpenAI client can pin to a source by setting `model` to its -`name/anything`; `AUTO` (or the gateway's `default_model: AUTO`) follows the -tiered AUTO chain. - -### AUTO chain (Priority page) - -AUTO scheduling is driven **only** by the rules saved on the Priority page. The -numeric `models[].priority` in source configs no longer participates in -scheduling and is no longer shown. **Chat and image generation are two -independent chains**, switched in the Priority page via the `Chat / Image` toggle: - -- **Chat chain** (`auto` field): serves `POST /v1/chat/completions` with - `model: AUTO`; only `kind: chat` models are honored (image slots are skipped). -- **Image chain** (`auto_image` field): serves `POST /v1/images/generations` with - `model: AUTO`; only `kind: image` models are honored. With no image chain - configured, AUTO image generation falls back to legacy discovery (all sources - exposing an image model, in registry order) for backward compatibility. - -Each rule is one "slot": `{ model, source, tier, token_quota, period, hours }`. - -- `tier` is the priority tier: models in the same tier sit side by side and - share it; tiers run high → low. -- The same model may appear in several slots (e.g. A low → B low → A high → - B high) and is tried in tier order. -- `token_quota` > 0 means the slot is skipped once its tokens within the reset - window are exhausted; `period` supports `hour` / `week` / `month` / `nhour` - (with `hours`); empty = unlimited. -- Within a tier, slots run by preference score; cooling / quota-exhausted / - hard-failed slots fall through to the next tier, and total failure returns - 503 with a per-tier summary. - -### Sources vs. adapters - -- A **source** describes a connection to an upstream: `name`, `base_url`, - `api_key`, model list and priorities. -- An **adapter** is the protocol translation logic (Lua script): converts the - unified OpenAI request to the upstream's native format and back. -- One adapter serves many sources (e.g. `openai.lua` for any OpenAI-compatible - site); one source can switch adapters via its `adapter` field. -- Sources decide *whom to talk to and which models to expose*; adapters decide - *how to talk*. They are linked by the `adapter` field inside each `sources[]` - entry. - -Loading flow (assembled by `Core` in `internal/core`): - -1. On startup `lua.NewVM(adapter_dir)` loads all adapters: built-in ones - (embedded at compile time) + same-name override files under `adapter_dir`. -2. `config.Load` reads `config.yaml`; `config.NewStore(runtime_file)` reads - WebUI-edited runtime sources; both are merged by name. -3. `rebuildRegistry` creates a `provider.Provider` per source (holding its - adapter), sizing the adapter's worker pool from the source's concurrency - limit. -4. Incoming requests are routed by `Registry` to a provider, which calls the - adapter's `transform_request` → HTTP call → `transform_response` / - `transform_stream_chunk`. - -"Add/Edit source", "Upload adapter", "AUTO chain edits" and "per-key model -scopes" in the WebUI take effect immediately (written to the runtime file / -`adapter_dir`, then reassembled — no restart). Directly editing `config.yaml` -or files under `adapter_dir` requires a process restart. - -Deleting a source/adapter in the WebUI truly removes runtime (UI-created) -sources and `.lua` files; base sources defined in `config.yaml` cannot rewrite -that file, so they are hidden via a deletion tombstone (still hidden after -restart) and can be restored by re-adding the same name in the UI. - -### Source templates (multi-key balancing) - -When several upstream keys share one config (URL / adapter / model list / -concurrency / rpm), copying the whole source block N times is wasteful. A -**source template** stores every source field except `name` and `api_key`, so you -create N key-only-different sources from one shared skeleton — natural multi-key -load balancing (each expanded source schedules, cools down, and reports health -independently). - -- WebUI Sources page, top-right **Templates** button: list all templates with - edit / delete / create. -- Add-source dialog has two header buttons: - - **From template** → pick a template → form auto-fills (name + key still - yours to enter); - - **As template** → save the current form's non-key/name fields as a template. -- Templates persist in the runtime file's `source_templates` field alongside - runtime sources (hot-managed, apply immediately). Templates hold no secret - keys, so they're safe to share / version-control. - -A template is just a recipe — it doesn't become a source by itself. Only the -key-bearing sources created from it carry real traffic. - -### Memory footprint (measured and tuned) - -Memory is not a constant: it scales with the **number of configured sources** -(each source owns an `http.Transport` with its connection pool plus per-model -health state), not with uptime. Measured on this machine (Linux x86_64, 12 cores): - -| Deployment shape | Startup RSS | Settled RSS | -|---|---|---| -| 1 source / 1 adapter (minimal) | ~4 MB | ~10 MB | -| 1 source + a 29 MB audit history | ~19 MB | ~20 MB | -| **16 sources / 13 adapters / 59 models (this host)** | ~28 MB | **~32-35 MB** | - -> For reference, the same production config used **~105 MB** before this round of -> work. The reduction comes from three places: raw audit records are no longer kept -> resident (~25 MB — the aggregates still scan the full history, but the scan -> releases as it goes), Lua state pools no longer grow monotonically, and the two -> runtime knobs below. - -Breakdown of the production instance (per-region, from `/proc//smaps`): - -| Region | RSS | Notes | -|---|---|---| -| Go heap | ~14 MB | provider/registry/scheduler state + connection-pool buffers | -| Other anonymous (thread stacks / LuaJIT chunks / runtime) | ~12 MB | scales with thread count and loaded adapters | -| Binary text+rodata | ~8 MB | mapped executable pages (read-only, reclaimable by the kernel) | -| Go runtime reservations | ~4 MB | the 2 GB+ you see in VSZ is address space, not physical memory | -| Shared libraries | ~3 MB | libc / libluajit / libm | - -**Two recommended deployment knobs** (environment only, no code change): - -```ini -# /etc/systemd/system/llmsproxy.service -Environment=GOGC=50 -Environment=MALLOC_ARENA_MAX=2 -``` - -- `MALLOC_ARENA_MAX=2`: LuaJIT allocates through cgo into glibc malloc, and glibc - allows up to `8 x nproc` per-thread arenas. Every OS thread that touches malloc - claims one (~1 MB each) and **never returns it to the OS**. Measured: 8-15 - arenas (7-12 MB) down to **0**. -- `GOGC=50`: halves the Go heap growth target. **It does nothing on its own** — - the heap it saves is immediately consumed by additional glibc arenas (measured - 20.3 -> 21.5 MB, i.e. slightly worse) — so it must be paired with - `MALLOC_ARENA_MAX`. Together they cut settled RSS by **~19%**. The gateway is - I/O bound (1min10s of CPU per 9 hours here), so the extra GC cycles are free. - -> The desktop build (Electron) already injects both when spawning the embedded -> core (`cmd/gui/main.js`); exporting either variable yourself overrides it. If you -> write your own systemd unit or container spec, add them there. - -Optional: `GOMEMLIMIT=48MiB` as a soft ceiling saves roughly 1 MB more, at the -cost of GC turning aggressive as the limit approaches. Not worth it for a relaxed -deployment. - -### disable_thinking - -With `"disable_thinking": true` in the request body, the gateway passes it to -each adapter; the DeepSeek adapter maps it to `extra_body.thinking.type = -"disabled"`, other sources follow their own protocol. - -### WebUI - -Built-in admin page at `GET /`; after login you can: - -- **Status**: source online state (live probe, error on hover), per - model/source/key usage stats, request records filtered by key, and CSV export - over a time range; click a model to pin the connection snippet to it. -- **Chat**: streaming / non-streaming debug. -- **Keys**: create/edit gateway keys, set per-key model scopes (model + source + - token quota + reset period); admins manage all keys, users see only their own. -- **Priority**: drag blocks to build the tiered AUTO chain. -- **Sources**: add/edit/delete upstream sources online (secrets stored - encrypted). -- **Adapters**: upload / delete Lua adapter scripts. - -Changes persist to `runtime_file` (survive restarts). - -## Lua adapter protocol - -Full API: **[Lua adapter docs](docs/lua-adapters.md)** -([中文](docs/lua-adapters-en.md)). - -Each adapter is a Lua script returning a table -(`internal/lua/adapters/.lua`), overridable by a same-name file under -`adapter_dir` — **no recompile needed**, restart to take effect; uploads via -the WebUI take effect immediately. - -```lua -return { - name = "mysrc", - version = "1.0.0", - endpoint = "/chat/completions", -- upstream path (source.endpoint overrides) - headers = { ["X-Static"] = "v" }, -- static default headers (fallback) - - -- request: convert unified OpenAI req -> upstream native format, return string - transform_request = function(raw_json) ... end, - - -- response: convert upstream response to unified format string - -- { content, reasoning_content, finish_reason, token_usage{...}, tool_calls[{...}] } - transform_response = function(raw_json) ... end, - - -- streaming chunk: convert upstream SSE data to { content, done, ... }; "" skips - transform_stream_chunk = function(raw_chunk) ... end, - - -- [optional] dynamic headers / signature hook - -- meta = { url, method, body, api_key, timestamp, source={ name, meta={...} } } - build_headers = function(meta) return { ["X-App-Sign"] = sign } end, -} -``` - -Shared helpers: `hmac_sha256_hex(key, data)`, `sha256_hex(data)`, -`base64_encode(s)`, `tohex(s)`, `json.encode/decode`, `log(level, msg)`. - -### Built-in adapters - -`openai` `deepseek` `anthropic` `gemini` `github` `groq` `mistral` `ollama` -`kimicode` `qwen38` `sensenova` `trae`. - -`anthropic`/`gemini`/`ollama` include multimodal conversion -(`image_url` → their native format); with `disable_thinking` the `deepseek` -adapter sets `extra_body.thinking.type` to `disabled`. - -**kimicode** demonstrates `build_headers`: the cloud validates the calling -app, so you HMAC-sign timestamp+URL+body with `meta.app_secret` and add -`X-App-Sign`-style headers. Configure `sources[].meta.{app_id, app_secret, -app_agent}`. - -## Desktop GUI (Electron) - -Optional standalone product for desktop users on your internal network: an -**embedded ModelRouter core** in a tray-style desktop app: one-click start/stop, -system tray, autostart, silent launch, and a pixel-for-pixel embedded full WebUI -(Status/Chat/Keys/Priority/Sources/Adapters — no login needed). Server users -keep using the plain Go binary. - -### Two installer kinds: Headless and Desktop - -Release installers ship in two flavors for different audiences: - -- **Headless (server)**: the plain core binary (no Electron), config-driven, - for servers / containers / systemd / unattended runs. Ships as `deb` + `rpm` - (systemd unit with memory tuning + example config) and `tar.gz` (binary + - `config.example.yaml` + `adapters/` + README), Windows as `.zip`. -- **Desktop (desktop app)**: the Electron GUI installer with an embedded core, - for personal daily desktop use. - -### GUI vs. plain backend - -| Scenario | Pick | Why | -| ---- | ---- | ---- | -| Server / intranet gateway / unattended long-running | **Headless** (single binary) | ~8 MB binary; RSS scales with the number of sources (~10 MB for one, ~32-35 MB for 16 — see [Memory footprint](#memory-footprint-measured-and-tuned)), zero-dep single process — drop it into systemd or any container, remote admin | -| Personal desktop daily use / multi-device intranet sharing | **Desktop** (GUI) | no-login embedded WebUI, tray one-click, autostart, silent background — for non-CLI users | -| Windows desktop | **Desktop** | plain backend needs manual service registration; GUI ships native tray/autostart | -| CI one-shot 3-platform installers | **Desktop packaging scripts** | `make gui-*` emits deb / AppImage / NSIS, drops straight into a release pipeline | - -Both share the exact same `llmsproxy` core (LuaJIT build) — configs and -adapters are fully compatible, freely interchangeable. - -### Build & run +Any OpenAI SDK works: point `base_url` at the gateway and use an authorized gateway +key as `api_key`. + +## Documentation + +The detailed documentation is currently written in Chinese. + +| Topic | Document | +|---|---| +| Implementation and the full feature list | [docs/features.md](docs/features.md) | +| Configuration: `config.yaml`, model discovery, multiple keys | [docs/configuration.md](docs/configuration.md) | +| Routing: model routing, AUTO chain, cooldown, sources vs adapters, source templates | [docs/routing.md](docs/routing.md) | +| Adapters: Lua protocol and built-in adapters | [docs/adapters.md](docs/adapters.md) · [docs/lua-adapters-en.md](docs/lua-adapters-en.md) (English) | +| Plugins: SDK contract, hooks, UI extensions, HTTP endpoints | [docs/plugins.md](docs/plugins.md) | +| Desktop app: Headless / Desktop artifacts, build and run | [docs/gui.md](docs/gui.md) | +| Performance: measured memory use, tuning, thinking budget | [docs/performance.md](docs/performance.md) | +| WebUI usage | [docs/webui.md](docs/webui.md) | +| Development: layout, tests, branch and release discipline | [docs/development.md](docs/development.md) · [docs/git-workflow-en.md](docs/git-workflow-en.md) (English) | +| Deployment and migration | [docs/deployment.md](docs/deployment.md) | + +## Renamed from llmsproxy + +The product name has been ModelRouter for a while; the binary, config directory, +service name and environment variables are now `modelrouter` as well: + +| Before | After | +|---|---| +| `llmsproxy` (binary) | `modelrouter` | +| `/etc/llmsproxy/` | `/etc/modelrouter/` | +| `llmsproxy.service` | `modelrouter.service` | +| `LLMSPROXY_*` | `MODELROUTER_*` | + +Existing installs migrate with a dry-run-by-default script (copies, never moves, +and can roll back): ```bash -cd cmd/gui -npm install -make gui # or npm run dev — dev run (needs make build first for bin/llmsproxy) -make gui-deb # local deb (share with other Linux users) -make gui-dist # deb + rpm + AppImage (linux; rpm needs system rpmbuild) -make core-dist # core deb + rpm + tar.gz (server edition; needs nfpm) -make gui-win # Windows NSIS (host mingw + wine) -make gui-win-docker # Windows NSIS, fully dockerized (wine in image; no host mingw/node/wine) +sudo ./packaging/migrate-to-modelrouter.sh # preview +sudo ./packaging/migrate-to-modelrouter.sh --apply # apply +sudo ./packaging/migrate-to-modelrouter.sh --rollback --apply ``` -Artifacts land in `cmd/build/gui-dist/`: `ModelRouter-.AppImage`, -`modelrouter-gui__amd64.deb`, `ModelRouter Setup .exe` (win); core -packages land in `cmd/build/dist/`: `llmsproxy__amd64.deb`, -`llmsproxy-.x86_64.rpm`. Both target families run -`packaging/verify-dist.sh` afterwards — a size floor that rejects degenerate -installers (1.3.0 shipped a 264 KB NSIS stub because the host wine was broken -and nothing caught it). At release time artifacts are renamed into the -`ModelRouter-Desktop-*` (GUI) and `ModelRouter-Headless-*` (plain backend) -families before upload to GitCode Releases. +Old paths and old environment variable names are still accepted on the read side, +so an upgrade cannot fail merely because of ordering. `deploy.sh` refuses to run +when it finds the old `/etc/llmsproxy` without a new config, because that would +deploy a gateway with no key and no sources. -#### Windows packaging (dockerized, recommended) - -`make gui-win-docker` does cross-compile + NSIS **entirely in docker**; the host -needs **no mingw, no wine, no node**. Wine (with the i386 runtime) lives inside - the image, fixing the broken-host-wine failure that produced the 264 KB stub -installer. - -- Image: `cmd/gui/docker/win-builder/Dockerfile` (`golang:1.25` + mingw-w64 + - node + wine32/wine64 + prebuilt LuaJIT for Windows: `lua51.dll` + import lib). - LuaJIT source comes from the gitcode mirror (reachable in CN) with github - fallback; Go modules go through `https://goproxy.cn`. -- One-shot release: `cmd/gui/scripts/dist-win-docker.sh` — core build + NSIS - packaging run inside the container (`USE_SYSTEM_WINE=true` uses the image's - wine); electron-builder's ~400 MB toolchain cache lives in a docker volume so - later builds reuse it. -- The `modelrouter/win-builder` image is auto-built on first run (~5-10 min) - and reused afterwards. -- After packaging, `packaging/verify-dist.sh` runs as a gate: an exe under - 5 MB (or a deb/rpm/7z under 10 MB) fails the build — a broken installer can - never reach a Release again. - -```bash -# Full Windows release (rebuild core + NSIS): -make gui-win-docker -``` - -### Features - -- **Embedded core**: auto-spawns the bundled `llmsproxy` (LuaJIT build); - config/keys/adapters live in `/profile/`. First run generates a - random admin key written into `keys` (not seed) and self-injects it — WebUI - needs no login and never nags about changing the initial key. Port changeable - in settings (default 8787). -- **System tray**: core status dot, autostart/silent toggles, start/stop/restart - core, quit. -- **Silent launch**: `--silent` flag or setting — starts to the tray only; - first run always shows the window. -- **Autostart**: `setLoginItemSettings` on Windows/macOS; writes - `~/.config/autostart/modelrouter-gui.desktop` on Linux, appending `--silent` - when silent mode is on. +The plugin SDK follows the same naming: the only public entry point is +`modelrouter` (`llmsproxy` remains a compatibility alias). See +[docs/plugins.md](docs/plugins.md). ## Layout -``` -cmd/llmsproxy # entry point -internal/config # YAML config loading/validation -internal/lua # LuaJIT VM + worker pool + AdapterCache + built-ins (embed) +```text +cmd/modelrouter # entry point (pure Go server) +cmd/gui # Electron desktop shell (embeds the same core) +internal/config # config parsing, sealed storage, runtime file +internal/core # scheduling, kernel actions, state and audit +internal/lua # LuaJIT VM + elastic worker pool + built-in adapters (embed) +internal/lua/plugins # optional plugin examples (not shipped in binary/archives) internal/provider # Provider(HTTP) + Registry(routing) -internal/gateway # OpenAI-compatible HTTP + auth + SDK/streaming + WebUI -internal/types # unified format & OpenAI wire types +internal/gateway # OpenAI-compatible HTTP service + auth + streaming + WebUI +internal/types # unified format and OpenAI wire types +packaging/ # deb/rpm/tar packaging, systemd unit, migration script +docs/ # topic documentation ``` -## Development & collaboration - -- **Git branching workflow**: main / feature / release-branch conventions in - **[docs/git-workflow-en.md](docs/git-workflow-en.md)** - ([中文](docs/git-workflow.md)). The line: `main` is always deployable → - features merge into `main` → cut `release/vX.Y.Z` and tag → release → hotfixes - land on the release branch and are cherry-picked back into `main`. -- **Lua adapter protocol**: see - **[docs/lua-adapters-en.md](docs/lua-adapters-en.md)** - ([中文](docs/lua-adapters.md)). - -## Tests +## Development and tests ```bash -go test -tags luajit ./... +go test -tags luajit ./... # everything +go test -tags luajit ./internal/lua # one package (required for LuaJIT changes) ``` -Covers: config validation, adapter load/transform, signature hooks, gateway -auth, SDK round trip, SSE streaming, model routing, multimodal passthrough and -disable_thinking. +- Branch and release discipline: [docs/git-workflow-en.md](docs/git-workflow-en.md). +- Plugins live in `plugin_dir`. The gateway never creates, overwrites or deletes + plugin files, and a desktop upgrade never touches them either. diff --git a/cmd/gui/main.js b/cmd/gui/main.js index fd01bc3..a57c959 100644 --- a/cmd/gui/main.js +++ b/cmd/gui/main.js @@ -1,5 +1,5 @@ // ModelRouter Desktop GUI — main process -// Tray-app style: bundles and manages an embedded llmsproxy core instance +// Desktop shell: bundles and manages an embedded modelrouter core instance // in a per-user profile dir (userData), auto-starts it, keeps a tray icon, // supports launch-at-login. Server-side users keep using the pure Go binary. const { @@ -50,11 +50,27 @@ const CONFIG_FILE = path.join(PROFILE_DIR, "config.yaml"); const PLUGINS_DIR = path.join(PROFILE_DIR, "plugins"); // Packaged: extraResource at process.resourcesPath/bin (outside asar, spawnable). // Dev: cmd/gui/bin, populated by prepare-core.js. -const CORE_EXE = path.join( +// The embedded core is named after the product. Installs produced before the +// llmsproxy -> modelrouter rename still carry the old file name inside their +// profile, so resolution falls back to it instead of declaring the core missing. +const CORE_BIN_DIR = path.join( app.isPackaged ? process.resourcesPath : __dirname, "bin", +); +const CORE_EXE = path.join( + CORE_BIN_DIR, + process.platform === "win32" ? "modelrouter.exe" : "modelrouter", +); +const LEGACY_CORE_EXE = path.join( + CORE_BIN_DIR, process.platform === "win32" ? "llmsproxy.exe" : "llmsproxy", ); + +function resolveCoreExe() { + if (fs.existsSync(CORE_EXE)) return CORE_EXE; + if (fs.existsSync(LEGACY_CORE_EXE)) return LEGACY_CORE_EXE; + return CORE_EXE; +} // Packaged window icon: must be a REAL file on disk (extraResource), not an // asar path — Electron/GTK pulls BrowserWindow icons from the filesystem and // crashes (SIGSEGV) when given an asar virtual path on Linux. @@ -163,7 +179,7 @@ function unsealViaCore() { let key = ""; try { const out = execFileSync( - CORE_EXE, + resolveCoreExe(), ["-config", CONFIG_FILE, "-show-secrets"], { encoding: "utf-8", @@ -215,7 +231,7 @@ function coreStarted() { function writeDefaultEmbeddedConfig() { // Generate a self-contained embedded profile with a fresh random admin key. // No upstream sources are preconfigured — the desktop user adds them via - // the WebUI's sources page (like a desktop app's empty-by-default profile). + // the WebUI's sources page (empty by default, like the WebUI profile). // If a config already exists we do NOT overwrite it (keep user's edits). if (fs.existsSync(CONFIG_FILE)) return; const key = "sk-gw-" + crypto.randomBytes(16).toString("hex"); @@ -272,8 +288,9 @@ function startCore() { console.log("core already running"); return; } - if (!fs.existsSync(CORE_EXE)) { - console.error("embedded core binary missing:", CORE_EXE); + const coreExe = resolveCoreExe(); + if (!fs.existsSync(coreExe)) { + console.error("embedded core binary missing:", coreExe); return; } // ensure profile dir + config + local adapters exist @@ -281,9 +298,8 @@ function startCore() { if (!fs.existsSync(CONFIG_FILE)) writeDefaultEmbeddedConfig(); pluginSync.ensurePluginDirConfigured(CONFIG_FILE, PLUGINS_DIR, log); fs.mkdirSync(PLUGINS_DIR, { recursive: true }); - pluginSync.syncBundledPluginsOnUpgrade(PROFILE_DIR, PLUGINS_DIR, app.getVersion(), log); - coreProc = spawn(CORE_EXE, ["-config", CONFIG_FILE], { + coreProc = spawn(coreExe, ["-config", CONFIG_FILE], { cwd: PROFILE_DIR, env: Object.assign({}, process.env, { LLMS_PROXY_GUI: "1", @@ -686,7 +702,7 @@ function createWindow() { }); } // Silent start: `--silent` CLI flag or settings.silentStart keeps the app - // in the tray without showing the window (tray-app style). First run + // in the tray without showing the window (desktop-shell behaviour). First run // always shows the window so the user sees the dashboard. mainWindow.once("ready-to-show", () => { const firstRun = !fs.existsSync( @@ -810,8 +826,8 @@ ipcMain.handle("core:state", () => ({ port: settings.port || DEFAULT_PORT, profileDir: PROFILE_DIR, configFile: CONFIG_FILE, - coreExe: CORE_EXE, - coreExists: fs.existsSync(CORE_EXE), + coreExe: resolveCoreExe(), + coreExists: fs.existsSync(resolveCoreExe()), })); ipcMain.handle("core:start", () => { app.isQuitting = false; @@ -883,7 +899,7 @@ if (gotLock) { createWindow(); createTray(); - // start embedded core if not already running (another instance of llmsproxy) + // start embedded core if not already running (another instance of modelrouter) setTimeout(async () => { const p = await probeEmbedded(); if (!p.ok) { diff --git a/cmd/gui/package.json b/cmd/gui/package.json index 0cec281..d3d1ba8 100644 --- a/cmd/gui/package.json +++ b/cmd/gui/package.json @@ -39,8 +39,8 @@ }, "extraResources": [ { - "from": "bin/llmsproxy", - "to": "bin/llmsproxy" + "from": "bin/modelrouter", + "to": "bin/modelrouter" }, { "from": "build/icon.png", @@ -66,8 +66,8 @@ "icon": "build/icon.ico", "extraResources": [ { - "from": "bin/llmsproxy.exe", - "to": "bin/llmsproxy.exe" + "from": "bin/modelrouter.exe", + "to": "bin/modelrouter.exe" }, { "from": "bin/lua51.dll", diff --git a/cmd/gui/plugin-sync.js b/cmd/gui/plugin-sync.js index c9b2318..c7b2ac9 100644 --- a/cmd/gui/plugin-sync.js +++ b/cmd/gui/plugin-sync.js @@ -1,18 +1,11 @@ -// Plugin-tree maintenance for the embedded core (no Electron imports, so this +// Plugin-tree maintenance for the desktop profile (no Electron imports, so this // is unit-testable with plain node). // -// Two problems this solves, both invisible from the gateway side: -// 1. The core deliberately does NOT default `plugin_dir` (an operator opts -// in). A desktop profile without it runs with ZERO plugins — the channels -// plugin (account login, credential vault, auto check-in) never appears. -// 2. The core seeds bundled plugins only when the file is ABSENT, so a -// desktop upgrade would keep the user's old channels.lua forever and -// silently skip every channel fix. On version change the shell must drop -// the built-in files so the new binary reseeds them. +// The desktop may configure an operator-selected plugin directory, but it never +// owns the files in that directory. In particular, upgrades must not seed, +// replace, rename, or delete plugins. Plugin installation is an explicit +// operator action (or an external package manager action). const fs = require("fs"); -const path = require("path"); - -const BUILTIN_PLUGINS = ["billing.lua", "channels.lua", "quota.lua", "usage.lua"]; // ensurePluginDirConfigured back-fills `plugin_dir` into an existing config. // Appending a top-level YAML key is safe for the shapes this file can have @@ -25,7 +18,7 @@ function ensurePluginDirConfigured(configFile, pluginsDir, log) { fs.writeFileSync( configFile, cur.replace(/\s*$/, "") + - "\n# added by ModelRouter Desktop: enables the plugin tree (channels etc.)\n" + + "\n# added by ModelRouter Desktop: enables the operator plugin directory\n" + "plugin_dir: " + JSON.stringify(pluginsDir) + "\n", "utf-8", ); @@ -37,46 +30,4 @@ function ensurePluginDirConfigured(configFile, pluginsDir, log) { } } -// syncBundledPluginsOnUpgrade drops built-in plugin files when the app version -// changed, so the core reseeds them from the binary it shipped with. Returns -// { changed, dropped:[...], kept: "prev-" }. -function syncBundledPluginsOnUpgrade(profileDir, pluginsDir, version, log) { - const stamp = path.join(profileDir, ".plugin-seed-version"); - let have = ""; - try { - have = fs.readFileSync(stamp, "utf-8").trim(); - } catch (e) { - /* first run */ - } - if (have === version) return { changed: false, dropped: [] }; - const dropped = []; - for (const name of BUILTIN_PLUGINS) { - const f = path.join(pluginsDir, name); - if (!fs.existsSync(f)) continue; - try { - if (have) fs.renameSync(f, f + ".prev-" + have); - else fs.unlinkSync(f); - dropped.push(name); - } catch (e) { - if (log) log("[gui] drop builtin plugin " + name + " failed: " + e.message); - } - } - try { - fs.writeFileSync(stamp, version + "\n", "utf-8"); - } catch (e) { - if (log) log("[gui] write plugin seed stamp failed: " + e.message); - } - if (dropped.length && log) { - log( - "[gui] plugins refreshed for " + version + " (" + dropped.length + - " builtin file(s) reseeded by core)", - ); - } - return { changed: true, dropped, kept: have ? "prev-" + have : "" }; -} - -module.exports = { - BUILTIN_PLUGINS, - ensurePluginDirConfigured, - syncBundledPluginsOnUpgrade, -}; +module.exports = { ensurePluginDirConfigured }; diff --git a/cmd/gui/plugin-sync.test.js b/cmd/gui/plugin-sync.test.js index e3d3c86..dfd0f04 100644 --- a/cmd/gui/plugin-sync.test.js +++ b/cmd/gui/plugin-sync.test.js @@ -1,26 +1,22 @@ -// Unit tests for the desktop plugin-tree maintenance (plain node, no Electron). -// Run: node --test cmd/gui/plugin-sync.test.js +// Unit tests for desktop plugin-directory configuration (plain node). const test = require("node:test"); const assert = require("node:assert"); const fs = require("fs"); const os = require("os"); const path = require("path"); -const { BUILTIN_PLUGINS, ensurePluginDirConfigured, syncBundledPluginsOnUpgrade } = require("./plugin-sync"); +const { ensurePluginDirConfigured } = require("./plugin-sync"); const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "plugin-sync-")); test("back-fills plugin_dir into an existing profile without it", () => { const dir = tmp(); const cfg = path.join(dir, "config.yaml"); - fs.writeFileSync(cfg, 'listen: 127.0.0.1:8787\ndefault_model: AUTO\nkeys:\n - key: k\n role: admin\n'); + fs.writeFileSync(cfg, "listen: 127.0.0.1:8787\ndefault_model: AUTO\nkeys:\n - key: k\n role: admin\n"); const plugins = path.join(dir, "plugins"); assert.equal(ensurePluginDirConfigured(cfg, plugins, null), "added"); const out = fs.readFileSync(cfg, "utf-8"); assert.match(out, /^plugin_dir: "/m); - assert.ok(out.includes(plugins.replace(/\\/g, "\\\\")) || out.includes(plugins), "path written"); - // the rest of the profile must survive untouched - assert.match(out, /^listen: 127\.0\.0\.1:8787$/m); - assert.match(out, /^ - key: k$/m); + assert.match(out, /^listen: 127.0.0.1:8787$/m); }); test("leaves a profile that already sets plugin_dir alone", () => { @@ -39,42 +35,14 @@ test("missing config is not created by the back-fill", () => { assert.equal(fs.existsSync(cfg), false); }); -test("upgrade drops builtin plugins (keeping a backup) but not user plugins", () => { +test("desktop upgrade never modifies operator plugin files", () => { const dir = tmp(); const plugins = path.join(dir, "plugins"); fs.mkdirSync(plugins, { recursive: true }); - for (const f of BUILTIN_PLUGINS) fs.writeFileSync(path.join(plugins, f), "OLD"); + fs.writeFileSync(path.join(plugins, "channels.lua"), "OPERATOR FILE"); fs.writeFileSync(path.join(plugins, "my-own.lua"), "MINE"); - fs.writeFileSync(path.join(dir, ".plugin-seed-version"), "1.0.0\n"); - - const r = syncBundledPluginsOnUpgrade(dir, plugins, "2.0.42", null); - assert.equal(r.changed, true); - assert.deepEqual(r.dropped.sort(), [...BUILTIN_PLUGINS].sort()); - for (const f of BUILTIN_PLUGINS) { - assert.equal(fs.existsSync(path.join(plugins, f)), false, f + " dropped"); - assert.equal(fs.readFileSync(path.join(plugins, f + ".prev-1.0.0"), "utf-8"), "OLD"); - } + // There is deliberately no version stamp and no upgrade synchronizer. + assert.equal(fs.readFileSync(path.join(plugins, "channels.lua"), "utf-8"), "OPERATOR FILE"); assert.equal(fs.readFileSync(path.join(plugins, "my-own.lua"), "utf-8"), "MINE"); - assert.equal(fs.readFileSync(path.join(dir, ".plugin-seed-version"), "utf-8").trim(), "2.0.42"); -}); - -test("same version is a no-op (nothing dropped)", () => { - const dir = tmp(); - const plugins = path.join(dir, "plugins"); - fs.mkdirSync(plugins, { recursive: true }); - fs.writeFileSync(path.join(plugins, "channels.lua"), "CURRENT"); - fs.writeFileSync(path.join(dir, ".plugin-seed-version"), "2.0.42\n"); - const r = syncBundledPluginsOnUpgrade(dir, plugins, "2.0.42", null); - assert.equal(r.changed, false); - assert.equal(fs.readFileSync(path.join(plugins, "channels.lua"), "utf-8"), "CURRENT"); -}); - -test("first run on a fresh profile just stamps the version", () => { - const dir = tmp(); - const plugins = path.join(dir, "plugins"); - fs.mkdirSync(plugins, { recursive: true }); - const r = syncBundledPluginsOnUpgrade(dir, plugins, "2.0.42", null); - assert.equal(r.changed, true); - assert.deepEqual(r.dropped, []); - assert.equal(fs.existsSync(path.join(dir, ".plugin-seed-version")), true); + assert.equal(fs.readdirSync(dir).includes(".plugin-seed-version"), false); }); diff --git a/cmd/gui/renderer/app.js b/cmd/gui/renderer/app.js index ec80b49..7797315 100644 --- a/cmd/gui/renderer/app.js +++ b/cmd/gui/renderer/app.js @@ -1,5 +1,5 @@ // ModelRouter Desktop GUI — renderer -// Tray-app style: the content area is a full-screen iframe loading the +// Desktop shell: the content area is a full-screen iframe loading the // real WebUI 1:1 (status / chat / keys / priority / sources / adapters). // The shell only adds the titlebar, settings overlay and tray integration. const state = { diff --git a/cmd/gui/renderer/index.html b/cmd/gui/renderer/index.html index 9482708..4f42edc 100644 --- a/cmd/gui/renderer/index.html +++ b/cmd/gui/renderer/index.html @@ -83,7 +83,7 @@ - +