mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-11-09 18:57:21 +00:00
Compare commits
256 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 190c2d23d6 | |||
| 1ae9558547 | |||
| b09d19d8f4 | |||
| 8f2c5572e7 | |||
| 449e59dd52 | |||
| c0d7c79d0d | |||
| 1496255488 | |||
| e2ced5ed66 | |||
| f67bcf31b0 | |||
| a708eb10c9 | |||
| f32c10d263 | |||
| 9cdf112de4 | |||
| f7e926ccb7 | |||
| 688dce88d1 | |||
| de7553eaa2 | |||
| 97e4ab582d | |||
| 3ef918a87a | |||
| a073423867 | |||
| 739d5670ba | |||
| 6ec12a3751 | |||
| db1cb7182d | |||
| 79bc55cfbe | |||
| 4b9fb7b056 | |||
| 7941b4110a | |||
| 5ccf068db3 | |||
| b4c5f0f0fc | |||
| 442fa5e3f6 | |||
| 0caa37ea6a | |||
| 33cb2bdb06 | |||
| b24b69719d | |||
| 24e1dee960 | |||
| b57b442c0a | |||
| 0ae7d2c8be | |||
| 02c01582f1 | |||
| 6454b094bc | |||
| 771b95a133 | |||
| 441de434f5 | |||
| 9feef8f4cb | |||
| d138ef80a5 | |||
| 2c278d481d | |||
| b5c89fd3d7 | |||
| 7f193a997d | |||
| 3944a578c7 | |||
| abd0e1d943 | |||
| 055705c134 | |||
| 401a7ead27 | |||
| 2924da0032 | |||
| 411503aea5 | |||
| 9c7b5f8556 | |||
| 662f8f02b6 | |||
| b0f33c188f | |||
| 711ccdeea8 | |||
| ff7051d608 | |||
| efc9b37965 | |||
| 2113bfc125 | |||
| c9f34538ec | |||
| 12bfd53f16 | |||
| 25970a2135 | |||
| dcf327d3c5 | |||
| 6f2d4dc6eb | |||
| fc5326402f | |||
| aff8354c8a | |||
| 56ea269f98 | |||
| 091f790734 | |||
| aea1a2c48d | |||
| 6eeb5832a3 | |||
| 9e5a5f550b | |||
| a189c4f61c | |||
| 8a4153d18a | |||
| 58073a6cbe | |||
| 59902082c6 | |||
| a5923b3797 | |||
| dda537cda9 | |||
| 91f6d1f32f | |||
| 2d7388ad35 | |||
| c1048dcd4e | |||
| 00e59a7e1d | |||
| 2b8fcaf434 | |||
| c0e08f907a | |||
| 47fe759025 | |||
| 3ac4b35eb1 | |||
| d795266572 | |||
| 969f3138fd | |||
| a82ae2b60e | |||
| 00db19eb77 | |||
| 2d2c27c430 | |||
| 38490690c5 | |||
| 0f83fc048a | |||
| 969c72bfb1 | |||
| 1f29be43db | |||
| 423dd5cf22 | |||
| 8109acf7cd | |||
| eeba1e49f8 | |||
| 7c82c7ff6b | |||
| ade9c122eb | |||
| 1f18643d1f | |||
| 71cc142615 | |||
| e6fa2a678f | |||
| 33bb7a98b2 | |||
| b871c0c713 | |||
| 1e6b4864ec | |||
| 8b318a8507 | |||
| 89d759ece7 | |||
| 69da3c9b92 | |||
| 8d23005f7d | |||
| 1be34557dd | |||
| ddbafe822c | |||
| 9e5fec08b8 | |||
| 68446fb6fe | |||
| 482a65958b | |||
| 796b30f5b0 | |||
| 7e0a428c23 | |||
| 7a8d97001c | |||
| 25a525be84 | |||
| 850920155a | |||
| ea8a9ac4c6 | |||
| eaac0ef6b3 | |||
| 8f73f77ec2 | |||
| 226493b5ae | |||
| c36a794aa8 | |||
| d3a0eb6311 | |||
| 0f263f895a | |||
| cbad61a3ea | |||
| 044ff1129b | |||
| 2a2067284d | |||
| 742e2b5ed3 | |||
| 81b3661336 | |||
| 9521a35a03 | |||
| 9bec158d2e | |||
| 252e3486bd | |||
| 0c56392d29 | |||
| f813ec714f | |||
| 7a6e03692d | |||
| 420379784b | |||
| 92c240cc25 | |||
| 5720e60fc9 | |||
| 42ec552c6d | |||
| 67e2ba070f | |||
| 29c87ce015 | |||
| 583976f7da | |||
| a020247627 | |||
| af8b93bd7c | |||
| 3819b00ded | |||
| 10ad4421f9 | |||
| f55d514e85 | |||
| faa3be5c37 | |||
| 8e2d384a7b | |||
| ff9b460eba | |||
| 1f54143956 | |||
| a7a0c6adee | |||
| 741edfea98 | |||
| ed1e250452 | |||
| fe9d5c6054 | |||
| 3e94047889 | |||
| 4a90099fb2 | |||
| 59229990ac | |||
| 6238265502 | |||
| 4bdde9cd61 | |||
| 70c24ba158 | |||
| c1733feeaa | |||
| 5b433b3266 | |||
| d1b7d94043 | |||
| 1586a051da | |||
| 41ff6d1be3 | |||
| 6fe6014d92 | |||
| 5ab047e48b | |||
| 0004545190 | |||
| ec5e6dd57c | |||
| 1dda9e4477 | |||
| 705e079285 | |||
| 04f12fae71 | |||
| 8949405746 | |||
| cf53d72af4 | |||
| d86514c474 | |||
| d4ae2a1d13 | |||
| 652a035d47 | |||
| 6bec260513 | |||
| 03430bcef5 | |||
| 192e545d40 | |||
| ed954b77bd | |||
| cb3ea25586 | |||
| de9d7c8008 | |||
| 8a3059740f | |||
| 0232b9cd24 | |||
| 028e42b036 | |||
| a07449d32e | |||
| 84c9066bed | |||
| 84a9fccd9a | |||
| 63549db832 | |||
| d6d41209d9 | |||
| 3c59c33c58 | |||
| 5948ad19ca | |||
| 6b6f9526ff | |||
| 4d0efab86b | |||
| b6e687b567 | |||
| 2353d43c16 | |||
| 893857464d | |||
| 2f5cde7fea | |||
| da0282a3db | |||
| 6f2b6b8d3d | |||
| 96d97b0e0e | |||
| 6de9f28269 | |||
| 50997a4e3a | |||
| 74ec1e7874 | |||
| fa50ec8bc7 | |||
| c3b858d5ba | |||
| a59295e826 | |||
| dafc3b3fc3 | |||
| 66786bb0e3 | |||
| f4d5e779a5 | |||
| 61f402beef | |||
| 57fada3b0d | |||
| b6148e6f79 | |||
| 24e00cdc50 | |||
| 9016eda905 | |||
| 4d1407bbb4 | |||
| 5f6939b590 | |||
| a0ab9ddda5 | |||
| e4111337cd | |||
| 93eeba3762 | |||
| 958f1f48b5 | |||
| 9d6653a573 | |||
| 1c6a2821e4 | |||
| 3ad46ea23e | |||
| 6113924655 | |||
| d3203d88fc | |||
| 7836b95a84 | |||
| b6d0358079 | |||
| 0858a747ca | |||
| 7934507179 | |||
| f042bbf4af | |||
| 97a6cf9224 | |||
| d92db3a9fd | |||
| 0e747b0891 | |||
| be8d753e36 | |||
| 438629256d | |||
| a873fe4724 | |||
| 31bc3968c5 | |||
| c40b5b5981 | |||
| 7ce0378e8b | |||
| ae9f56ddb3 | |||
| 3860c7381f | |||
| 94c1e7688f | |||
| b8386465ea | |||
| f682205d26 | |||
| ba81a4d7e1 | |||
| e30199b29d | |||
| a1fddf72b7 | |||
| b9a025efba | |||
| 7c7cf3e078 | |||
| 18525ae4c0 | |||
| c4b37d4322 | |||
| 33b778faee | |||
| 345cbb30d6 | |||
| 501ad6aac9 | |||
| 39cda89def |
24
.snapshots/chat_nonstream.json
Normal file
24
.snapshots/chat_nonstream.json
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"finish_reason": "stop",
|
||||||
|
"index": 0,
|
||||||
|
"message": {
|
||||||
|
"content": "pong",
|
||||||
|
"role": "assistant"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"created": <TS>,
|
||||||
|
"id": "<ID>",
|
||||||
|
"model": "good-m",
|
||||||
|
"object": "chat.completion",
|
||||||
|
"usage": {
|
||||||
|
"completion": 1,
|
||||||
|
"completion_tokens": 1,
|
||||||
|
"prompt": 3,
|
||||||
|
"prompt_tokens": 3,
|
||||||
|
"total": 4,
|
||||||
|
"total_tokens": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
95
.snapshots/chat_stream.txt
Normal file
95
.snapshots/chat_stream.txt
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
{
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"delta": {
|
||||||
|
"content": "",
|
||||||
|
"role": "assistant"
|
||||||
|
},
|
||||||
|
"finish_reason": null,
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"created": <TS>,
|
||||||
|
"id": "<ID>",
|
||||||
|
"model": "good-m",
|
||||||
|
"object": "chat.completion.chunk"
|
||||||
|
}
|
||||||
|
{
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"delta": {
|
||||||
|
"content": "Hel",
|
||||||
|
"role": "assistant"
|
||||||
|
},
|
||||||
|
"finish_reason": null,
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"created": <TS>,
|
||||||
|
"id": "<ID>",
|
||||||
|
"model": "good-m",
|
||||||
|
"object": "chat.completion.chunk"
|
||||||
|
}
|
||||||
|
{
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"delta": {
|
||||||
|
"content": "lo",
|
||||||
|
"role": "assistant"
|
||||||
|
},
|
||||||
|
"finish_reason": null,
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"created": <TS>,
|
||||||
|
"id": "<ID>",
|
||||||
|
"model": "good-m",
|
||||||
|
"object": "chat.completion.chunk"
|
||||||
|
}
|
||||||
|
{
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"delta": {
|
||||||
|
"content": "",
|
||||||
|
"role": "assistant"
|
||||||
|
},
|
||||||
|
"finish_reason": "stop",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"created": <TS>,
|
||||||
|
"id": "<ID>",
|
||||||
|
"model": "good-m",
|
||||||
|
"object": "chat.completion.chunk"
|
||||||
|
}
|
||||||
|
{
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"delta": {
|
||||||
|
"content": ""
|
||||||
|
},
|
||||||
|
"finish_reason": "stop",
|
||||||
|
"index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"created": <TS>,
|
||||||
|
"id": "<ID>",
|
||||||
|
"model": "good-m",
|
||||||
|
"object": "chat.completion.chunk"
|
||||||
|
}
|
||||||
|
{
|
||||||
|
"choices": [],
|
||||||
|
"created": <TS>,
|
||||||
|
"id": "<ID>",
|
||||||
|
"model": "good-m",
|
||||||
|
"object": "chat.completion.chunk",
|
||||||
|
"usage": {
|
||||||
|
"completion": 1,
|
||||||
|
"completion_tokens": 1,
|
||||||
|
"prompt": 16,
|
||||||
|
"prompt_tokens": 16,
|
||||||
|
"total": 17,
|
||||||
|
"total_tokens": 17
|
||||||
|
}
|
||||||
|
}
|
||||||
|
[DONE]
|
||||||
8
.snapshots/image_generation.json
Normal file
8
.snapshots/image_generation.json
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"created": <TS>,
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"b64_json": "QUJD"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
501
HANDOFF_2026-10-19.md
Normal file
501
HANDOFF_2026-10-19.md
Normal file
@ -0,0 +1,501 @@
|
|||||||
|
# ModelRouter / llmsproxy 交接文档
|
||||||
|
|
||||||
|
> **历史文档**:本文记录的渠道能力(channels / quota 插件与 4 个私有协议适配器)
|
||||||
|
> 已于 v3.0.2 按合规要求整体移出仓库。保留原文仅作历史记录,其中的路径、
|
||||||
|
> 模块名与适配器清单不再代表当前代码。
|
||||||
|
|
||||||
|
更新时间:2026-10-19
|
||||||
|
工作目录:`/home/program/llmsproxy`
|
||||||
|
当前分支:`release/v2.0.x`
|
||||||
|
|
||||||
|
## 1. 当前 Git 状态
|
||||||
|
|
||||||
|
最近提交:
|
||||||
|
|
||||||
|
- `c9f3453 fix(codearts): 识别 InferHub 未注册模型`
|
||||||
|
- `12bfd53 fix: 收口渠道源迁移与测试页合并`
|
||||||
|
- `25970a2 refactor(scheduler)!: 移除 priority 废弃字段 + 一通道一源 + discover 复用凭据`
|
||||||
|
|
||||||
|
`c9f3453` 已推送到 `origin/release/v2.0.x`。
|
||||||
|
|
||||||
|
注意:当前工作树在最后一轮添加 provider 回归测试时出现了一个**未完成且可能无法编译的测试编辑**。最后一次工具报错:
|
||||||
|
|
||||||
|
```text
|
||||||
|
internal/provider/provider_test.go:99: missing ',' in argument list
|
||||||
|
```
|
||||||
|
|
||||||
|
下一位 agent 接手后的第一步必须是:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/program/llmsproxy
|
||||||
|
git status --short
|
||||||
|
gofmt -w internal/provider/provider_test.go 2>&1 || true
|
||||||
|
go test -tags luajit ./internal/provider/ 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
如果测试文件处于半编辑状态,先检查 `TestProviderChatStreamNormalizesStopAfterToolCall`,修正 Go 字符串转义或删除该未完成测试后再继续。不要直接部署当前工作树。
|
||||||
|
|
||||||
|
## 2. 生产服务状态
|
||||||
|
|
||||||
|
生产服务:`llmsproxy.service`
|
||||||
|
|
||||||
|
- 当前服务 active
|
||||||
|
- 监听:`0.0.0.0:8081`
|
||||||
|
- 二进制已通过 `./deploy.sh` 原子替换并重启
|
||||||
|
- 最近一次部署健康检查通过
|
||||||
|
- 当前生产插件 SHA256 前缀曾核对为:`1bf26cbb3abcb107`
|
||||||
|
- 当前生产二进制在最近部署后已更新
|
||||||
|
|
||||||
|
不要把密钥写入交接记录。生产 admin key 保存在本机运维环境中,不应复制到聊天、文档或提交。
|
||||||
|
|
||||||
|
## 3. Channel 源当前状态
|
||||||
|
|
||||||
|
生产源已经从旧的“一账号一源”收口成“一渠道一源”:
|
||||||
|
|
||||||
|
```text
|
||||||
|
channels-trae
|
||||||
|
channels-minimax
|
||||||
|
channels-lobsterai
|
||||||
|
channels-buddy
|
||||||
|
channels-codearts
|
||||||
|
channels-zcode
|
||||||
|
```
|
||||||
|
|
||||||
|
旧形态源已经清理:
|
||||||
|
|
||||||
|
```text
|
||||||
|
channels-<provider>-<account_id>
|
||||||
|
```
|
||||||
|
|
||||||
|
此前旧源和新源并存,造成同一渠道同一模型出现多份,只能靠账号 hex 区分。
|
||||||
|
|
||||||
|
根因已修复:内核 `list_sources` 返回裸数组,但旧插件代码按 `{sources = [...]}` 信封读取,导致迁移清理循环永远为空。当前 `channels.lua` 已兼容:
|
||||||
|
|
||||||
|
- 裸数组
|
||||||
|
- `{sources = [...]}`
|
||||||
|
- `{json = {sources = [...]}}`
|
||||||
|
|
||||||
|
相关代码:
|
||||||
|
|
||||||
|
```text
|
||||||
|
internal/lua/plugins/channels.lua: syncProviderSource
|
||||||
|
internal/core/channels_legacy_source_cleanup_test.go
|
||||||
|
```
|
||||||
|
|
||||||
|
当前生产源列表已实测只有 6 个新形态源。
|
||||||
|
|
||||||
|
## 4. CodeArts 当前状态
|
||||||
|
|
||||||
|
### 已确认可用的部分
|
||||||
|
|
||||||
|
CodeArts 使用:
|
||||||
|
|
||||||
|
```text
|
||||||
|
adapter = codearts
|
||||||
|
base_url = https://snap-access.cn-north-4.myhuaweicloud.com
|
||||||
|
```
|
||||||
|
|
||||||
|
CodeArts 的 chat 请求需要完整 AK/SK/security token 凭据,由插件适配器签名,不是裸 Bearer token。
|
||||||
|
|
||||||
|
真实流式工具调用测试已经成功:
|
||||||
|
|
||||||
|
请求形状:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"model": "channels-codearts:openpangu-2.0-flash",
|
||||||
|
"messages": [{"role": "user", "content": "Use the weather tool for Beijing."}],
|
||||||
|
"tools": [{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "get_weather",
|
||||||
|
"description": "Get weather",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"city": {"type": "string"}},
|
||||||
|
"required": ["city"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}],
|
||||||
|
"tool_choice": "auto",
|
||||||
|
"stream": true,
|
||||||
|
"max_tokens": 64
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
结果:
|
||||||
|
|
||||||
|
- HTTP 200
|
||||||
|
- `Content-Type: text/event-stream`
|
||||||
|
- 返回结构化 `tool_calls`
|
||||||
|
- 函数名 `get_weather`
|
||||||
|
- 参数最终为 `{"city":"Beijing"}`
|
||||||
|
- `finish_reason = tool_calls`
|
||||||
|
|
||||||
|
因此 CodeArts 的流式工具调用链路当前是通的。
|
||||||
|
|
||||||
|
### CodeArts 模型刷新
|
||||||
|
|
||||||
|
生产调用:
|
||||||
|
|
||||||
|
```http
|
||||||
|
POST /api/plugins/http/channels/model/refresh
|
||||||
|
{"platform":"codearts"}
|
||||||
|
```
|
||||||
|
|
||||||
|
实测返回:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"results": [
|
||||||
|
{"platform": "codearts", "ok": true, "count": 11}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
CodeArts 模型目录来自两个签名端点并合并:
|
||||||
|
|
||||||
|
1. `opengw.developer.huaweicloud.com/api/v1/gateway/config`
|
||||||
|
- `result.models`
|
||||||
|
2. `snap-access.cn-north-4.myhuaweicloud.com/v1/model/builtin`
|
||||||
|
- `builtinModels`
|
||||||
|
|
||||||
|
相关解析:
|
||||||
|
|
||||||
|
```text
|
||||||
|
internal/lua/plugins/channels.lua: parseRemoteModels
|
||||||
|
internal/lua/plugins/channels.lua: refreshRemoteModels
|
||||||
|
internal/core/channels_codearts_catalog_test.go
|
||||||
|
```
|
||||||
|
|
||||||
|
### 已确认失效并禁用的 CodeArts 模型
|
||||||
|
|
||||||
|
下面 4 个模型曾收到明确的上游 404:
|
||||||
|
|
||||||
|
```text
|
||||||
|
CodeArts-Pangu
|
||||||
|
deepseek-v4-flash-0731
|
||||||
|
deepseek-v4-pro-0813
|
||||||
|
glm-5.3-flash
|
||||||
|
```
|
||||||
|
|
||||||
|
生产上已通过 `channels/model/set_disabled` 禁用。它们仍可能在 `model/list` 中出现,但 `disabled=true`,不会作为可用模型路由。
|
||||||
|
|
||||||
|
新增的死模型判据:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
model is not registered
|
||||||
|
```
|
||||||
|
|
||||||
|
对应提交:`c9f3453`。
|
||||||
|
|
||||||
|
注意:当前插件的 dead-model 机制只在收到真实请求失败后记录模型;刷新目录不会自动根据“远端目录没有该模型”删除模型,这是刻意设计,避免误删远端目录不完整的平台模型。
|
||||||
|
|
||||||
|
CodeArts 当前仍有一些模型返回 429/TPM 限流,例如:
|
||||||
|
|
||||||
|
```text
|
||||||
|
deepseek-v4-flash
|
||||||
|
deepseek-v4.1-flash
|
||||||
|
```
|
||||||
|
|
||||||
|
429 不是模型失效,不能禁用。
|
||||||
|
|
||||||
|
## 5. 其他 Channel 实测结果
|
||||||
|
|
||||||
|
### Trae
|
||||||
|
|
||||||
|
真实流式 + tools 请求:
|
||||||
|
|
||||||
|
```text
|
||||||
|
channels-trae:DeepSeek-V4-Flash
|
||||||
|
```
|
||||||
|
|
||||||
|
结果:
|
||||||
|
|
||||||
|
- HTTP 200
|
||||||
|
- SSE 正常返回
|
||||||
|
- 结构化 tool call 返回
|
||||||
|
- 但上游最后的结束原因可能是 `stop`,即使前面已经出现 tool-call fragments
|
||||||
|
|
||||||
|
这会让 Agent 把工具调用误判成普通文本完成。
|
||||||
|
|
||||||
|
当前准备中的修复在统一 provider 流式层:
|
||||||
|
|
||||||
|
```text
|
||||||
|
internal/provider/provider.go
|
||||||
|
```
|
||||||
|
|
||||||
|
逻辑是:只要本次流出现真实 tool-call 片段,后续 `FinishReason == "stop"` 就归一成 `tool_calls`。
|
||||||
|
|
||||||
|
另外,Trae 的非流式接口偶尔返回 SSE。当前 `trae.lua` 对非流式 SSE 只取最后一条 data,不能完整合并多条工具调用/推理片段。这是后续仍需处理的协议问题,但不能用非流式手工请求结果推断 Pi 的真实请求模式。
|
||||||
|
|
||||||
|
### Buddy
|
||||||
|
|
||||||
|
真实流式 + tools 请求:
|
||||||
|
|
||||||
|
```text
|
||||||
|
channels-buddy:glm-5.3-flash
|
||||||
|
```
|
||||||
|
|
||||||
|
结果:
|
||||||
|
|
||||||
|
- HTTP 200
|
||||||
|
- SSE 正常返回
|
||||||
|
- tool-call 参数分片可见
|
||||||
|
- 最终有 `finish_reason = tool_calls`
|
||||||
|
- 但某些上游 chunk 的 `tool_calls` 可能是空对象 `{}`
|
||||||
|
|
||||||
|
空对象不是合法的 OpenAI tool-call 数组,部分 Agent 客户端会拒绝。
|
||||||
|
|
||||||
|
当前准备中的修复在:
|
||||||
|
|
||||||
|
```text
|
||||||
|
internal/lua/adapters/buddy.lua
|
||||||
|
```
|
||||||
|
|
||||||
|
只在 `type(delta.tool_calls) == "table" and #delta.tool_calls > 0` 时输出 `tool_calls`,避免 Lua 空表被编码为 `{}`。
|
||||||
|
|
||||||
|
Buddy 的非流式错误:
|
||||||
|
|
||||||
|
```text
|
||||||
|
11101: Non-stream chat request is currently not supported
|
||||||
|
```
|
||||||
|
|
||||||
|
之前出现的该错误来自手工 `curl` 测试中显式设置 `stream:false`,不能归因到 Pi。真实 `stream:true + tools` 已经证明 Buddy 流式工具调用可以工作。
|
||||||
|
|
||||||
|
### MiniMax
|
||||||
|
|
||||||
|
生产日志反复出现:
|
||||||
|
|
||||||
|
```text
|
||||||
|
adapter "anthropic": status=401 body={"error":"invalid access token"}
|
||||||
|
provider channels-minimax: api error 401
|
||||||
|
```
|
||||||
|
|
||||||
|
这是凭据失效问题,不是 tools 形状问题。
|
||||||
|
|
||||||
|
当前源:
|
||||||
|
|
||||||
|
```text
|
||||||
|
channels-minimax
|
||||||
|
adapter = anthropic
|
||||||
|
endpoint = /mavis/api/v1/llm/v1/messages
|
||||||
|
```
|
||||||
|
|
||||||
|
需要用户重新登录/提供有效凭据。不要擅自刷新一次性 refresh token,之前已经发生过 refresh token 被消耗后无法再次使用的事故。
|
||||||
|
|
||||||
|
## 6. 当前未提交的代码改动
|
||||||
|
|
||||||
|
最后一轮已经改动但尚未完成提交:
|
||||||
|
|
||||||
|
```text
|
||||||
|
internal/provider/provider.go
|
||||||
|
internal/lua/adapters/trae.lua
|
||||||
|
internal/lua/adapters/buddy.lua
|
||||||
|
internal/provider/provider_test.go # 可能有半编辑、需先修复
|
||||||
|
```
|
||||||
|
|
||||||
|
已完成的意图:
|
||||||
|
|
||||||
|
1. provider 流式层记录 `sawToolCalls`
|
||||||
|
2. 看到真实 tool-call 后,把后续 `stop` 归一为 `tool_calls`
|
||||||
|
3. Trae 不输出空工具调用表
|
||||||
|
4. Buddy 不输出空工具调用表
|
||||||
|
5. provider 回归测试模拟“tool fragment + stop”场景
|
||||||
|
|
||||||
|
但 provider 测试最后一次编辑工具报错:
|
||||||
|
|
||||||
|
```text
|
||||||
|
internal/provider/provider_test.go:99: missing ',' in argument list
|
||||||
|
```
|
||||||
|
|
||||||
|
因此必须先修测试文件,再提交。
|
||||||
|
|
||||||
|
## 7. 推荐接手顺序
|
||||||
|
|
||||||
|
### 第一步:修复工作树
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/program/llmsproxy
|
||||||
|
git status --short
|
||||||
|
sed -n '1,150p' internal/provider/provider_test.go
|
||||||
|
gofmt -w internal/provider/provider_test.go 2>&1 || true
|
||||||
|
go test -tags luajit ./internal/provider/ 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
若测试函数字符串转义难以修复,先删除半成品测试,再重新用 raw string 写测试。
|
||||||
|
|
||||||
|
### 第二步:跑定向验证
|
||||||
|
|
||||||
|
```bash
|
||||||
|
luac -p internal/lua/adapters/trae.lua
|
||||||
|
luac -p internal/lua/adapters/buddy.lua
|
||||||
|
go test -tags luajit ./internal/provider/
|
||||||
|
go test -tags luajit ./internal/lua/ -run 'Adapter|Buddy|Trae|Stream'
|
||||||
|
```
|
||||||
|
|
||||||
|
### 第三步:构建并部署临时验证版
|
||||||
|
|
||||||
|
修复确认后:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go build ./...
|
||||||
|
./deploy.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
部署前确认:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sha256sum internal/lua/plugins/channels.lua
|
||||||
|
sha256sum /etc/llmsproxy/plugins/channels.lua
|
||||||
|
systemctl is-active llmsproxy
|
||||||
|
```
|
||||||
|
|
||||||
|
### 第四步:真实流式回归
|
||||||
|
|
||||||
|
不要用 `stream:false` 测试流式问题。使用:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
KEY='<从本机安全环境读取,不写入文档>'
|
||||||
|
TOOL='{"type":"function","function":{"name":"get_weather","description":"Get weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}'
|
||||||
|
|
||||||
|
curl --no-buffer --max-time 90 \
|
||||||
|
-H "Authorization: Bearer $KEY" \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d "{\"model\":\"channels-codearts:openpangu-2.0-flash\",\"messages\":[{\"role\":\"user\",\"content\":\"Use the weather tool for Beijing.\"}],\"tools\":[$TOOL],\"tool_choice\":\"auto\",\"stream\":true,\"max_tokens\":64}" \
|
||||||
|
http://127.0.0.1:8081/v1/chat/completions
|
||||||
|
```
|
||||||
|
|
||||||
|
分别将 model 替换为:
|
||||||
|
|
||||||
|
```text
|
||||||
|
channels-codearts:openpangu-2.0-flash
|
||||||
|
channels-trae:DeepSeek-V4-Flash
|
||||||
|
channels-buddy:glm-5.3-flash
|
||||||
|
```
|
||||||
|
|
||||||
|
验收:
|
||||||
|
|
||||||
|
- HTTP 200
|
||||||
|
- `Content-Type: text/event-stream`
|
||||||
|
- 不能出现 `tool_calls: {}`
|
||||||
|
- 有 tool-call fragments 时最终必须是 `finish_reason: tool_calls`
|
||||||
|
- CodeArts 当前应继续返回结构化工具调用
|
||||||
|
|
||||||
|
### 第五步:CodeArts 刷新验收
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -sS -X POST \
|
||||||
|
-H "Authorization: Bearer $KEY" \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d '{"platform":"codearts"}' \
|
||||||
|
http://127.0.0.1:8081/api/plugins/http/channels/model/refresh
|
||||||
|
```
|
||||||
|
|
||||||
|
预期:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"ok":true,"results":[{"platform":"codearts","ok":true,"count":11}]}
|
||||||
|
```
|
||||||
|
|
||||||
|
再检查:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -sS -X POST \
|
||||||
|
-H "Authorization: Bearer $KEY" \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d '{"platform":"codearts"}' \
|
||||||
|
http://127.0.0.1:8081/api/plugins/http/channels/model/list
|
||||||
|
```
|
||||||
|
|
||||||
|
预期:远端目录 11 个 + 兜底补位,已确认 404 的 4 个仍保持 `disabled=true`。
|
||||||
|
|
||||||
|
## 8. 发布/安装包状态
|
||||||
|
|
||||||
|
目前没有完成“全部安装包”发布闭环。
|
||||||
|
|
||||||
|
已经发生的事情:
|
||||||
|
|
||||||
|
- 核心包构建成功,但脚本实际使用了 `git describe`,产物版本类似:
|
||||||
|
`2.0.40~4-gc9f3453`
|
||||||
|
- Linux GUI 包构建成功,版本为 `1.9.7`
|
||||||
|
- Windows 主机 Wine 构建失败,生成了一个 264KB 的坏 `ModelRouter Setup 1.9.7.exe`,绝对不能发布
|
||||||
|
- Docker Windows 构建也失败在拉取 Docker Hub 基础镜像 `golang:1.25` 网络超时
|
||||||
|
- apt 无法找到 `wine32:i386`
|
||||||
|
- 旧的 1.9.5/1.9.6 Windows 包是历史产物,不能作为当前版本
|
||||||
|
|
||||||
|
不要把旧 Windows exe 当作新包发布。
|
||||||
|
|
||||||
|
仓库已有 Docker 方案:
|
||||||
|
|
||||||
|
```text
|
||||||
|
cmd/gui/scripts/dist-win-docker.sh
|
||||||
|
cmd/gui/docker/win-builder/Dockerfile
|
||||||
|
```
|
||||||
|
|
||||||
|
但当前 Docker Hub 网络不可用。若要继续做 Windows 包:
|
||||||
|
|
||||||
|
1. 在可访问 Docker Hub 的构建机/CI 执行 Docker 方案;或
|
||||||
|
2. 提供包含 Wine32/Wine64 的完整构建环境;或
|
||||||
|
3. 暂时只发布 Linux/core 包,并明确说明 Windows 包未完成。
|
||||||
|
|
||||||
|
版本脚本注意:
|
||||||
|
|
||||||
|
- 核心包脚本使用位置参数:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./packaging/core-dist.sh 2.0.41
|
||||||
|
```
|
||||||
|
|
||||||
|
不是 `VERSION=2.0.41 ./packaging/core-dist.sh`。
|
||||||
|
- GUI 版本来自 `cmd/gui/package.json`,环境变量 `VERSION` 不会自动修改 Electron 产品版本。
|
||||||
|
- 清理旧产物后再打包,并在全部包完成后统一计算 `SHA256SUMS`。
|
||||||
|
|
||||||
|
## 9. 参考插件状态
|
||||||
|
|
||||||
|
用户要求按:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://github.com/Ebony-Vinyl/dsh-our-free-model
|
||||||
|
```
|
||||||
|
|
||||||
|
逐厂商对照参考插件。
|
||||||
|
|
||||||
|
当前本地路径:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/tmp/pi-github-repos/Ebony-Vinyl/dsh-our-free-model
|
||||||
|
```
|
||||||
|
|
||||||
|
但该目录当前不是有效可读源码副本:
|
||||||
|
|
||||||
|
- `vendor/channel-pack/src` 下面只有空目录;
|
||||||
|
- `.git` 没有有效 `HEAD`/refs;
|
||||||
|
- 无法从本地 Git 对象恢复源码;
|
||||||
|
- 本地也没有 `buddy.ts`、`trae.ts`、`codearts.ts` 等文件。
|
||||||
|
|
||||||
|
因此不能声称当前适配器是逐行照参考插件移植的。若要继续按用户要求完成,必须先重新 clone/恢复该 GitHub 仓库源码,确认以下文件实际存在后再对照:
|
||||||
|
|
||||||
|
```text
|
||||||
|
vendor/channel-pack/src/trae.ts
|
||||||
|
vendor/channel-pack/src/buddy.ts
|
||||||
|
vendor/channel-pack/src/codearts.ts
|
||||||
|
vendor/channel-pack/src/*adapter*
|
||||||
|
vendor/channel-pack/src/*model*
|
||||||
|
```
|
||||||
|
|
||||||
|
当前仓库代码只能作为临时协议修复基础,不能代替参考插件源码。
|
||||||
|
|
||||||
|
## 10. 关键结论
|
||||||
|
|
||||||
|
- Pi 是否发送流式请求,历史日志没有保存入站 `stream/tools` 字段,不能靠旧日志判断。
|
||||||
|
- 本次真实 `stream:true + tools` 测试证明 ModelRouter 流式路径能工作:
|
||||||
|
- CodeArts:通过;
|
||||||
|
- Trae:通过但结束原因需归一;
|
||||||
|
- Buddy:通过但需过滤空 `{}`;
|
||||||
|
- MiniMax:凭据 401,独立问题。
|
||||||
|
- 不要再用 `stream:false` 的手工请求推断 Pi 的问题。
|
||||||
|
- CodeArts 刷新接口当前真实返回 `ok=true,count=11`,刷新链路不是当前阻塞点。
|
||||||
|
- 当前最紧急的代码状态是修复 `provider_test.go` 的半编辑,然后验证未提交的统一流式修复。
|
||||||
|
- 未经测试、未重新构建、未重新部署,不要继续改生产账号或刷新一次性凭据。
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 HeNanMU-EEA
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
14
Makefile
14
Makefile
@ -1,8 +1,8 @@
|
|||||||
.PHONY: all build test gui gui-dev gui-dist gui-deb gui-win gui-win-docker help clean
|
.PHONY: all build test gui gui-dev gui-dist gui-deb gui-win gui-win-docker help clean gui-test
|
||||||
|
|
||||||
# ModelRouter - single-binary gateway + optional Electron desktop GUI (embedded core)
|
# ModelRouter - single-binary gateway + optional Electron desktop GUI (embedded core)
|
||||||
export BINARY
|
export BINARY
|
||||||
BINARY=llmsproxy
|
BINARY=modelrouter
|
||||||
export BUILD_DIR
|
export BUILD_DIR
|
||||||
BUILD_DIR=build
|
BUILD_DIR=build
|
||||||
export GUI_DIR
|
export GUI_DIR
|
||||||
@ -20,7 +20,7 @@ all: build
|
|||||||
# ---- core binary (server users / embedded core) ----
|
# ---- core binary (server users / embedded core) ----
|
||||||
build:
|
build:
|
||||||
@mkdir -p $(BUILD_DIR)
|
@mkdir -p $(BUILD_DIR)
|
||||||
CGO_ENABLED=1 go build -tags luajit -trimpath -o $(BUILD_DIR)/$(BINARY) ./cmd/llmsproxy
|
CGO_ENABLED=1 go build -tags luajit -trimpath -o $(BUILD_DIR)/$(BINARY) ./cmd/modelrouter
|
||||||
@echo "Built: $(BUILD_DIR)/$(BINARY) ($(VERSION))"
|
@echo "Built: $(BUILD_DIR)/$(BINARY) ($(VERSION))"
|
||||||
|
|
||||||
# ---- core distribution packages (deb + rpm + tar.gz) -----
|
# ---- core distribution packages (deb + rpm + tar.gz) -----
|
||||||
@ -65,6 +65,13 @@ gui-dist-dir:
|
|||||||
test:
|
test:
|
||||||
go test ./...
|
go test ./...
|
||||||
|
|
||||||
|
# Desktop shell unit tests (no Electron needed): plugin-tree seeding/migration.
|
||||||
|
# These guard the two things that silently break the desktop build:
|
||||||
|
# - plugin_dir missing from an embedded profile -> channels plugin invisible
|
||||||
|
# - bundled plugin files never refreshed -> old channels.lua survives upgrades
|
||||||
|
gui-test:
|
||||||
|
@cd $(GUI_DIR) && node --test plugin-sync.test.js
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf $(BUILD_DIR) cmd/build cmd/gui/dist cmd/gui/build/gui-dist cmd/gui/bin
|
rm -rf $(BUILD_DIR) cmd/build cmd/gui/dist cmd/gui/build/gui-dist cmd/gui/bin
|
||||||
|
|
||||||
@ -75,6 +82,7 @@ help:
|
|||||||
@echo " make gui run desktop GUI (dev)"
|
@echo " make gui run desktop GUI (dev)"
|
||||||
@echo " make gui-dev run GUI with devtools"
|
@echo " make gui-dev run GUI with devtools"
|
||||||
@echo " make gui-dist build deb + rpm + AppImage (linux)"
|
@echo " make gui-dist build deb + rpm + AppImage (linux)"
|
||||||
|
@echo " make gui-test desktop shell unit tests (plugin tree seeding)"
|
||||||
@echo " make gui-deb build deb only"
|
@echo " make gui-deb build deb only"
|
||||||
@echo " make gui-win build Windows nsis (needs host mingw + wine)"
|
@echo " make gui-win build Windows nsis (needs host mingw + wine)"
|
||||||
@echo " make gui-win-docker build Windows nsis all-in-docker (no host mingw needed)"
|
@echo " make gui-win-docker build Windows nsis all-in-docker (no host mingw needed)"
|
||||||
|
|||||||
626
README.md
626
README.md
@ -1,104 +1,51 @@
|
|||||||
# ModelRouter
|
# ModelRouter
|
||||||
|
|
||||||
|
> **仓库**: <https://atomgit.com/HeNanMU-EEA/ModelRouter>
|
||||||
|
>
|
||||||
> **English**: [README_EN.md](./README_EN.md)
|
> **English**: [README_EN.md](./README_EN.md)
|
||||||
>
|
>
|
||||||
> **AI-assisted**: 本项目使用 AI 辅助编程(代码与文档由 AI 协作完成,经人工复核)。
|
> **AI-assisted**: 本项目使用 AI 辅助编程(代码与文档由 AI 协作完成,经人工复核)。
|
||||||
|
|
||||||
面向多 llms 订阅者,部署在内网,实现一次配置多个服务共同使用的效果。支持指定模型或 auto 模式,按照配置的优先级选择可用模型提供服务。
|
面向多 LLM 订阅者的内网网关:一次配置,多个服务共用一组上游额度。支持指定模型或
|
||||||
|
`AUTO` 模式,按配置的优先级选择可用模型提供服务。
|
||||||
|
|
||||||
> **轻量且增源无需重编译**:网关本体是单 Go 二进制(约 10MB,零运行时依赖)。新增/切换上游
|
网关本体是单个 Go 二进制(约 10 MB,零运行时依赖)。新增或切换上游只需在
|
||||||
> 只需在 `config.yaml`(或 WebUI)加一个 `sources` 条目或挂一个 `.lua` 适配器——**不改 Go、
|
`config.yaml`(或 WebUI)加一个 `sources` 条目、或挂一个 `.lua` 适配器——**不改 Go、
|
||||||
> 不重编译**。源与适配器的改动经 WebUI 提交时即时生效(热更新);直接编辑
|
不重编译**。源与适配器的改动经 WebUI 提交时即时生效(热更新);直接编辑
|
||||||
> `config.yaml` 或 `adapter_dir` 下的 `.lua` 文件则需要重启进程生效。
|
`config.yaml` 或 `adapter_dir` 下的 `.lua` 文件需要重启进程生效。
|
||||||
|
|
||||||
## 核心优势
|
## 核心能力
|
||||||
|
|
||||||
### 极致轻量
|
- **统一接口**:对内暴露 OpenAI 兼容的 `/v1/chat/completions`、`/v1/models`、
|
||||||
|
`/v1/images/generations`;所有源对外都是同一套格式(含 `reasoning_content`、
|
||||||
- **单二进制**:编译后 8~12MB(`-s -w` strip 后约 8MB),零运行时依赖(仅依赖系统 libc),部署即用
|
`tool_calls`、`usage`)。
|
||||||
- **低内存占用**:与源数量相关,非与运行时长相关——单源 ~10MB、16 源生产实例 ~32–35MB([实测分解与调优](#内存占用实测与调优))
|
- **多源 + AUTO 链**:一个进程内任意多个上游源;`default_model: AUTO` 时按优先级页
|
||||||
- **日志按需加载**:统计数字来自**全量**审计日志(启动时流式扫一遍即释放,29MB/22 万行约 260ms),但原始记录不常驻内存——默认只加载首屏,下滚自动分页,CSV 导出流式写出(O(1) 内存),离页即释放
|
保存的档位逐档调度,源故障自动降级并冷却,恢复后半冷却探测自愈。
|
||||||
- **零运行时依赖**:纯 Go + LuaJIT 静态链接,无需安装 Python/Node/Java 等运行时
|
- **协议适配在 Lua 层**:每个源挂一个 `.lua` 适配器,完成请求/响应/流式分片的双向
|
||||||
- **启动极快**:冷启动 < 200ms,热重载配置 < 10ms
|
转换,并可用 `build_headers(meta)` 注入签名头。协议差异不进 Go 内核。
|
||||||
|
- **插件独立**:业务扩展(计费、用量统计等)是外置 Lua 插件,由插件目录加载,
|
||||||
### 强大的多租户调度能力
|
不编进二进制、不随发行包分发;插件只通过 `modelrouter` SDK 与内核交互。
|
||||||
|
- **多租户密钥**:网关 key 与上游 key 相互独立,按 key 限定模型范围、AUTO 链与配额。
|
||||||
- **多密钥多租户**:支持无限密钥,每个密钥独立角色、模型范围、Token 配额、重置周期
|
- **加密存储**:上游 key、自定义请求头、网关 key 均以 AES-256-GCM 落盘,
|
||||||
- **按模型配额**:每把 key 的每个模型单独配 token + 请求数配额与重置周期(小时/周/月/自定义 N 小时);一个模型用满只影响该模型,同 key 其它模型照常;耗尽返 429 + `Retry-After` 并点名模型,admin key 永不受限
|
`master.key` 单独 0600 权限(或由 `MODELROUTER_MASTER_KEY` 注入)。
|
||||||
- **AUTO 智能调度**:基于优先级档位的分级调度,同优先级源自动轮询负载均衡,故障自动毫秒级故障转移
|
- **可观测**:状态页实时探测源可达性;HTTP 访问日志、配置变更审计、请求与用量 CSV 导出;
|
||||||
- **自愈冷却**:冷却上限 5 分钟,过半后放行 1 个探测请求,上游/额度恢复即刻回归轮询,无需等满冷却窗口
|
写操作不落在请求热路径上。
|
||||||
- **Token 配额管理**:精确到模型级别的 Token 配额控制,支持小时/周/月/自定义小时周期自动重置
|
|
||||||
- **同优先级源负载均衡**:同一优先级档位的多个源,请求自动 Round-Robin 均匀分发,故障毫秒级故障转移
|
|
||||||
- **Source-Model 前缀路由**:支持 `source-model`/`source:model`/`source/model` 精确指定上游源
|
|
||||||
|
|
||||||
### 生产级可靠性
|
|
||||||
|
|
||||||
- **热加载配置**:WebUI 修改源/密钥/适配器/AUTO链即时生效,无需重启
|
|
||||||
- **密钥加密落盘**:AES-256-GCM 加密存储 `api_key`、网关密钥、请求头,master.key 0600 权限保护
|
|
||||||
- **实时源探测**:`GET {base}/models` 定期探测,毫秒级感知上游状态,不污染调度退避状态
|
|
||||||
- **全链路审计**:HTTP 访问日志、登录/配置变更审计、请求记录 CSV 导出、密钥用量统计 CSV 导出
|
|
||||||
|
|
||||||
### 灵活的协议适配
|
|
||||||
|
|
||||||
- **LuaJIT VM**:每个适配器独立 VM + **弹性 worker 池**(按需扩缩容,空闲时自动回收),安全并发,Lua 脚本热加载无需重启
|
|
||||||
- **Lua 适配器协议**:`transform_request` / `transform_response` / `transform_stream_chunk` 双向转换
|
|
||||||
- **动态请求头钩子**:`build_headers(meta)` 支持 HMAC 签名、动态 Header 注入
|
|
||||||
- **多模态透传**:`image_url` 等多模态内容在多源间无损透传,Anthropic/Gemini/Ollama 自动转换
|
|
||||||
|
|
||||||
## 实现
|
|
||||||
|
|
||||||
统一 OpenAI 兼容网关:把多个上游 LLM 源(DeepSeek、Qijiar、OpenAI、Anthropic、Gemini、
|
|
||||||
Groq、Mistral、Ollama、KimiCode…)通过 **Lua 适配器** 做协议转换,对内网暴露
|
|
||||||
一个标准 `OpenAI Chat Completions` 接口(`/v1/chat/completions` + `/v1/models`),
|
|
||||||
支持单次结算、SSE 流式、**AUTO 模型路由**与**多模态**透传。
|
|
||||||
|
|
||||||
从 [HomeAgent](https://gitcode.com/JianFeeeee/HomeAgent) 的多源 LLM 适配层
|
|
||||||
(`internal/agent/api/provider.go` + `internal/lua/adapters/*`)抽离而来并独立演进。
|
|
||||||
|
|
||||||
## 特性
|
|
||||||
|
|
||||||
- **多源**:一个进程内配置任意多个上游源,按请求的 `model` 自动路由。
|
|
||||||
- **AUTO 模式**:`default_model: AUTO` 时按优先级页保存的 AUTO 链档位逐档调度并发请求。
|
|
||||||
- **统一输出**:所有源对外都是 OpenAI 格式(含 `reasoning_content`、`tool_calls`、`usage`)。
|
|
||||||
- **生图**:`POST /v1/images/generations`,`kind: image` 的模型独立路由。
|
|
||||||
- **多模态**:`content` 数组(`image_url` 等)在多源间无损透传,Anthropic/Gemini/Ollama 自动转换。
|
|
||||||
- **LuaJIT VM**:基于 golua 绑定的 LuaJIT,每个适配器独立 VM + worker 池,安全并发。
|
|
||||||
池是 **弹性** 的:`Σ max_concurrent` 只是上限而非预分配,状态按需创建、空闲时自动回收,
|
|
||||||
因此空载网关几乎不持有 Lua 状态(启动 0 个)。
|
|
||||||
- 扩容步长由适配器最大并发决定:`clamp(ceil(max/8), 1, 8)`,且仅在**真实并发争用**
|
|
||||||
(所有现有状态均已占用)时批量预热,顺序流量始终只用 1 个状态。
|
|
||||||
- 缩容步长由当前连接数决定:`clamp(ceil(冗余/(1+占用)), 1, 冗余)`——无连接时一轮收到
|
|
||||||
常驻下限,忙时每轮只释放 1 个,保护热路径。WebUI「适配器」页展示实时池指标。
|
|
||||||
- **disable_thinking**:请求 `disable_thinking:true`(或上游对应字段)动态开关推理。
|
|
||||||
- **Lua 适配协议**:每个源挂一个 `.lua` 适配器,完成 `transform_request` /
|
|
||||||
`transform_response` / `transform_stream_chunk` 双向转换,协议差异全在 Lua 层。
|
|
||||||
- **签名 / 请求头钩子**:适配器可定义 `build_headers(meta)`,在 Go 发 HTTP 前
|
|
||||||
动态注入/签名请求头——用于云端 API 校验调用方 app(如 KimiCode 只放行特定
|
|
||||||
agent)。提供 `hmac_sha256_hex` / `sha256_hex` / `base64_encode` 等签名辅助。
|
|
||||||
- **WebUI**:内置管理界面,可在线查看/新增/编辑上游源与模型、配置 AUTO 优先级链、
|
|
||||||
管理密钥模型范围,写入运行时文件持久化。
|
|
||||||
- **密钥加密存储**:运行时文件中的上游 `api_key`、自定义请求头值、网关 key 均以
|
|
||||||
AES-256-GCM 加密落盘(`master.key` 独立 0600,或 `LLMS_PROXY_MASTER_KEY`)。
|
|
||||||
- **实时源探测**:状态页探测各源可达性(`GET {base}/models`,失败回退最小请求),
|
|
||||||
不污染正常调度的退避状态,错误信息在 UI 可悬停查看。
|
|
||||||
- **鉴权**:网关自身用 `gateway_keys` 校验客户端 Bearer key;与上游各自的 key 相互独立。
|
|
||||||
- **流式**:SSE `chat.completion.chunk`,含角色首包与 `[DONE]` 收尾。
|
|
||||||
- **全链路审计**:HTTP 访问日志、登录/配置变更审计、请求记录 CSV 导出、密钥用量统计 CSV 导出
|
|
||||||
- **AES-256-GCM 加密存储**:运行时文件敏感字段加密落盘,master.key 0600 权限,支持环境变量注入主密钥
|
|
||||||
|
|
||||||
## 快速开始
|
## 快速开始
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
GOMODCACHE=... GOPROXY=off go build -tags luajit -o llmsproxy ./cmd/llmsproxy
|
# 需要 LuaJIT 开发库;-tags luajit 是必须的
|
||||||
./llmsproxy -config config.yaml # 首次运行自动生成默认配置并打印随机 admin key
|
GOMODCACHE=... GOPROXY=off go build -tags luajit -o modelrouter ./cmd/modelrouter
|
||||||
|
|
||||||
|
# 首次运行会生成默认配置,并在日志里打印随机 admin key
|
||||||
|
./modelrouter -config config.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
> 依赖 [golua](https://github.com/aarzilli/golua)(LuaJIT 绑定)。**必须**带 `-tags luajit`
|
> 依赖 [golua](https://github.com/aarzilli/golua)(LuaJIT 绑定)。不带 `-tags luajit`
|
||||||
> 构建,否则默认走内置超集 gopher-lua 路径(行为略有差异)。
|
> 会走 gopher-lua 路径,行为略有差异。
|
||||||
|
|
||||||
> **配置不入库**:仓库不携带任何 config 文件(配置文件含密钥)。二进制首次运行时
|
> **配置不入库**:仓库不携带任何 config 文件(其中含密钥)。默认配置只绑定
|
||||||
> 会在 `-config` 指定的位置生成默认配置:随机 admin key(打印在启动日志中)、
|
> `127.0.0.1:8080`,首次登录后请在 WebUI「密钥」页更换管理员密钥。
|
||||||
> 仅绑定 `127.0.0.1:8080`。首次登录后请在 WebUI「密钥」页更换管理员密钥。
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 无 key -> 401
|
# 无 key -> 401
|
||||||
@ -115,474 +62,77 @@ curl -N -H "Authorization: Bearer $KEY" \
|
|||||||
http://127.0.0.1:8080/v1/chat/completions
|
http://127.0.0.1:8080/v1/chat/completions
|
||||||
```
|
```
|
||||||
|
|
||||||
任何 OpenAI SDK 把 `base_url` 指到网关地址、`api_key` 用任意一个已创建并授权的网关 key 即可。
|
任何 OpenAI SDK 把 `base_url` 指向网关、`api_key` 换成任一已授权的网关 key 即可。
|
||||||
|
|
||||||
## 配置
|
## 文档
|
||||||
|
|
||||||
首次运行在 `-config` 路径自动生成默认配置(默认 `./config.yaml`),核心字段:
|
| 主题 | 文档 |
|
||||||
|
|---|---|
|
||||||
|
| 实现与完整特性清单 | [docs/features.md](docs/features.md) |
|
||||||
|
| 配置:`config.yaml`、自动探测模型、多密钥 | [docs/configuration.md](docs/configuration.md) |
|
||||||
|
| 路由:模型路由、AUTO 链、冷却与半冷却、源与适配器、源模板 | [docs/routing.md](docs/routing.md) |
|
||||||
|
| 适配器:Lua 协议、内置适配器清单 | [docs/adapters.md](docs/adapters.md) · [docs/lua-adapters.md](docs/lua-adapters.md)([EN](docs/lua-adapters-en.md)) |
|
||||||
|
| 插件:SDK 契约、钩子、UI 扩展、HTTP 端点 | [docs/plugins.md](docs/plugins.md) |
|
||||||
|
| 桌面端:Headless / Desktop 两种产物、构建与运行 | [docs/gui.md](docs/gui.md) |
|
||||||
|
| 性能:实测内存占用与调优、thinking 预算 | [docs/performance.md](docs/performance.md) |
|
||||||
|
| WebUI 使用 | [docs/webui.md](docs/webui.md) |
|
||||||
|
| 开发:目录职责、测试、分支与发布纪律 | [docs/development.md](docs/development.md) · [docs/git-workflow.md](docs/git-workflow.md)([EN](docs/git-workflow-en.md)) |
|
||||||
|
| 部署与迁移 | [docs/deployment.md](docs/deployment.md) |
|
||||||
|
|
||||||
```yaml
|
## 从 llmsproxy 改名
|
||||||
listen: 127.0.0.1:8080 # 网关监听地址(建议绑内网/回环)
|
|
||||||
gateway_keys: [sk-gw-<随机>] # 初始 admin 密钥(每次安装随机生成,仅首启写入运行时存储用)
|
|
||||||
default_model: AUTO # model 无法路由时自动按优先级链选源
|
|
||||||
adapter_dir: adapters # Lua 适配目录;目录不存在时首启创建并 seed 内置,存在则只读
|
|
||||||
runtime_file: runtime.json # WebUI 编辑的源/密钥/AUTO 链持久化到此文件
|
|
||||||
|
|
||||||
sources:
|
产品名早已是 ModelRouter,二进制、配置目录、服务名与环境变量现已统一为
|
||||||
- name: deepseek
|
`modelrouter`:
|
||||||
base_url: https://api.deepseek.com
|
|
||||||
api_key: sk-... # 也可用 api_key_env: SOME_ENV 引用环境变量(不落盘明文)
|
|
||||||
adapter: deepseek
|
|
||||||
max_concurrent: 8
|
|
||||||
models:
|
|
||||||
- id: deepseek-v4-flash
|
|
||||||
priority: 100 # YAML 源首次启动会 seed 进 AUTO 链的初值
|
|
||||||
kind: chat
|
|
||||||
- id: deepseek-v4-pro
|
|
||||||
priority: 60
|
|
||||||
kind: chat
|
|
||||||
# 静态请求头(比适配器默认优先)
|
|
||||||
headers: { X-Tenant: prod }
|
|
||||||
# 传给 Lua build_headers 的透传元数据
|
|
||||||
meta: { app_id: x, app_secret: y }
|
|
||||||
temperature: 0.7
|
|
||||||
max_tokens: 4096
|
|
||||||
timeout: 120s # 请求超时,默认 120s
|
|
||||||
```
|
|
||||||
|
|
||||||
`config.yaml`、`runtime.json`、`master.key`、`adapters/` 均不提交版本库(见
|
| 旧 | 新 |
|
||||||
`.gitignore`),请随备份带走——`master.key` 丢失后运行时密文无法解密。
|
|---|---|
|
||||||
|
| `llmsproxy`(二进制) | `modelrouter` |
|
||||||
|
| `/etc/llmsproxy/` | `/etc/modelrouter/` |
|
||||||
|
| `llmsproxy.service` | `modelrouter.service` |
|
||||||
|
| `LLMSPROXY_*` | `MODELROUTER_*` |
|
||||||
|
|
||||||
运行文件生成在 `-config` 所在目录:`adapter_dir` / `runtime_file` 自动指向
|
已部署的机器用迁移脚本升级(**默认 dry-run**,复制不移动,可回滚):
|
||||||
配置文件旁边的 `adapters/` 与 `runtime.json`,从任意工作目录启动都可用。
|
|
||||||
|
|
||||||
运行时文件(`runtime_file`)中的敏感字段自动加密:
|
|
||||||
|
|
||||||
- 加密算法 AES-256-GCM,格式 `enc:v1:<base64>`。
|
|
||||||
- 主密钥来源:环境变量 `LLMS_PROXY_MASTER_KEY`(64 位 hex);否则读取
|
|
||||||
`runtime_file` 同目录的 `master.key`;都不存在时首次启动自动生成 `master.key`(0600)。
|
|
||||||
- 升级时旧明文文件自动兼容:首次运行正常读取,任何 UI 保存操作触发全文件加密迁移。
|
|
||||||
- 注意:**master.key 丢失后密文无法解密**,请随配置一起备份;切勿提交到版本库。
|
|
||||||
|
|
||||||
### 网关密钥(多密钥)
|
|
||||||
|
|
||||||
网关鉴权采用「多密钥 + 角色 + 模型范围」架构,密钥持久化在 `runtime_file` 的
|
|
||||||
`keys` 字段(加密存储):
|
|
||||||
|
|
||||||
- `gateway_keys` 配置只是**初始 admin 密钥种子**:首次启动迁移为运行时 admin
|
|
||||||
key,之后不再参与鉴权管理。
|
|
||||||
- **重要:首次启动后请在 WebUI「密钥」页更换管理员密钥**——初始密钥明文写在
|
|
||||||
`config.yaml` 里(每次安装随机生成),继续使用存在被盗风险;用新密钥登录后删除初始密钥。
|
|
||||||
- WebUI **密钥页**可创建/删除密钥;每个密钥可指定 `admin`(管理全部)或
|
|
||||||
`user`(仅看自己的 key)角色,并配置**模型范围**(模型 + 源 + token 配额 +
|
|
||||||
重置周期)。
|
|
||||||
- 客户端用任意一个已授权的密钥明文作为 Bearer(`Authorization: Bearer <key>`)。
|
|
||||||
- 删除密钥即从运行时存储移除,立即失效。
|
|
||||||
|
|
||||||
#### 密钥用量配额
|
|
||||||
|
|
||||||
**配额按模型单独设置**:每个密钥的 `models[]` 里,每一条模型范围各自带一份
|
|
||||||
token 配额与请求数配额。一个模型用满只影响该模型,同一密钥的其它模型照常工作。
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
keys:
|
|
||||||
- key: sk-gw-<hex>
|
|
||||||
role: user
|
|
||||||
name: agent-alice
|
|
||||||
models:
|
|
||||||
- model: deepseek-v4-flash
|
|
||||||
token_quota: 1000000 # 本周期内该 key 用这个模型的 token 预算
|
|
||||||
req_quota: 20000 # 本周期内的请求次数
|
|
||||||
period: nhour # "" | hour | week | month | nhour
|
|
||||||
hours: 6 # 仅 nhour
|
|
||||||
- model: AUTO
|
|
||||||
token_quota: 5000000 # AUTO 也是一条独立配额
|
|
||||||
period: day # ← 这种写法会被拒绝(词表只有 hour/week/month/nhour)
|
|
||||||
- model: kimi-k3 # 未列配额 = 无限
|
|
||||||
```
|
|
||||||
|
|
||||||
- **没有「整钥总配额」**:这是刻意的设计。整钥总额会让「把 A 模型的额度挪给
|
|
||||||
B 模型」变成一次全局重分配;按模型独立计费则每个模型各自可控,运维可以
|
|
||||||
看出哪个模型吃掉了预算。
|
|
||||||
- `period` 词表:空 = 永不过期(累计总量),`hour` / `week` / `month` = 固定窗口,
|
|
||||||
`nhour` + `hours` = 自定义小时数。**拼错的周期在写入时就被拒**,不会静默变成
|
|
||||||
永不过期。
|
|
||||||
- 配额严格按密钥隔离,且**同一密钥内按模型隔离**:A 密钥用满 `m1` 不会消耗
|
|
||||||
B 密钥的额度,同一密钥的 `m2` 也不受影响。
|
|
||||||
- 配额统计含聊天、流式、生图,跨重启从审计日志回放(保留 40 天,覆盖最长的
|
|
||||||
month 窗口)。
|
|
||||||
- 配额耗尽返回 **429 + `Retry-After`**(`rate_limit_exceeded`),消息里点名是哪个
|
|
||||||
模型用满了,客户端可等窗口重置后自动恢复;模型越权才是 403。
|
|
||||||
**admin 密钥永不受配额限制**(其 scope 上的配额也不强制),避免把管理员
|
|
||||||
锁在门外。
|
|
||||||
- 窗口用量按整点小时分桶统计,实际释放比配置窗口最多晚 1 小时(配额宁可晚释放
|
|
||||||
也不超发)。
|
|
||||||
- `PUT /api/keys/{key}` 提交 `models` 即同时提交它们的配额(配额就是 scope 的一部分,
|
|
||||||
不存在会与模型列表脱节的第二份预算)。显式 `0` = 解除该模型的限制。
|
|
||||||
|
|
||||||
##### 配额拒绝 vs 容量拒绝:两种「拒绝」含义不同
|
|
||||||
|
|
||||||
配额在**入口**检查,不占用任何上游槽位,因此是廉价拒绝;容量不足则要走
|
|
||||||
`busyWait` 有界等待后才返回 503。实测(容量 8、0.6s/请求、100 并发):
|
|
||||||
|
|
||||||
| 情况 | 状态码 | 延迟 | 是否打到上游 |
|
|
||||||
|---|---|---|---|
|
|
||||||
| 配额用尽(`req_quota` 50 → 100 并发) | `429 rate_limit_exceeded` + `Retry-After` | **19–21ms** | 否 |
|
|
||||||
| 容量打满(同一拓扑,未限配额) | `503 upstream_error` | ~2.6s(`busyWait`) | 超出部分否 |
|
|
||||||
|
|
||||||
客户端因此可以区分「等窗口重置」与「等上游腾出容量」:前者只需等待,后者
|
|
||||||
通常该降并发或换源。
|
|
||||||
|
|
||||||
##### 配额的开销
|
|
||||||
|
|
||||||
配额桶按 (密钥, 模型, 整点小时) 分桶保留 40 天,实测(AMD 7840HS):
|
|
||||||
|
|
||||||
- 窗口查询按窗口长度而非保留总量扫描:1h 窗口 49ns、24h 窗口 55ns、
|
|
||||||
30d 窗口 3.9μs(此前全扫保留总量,960 桶时 5.9μs)。
|
|
||||||
- 记录一条请求:283ns、3 分配(与引入配额前相同,分配来自 ring buffer)。
|
|
||||||
- 内存:生产形态(7 密钥 × 8 模型 × 2 源 × 满 40 天 retention)约 **3.7MB**。
|
|
||||||
按源 pin 的 `source::model` 桶**惰性创建**——只有当某条配额真的 pin 了
|
|
||||||
某个源时才维护,否则每条记录多写一份桶,在 20 密钥 × 8 模型 × 3 源下会
|
|
||||||
多占 18MB。
|
|
||||||
代价:配置 pinned 配额之前的历史用量无法事后按源拆分,所以 **pinned 配额的
|
|
||||||
首个窗口可能少算**。
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 配额耗尽时客户端看到(点名了具体模型)
|
sudo ./packaging/migrate-to-modelrouter.sh # 预演
|
||||||
HTTP/1.1 429 Too Many Requests
|
sudo ./packaging/migrate-to-modelrouter.sh --apply # 执行
|
||||||
Retry-After: 2100
|
sudo ./packaging/migrate-to-modelrouter.sh --rollback --apply
|
||||||
{"error":{"type":"rate_limit_exceeded","message":"token quota exceeded for \"deepseek-v4-flash\" (5000000/5000000)"}}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 模型路由
|
旧路径与旧环境变量在读取侧仍被接受,因此升级不会因为顺序问题立刻失效。
|
||||||
|
`deploy.sh` 在检测到旧 `/etc/llmsproxy` 而新配置不存在时会**中止**,避免部署出
|
||||||
`/v1/chat/completions` 的 `model` 解析顺序:
|
一个「能启动但没有 key / 没有源」的空实例。
|
||||||
|
|
||||||
1. `source/model` 或 `source:model` 前缀 → 指定源;
|
插件 SDK 同名调整:唯一公开入口是 `modelrouter`(`llmsproxy` 仅作兼容别名),
|
||||||
2. 精确匹配某个源的 `model`;
|
详见 [docs/plugins.md](docs/plugins.md)。
|
||||||
3. 配置 `default_model: AUTO` 时 → 走优先级页保存的 AUTO 链(见下);
|
|
||||||
4. 否则回落(返回错误)。
|
## 目录结构
|
||||||
|
|
||||||
任何 OpenAI 客户端,只要 `model` 设为某个源的 `name/任意名`,即可锁定走该源;
|
```text
|
||||||
设为 `AUTO`(或网关配了 `default_model: AUTO`)即自动按优先级链选源。
|
cmd/modelrouter # 入口(纯 Go 服务端)
|
||||||
|
cmd/gui # Electron 桌面壳(内嵌同一份核心)
|
||||||
### AUTO 链(优先级页)
|
internal/config # 配置解析、加密存储、运行时文件
|
||||||
|
internal/core # 调度、内核动作、状态与审计
|
||||||
AUTO 调度**只**由优先级页保存的规则决定,源配置里的 `models[].priority` 数字不再
|
internal/lua # LuaJIT VM + 弹性 worker 池 + 内置适配器 (embed)
|
||||||
参与调度、也不再显示。**聊天与生图是两套独立的链**,在优先级页面通过「聊天 / 生图」
|
internal/lua/plugins # 可选插件示例(不随二进制/发行包分发)
|
||||||
按钮切换编辑:
|
|
||||||
|
|
||||||
- **聊天链**(`runtime_file` 的 `auto` 字段):服务 `POST /v1/chat/completions`
|
|
||||||
的 `model: AUTO` 请求;只接受 `kind: chat` 模型,配置在链里的生图槽位会被忽略。
|
|
||||||
- **生图链**(`runtime_file` 的 `auto_image` 字段):服务 `POST /v1/images/generations`
|
|
||||||
的 `model: AUTO` 请求;只接受 `kind: image` 模型。未配置生图链时,AUTO 生图回落
|
|
||||||
到「所有含生图模型的源、按注册顺序」的旧行为,兼容旧配置。
|
|
||||||
|
|
||||||
每条规则 = 一个「槽位」:`{ model, source, tier, token_quota, period, hours }`。
|
|
||||||
|
|
||||||
- `tier` 表示优先级档位:同一档的模型并排、共享该优先级;档位从上到下递减。
|
|
||||||
- 同一模型可配置多个槽位(如 A 源低配 → B 源低配 → A 源高配 → B 源高配),按档位顺延。
|
|
||||||
- `token_quota` > 0 时该槽位在重置周期内用满即顺延到下一槽位;`period` 支持
|
|
||||||
`hour` / `week` / `month` / `nhour`(配合 `hours`),空 = 不限。
|
|
||||||
- 同 tier 内按「偏好分」排序,失败槽位冷却后自动跳过;冷却 / 配额耗尽 / hard
|
|
||||||
错误会顺延到下一 tier,全部失败时返回 503 并附上每档失败摘要。
|
|
||||||
|
|
||||||
### 冷却与半冷却探测(自愈调度)
|
|
||||||
|
|
||||||
槽位失败后进入指数退避冷却(5s → 10s → 20s …,上限 **5 分钟**)。冷却不是全时段
|
|
||||||
硬闸:
|
|
||||||
|
|
||||||
- **前半段完全静默**:刚失败的槽位不接任何流量,避免对故障上游持续施压。
|
|
||||||
- **后半段放行 1 个探测**:窗口过半后,同一 `(源, 模型)` 最多允许 **一个** 请求作为
|
|
||||||
探测通过。探测槽数 = `clamp(max_concurrent / 10, 1, 2)`——`max_concurrent: 10`
|
|
||||||
的源恰好放行 1 个探测请求。
|
|
||||||
- **探测排在最后**:同 tier 内探测候选永远排在健康候选之后,只有没有正常槽可用时
|
|
||||||
才承接真实流量,因此探测不会抢走可用容量。
|
|
||||||
- **探测即真实请求**:成功则立即清零失败计数与冷却,槽位当场回到正常轮询——
|
|
||||||
上游恢复/额度恢复后不必等满整个冷却窗口。失败则重开一个新窗口,下次探测顺延到
|
|
||||||
新窗口的中点,不会连续重试。
|
|
||||||
|
|
||||||
分类冷却策略:
|
|
||||||
|
|
||||||
| 情况 | 冷却 | 说明 |
|
|
||||||
|---|---|---|
|
|
||||||
| 传输错误 / 5xx | 5s→10s→20s…,上限 5min | 指数退避 |
|
|
||||||
| 401 / 403 凭据错误 | 10min | 换 key 后由下一次探测自动接回 |
|
|
||||||
| 429 限流 | 30s 固定 | 「太快」不等于「坏了」,不进指数阶梯 |
|
|
||||||
| 配额耗尽(`insufficient_quota` / `余额不足` 等) | 对齐配额窗口,上限 30min | **不进指数阶梯、不涨失败计数**,额度恢复即可用 |
|
|
||||||
|
|
||||||
WebUI「优先级」页的槽位健康标签会区分 `冷却` / `待探测` / `探测中`,悬停可看窗口
|
|
||||||
起点、探测放行时刻与完全恢复时刻。
|
|
||||||
|
|
||||||
### 源(Source)与适配器(Adapter)的关系
|
|
||||||
|
|
||||||
- **源** 是到某个上游的连接描述:`name`、`base_url`、`api_key`、模型列表与优先级。
|
|
||||||
- **适配器** 是协议转换逻辑(Lua 脚本):把统一 OpenAI 格式请求转成上游原生格式,
|
|
||||||
再把上游响应/流式分块转回统一格式。
|
|
||||||
- 一个适配器可被多个源复用(如 `openai.lua` 同时服务多个 OpenAI 兼容站点);
|
|
||||||
同一个源也可以切换不同适配器(改 `adapter` 字段即可)。
|
|
||||||
- 源决定“连谁、暴露哪些模型”,适配器决定“怎么对话”——二者在 `sources[]` 条目中
|
|
||||||
通过 `adapter` 字段关联。
|
|
||||||
|
|
||||||
加载流程(`internal/core` 的 `Core` 负责装配):
|
|
||||||
|
|
||||||
1. 启动时 `lua.NewVM(adapter_dir)` 加载全部适配器:内置适配器(编译期 embed)+
|
|
||||||
`adapter_dir` 下的同名覆盖文件;内置文件写在代码内,覆盖文件要求更高优先级。
|
|
||||||
2. `config.Load` 读取 `config.yaml`,`config.NewStore(runtime_file)` 读 WebUI 改动的
|
|
||||||
运行时源,二者按名称合并成完整源列表。
|
|
||||||
3. `rebuildRegistry` 为每个源创建 `provider.Provider`(持有目标适配器),并按源的
|
|
||||||
并发上限配置适配器 worker 池**上限**(不预分配,见下)。
|
|
||||||
4. 请求进来时 `Registry` 按 `model` 路由到 Provider,Provider 调适配器
|
|
||||||
`transform_request` → HTTP 发送 → `transform_response` / `transform_stream_chunk`。
|
|
||||||
|
|
||||||
WebUI 上的"新增/编辑源"、"上传 Lua 适配器"、"改 AUTO 优先级链"与"密钥模型范围"
|
|
||||||
都即时生效(写入运行时文件或 `adapter_dir` 后重新装配,无需重启);直接编辑
|
|
||||||
`config.yaml` / `adapter_dir` 下的文件则需要重启进程才会重新加载。
|
|
||||||
|
|
||||||
在 WebUI 删除源/适配器时:运行时(WebUI 创建)的源与 `.lua` 文件会真正移除;
|
|
||||||
`config.yaml` 中定义的基源无法改写配置文件,采用删除标记隐藏(重启后仍隐藏),
|
|
||||||
在 UI 里重新添加同名源即可恢复。
|
|
||||||
|
|
||||||
### 源模板(多 key 负载均衡)
|
|
||||||
|
|
||||||
多个上游 key 共享同一套配置(URL / 适配器 / 模型列表 / 并发 / rpm)时,逐一复制
|
|
||||||
整段 source 太冗余。**源模板**保存除 `name` 和 `api_key` 以外的全部源字段,
|
|
||||||
从模板一键创建多个仅 key 不同的源,天然支持多 key 负载均衡(每个展开源独立调度、
|
|
||||||
独立冷却、独立健康状态)。
|
|
||||||
|
|
||||||
- WebUI「源」页右上角「**模板管理**」按钮:列出全部模板,可编辑 / 删除 / 新建。
|
|
||||||
- 添加源弹窗右上角两个按钮:
|
|
||||||
- **从模板创建** → 选模板 → 表单自动填充(name + key 仍需你填);
|
|
||||||
- **存为模板** → 把当前表单非 key/name 字段存为新模板。
|
|
||||||
- 模板持久化在 `runtime_file` 的 `source_templates` 字段,与运行时源一样热管理,
|
|
||||||
改完即时生效。模板本身不含密钥,可以安全共享 / 纳管。
|
|
||||||
|
|
||||||
模板不会自己变成源——它只是「配方」。真正承担流量的是从模板创建出来的、带 key
|
|
||||||
的具体源。
|
|
||||||
|
|
||||||
### 内存占用(实测与调优)
|
|
||||||
|
|
||||||
内存不是一个定数:它随**配置的源数量**增长(每个源一个 `http.Transport` 与连接池、
|
|
||||||
一组健康状态),与运行时长无关。本机实测(Linux x86_64,12 核):
|
|
||||||
|
|
||||||
| 部署形态 | 启动 RSS | 稳态 RSS |
|
|
||||||
|---|---|---|
|
|
||||||
| 1 源 / 1 适配器(最小配置) | ~4 MB | ~10 MB |
|
|
||||||
| 1 源 + 29 MB 历史审计日志 | ~19 MB | ~20 MB |
|
|
||||||
| **16 源 / 13 适配器 / 59 模型(本机生产)** | ~28 MB | **~32–35 MB** |
|
|
||||||
|
|
||||||
> 历史参考:本项优化前同一生产配置为 **~105 MB**。降幅来自三处:审计日志的**原始记录**
|
|
||||||
> 不再常驻内存(约 25 MB;统计聚合仍扫全量,但扫完即释放)、Lua 状态池不再单调增长、
|
|
||||||
> 以及下面两个运行时开关。
|
|
||||||
|
|
||||||
内存构成(生产实例分段测量,`/proc/<pid>/smaps`):
|
|
||||||
|
|
||||||
| 区域 | RSS | 说明 |
|
|
||||||
|---|---|---|
|
|
||||||
| Go 堆 | ~14 MB | provider/registry/scheduler 结构 + 连接池缓冲 |
|
|
||||||
| 其他匹名(线程栈 / LuaJIT chunk / runtime) | ~12 MB | 与线程数、已加载适配器数相关 |
|
|
||||||
| 二进制 text+rodata | ~8 MB | 映射的可执行文件页(只读、可被内核回收) |
|
|
||||||
| Go runtime 预留 | ~4 MB | VSZ 上看到的 2 GB+ 是地址空间预留,不占物理内存 |
|
|
||||||
| 共享库 | ~3 MB | libc / libluajit / libm |
|
|
||||||
|
|
||||||
**两个推荐的部署开关**(只适用于环境变量,无需改代码):
|
|
||||||
|
|
||||||
```ini
|
|
||||||
# /etc/systemd/system/llmsproxy.service
|
|
||||||
Environment=GOGC=50
|
|
||||||
Environment=MALLOC_ARENA_MAX=2
|
|
||||||
```
|
|
||||||
|
|
||||||
- `MALLOC_ARENA_MAX=2`:LuaJIT 的分配走 cgo → glibc malloc,glibc 默认允许 `8×nproc`
|
|
||||||
个 per-thread arena,每个碰到 malloc 的 OS 线程会占用一个(各约 1 MB,且**不归还给系统**)。
|
|
||||||
实测从 8–15 个 arena(约 7–12 MB)降到 **0**。
|
|
||||||
- `GOGC=50`:把 Go 堆增长目标减半。**单独使用无效**(省下的堆会立即被更多 glibc
|
|
||||||
arena 吃掉,实测 20.3 → 21.5 MB 反而变大),必须与 `MALLOC_ARENA_MAX` 配合,
|
|
||||||
两者同时开启才降 **~19%**。网关是 I/O 密集型(本机 9 小时仅消耗 1min10s CPU),
|
|
||||||
多出的 GC 周期与它的空闲 CPU 相比可忽略。
|
|
||||||
|
|
||||||
> 桌面版(Electron)已在 `cmd/gui/main.js` 里默认为内嵌核心注入这两个开关;手动 export 同名
|
|
||||||
> 环境变量可覆盖。如果你用自己的 systemd unit / 容器,建议照上面加上。
|
|
||||||
|
|
||||||
可选:`GOMEMLIMIT=48MiB` 作软上限,实测再省 ~1 MB;代价是逐近上限时 GC 转激进,
|
|
||||||
温和场景不必开。
|
|
||||||
|
|
||||||
### disable_thinking
|
|
||||||
|
|
||||||
请求体带 `"disable_thinking": true` 时,网关透传给各适配器;DeepSeek 适配器将其
|
|
||||||
映射为 `extra_body.thinking.type = "disabled"` 关闭推理,其余源按各自协议处理。
|
|
||||||
|
|
||||||
### WebUI
|
|
||||||
|
|
||||||
内置管理界面(`GET /`),登录后可在浏览器完成:
|
|
||||||
|
|
||||||
- **状态页**:源在线状态(实时探测 + 悬停看错误)、模型/源/key 用量统计、请求记录,
|
|
||||||
支持按时间范围导出 CSV;点击模型可生成 pin 到该模型的连接配置。
|
|
||||||
- **对话页**:流式/非流式调试。
|
|
||||||
- **密钥页**:创建/编辑网关 key,为每个 key 配模型范围(模型 + 源 + token 配额 + 周期),
|
|
||||||
管理员管理全部 key,用户只看到自己的 key。每个模型砖块显示自己的配额徽标
|
|
||||||
(如 `1.0K 77×·1h`,未设配额显示 `∞`),点开可编辑该模型的 token 配额、
|
|
||||||
请求数配额与重置周期 —— 配额按模型独立生效,一个用满不影响同一 key 的其它模型。
|
|
||||||
「我的密钥」页对用户展示本 key 的模型范围与各自配额。
|
|
||||||
- **优先级页**:拖拽积木配置 AUTO 链档位。
|
|
||||||
- **源页**:在线增删改上游源(API key 等敏感字段加密落盘)。
|
|
||||||
- **适配器页**:上传 / 删除 Lua 适配器脚本。
|
|
||||||
|
|
||||||
改动写入 `runtime_file`(重启仍生效)。
|
|
||||||
|
|
||||||
## Lua 适配器协议
|
|
||||||
|
|
||||||
完整 API 见 **[Lua 适配器 API 文档](docs/lua-adapters.md)**([English](docs/lua-adapters-en.md))。
|
|
||||||
|
|
||||||
每个适配器是一个返回 table 的 Lua 脚本(`internal/lua/adapters/<name>.lua`),
|
|
||||||
加载时可被 `adapter_dir` 下的同名脚本覆盖——**改 Lua 脚本无需重编译**,
|
|
||||||
重启即生效;通过 WebUI 上传的适配器与在线编辑的源配置则即时生效。
|
|
||||||
|
|
||||||
```lua
|
|
||||||
return {
|
|
||||||
name = "mysrc",
|
|
||||||
version = "1.0.0",
|
|
||||||
endpoint = "/chat/completions", -- 上游路径(可被 source.endpoint 覆盖)
|
|
||||||
headers = { ["X-Static"] = "v" }, -- 静态默认请求头(build_headers 缺省时使用)
|
|
||||||
|
|
||||||
-- 请求转换:把 OpenAI 格式 req 转成上游原生格式,返回字符串
|
|
||||||
transform_request = function(raw_json) ... end,
|
|
||||||
|
|
||||||
-- 响应转换:把上游原生响应转成统一格式字符串
|
|
||||||
-- { content, reasoning_content, finish_reason, token_usage{...}, tool_calls[{...}] }
|
|
||||||
transform_response = function(raw_json) ... end,
|
|
||||||
|
|
||||||
-- 流式分块转换:把上游 SSE data 转成 { content, done, ... },返回 "" 则跳过
|
|
||||||
transform_stream_chunk = function(raw_chunk) ... end,
|
|
||||||
|
|
||||||
-- [可选] 动态请求头/签名钩子
|
|
||||||
-- meta = { url, method, body, api_key, timestamp, source={ name, meta={...} } }
|
|
||||||
build_headers = function(meta) return { ["X-App-Sign"] = sign } end,
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
内置信号辅助:`hmac_sha256_hex(key, data)`、`sha256_hex(data)`、`base64_encode(s)`、
|
|
||||||
`tohex(s)`、`json.encode/decode`、`log(level, msg)`。
|
|
||||||
|
|
||||||
### 内置适配器
|
|
||||||
|
|
||||||
`openai` `deepseek` `anthropic` `gemini` `github` `groq` `mistral` `ollama` `kimicode`。
|
|
||||||
|
|
||||||
`anthropic`/`gemini`/`ollama` 适配器内置多模态转换(`image_url` → 各自上游格式);若
|
|
||||||
源启用 `disable_thinking`,`deepseek` 适配器会把 `extra_body.thinking.type` 置为
|
|
||||||
`disabled`。
|
|
||||||
|
|
||||||
**kimicode** 是展示 `build_headers` 的样例:云端校验调用方 app,需要按
|
|
||||||
`meta.app_secret` 对时间戳+URL+请求体哈希做 HMAC 签名并附 `X-App-Sign` 等头。
|
|
||||||
配好 `sources[].meta.{app_id, app_secret, app_agent}` 即可。
|
|
||||||
|
|
||||||
## 桌面 GUI(Electron)
|
|
||||||
|
|
||||||
可选独立产物,面向内网桌面端用户:**内嵌 ModelRouter 核心**(Clash Verge 形态),
|
|
||||||
一键启停、系统托盘常驻、开机自启、静默启动,内容区一比一内嵌完整 WebUI
|
|
||||||
(状态/对话/密钥/优先级/源/适配器 六个页面,免登录)。服务器用户继续使用纯 Go 二进制。
|
|
||||||
|
|
||||||
### 两种安装包:Headless 与 Desktop
|
|
||||||
|
|
||||||
发布到 Release 的安装包分为两类,面向不同用户:
|
|
||||||
|
|
||||||
- **Headless(服务器版)**:纯核心二进制(无 Electron),配置驱动,适合服务器、
|
|
||||||
容器、systemd 等无人值守场景。发布包含 `deb` + `rpm`(含 systemd unit、内存调优
|
|
||||||
与示例配置)与 `tar.gz`(二进制 + `config.example.yaml` + `adapters/` + README),
|
|
||||||
Windows 为 `.zip`。
|
|
||||||
- **Desktop(桌面版)**:Electron GUI 安装包,内嵌核心,适合个人桌面日常使用。
|
|
||||||
|
|
||||||
### 选择 GUI 还是纯后端
|
|
||||||
|
|
||||||
| 场景 | 推荐 | 理由 |
|
|
||||||
| ------ | ------ | ------ |
|
|
||||||
| 服务器 / 内网网关 / 无人值守常驻 | **Headless**(单二进制) | 极轻量(二进制 ~8MB;RSS 随源数量,单源 ~10MB、16 源 ~32–35MB,见[内存占用](#内存占用实测与调优)),零依赖单进程,直接跑在 systemd/任意容器里,远程管理 |
|
|
||||||
| 个人桌面日常使用 / 多设备内网共享 | **Desktop**(GUI) | 免登录内嵌 WebUI、系统托盘一键启停、开机自启、静默后台,适合不懂命令行的使用者 |
|
|
||||||
| Windows 桌面 | **Desktop** | 纯后端在 Windows 上需自行注册服务,GUI 提供原生托盘/自启体验 |
|
|
||||||
| CI 一键出三平台安装包 | **Desktop 打包脚本** | `make gui-*` 系列出 deb/AppImage/NSIS,可进发行流水线 |
|
|
||||||
|
|
||||||
两者完全同源:GUI 内嵌的就是纯后端的同一份 `llmsproxy` 核心(LuaJIT 版),
|
|
||||||
配置/适配器格式完全一致,可随时互换。
|
|
||||||
|
|
||||||
### 构建与运行
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd cmd/gui
|
|
||||||
npm install
|
|
||||||
make gui # 或 npm run dev —— 开发运行(需先 make build 生成 bin/llmsproxy)
|
|
||||||
make gui-deb # 本机构建 deb(分享给其它 Linux 用户)
|
|
||||||
make gui-dist # 构建 deb + rpm + AppImage(linux,rpm 需系统 rpmbuild)
|
|
||||||
make core-dist # 构建核心 deb + rpm + tar.gz(服务器版,需 nfpm)
|
|
||||||
make gui-win # 构建 Windows nsis 安装包(本机 mingw + wine)
|
|
||||||
make gui-win-docker # 构建 Windows nsis 安装包(全 docker 自足,含 wine)
|
|
||||||
make gui-dist-dir # 仅解包产物(调试用,不产安装包)
|
|
||||||
```
|
|
||||||
|
|
||||||
产物在 `cmd/build/gui-dist/`:`ModelRouter-<ver>.AppImage`、
|
|
||||||
`modelrouter-gui_<ver>_amd64.deb`、`ModelRouter Setup <ver>.exe`(win);
|
|
||||||
核心包在 `cmd/build/dist/`:`llmsproxy_<ver>_amd64.deb`、`llmsproxy-<ver>.x86_64.rpm`。
|
|
||||||
rpm 需系统 `rpmbuild`。每次打包后都会跑 `packaging/verify-dist.sh` 产物体检
|
|
||||||
(大小下限门禁——1.3.0 曾因宿主 wine 损坏产出 264KB 的残缺 exe 却没有被拦住)。
|
|
||||||
发布时会把产物重命名为 `ModelRouter-Desktop-*`(GUI)与 `ModelRouter-Headless-*`(纯后端)
|
|
||||||
两类上传到 GitCode Release。
|
|
||||||
|
|
||||||
#### Windows 打包(docker 固化方案,推荐)
|
|
||||||
|
|
||||||
`make gui-win-docker` 一步完成:交叉编译核心 + NSIS 打包**全在 docker 内**,宿主机
|
|
||||||
**无需 mingw、无需 wine、无需 node**。镜像内置 wine(含 i386 运行时),修复了
|
|
||||||
此前宿主 wine 损坏(缺 syswow64 → `c0000135`)导致产出 264KB 残缺 exe 的问题。
|
|
||||||
|
|
||||||
- 构建镜像:`cmd/gui/docker/win-builder/Dockerfile`(`golang:1.25` + mingw-w64 +
|
|
||||||
node + wine32/wine64 + 预编译 LuaJIT windows 版 `lua51.dll` 与 import lib)。
|
|
||||||
LuaJIT 源码取自 gitcode 镜像(国内可达),github 兜底;Go 依赖走 `https://goproxy.cn`。
|
|
||||||
- 一键发布:`cmd/gui/scripts/dist-win-docker.sh` —— 核心编译 + electron-builder
|
|
||||||
NSIS 全部在容器内执行(`USE_SYSTEM_WINE=true` 走镜像内的 wine);electron-builder
|
|
||||||
的 ~400MB 工具链缓存挂到 docker volume,二次构建增量复用。
|
|
||||||
- 首次运行自动构建 `modelrouter/win-builder` 镜像(约 5–10 分钟);已存在则直接复用。
|
|
||||||
- 打包完成后自动跑 `packaging/verify-dist.sh` 门禁:exe 低于 5MB / deb·rpm·7z 低于
|
|
||||||
10MB 即失败——残缺安装包不可能再溜进 Release。
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 全 Windows 发布(核心重编 + NSIS):
|
|
||||||
make gui-win-docker
|
|
||||||
```
|
|
||||||
|
|
||||||
### 特性
|
|
||||||
|
|
||||||
- **内嵌核心**:自动拉起捆绑的 `llmsproxy`(LuaJIT 版),配置/密钥/adapter 存于
|
|
||||||
用户数据目录 `<userData>/profile/`,首次运行自动生成随机 admin key 写入 `keys`
|
|
||||||
(非 seed)并自注入,WebUI 免登录、不会提示"更换初始密钥"。端口可在设置中更改(默认 8787)。
|
|
||||||
- **系统托盘**:内核状态点、开机自启/静默启动开关、启停/重启内核、退出。
|
|
||||||
- **静默启动**:`--silent` 参数或设置项,启动仅驻托盘不弹窗;首次运行始终显示窗口。
|
|
||||||
- **开机自启**:Windows/macOS 用 `setLoginItemSettings`;Linux 写
|
|
||||||
`~/.config/autostart/modelrouter-gui.desktop`,静默开启时自动附加 `--silent`。
|
|
||||||
|
|
||||||
## 目录
|
|
||||||
|
|
||||||
```
|
|
||||||
cmd/llmsproxy # 入口(纯 Go 服务端)
|
|
||||||
cmd/gui # Electron 桌面 GUI(内嵌核心 + 托盘 + 打包)
|
|
||||||
internal/config # YAML 配置加载/校验
|
|
||||||
internal/lua # LuaJIT VM + worker 池 + AdapterCache + 内置适配器 (embed)
|
|
||||||
internal/provider # Provider(HTTP) + Registry(路由)
|
internal/provider # Provider(HTTP) + Registry(路由)
|
||||||
internal/gateway # OpenAI 兼容 HTTP 服务 + 鉴权 + SDK/流式 + WebUI
|
internal/gateway # OpenAI 兼容 HTTP 服务 + 鉴权 + 流式 + WebUI
|
||||||
internal/types # 统一格式 & OpenAI wire 类型
|
internal/types # 统一格式 & OpenAI wire 类型
|
||||||
|
packaging/ # deb/rpm/tar 打包、systemd unit、迁移脚本
|
||||||
|
docs/ # 主题文档
|
||||||
```
|
```
|
||||||
|
|
||||||
## 开发协作
|
## 开发与测试
|
||||||
|
|
||||||
- **Git 分支工作流**:主分支 / 特性分支 / 发布分支的规范见
|
|
||||||
**[docs/git-workflow.md](docs/git-workflow.md)**([English](docs/git-workflow-en.md))。
|
|
||||||
主线:`main` 永可部署 → 特性合入 `main` → 切 `release/vX.Y.Z` 打 tag 发布 →
|
|
||||||
hotfix 提交发布分支并 cherry-pick 回 `main`。
|
|
||||||
- **Lua 适配器协议**:见 **[docs/lua-adapters.md](docs/lua-adapters.md)**
|
|
||||||
([English](docs/lua-adapters-en.md))。
|
|
||||||
|
|
||||||
## 测试
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go test -tags luajit ./...
|
go test -tags luajit ./... # 全量
|
||||||
|
go test -tags luajit ./internal/lua # 单包(LuaJIT 相关改动必跑)
|
||||||
```
|
```
|
||||||
|
|
||||||
覆盖:配置校验、适配器加载/变换、签名钩子、Gateway 鉴权、SDK 结算、SSE 流式、
|
- 分支与发布纪律见 [docs/git-workflow.md](docs/git-workflow.md)。
|
||||||
模型路由、多模态透传与 disable_thinking。
|
- 插件放置在 `plugin_dir`,网关启动时**不会**创建、覆盖或删除任何插件文件;
|
||||||
|
升级桌面端也不会动它们。
|
||||||
|
|
||||||
|
## 许可
|
||||||
|
|
||||||
|
MIT License,见 [LICENSE](./LICENSE)。
|
||||||
|
|||||||
695
README_EN.md
695
README_EN.md
@ -1,626 +1,155 @@
|
|||||||
# ModelRouter
|
# ModelRouter
|
||||||
|
|
||||||
|
> **Repository**: <https://atomgit.com/HeNanMU-EEA/ModelRouter>
|
||||||
|
>
|
||||||
> **中文**: [README.md](./README.md)
|
> **中文**: [README.md](./README.md)
|
||||||
>
|
>
|
||||||
> **AI-assisted**: This project is developed with AI-assisted coding (code and docs are produced in collaboration with AI, reviewed by humans).
|
> **AI-assisted**: code and documentation were produced with AI assistance and
|
||||||
|
> reviewed by a human.
|
||||||
|
|
||||||
A lightweight, unified OpenAI-compatible LLM gateway for internal networks —
|
An intranet gateway for people with several LLM subscriptions: configure once and
|
||||||
configure once, and let multiple services (agents, SDKs, bots) share many
|
let multiple services share the upstream capacity. Pick a model explicitly, or let
|
||||||
upstreams (DeepSeek, Qijiar, OpenAI, Anthropic, Gemini, Groq, Mistral, Ollama,
|
`AUTO` choose by priority among the sources that are currently healthy.
|
||||||
KimiCode…) behind a single endpoint. Pick a specific model or use **AUTO** mode,
|
|
||||||
which routes to the best healthy upstream by configured priority.
|
|
||||||
|
|
||||||
> **Lightweight, no recompile to add sources**: the gateway is a single Go
|
The gateway itself is a single Go binary (~10 MB, no runtime dependencies). Adding
|
||||||
> binary (8-12 MB, ~8 MB stripped; zero runtime dependencies). Adding or switching an upstream is
|
or switching an upstream is a `sources` entry in `config.yaml` (or the WebUI), or a
|
||||||
> just a `sources` entry in `config.yaml` (or via the WebUI) or a `.lua`
|
dropped-in `.lua` adapter — **no Go changes, no rebuild**. Changes submitted through
|
||||||
> adapter — **no Go changes, no recompile**. Changes submitted through the
|
the WebUI take effect immediately; editing `config.yaml` or a `.lua` adapter file
|
||||||
> WebUI take effect immediately (hot reload); editing `config.yaml` or a `.lua`
|
directly requires a restart.
|
||||||
> file under `adapter_dir` by hand requires a process restart.
|
|
||||||
|
|
||||||
## Overview
|
## What it does
|
||||||
|
|
||||||
Exposes a standard `OpenAI Chat Completions` API (`/v1/chat/completions` +
|
- **One OpenAI-compatible API**: `/v1/chat/completions`, `/v1/models`,
|
||||||
`/v1/models`) to your internal network, translating between multiple upstream
|
`/v1/images/generations`. Every source speaks the same shape outwards, including
|
||||||
protocols via **Lua adapters**. Supports one-shot calls, SSE streaming,
|
`reasoning_content`, `tool_calls` and `usage`.
|
||||||
**AUTO model routing**, multimodal passthrough, and tool calls.
|
- **Many sources, one AUTO chain**: configure any number of upstreams; `AUTO`
|
||||||
|
follows the priority tiers saved in the WebUI, fails over between them, cools
|
||||||
Extracted and independently evolved from the multi-source LLM adapter layer of
|
down broken sources and probes them back to health at half-cool.
|
||||||
[HomeAgent](https://gitcode.com/JianFeeeee/HomeAgent)
|
- **Protocol differences live in Lua**: each source binds a `.lua` adapter that
|
||||||
(`internal/agent/api/provider.go` + `internal/lua/adapters/*`).
|
transforms requests, responses and stream chunks both ways, and can sign headers
|
||||||
|
via `build_headers(meta)`. Vendor knowledge stays out of the Go kernel.
|
||||||
## Features
|
- **Plugins are independent**: business extensions (channel login, billing, quota
|
||||||
|
probing) are external Lua plugins loaded from `plugin_dir`. They are not compiled
|
||||||
- **Multi-source**: any number of upstream sources in one process, routed by the
|
into the binary and not shipped in release archives, and they talk to the kernel
|
||||||
request's `model`.
|
only through the `modelrouter` SDK.
|
||||||
- **AUTO mode**: with `default_model: AUTO`, scheduling follows the tiered AUTO
|
- **Multi-tenant keys**: gateway keys are separate from upstream keys, and can be
|
||||||
chain saved on the Priority page (see below).
|
scoped to a model set, an AUTO chain and a quota.
|
||||||
- **Unified output**: every source speaks OpenAI format (incl.
|
- **Encrypted at rest**: upstream keys, custom header values and gateway keys are
|
||||||
`reasoning_content`, `tool_calls`, `usage`).
|
sealed with AES-256-GCM; `master.key` is 0600, or injected via
|
||||||
- **Image generation**: `POST /v1/images/generations`, routed to models with
|
`MODELROUTER_MASTER_KEY`.
|
||||||
`kind: image`.
|
- **Observability**: live source probing, HTTP access log, config-change audit,
|
||||||
- **Per-model quota**: each key gives every model its own token and request caps
|
request and usage CSV export. Writes are kept off the request hot path.
|
||||||
plus a reset period (hour/week/month/custom N hours). One model running out
|
|
||||||
affects only that model — the key's other models keep working. Exhaustion
|
|
||||||
answers 429 + `Retry-After` naming the model, so a client resumes when the
|
|
||||||
window rolls over; admin keys are never capped.
|
|
||||||
- **Multimodal**: `content` arrays (`image_url` etc.) pass through losslessly;
|
|
||||||
Anthropic/Gemini/Ollama are translated automatically.
|
|
||||||
- **LuaJIT VM**: golua-binding LuaJIT; each adapter has its own VM + worker
|
|
||||||
pool for safe concurrency. Pools are **elastic**: the sum of `max_concurrent`
|
|
||||||
over an adapter's sources is a ceiling, not a preallocation — states are booted
|
|
||||||
on demand and reclaimed when demand drops, so an idle gateway holds close to
|
|
||||||
zero Lua states. The grow step follows the adapter's max concurrency
|
|
||||||
(`clamp(ceil(max/8), 1, 8)`, and never exceeds the number of queued callers);
|
|
||||||
the shrink step follows the live connection count
|
|
||||||
(`clamp(ceil(slack/(1+in_use)), 1, slack)`), so an idle pool collapses in one
|
|
||||||
round while a busy one gives up a single state at a time.
|
|
||||||
- **On-demand request logs**: dashboard totals are computed from the **full**
|
|
||||||
audit history (streamed once at startup and released — 29 MB / 221k lines in
|
|
||||||
~260 ms), while the raw records are never held in memory: the dashboard loads
|
|
||||||
one screen, scrolling pages the rest straight off disk, CSV export streams in
|
|
||||||
O(1) memory, and leaving the page releases everything.
|
|
||||||
- **Self-healing cooldown**: cooldown is capped at 5 minutes and, past the
|
|
||||||
window's midpoint, exactly one probe request is allowed through; a recovered
|
|
||||||
upstream (or a reset quota) returns to full rotation on that probe instead of
|
|
||||||
waiting out the window.
|
|
||||||
- **disable_thinking**: `disable_thinking: true` toggles reasoning per-request.
|
|
||||||
- **Lua adapter protocol**: each source mounts a `.lua` adapter with
|
|
||||||
`transform_request` / `transform_response` / `transform_stream_chunk` — all
|
|
||||||
protocol differences live in Lua, Go only schedules and proxies.
|
|
||||||
- **Signature / header hooks**: adapters may define `build_headers(meta)` to
|
|
||||||
inject/sign request headers before the HTTP call (e.g. KimiCode style
|
|
||||||
app-validation), with helpers like `hmac_sha256_hex`, `sha256_hex`,
|
|
||||||
`base64_encode`.
|
|
||||||
- **WebUI**: built-in management page to view/add/edit sources & models, edit
|
|
||||||
the AUTO chain, and manage per-key model scopes, persisted to a runtime file.
|
|
||||||
- **Encrypted secrets at rest**: upstream `api_key`, custom header values and
|
|
||||||
gateway keys are stored AES-256-GCM encrypted (`master.key` 0600 next to the
|
|
||||||
runtime file, or `LLMS_PROXY_MASTER_KEY`).
|
|
||||||
- **Live source probing**: the status page probes each source
|
|
||||||
(`GET {base}/models`, falling back to a minimal chat call) without touching
|
|
||||||
the scheduler's backoff state; errors are shown on hover.
|
|
||||||
- **Auth**: the gateway validates client keys via `gateway_keys`; independent
|
|
||||||
from each upstream's own key.
|
|
||||||
- **Streaming**: SSE `chat.completion.chunk` with a role-first chunk and
|
|
||||||
`[DONE]` terminator.
|
|
||||||
|
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
GOMODCACHE=... GOPROXY=off go build -tags luajit -o llmsproxy ./cmd/llmsproxy
|
# LuaJIT development libraries are required; -tags luajit is mandatory
|
||||||
./llmsproxy -config config.yaml # first run generates a default config and prints a random admin key
|
GOMODCACHE=... GOPROXY=off go build -tags luajit -o modelrouter ./cmd/modelrouter
|
||||||
|
|
||||||
|
# First run generates a default config and prints a random admin key
|
||||||
|
./modelrouter -config config.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
> Depends on [golua](https://github.com/aarzilli/golua) (LuaJIT bindings).
|
> Depends on [golua](https://github.com/aarzilli/golua) (LuaJIT bindings). Without
|
||||||
> You **must** build with `-tags luajit`; otherwise the built-in superset
|
> `-tags luajit` the build falls back to gopher-lua, which behaves slightly
|
||||||
> gopher-lua path is used (behavior differs slightly).
|
> differently.
|
||||||
|
|
||||||
> **No config is shipped in the repo** (config files carry real keys). On
|
> **No config in the repo**: the repository ships no config file (they contain
|
||||||
> first run the binary generates a default config at the `-config` path: a
|
> keys). The generated default binds `127.0.0.1:8080` only; rotate the admin key in
|
||||||
> random admin key (printed to the startup log), loopback-only `127.0.0.1:8080`,
|
> the WebUI after the first login.
|
||||||
> and a keyless source ready to chat. Rotate the admin key in the
|
|
||||||
> WebUI after first login.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# no key -> 401
|
# No key -> 401
|
||||||
curl http://127.0.0.1:8080/v1/models
|
curl http://127.0.0.1:8080/v1/models
|
||||||
|
|
||||||
# one-shot ($KEY = the admin key printed at first startup)
|
# Non-streaming ($KEY = the admin key printed at first start)
|
||||||
curl -H "Authorization: Bearer $KEY" \
|
curl -H "Authorization: Bearer $KEY" \
|
||||||
-d '{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}' \
|
-d '{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}' \
|
||||||
http://127.0.0.1:8080/v1/chat/completions
|
http://127.0.0.1:8080/v1/chat/completions
|
||||||
|
|
||||||
# streaming
|
# Streaming
|
||||||
curl -N -H "Authorization: Bearer $KEY" \
|
curl -N -H "Authorization: Bearer $KEY" \
|
||||||
-d '{"model":"AUTO","stream":true,"messages":[{"role":"user","content":"hi"}]}' \
|
-d '{"model":"AUTO","stream":true,"messages":[{"role":"user","content":"hi"}]}' \
|
||||||
http://127.0.0.1:8080/v1/chat/completions
|
http://127.0.0.1:8080/v1/chat/completions
|
||||||
```
|
```
|
||||||
|
|
||||||
Any OpenAI SDK works: point `base_url` at the gateway, use any authorized
|
Any OpenAI SDK works: point `base_url` at the gateway and use an authorized gateway
|
||||||
gateway key as the API key.
|
key as `api_key`.
|
||||||
|
|
||||||
## Configuration
|
## Documentation
|
||||||
|
|
||||||
Generated on first run at the `-config` path (default `./config.yaml`). Core fields:
|
The detailed documentation is currently written in Chinese.
|
||||||
|
|
||||||
```yaml
|
| Topic | Document |
|
||||||
listen: 127.0.0.1:8080 # bind address (keep internal/loopback)
|
|---|---|
|
||||||
gateway_keys: [sk-gw-<random>] # initial admin key seed (random per install), migrated to the store on first start
|
| Implementation and the full feature list | [docs/features.md](docs/features.md) |
|
||||||
default_model: AUTO # when model is unroutable, follow the AUTO chain
|
| Configuration: `config.yaml`, model discovery, multiple keys | [docs/configuration.md](docs/configuration.md) |
|
||||||
adapter_dir: adapters # Lua adapter dir; created+seeded if missing, read-only otherwise
|
| Routing: model routing, AUTO chain, cooldown, sources vs adapters, source templates | [docs/routing.md](docs/routing.md) |
|
||||||
runtime_file: runtime.json # WebUI-edited sources/keys/AUTO chain persist here
|
| Adapters: Lua protocol and built-in adapters | [docs/adapters.md](docs/adapters.md) · [docs/lua-adapters-en.md](docs/lua-adapters-en.md) (English) |
|
||||||
|
| Plugins: SDK contract, hooks, UI extensions, HTTP endpoints | [docs/plugins.md](docs/plugins.md) |
|
||||||
sources:
|
| Desktop app: Headless / Desktop artifacts, build and run | [docs/gui.md](docs/gui.md) |
|
||||||
- name: deepseek
|
| Performance: measured memory use, tuning, thinking budget | [docs/performance.md](docs/performance.md) |
|
||||||
base_url: https://api.deepseek.com
|
| WebUI usage | [docs/webui.md](docs/webui.md) |
|
||||||
api_key: sk-... # or api_key_env: SOME_ENV to read from an env var
|
| Development: layout, tests, branch and release discipline | [docs/development.md](docs/development.md) · [docs/git-workflow-en.md](docs/git-workflow-en.md) (English) |
|
||||||
adapter: deepseek
|
| Deployment and migration | [docs/deployment.md](docs/deployment.md) |
|
||||||
max_concurrent: 8
|
|
||||||
models:
|
## Renamed from llmsproxy
|
||||||
- id: deepseek-v4-flash
|
|
||||||
priority: 100 # YAML sources seed the AUTO chain on first start
|
The product name has been ModelRouter for a while; the binary, config directory,
|
||||||
kind: chat
|
service name and environment variables are now `modelrouter` as well:
|
||||||
- id: deepseek-v4-pro
|
|
||||||
priority: 60
|
| Before | After |
|
||||||
kind: chat
|
|---|---|
|
||||||
# static headers (take precedence over adapter defaults)
|
| `llmsproxy` (binary) | `modelrouter` |
|
||||||
headers: { X-Tenant: prod }
|
| `/etc/llmsproxy/` | `/etc/modelrouter/` |
|
||||||
# passthrough metadata for the Lua build_headers hook
|
| `llmsproxy.service` | `modelrouter.service` |
|
||||||
meta: { app_id: x, app_secret: y }
|
| `LLMSPROXY_*` | `MODELROUTER_*` |
|
||||||
temperature: 0.7
|
|
||||||
max_tokens: 4096
|
Existing installs migrate with a dry-run-by-default script (copies, never moves,
|
||||||
timeout: 120s # request timeout, default 120s
|
and can roll back):
|
||||||
```
|
|
||||||
|
|
||||||
Sensitive fields in the runtime file (`runtime_file`) are encrypted at rest:
|
|
||||||
|
|
||||||
- AES-256-GCM, stored as `enc:v1:<base64>`.
|
|
||||||
- Master key source: env var `LLMS_PROXY_MASTER_KEY` (64 hex chars), else a
|
|
||||||
`master.key` file next to the runtime file; if neither exists it is generated
|
|
||||||
on first start (mode 0600).
|
|
||||||
- Old plaintext files still load; the first UI save migrates the whole file to
|
|
||||||
ciphertext.
|
|
||||||
- Back up `master.key` with your config — losing it makes the secrets
|
|
||||||
undecryptable. Do not commit it.
|
|
||||||
|
|
||||||
### Gateway keys (multi-key)
|
|
||||||
|
|
||||||
Gateway auth uses a "multi-key + role + model scope" model. Keys are persisted
|
|
||||||
under the `keys` field of the runtime file (encrypted at rest):
|
|
||||||
|
|
||||||
- The `gateway_keys` config is only an **initial admin key seed** — it is
|
|
||||||
migrated into the runtime store on first start and no longer drives auth.
|
|
||||||
- **Important: after first start, replace the admin key via the WebUI Keys
|
|
||||||
page.** The seed key is written in plaintext in `config.yaml`, so keeping it
|
|
||||||
active is a security risk; create a new admin key, log in with it, then
|
|
||||||
delete the seed key.
|
|
||||||
- The WebUI **Keys page** creates/deletes keys. Each key has a role (`admin`
|
|
||||||
manages everything, `user` sees only its own key) and an optional **model
|
|
||||||
scope** (model + source + token quota + reset period). Each model brick shows
|
|
||||||
its own budget badge (e.g. `1.0K 77×·1h`, `∞` when uncapped); clicking it
|
|
||||||
edits that model's token quota, request quota and reset period. Quotas apply
|
|
||||||
per model, so one model running out never blocks the key's others. The
|
|
||||||
"My key" view shows a user its model scopes and their budgets.
|
|
||||||
- Clients authenticate with any authorized key's plaintext as
|
|
||||||
`Authorization: Bearer <key>`.
|
|
||||||
- Deleting a key removes it from the store immediately.
|
|
||||||
|
|
||||||
#### Per-key usage quota
|
|
||||||
|
|
||||||
**Quotas are per model.** Each key's `models[]` list gives every model its own
|
|
||||||
token budget and request budget. One model running out affects only that model
|
|
||||||
— the key's other models keep working.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
keys:
|
|
||||||
- key: sk-gw-<hex>
|
|
||||||
role: user
|
|
||||||
name: agent-alice
|
|
||||||
models:
|
|
||||||
- model: deepseek-v4-flash
|
|
||||||
token_quota: 1000000 # this key's token budget for this model
|
|
||||||
req_quota: 20000 # requests within the window
|
|
||||||
period: nhour # "" | hour | week | month | nhour
|
|
||||||
hours: 6 # n-hour only
|
|
||||||
- model: AUTO
|
|
||||||
token_quota: 5000000 # AUTO is a quota entry like any other
|
|
||||||
period: hour
|
|
||||||
- model: kimi-k3 # no quota listed = unlimited
|
|
||||||
```
|
|
||||||
|
|
||||||
- **There is deliberately no key-wide total.** A key-wide cap would make
|
|
||||||
"move A's budget to B" a global reallocation; per-model budgets keep each
|
|
||||||
model independently controllable, so it stays visible which model is
|
|
||||||
actually consuming the spend.
|
|
||||||
- `period`: empty = never resets (lifetime total); `hour` / `week` / `month` =
|
|
||||||
fixed windows; `nhour` + `hours` = a custom hour count. **A misspelled
|
|
||||||
period is rejected at write time** rather than silently becoming a
|
|
||||||
never-resetting quota.
|
|
||||||
- Quotas are isolated per key *and* per model within a key: one key exhausting
|
|
||||||
`m1` never draws on another key's budget, and never blocks the same key's
|
|
||||||
`m2`.
|
|
||||||
- Usage counts chat, streaming and image requests, and survives a restart by
|
|
||||||
replaying the audit log (40 days retained, covering the longest `month`
|
|
||||||
window).
|
|
||||||
- An exhausted quota returns **429 + `Retry-After`**
|
|
||||||
(`rate_limit_exceeded`) and the message names the model that ran out, so a
|
|
||||||
client resumes when the window rolls over; a model the key may not use stays
|
|
||||||
403. **Admin keys are never capped** (quotas on their scopes are not
|
|
||||||
enforced either), so a cap can never lock the operator out.
|
|
||||||
- Buckets are whole unix hours, so a window frees up at most an hour late
|
|
||||||
(deliberately freeing late rather than overspending).
|
|
||||||
- `PUT /api/keys/{key}` submits quotas by submitting `models` — the caps are
|
|
||||||
part of the scope, so there is no second budget that can drift out of sync
|
|
||||||
with the model list. An explicit `0` lifts that model's cap.
|
|
||||||
|
|
||||||
##### Quota rejection vs capacity rejection
|
|
||||||
|
|
||||||
A quota is checked at the entry point and occupies no upstream slot, so it is a
|
|
||||||
cheap rejection. Running out of capacity instead waits out the bounded
|
|
||||||
`busyWait` before answering 503. Measured (capacity 8, 0.6 s/request, 100
|
|
||||||
concurrent clients):
|
|
||||||
|
|
||||||
| Case | Status | Latency | Reaches upstream |
|
|
||||||
|---|---|---|---|
|
|
||||||
| Quota spent (`req_quota` 50, 100 concurrent) | `429 rate_limit_exceeded` + `Retry-After` | **19–21 ms** | no |
|
|
||||||
| Capacity exhausted (same topology, no quota) | `503 upstream_error` | ~2.6 s (`busyWait`) | overflow: no |
|
|
||||||
|
|
||||||
So a client can tell "wait for the window to roll over" apart from "wait for
|
|
||||||
upstream capacity" — the first only needs patience, the second usually means
|
|
||||||
back off concurrency or switch sources.
|
|
||||||
|
|
||||||
##### What the quota costs
|
|
||||||
|
|
||||||
Buckets are kept per (key, model, whole unix hour) for 40 days. Measured on an
|
|
||||||
AMD 7840HS:
|
|
||||||
|
|
||||||
- A window query scans the window, not the whole retention: 49 ns for 1 h,
|
|
||||||
55 ns for 24 h, 3.9 µs for 30 d.
|
|
||||||
- Recording one request: 283 ns, 3 allocations (unchanged from before this
|
|
||||||
feature; the allocations come from the record ring buffer).
|
|
||||||
- Memory: the production shape (7 keys x 8 models x 2 sources at full 40-day
|
|
||||||
retention) costs about **3.7 MB**. The `source::model` bucket used by a
|
|
||||||
source-pinned quota is created **lazily** — it is maintained only once some
|
|
||||||
quota actually pins that source, because writing it on every record would
|
|
||||||
cost an extra 18 MB at 20 keys x 8 models x 3 sources.
|
|
||||||
The trade-off: usage recorded before a pinned quota was configured cannot be
|
|
||||||
split by source afterwards, so **a pinned quota may under-count its first
|
|
||||||
window**.
|
|
||||||
|
|
||||||
```
|
|
||||||
HTTP/1.1 429 Too Many Requests
|
|
||||||
Retry-After: 2100
|
|
||||||
{"error":{"type":"rate_limit_exceeded","message":"token quota exceeded for \"deepseek-v4-flash\" (5000000/5000000)"}}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Model routing
|
|
||||||
|
|
||||||
`/v1/chat/completions` `model` resolution order:
|
|
||||||
|
|
||||||
1. `source/model` or `source:model` prefix → pinned source;
|
|
||||||
2. exact match of a source's `model`;
|
|
||||||
3. with `default_model: AUTO` → follow the AUTO chain from the Priority page
|
|
||||||
(see below);
|
|
||||||
4. otherwise the request errors.
|
|
||||||
|
|
||||||
Any OpenAI client can pin to a source by setting `model` to its
|
|
||||||
`name/anything`; `AUTO` (or the gateway's `default_model: AUTO`) follows the
|
|
||||||
tiered AUTO chain.
|
|
||||||
|
|
||||||
### AUTO chain (Priority page)
|
|
||||||
|
|
||||||
AUTO scheduling is driven **only** by the rules saved on the Priority page. The
|
|
||||||
numeric `models[].priority` in source configs no longer participates in
|
|
||||||
scheduling and is no longer shown. **Chat and image generation are two
|
|
||||||
independent chains**, switched in the Priority page via the `Chat / Image` toggle:
|
|
||||||
|
|
||||||
- **Chat chain** (`auto` field): serves `POST /v1/chat/completions` with
|
|
||||||
`model: AUTO`; only `kind: chat` models are honored (image slots are skipped).
|
|
||||||
- **Image chain** (`auto_image` field): serves `POST /v1/images/generations` with
|
|
||||||
`model: AUTO`; only `kind: image` models are honored. With no image chain
|
|
||||||
configured, AUTO image generation falls back to legacy discovery (all sources
|
|
||||||
exposing an image model, in registry order) for backward compatibility.
|
|
||||||
|
|
||||||
Each rule is one "slot": `{ model, source, tier, token_quota, period, hours }`.
|
|
||||||
|
|
||||||
- `tier` is the priority tier: models in the same tier sit side by side and
|
|
||||||
share it; tiers run high → low.
|
|
||||||
- The same model may appear in several slots (e.g. A low → B low → A high →
|
|
||||||
B high) and is tried in tier order.
|
|
||||||
- `token_quota` > 0 means the slot is skipped once its tokens within the reset
|
|
||||||
window are exhausted; `period` supports `hour` / `week` / `month` / `nhour`
|
|
||||||
(with `hours`); empty = unlimited.
|
|
||||||
- Within a tier, slots run by preference score; cooling / quota-exhausted /
|
|
||||||
hard-failed slots fall through to the next tier, and total failure returns
|
|
||||||
503 with a per-tier summary.
|
|
||||||
|
|
||||||
### Sources vs. adapters
|
|
||||||
|
|
||||||
- A **source** describes a connection to an upstream: `name`, `base_url`,
|
|
||||||
`api_key`, model list and priorities.
|
|
||||||
- An **adapter** is the protocol translation logic (Lua script): converts the
|
|
||||||
unified OpenAI request to the upstream's native format and back.
|
|
||||||
- One adapter serves many sources (e.g. `openai.lua` for any OpenAI-compatible
|
|
||||||
site); one source can switch adapters via its `adapter` field.
|
|
||||||
- Sources decide *whom to talk to and which models to expose*; adapters decide
|
|
||||||
*how to talk*. They are linked by the `adapter` field inside each `sources[]`
|
|
||||||
entry.
|
|
||||||
|
|
||||||
Loading flow (assembled by `Core` in `internal/core`):
|
|
||||||
|
|
||||||
1. On startup `lua.NewVM(adapter_dir)` loads all adapters: built-in ones
|
|
||||||
(embedded at compile time) + same-name override files under `adapter_dir`.
|
|
||||||
2. `config.Load` reads `config.yaml`; `config.NewStore(runtime_file)` reads
|
|
||||||
WebUI-edited runtime sources; both are merged by name.
|
|
||||||
3. `rebuildRegistry` creates a `provider.Provider` per source (holding its
|
|
||||||
adapter), sizing the adapter's worker pool from the source's concurrency
|
|
||||||
limit.
|
|
||||||
4. Incoming requests are routed by `Registry` to a provider, which calls the
|
|
||||||
adapter's `transform_request` → HTTP call → `transform_response` /
|
|
||||||
`transform_stream_chunk`.
|
|
||||||
|
|
||||||
"Add/Edit source", "Upload adapter", "AUTO chain edits" and "per-key model
|
|
||||||
scopes" in the WebUI take effect immediately (written to the runtime file /
|
|
||||||
`adapter_dir`, then reassembled — no restart). Directly editing `config.yaml`
|
|
||||||
or files under `adapter_dir` requires a process restart.
|
|
||||||
|
|
||||||
Deleting a source/adapter in the WebUI truly removes runtime (UI-created)
|
|
||||||
sources and `.lua` files; base sources defined in `config.yaml` cannot rewrite
|
|
||||||
that file, so they are hidden via a deletion tombstone (still hidden after
|
|
||||||
restart) and can be restored by re-adding the same name in the UI.
|
|
||||||
|
|
||||||
### Source templates (multi-key balancing)
|
|
||||||
|
|
||||||
When several upstream keys share one config (URL / adapter / model list /
|
|
||||||
concurrency / rpm), copying the whole source block N times is wasteful. A
|
|
||||||
**source template** stores every source field except `name` and `api_key`, so you
|
|
||||||
create N key-only-different sources from one shared skeleton — natural multi-key
|
|
||||||
load balancing (each expanded source schedules, cools down, and reports health
|
|
||||||
independently).
|
|
||||||
|
|
||||||
- WebUI Sources page, top-right **Templates** button: list all templates with
|
|
||||||
edit / delete / create.
|
|
||||||
- Add-source dialog has two header buttons:
|
|
||||||
- **From template** → pick a template → form auto-fills (name + key still
|
|
||||||
yours to enter);
|
|
||||||
- **As template** → save the current form's non-key/name fields as a template.
|
|
||||||
- Templates persist in the runtime file's `source_templates` field alongside
|
|
||||||
runtime sources (hot-managed, apply immediately). Templates hold no secret
|
|
||||||
keys, so they're safe to share / version-control.
|
|
||||||
|
|
||||||
A template is just a recipe — it doesn't become a source by itself. Only the
|
|
||||||
key-bearing sources created from it carry real traffic.
|
|
||||||
|
|
||||||
### Memory footprint (measured and tuned)
|
|
||||||
|
|
||||||
Memory is not a constant: it scales with the **number of configured sources**
|
|
||||||
(each source owns an `http.Transport` with its connection pool plus per-model
|
|
||||||
health state), not with uptime. Measured on this machine (Linux x86_64, 12 cores):
|
|
||||||
|
|
||||||
| Deployment shape | Startup RSS | Settled RSS |
|
|
||||||
|---|---|---|
|
|
||||||
| 1 source / 1 adapter (minimal) | ~4 MB | ~10 MB |
|
|
||||||
| 1 source + a 29 MB audit history | ~19 MB | ~20 MB |
|
|
||||||
| **16 sources / 13 adapters / 59 models (this host)** | ~28 MB | **~32-35 MB** |
|
|
||||||
|
|
||||||
> For reference, the same production config used **~105 MB** before this round of
|
|
||||||
> work. The reduction comes from three places: raw audit records are no longer kept
|
|
||||||
> resident (~25 MB — the aggregates still scan the full history, but the scan
|
|
||||||
> releases as it goes), Lua state pools no longer grow monotonically, and the two
|
|
||||||
> runtime knobs below.
|
|
||||||
|
|
||||||
Breakdown of the production instance (per-region, from `/proc/<pid>/smaps`):
|
|
||||||
|
|
||||||
| Region | RSS | Notes |
|
|
||||||
|---|---|---|
|
|
||||||
| Go heap | ~14 MB | provider/registry/scheduler state + connection-pool buffers |
|
|
||||||
| Other anonymous (thread stacks / LuaJIT chunks / runtime) | ~12 MB | scales with thread count and loaded adapters |
|
|
||||||
| Binary text+rodata | ~8 MB | mapped executable pages (read-only, reclaimable by the kernel) |
|
|
||||||
| Go runtime reservations | ~4 MB | the 2 GB+ you see in VSZ is address space, not physical memory |
|
|
||||||
| Shared libraries | ~3 MB | libc / libluajit / libm |
|
|
||||||
|
|
||||||
**Two recommended deployment knobs** (environment only, no code change):
|
|
||||||
|
|
||||||
```ini
|
|
||||||
# /etc/systemd/system/llmsproxy.service
|
|
||||||
Environment=GOGC=50
|
|
||||||
Environment=MALLOC_ARENA_MAX=2
|
|
||||||
```
|
|
||||||
|
|
||||||
- `MALLOC_ARENA_MAX=2`: LuaJIT allocates through cgo into glibc malloc, and glibc
|
|
||||||
allows up to `8 x nproc` per-thread arenas. Every OS thread that touches malloc
|
|
||||||
claims one (~1 MB each) and **never returns it to the OS**. Measured: 8-15
|
|
||||||
arenas (7-12 MB) down to **0**.
|
|
||||||
- `GOGC=50`: halves the Go heap growth target. **It does nothing on its own** —
|
|
||||||
the heap it saves is immediately consumed by additional glibc arenas (measured
|
|
||||||
20.3 -> 21.5 MB, i.e. slightly worse) — so it must be paired with
|
|
||||||
`MALLOC_ARENA_MAX`. Together they cut settled RSS by **~19%**. The gateway is
|
|
||||||
I/O bound (1min10s of CPU per 9 hours here), so the extra GC cycles are free.
|
|
||||||
|
|
||||||
> The desktop build (Electron) already injects both when spawning the embedded
|
|
||||||
> core (`cmd/gui/main.js`); exporting either variable yourself overrides it. If you
|
|
||||||
> write your own systemd unit or container spec, add them there.
|
|
||||||
|
|
||||||
Optional: `GOMEMLIMIT=48MiB` as a soft ceiling saves roughly 1 MB more, at the
|
|
||||||
cost of GC turning aggressive as the limit approaches. Not worth it for a relaxed
|
|
||||||
deployment.
|
|
||||||
|
|
||||||
### disable_thinking
|
|
||||||
|
|
||||||
With `"disable_thinking": true` in the request body, the gateway passes it to
|
|
||||||
each adapter; the DeepSeek adapter maps it to `extra_body.thinking.type =
|
|
||||||
"disabled"`, other sources follow their own protocol.
|
|
||||||
|
|
||||||
### WebUI
|
|
||||||
|
|
||||||
Built-in admin page at `GET /`; after login you can:
|
|
||||||
|
|
||||||
- **Status**: source online state (live probe, error on hover), per
|
|
||||||
model/source/key usage stats, request records filtered by key, and CSV export
|
|
||||||
over a time range; click a model to pin the connection snippet to it.
|
|
||||||
- **Chat**: streaming / non-streaming debug.
|
|
||||||
- **Keys**: create/edit gateway keys, set per-key model scopes (model + source +
|
|
||||||
token quota + reset period); admins manage all keys, users see only their own.
|
|
||||||
- **Priority**: drag blocks to build the tiered AUTO chain.
|
|
||||||
- **Sources**: add/edit/delete upstream sources online (secrets stored
|
|
||||||
encrypted).
|
|
||||||
- **Adapters**: upload / delete Lua adapter scripts.
|
|
||||||
|
|
||||||
Changes persist to `runtime_file` (survive restarts).
|
|
||||||
|
|
||||||
## Lua adapter protocol
|
|
||||||
|
|
||||||
Full API: **[Lua adapter docs](docs/lua-adapters.md)**
|
|
||||||
([中文](docs/lua-adapters-en.md)).
|
|
||||||
|
|
||||||
Each adapter is a Lua script returning a table
|
|
||||||
(`internal/lua/adapters/<name>.lua`), overridable by a same-name file under
|
|
||||||
`adapter_dir` — **no recompile needed**, restart to take effect; uploads via
|
|
||||||
the WebUI take effect immediately.
|
|
||||||
|
|
||||||
```lua
|
|
||||||
return {
|
|
||||||
name = "mysrc",
|
|
||||||
version = "1.0.0",
|
|
||||||
endpoint = "/chat/completions", -- upstream path (source.endpoint overrides)
|
|
||||||
headers = { ["X-Static"] = "v" }, -- static default headers (fallback)
|
|
||||||
|
|
||||||
-- request: convert unified OpenAI req -> upstream native format, return string
|
|
||||||
transform_request = function(raw_json) ... end,
|
|
||||||
|
|
||||||
-- response: convert upstream response to unified format string
|
|
||||||
-- { content, reasoning_content, finish_reason, token_usage{...}, tool_calls[{...}] }
|
|
||||||
transform_response = function(raw_json) ... end,
|
|
||||||
|
|
||||||
-- streaming chunk: convert upstream SSE data to { content, done, ... }; "" skips
|
|
||||||
transform_stream_chunk = function(raw_chunk) ... end,
|
|
||||||
|
|
||||||
-- [optional] dynamic headers / signature hook
|
|
||||||
-- meta = { url, method, body, api_key, timestamp, source={ name, meta={...} } }
|
|
||||||
build_headers = function(meta) return { ["X-App-Sign"] = sign } end,
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Shared helpers: `hmac_sha256_hex(key, data)`, `sha256_hex(data)`,
|
|
||||||
`base64_encode(s)`, `tohex(s)`, `json.encode/decode`, `log(level, msg)`.
|
|
||||||
|
|
||||||
### Built-in adapters
|
|
||||||
|
|
||||||
`openai` `deepseek` `anthropic` `gemini` `github` `groq` `mistral` `ollama`
|
|
||||||
`kimicode`.
|
|
||||||
|
|
||||||
`anthropic`/`gemini`/`ollama` include multimodal conversion
|
|
||||||
(`image_url` → their native format); with `disable_thinking` the `deepseek`
|
|
||||||
adapter sets `extra_body.thinking.type` to `disabled`.
|
|
||||||
|
|
||||||
**kimicode** demonstrates `build_headers`: the cloud validates the calling
|
|
||||||
app, so you HMAC-sign timestamp+URL+body with `meta.app_secret` and add
|
|
||||||
`X-App-Sign`-style headers. Configure `sources[].meta.{app_id, app_secret,
|
|
||||||
app_agent}`.
|
|
||||||
|
|
||||||
## Desktop GUI (Electron)
|
|
||||||
|
|
||||||
Optional standalone product for desktop users on your internal network: an
|
|
||||||
**embedded ModelRouter core** (Clash Verge-style): one-click start/stop, system
|
|
||||||
tray, autostart, silent launch, and a pixel-for-pixel embedded full WebUI
|
|
||||||
(Status/Chat/Keys/Priority/Sources/Adapters — no login needed). Server users
|
|
||||||
keep using the plain Go binary.
|
|
||||||
|
|
||||||
### Two installer kinds: Headless and Desktop
|
|
||||||
|
|
||||||
Release installers ship in two flavors for different audiences:
|
|
||||||
|
|
||||||
- **Headless (server)**: the plain core binary (no Electron), config-driven,
|
|
||||||
for servers / containers / systemd / unattended runs. Ships as `deb` + `rpm`
|
|
||||||
(systemd unit with memory tuning + example config) and `tar.gz` (binary +
|
|
||||||
`config.example.yaml` + `adapters/` + README), Windows as `.zip`.
|
|
||||||
- **Desktop (desktop app)**: the Electron GUI installer with an embedded core,
|
|
||||||
for personal daily desktop use.
|
|
||||||
|
|
||||||
### GUI vs. plain backend
|
|
||||||
|
|
||||||
| Scenario | Pick | Why |
|
|
||||||
| ---- | ---- | ---- |
|
|
||||||
| Server / intranet gateway / unattended long-running | **Headless** (single binary) | ~8 MB binary; RSS scales with the number of sources (~10 MB for one, ~32-35 MB for 16 — see [Memory footprint](#memory-footprint-measured-and-tuned)), zero-dep single process — drop it into systemd or any container, remote admin |
|
|
||||||
| Personal desktop daily use / multi-device intranet sharing | **Desktop** (GUI) | no-login embedded WebUI, tray one-click, autostart, silent background — for non-CLI users |
|
|
||||||
| Windows desktop | **Desktop** | plain backend needs manual service registration; GUI ships native tray/autostart |
|
|
||||||
| CI one-shot 3-platform installers | **Desktop packaging scripts** | `make gui-*` emits deb / AppImage / NSIS, drops straight into a release pipeline |
|
|
||||||
|
|
||||||
Both share the exact same `llmsproxy` core (LuaJIT build) — configs and
|
|
||||||
adapters are fully compatible, freely interchangeable.
|
|
||||||
|
|
||||||
### Build & run
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd cmd/gui
|
sudo ./packaging/migrate-to-modelrouter.sh # preview
|
||||||
npm install
|
sudo ./packaging/migrate-to-modelrouter.sh --apply # apply
|
||||||
make gui # or npm run dev — dev run (needs make build first for bin/llmsproxy)
|
sudo ./packaging/migrate-to-modelrouter.sh --rollback --apply
|
||||||
make gui-deb # local deb (share with other Linux users)
|
|
||||||
make gui-dist # deb + rpm + AppImage (linux; rpm needs system rpmbuild)
|
|
||||||
make core-dist # core deb + rpm + tar.gz (server edition; needs nfpm)
|
|
||||||
make gui-win # Windows NSIS (host mingw + wine)
|
|
||||||
make gui-win-docker # Windows NSIS, fully dockerized (wine in image; no host mingw/node/wine)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Artifacts land in `cmd/build/gui-dist/`: `ModelRouter-<ver>.AppImage`,
|
Old paths and old environment variable names are still accepted on the read side,
|
||||||
`modelrouter-gui_<ver>_amd64.deb`, `ModelRouter Setup <ver>.exe` (win); core
|
so an upgrade cannot fail merely because of ordering. `deploy.sh` refuses to run
|
||||||
packages land in `cmd/build/dist/`: `llmsproxy_<ver>_amd64.deb`,
|
when it finds the old `/etc/llmsproxy` without a new config, because that would
|
||||||
`llmsproxy-<ver>.x86_64.rpm`. Both target families run
|
deploy a gateway with no key and no sources.
|
||||||
`packaging/verify-dist.sh` afterwards — a size floor that rejects degenerate
|
|
||||||
installers (1.3.0 shipped a 264 KB NSIS stub because the host wine was broken
|
|
||||||
and nothing caught it). At release time artifacts are renamed into the
|
|
||||||
`ModelRouter-Desktop-*` (GUI) and `ModelRouter-Headless-*` (plain backend)
|
|
||||||
families before upload to GitCode Releases.
|
|
||||||
|
|
||||||
#### Windows packaging (dockerized, recommended)
|
The plugin SDK follows the same naming: the only public entry point is
|
||||||
|
`modelrouter` (`llmsproxy` remains a compatibility alias). See
|
||||||
`make gui-win-docker` does cross-compile + NSIS **entirely in docker**; the host
|
[docs/plugins.md](docs/plugins.md).
|
||||||
needs **no mingw, no wine, no node**. Wine (with the i386 runtime) lives inside
|
|
||||||
the image, fixing the broken-host-wine failure that produced the 264 KB stub
|
|
||||||
installer.
|
|
||||||
|
|
||||||
- Image: `cmd/gui/docker/win-builder/Dockerfile` (`golang:1.25` + mingw-w64 +
|
|
||||||
node + wine32/wine64 + prebuilt LuaJIT for Windows: `lua51.dll` + import lib).
|
|
||||||
LuaJIT source comes from the gitcode mirror (reachable in CN) with github
|
|
||||||
fallback; Go modules go through `https://goproxy.cn`.
|
|
||||||
- One-shot release: `cmd/gui/scripts/dist-win-docker.sh` — core build + NSIS
|
|
||||||
packaging run inside the container (`USE_SYSTEM_WINE=true` uses the image's
|
|
||||||
wine); electron-builder's ~400 MB toolchain cache lives in a docker volume so
|
|
||||||
later builds reuse it.
|
|
||||||
- The `modelrouter/win-builder` image is auto-built on first run (~5-10 min)
|
|
||||||
and reused afterwards.
|
|
||||||
- After packaging, `packaging/verify-dist.sh` runs as a gate: an exe under
|
|
||||||
5 MB (or a deb/rpm/7z under 10 MB) fails the build — a broken installer can
|
|
||||||
never reach a Release again.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Full Windows release (rebuild core + NSIS):
|
|
||||||
make gui-win-docker
|
|
||||||
```
|
|
||||||
|
|
||||||
### Features
|
|
||||||
|
|
||||||
- **Embedded core**: auto-spawns the bundled `llmsproxy` (LuaJIT build);
|
|
||||||
config/keys/adapters live in `<userData>/profile/`. First run generates a
|
|
||||||
random admin key written into `keys` (not seed) and self-injects it — WebUI
|
|
||||||
needs no login and never nags about changing the initial key. Port changeable
|
|
||||||
in settings (default 8787).
|
|
||||||
- **System tray**: core status dot, autostart/silent toggles, start/stop/restart
|
|
||||||
core, quit.
|
|
||||||
- **Silent launch**: `--silent` flag or setting — starts to the tray only;
|
|
||||||
first run always shows the window.
|
|
||||||
- **Autostart**: `setLoginItemSettings` on Windows/macOS; writes
|
|
||||||
`~/.config/autostart/modelrouter-gui.desktop` on Linux, appending `--silent`
|
|
||||||
when silent mode is on.
|
|
||||||
|
|
||||||
## Layout
|
## Layout
|
||||||
|
|
||||||
```
|
```text
|
||||||
cmd/llmsproxy # entry point
|
cmd/modelrouter # entry point (pure Go server)
|
||||||
internal/config # YAML config loading/validation
|
cmd/gui # Electron desktop shell (embeds the same core)
|
||||||
internal/lua # LuaJIT VM + worker pool + AdapterCache + built-ins (embed)
|
internal/config # config parsing, sealed storage, runtime file
|
||||||
|
internal/core # scheduling, kernel actions, state and audit
|
||||||
|
internal/lua # LuaJIT VM + elastic worker pool + built-in adapters (embed)
|
||||||
|
internal/lua/plugins # optional plugin examples (not shipped in binary/archives)
|
||||||
internal/provider # Provider(HTTP) + Registry(routing)
|
internal/provider # Provider(HTTP) + Registry(routing)
|
||||||
internal/gateway # OpenAI-compatible HTTP + auth + SDK/streaming + WebUI
|
internal/gateway # OpenAI-compatible HTTP service + auth + streaming + WebUI
|
||||||
internal/types # unified format & OpenAI wire types
|
internal/types # unified format and OpenAI wire types
|
||||||
|
packaging/ # deb/rpm/tar packaging, systemd unit, migration script
|
||||||
|
docs/ # topic documentation
|
||||||
```
|
```
|
||||||
|
|
||||||
## Development & collaboration
|
## Development and tests
|
||||||
|
|
||||||
- **Git branching workflow**: main / feature / release-branch conventions in
|
|
||||||
**[docs/git-workflow-en.md](docs/git-workflow-en.md)**
|
|
||||||
([中文](docs/git-workflow.md)). The line: `main` is always deployable →
|
|
||||||
features merge into `main` → cut `release/vX.Y.Z` and tag → release → hotfixes
|
|
||||||
land on the release branch and are cherry-picked back into `main`.
|
|
||||||
- **Lua adapter protocol**: see
|
|
||||||
**[docs/lua-adapters-en.md](docs/lua-adapters-en.md)**
|
|
||||||
([中文](docs/lua-adapters.md)).
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go test -tags luajit ./...
|
go test -tags luajit ./... # everything
|
||||||
|
go test -tags luajit ./internal/lua # one package (required for LuaJIT changes)
|
||||||
```
|
```
|
||||||
|
|
||||||
Covers: config validation, adapter load/transform, signature hooks, gateway
|
- Branch and release discipline: [docs/git-workflow-en.md](docs/git-workflow-en.md).
|
||||||
auth, SDK round trip, SSE streaming, model routing, multimodal passthrough and
|
- Plugins live in `plugin_dir`. The gateway never creates, overwrites or deletes
|
||||||
disable_thinking.
|
plugin files, and a desktop upgrade never touches them either.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT License, see [LICENSE](./LICENSE).
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
// ModelRouter Desktop GUI — main process
|
// ModelRouter Desktop GUI — main process
|
||||||
// Clash-Verge-style: bundles and manages an embedded llmsproxy core instance
|
// Desktop shell: bundles and manages an embedded modelrouter core instance
|
||||||
// in a per-user profile dir (userData), auto-starts it, keeps a tray icon,
|
// in a per-user profile dir (userData), auto-starts it, keeps a tray icon,
|
||||||
// supports launch-at-login. Server-side users keep using the pure Go binary.
|
// supports launch-at-login. Server-side users keep using the pure Go binary.
|
||||||
const {
|
const {
|
||||||
@ -18,6 +18,7 @@ const http = require("http");
|
|||||||
const os = require("os");
|
const os = require("os");
|
||||||
const crypto = require("crypto");
|
const crypto = require("crypto");
|
||||||
const { spawn, execFileSync } = require("child_process");
|
const { spawn, execFileSync } = require("child_process");
|
||||||
|
const pluginSync = require("./plugin-sync");
|
||||||
|
|
||||||
const LOG_FILE = path.join(app.getPath("userData"), "gui.log");
|
const LOG_FILE = path.join(app.getPath("userData"), "gui.log");
|
||||||
function log(msg) {
|
function log(msg) {
|
||||||
@ -40,13 +41,36 @@ console.error = function () {
|
|||||||
// Per-user profile dir: <userData>/profile
|
// Per-user profile dir: <userData>/profile
|
||||||
const PROFILE_DIR = path.join(app.getPath("userData"), "profile");
|
const PROFILE_DIR = path.join(app.getPath("userData"), "profile");
|
||||||
const CONFIG_FILE = path.join(PROFILE_DIR, "config.yaml");
|
const CONFIG_FILE = path.join(PROFILE_DIR, "config.yaml");
|
||||||
|
// Plugin tree for the embedded core. The core deliberately does NOT default
|
||||||
|
// plugin_dir (an operator opts in), so the desktop build must set it — without
|
||||||
|
// it the whole plugin system stays off and the channels plugin (multi-platform
|
||||||
|
// account login + credential vault + auto check-in) is invisible in the WebUI.
|
||||||
|
// Seeding is the core's job (SeedBundled fills MISSING files only), so a user's
|
||||||
|
// edited copy always wins.
|
||||||
|
const PLUGINS_DIR = path.join(PROFILE_DIR, "plugins");
|
||||||
// Packaged: extraResource at process.resourcesPath/bin (outside asar, spawnable).
|
// Packaged: extraResource at process.resourcesPath/bin (outside asar, spawnable).
|
||||||
// Dev: cmd/gui/bin, populated by prepare-core.js.
|
// Dev: cmd/gui/bin, populated by prepare-core.js.
|
||||||
const CORE_EXE = path.join(
|
// The embedded core is named after the product. Installs produced before the
|
||||||
|
// llmsproxy -> modelrouter rename still carry the old file name inside their
|
||||||
|
// profile, so resolution falls back to it instead of declaring the core missing.
|
||||||
|
const CORE_BIN_DIR = path.join(
|
||||||
app.isPackaged ? process.resourcesPath : __dirname,
|
app.isPackaged ? process.resourcesPath : __dirname,
|
||||||
"bin",
|
"bin",
|
||||||
|
);
|
||||||
|
const CORE_EXE = path.join(
|
||||||
|
CORE_BIN_DIR,
|
||||||
|
process.platform === "win32" ? "modelrouter.exe" : "modelrouter",
|
||||||
|
);
|
||||||
|
const LEGACY_CORE_EXE = path.join(
|
||||||
|
CORE_BIN_DIR,
|
||||||
process.platform === "win32" ? "llmsproxy.exe" : "llmsproxy",
|
process.platform === "win32" ? "llmsproxy.exe" : "llmsproxy",
|
||||||
);
|
);
|
||||||
|
|
||||||
|
function resolveCoreExe() {
|
||||||
|
if (fs.existsSync(CORE_EXE)) return CORE_EXE;
|
||||||
|
if (fs.existsSync(LEGACY_CORE_EXE)) return LEGACY_CORE_EXE;
|
||||||
|
return CORE_EXE;
|
||||||
|
}
|
||||||
// Packaged window icon: must be a REAL file on disk (extraResource), not an
|
// Packaged window icon: must be a REAL file on disk (extraResource), not an
|
||||||
// asar path — Electron/GTK pulls BrowserWindow icons from the filesystem and
|
// asar path — Electron/GTK pulls BrowserWindow icons from the filesystem and
|
||||||
// crashes (SIGSEGV) when given an asar virtual path on Linux.
|
// crashes (SIGSEGV) when given an asar virtual path on Linux.
|
||||||
@ -155,7 +179,7 @@ function unsealViaCore() {
|
|||||||
let key = "";
|
let key = "";
|
||||||
try {
|
try {
|
||||||
const out = execFileSync(
|
const out = execFileSync(
|
||||||
CORE_EXE,
|
resolveCoreExe(),
|
||||||
["-config", CONFIG_FILE, "-show-secrets"],
|
["-config", CONFIG_FILE, "-show-secrets"],
|
||||||
{
|
{
|
||||||
encoding: "utf-8",
|
encoding: "utf-8",
|
||||||
@ -207,7 +231,7 @@ function coreStarted() {
|
|||||||
function writeDefaultEmbeddedConfig() {
|
function writeDefaultEmbeddedConfig() {
|
||||||
// Generate a self-contained embedded profile with a fresh random admin key.
|
// Generate a self-contained embedded profile with a fresh random admin key.
|
||||||
// No upstream sources are preconfigured — the desktop user adds them via
|
// No upstream sources are preconfigured — the desktop user adds them via
|
||||||
// the WebUI's sources page (like Clash Verge's empty-by-default profile).
|
// the WebUI's sources page (empty by default, like the WebUI profile).
|
||||||
// If a config already exists we do NOT overwrite it (keep user's edits).
|
// If a config already exists we do NOT overwrite it (keep user's edits).
|
||||||
if (fs.existsSync(CONFIG_FILE)) return;
|
if (fs.existsSync(CONFIG_FILE)) return;
|
||||||
const key = "sk-gw-" + crypto.randomBytes(16).toString("hex");
|
const key = "sk-gw-" + crypto.randomBytes(16).toString("hex");
|
||||||
@ -216,6 +240,9 @@ function writeDefaultEmbeddedConfig() {
|
|||||||
"listen: 127.0.0.1:" + (settings.port || DEFAULT_PORT),
|
"listen: 127.0.0.1:" + (settings.port || DEFAULT_PORT),
|
||||||
"default_model: AUTO",
|
"default_model: AUTO",
|
||||||
"adapter_dir: " + JSON.stringify(path.join(PROFILE_DIR, "adapters")),
|
"adapter_dir: " + JSON.stringify(path.join(PROFILE_DIR, "adapters")),
|
||||||
|
// Plugin tree: channels/billing/quota/usage ship inside the core binary and
|
||||||
|
// are seeded into this dir on first start. Omitting it disables plugins.
|
||||||
|
"plugin_dir: " + JSON.stringify(PLUGINS_DIR),
|
||||||
"runtime_file: " + JSON.stringify(path.join(PROFILE_DIR, "runtime.json")),
|
"runtime_file: " + JSON.stringify(path.join(PROFILE_DIR, "runtime.json")),
|
||||||
"",
|
"",
|
||||||
// Admin key as a plain `keys` entry (no seed flag) so the web UI never
|
// Admin key as a plain `keys` entry (no seed flag) so the web UI never
|
||||||
@ -261,15 +288,18 @@ function startCore() {
|
|||||||
console.log("core already running");
|
console.log("core already running");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!fs.existsSync(CORE_EXE)) {
|
const coreExe = resolveCoreExe();
|
||||||
console.error("embedded core binary missing:", CORE_EXE);
|
if (!fs.existsSync(coreExe)) {
|
||||||
|
console.error("embedded core binary missing:", coreExe);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// ensure profile dir + config + local adapters exist
|
// ensure profile dir + config + local adapters exist
|
||||||
fs.mkdirSync(PROFILE_DIR, { recursive: true });
|
fs.mkdirSync(PROFILE_DIR, { recursive: true });
|
||||||
if (!fs.existsSync(CONFIG_FILE)) writeDefaultEmbeddedConfig();
|
if (!fs.existsSync(CONFIG_FILE)) writeDefaultEmbeddedConfig();
|
||||||
|
pluginSync.ensurePluginDirConfigured(CONFIG_FILE, PLUGINS_DIR, log);
|
||||||
|
fs.mkdirSync(PLUGINS_DIR, { recursive: true });
|
||||||
|
|
||||||
coreProc = spawn(CORE_EXE, ["-config", CONFIG_FILE], {
|
coreProc = spawn(coreExe, ["-config", CONFIG_FILE], {
|
||||||
cwd: PROFILE_DIR,
|
cwd: PROFILE_DIR,
|
||||||
env: Object.assign({}, process.env, {
|
env: Object.assign({}, process.env, {
|
||||||
LLMS_PROXY_GUI: "1",
|
LLMS_PROXY_GUI: "1",
|
||||||
@ -622,12 +652,57 @@ function createWindow() {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
mainWindow.loadFile(path.join(__dirname, "renderer", "index.html"));
|
mainWindow.loadFile(path.join(__dirname, "renderer", "index.html"));
|
||||||
|
// External links ALWAYS go to the user's default browser, never an in-app
|
||||||
|
// window. Three paths have to be covered, and missing any one of them is why
|
||||||
|
// "click the login link and nothing happens / the app navigates away" shows up:
|
||||||
|
// 1. window.open / target=_blank -> setWindowOpenHandler
|
||||||
|
// 2. top-level navigation (same-tab click) -> will-navigate
|
||||||
|
// 3. navigation inside the WebUI <iframe> -> will-frame-navigate
|
||||||
|
// The embedded core's own origin (and file:// for the shell) must keep
|
||||||
|
// working, otherwise the WebUI and plugin pages stop loading.
|
||||||
|
const isInternal = (url) =>
|
||||||
|
!url ||
|
||||||
|
url === "about:blank" ||
|
||||||
|
url.startsWith("file://") ||
|
||||||
|
url.startsWith(embeddedBaseUrl()) ||
|
||||||
|
url.startsWith("devtools://");
|
||||||
|
const openExternally = (url) => {
|
||||||
|
if (url && (url.startsWith("http://") || url.startsWith("https://"))) {
|
||||||
|
log("[gui] open in default browser: " + url);
|
||||||
|
shell.openExternal(url);
|
||||||
|
}
|
||||||
|
};
|
||||||
mainWindow.webContents.setWindowOpenHandler(({ url }) => {
|
mainWindow.webContents.setWindowOpenHandler(({ url }) => {
|
||||||
if (url && url.startsWith("http")) shell.openExternal(url);
|
if (!isInternal(url)) openExternally(url);
|
||||||
return { action: "deny" };
|
return { action: "deny" };
|
||||||
});
|
});
|
||||||
|
mainWindow.webContents.on("will-navigate", (e, url) => {
|
||||||
|
if (isInternal(url)) return;
|
||||||
|
e.preventDefault();
|
||||||
|
openExternally(url);
|
||||||
|
});
|
||||||
|
// iframe navigations (the WebUI content area and the plugin pages inside it).
|
||||||
|
// The channels plugin renders its server-side login browser (noVNC) in an
|
||||||
|
// iframe on a LOOPBACK port, so loopback MUST stay in-app — blocking it would
|
||||||
|
// break VNC login. Everything non-loopback is external: open the default
|
||||||
|
// browser instead of silently replacing the gateway UI with a third-party page.
|
||||||
|
const isLoopback = (url) => {
|
||||||
|
try {
|
||||||
|
const h = new URL(url).hostname;
|
||||||
|
return h === "127.0.0.1" || h === "localhost" || h === "[::1]";
|
||||||
|
} catch (e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if (mainWindow.webContents.on) {
|
||||||
|
mainWindow.webContents.on("will-frame-navigate", (e) => {
|
||||||
|
if (isInternal(e.url) || isLoopback(e.url)) return;
|
||||||
|
e.preventDefault();
|
||||||
|
openExternally(e.url);
|
||||||
|
});
|
||||||
|
}
|
||||||
// Silent start: `--silent` CLI flag or settings.silentStart keeps the app
|
// Silent start: `--silent` CLI flag or settings.silentStart keeps the app
|
||||||
// in the tray without showing the window (Clash-Verge style). First run
|
// in the tray without showing the window (desktop-shell behaviour). First run
|
||||||
// always shows the window so the user sees the dashboard.
|
// always shows the window so the user sees the dashboard.
|
||||||
mainWindow.once("ready-to-show", () => {
|
mainWindow.once("ready-to-show", () => {
|
||||||
const firstRun = !fs.existsSync(
|
const firstRun = !fs.existsSync(
|
||||||
@ -751,8 +826,8 @@ ipcMain.handle("core:state", () => ({
|
|||||||
port: settings.port || DEFAULT_PORT,
|
port: settings.port || DEFAULT_PORT,
|
||||||
profileDir: PROFILE_DIR,
|
profileDir: PROFILE_DIR,
|
||||||
configFile: CONFIG_FILE,
|
configFile: CONFIG_FILE,
|
||||||
coreExe: CORE_EXE,
|
coreExe: resolveCoreExe(),
|
||||||
coreExists: fs.existsSync(CORE_EXE),
|
coreExists: fs.existsSync(resolveCoreExe()),
|
||||||
}));
|
}));
|
||||||
ipcMain.handle("core:start", () => {
|
ipcMain.handle("core:start", () => {
|
||||||
app.isQuitting = false;
|
app.isQuitting = false;
|
||||||
@ -824,7 +899,7 @@ if (gotLock) {
|
|||||||
createWindow();
|
createWindow();
|
||||||
createTray();
|
createTray();
|
||||||
|
|
||||||
// start embedded core if not already running (another instance of llmsproxy)
|
// start embedded core if not already running (another instance of modelrouter)
|
||||||
setTimeout(async () => {
|
setTimeout(async () => {
|
||||||
const p = await probeEmbedded();
|
const p = await probeEmbedded();
|
||||||
if (!p.ok) {
|
if (!p.ok) {
|
||||||
|
|||||||
18
cmd/gui/package-lock.json
generated
18
cmd/gui/package-lock.json
generated
@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "modelrouter-gui",
|
"name": "modelrouter-gui",
|
||||||
"version": "1.9.3",
|
"version": "3.0.2",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "modelrouter-gui",
|
"name": "modelrouter-gui",
|
||||||
"version": "1.9.3",
|
"version": "3.0.2",
|
||||||
"license": "Proprietary",
|
"license": "Proprietary",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"electron": "^33.0.0",
|
"electron": "^33.0.0",
|
||||||
@ -1245,7 +1245,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/chownr": {
|
"node_modules/chownr": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
|
||||||
"integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
|
"integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
@ -2288,7 +2288,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/global-agent": {
|
"node_modules/global-agent": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
|
||||||
"integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
|
"integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
@ -2532,7 +2532,7 @@
|
|||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
"node_modules/is-fullwidth-code-point": {
|
"node_modules/is-fullwidth-code-point": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||||
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
|
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
@ -2623,7 +2623,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/json-buffer": {
|
"node_modules/json-buffer": {
|
||||||
"version": "3.0.1",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
|
||||||
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
|
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
@ -2715,7 +2715,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/matcher": {
|
"node_modules/matcher": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
|
||||||
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
|
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
@ -3492,7 +3492,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/shebang-regex": {
|
"node_modules/shebang-regex": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
|
||||||
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
|
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
@ -3612,7 +3612,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/sumchecker": {
|
"node_modules/sumchecker": {
|
||||||
"version": "3.0.1",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz",
|
||||||
"integrity": "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==",
|
"integrity": "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "modelrouter-gui",
|
"name": "modelrouter-gui",
|
||||||
"version": "1.9.6",
|
"version": "3.0.2",
|
||||||
"description": "ModelRouter Desktop — embedded ModelRouter core with tray",
|
"description": "ModelRouter Desktop — embedded ModelRouter core with tray",
|
||||||
"author": "ModelRouter",
|
"author": "ModelRouter",
|
||||||
"main": "main.js",
|
"main": "main.js",
|
||||||
@ -39,8 +39,8 @@
|
|||||||
},
|
},
|
||||||
"extraResources": [
|
"extraResources": [
|
||||||
{
|
{
|
||||||
"from": "bin/llmsproxy",
|
"from": "bin/modelrouter",
|
||||||
"to": "bin/llmsproxy"
|
"to": "bin/modelrouter"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"from": "build/icon.png",
|
"from": "build/icon.png",
|
||||||
@ -66,8 +66,8 @@
|
|||||||
"icon": "build/icon.ico",
|
"icon": "build/icon.ico",
|
||||||
"extraResources": [
|
"extraResources": [
|
||||||
{
|
{
|
||||||
"from": "bin/llmsproxy.exe",
|
"from": "bin/modelrouter.exe",
|
||||||
"to": "bin/llmsproxy.exe"
|
"to": "bin/modelrouter.exe"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"from": "bin/lua51.dll",
|
"from": "bin/lua51.dll",
|
||||||
@ -81,6 +81,7 @@
|
|||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
"main.js",
|
"main.js",
|
||||||
|
"plugin-sync.js",
|
||||||
"preload.js",
|
"preload.js",
|
||||||
"renderer/**/*",
|
"renderer/**/*",
|
||||||
"build/**/*",
|
"build/**/*",
|
||||||
@ -88,5 +89,5 @@
|
|||||||
],
|
],
|
||||||
"asar": true
|
"asar": true
|
||||||
},
|
},
|
||||||
"homepage": "https://gitcode.com/JianFeeeee/ModelRouter"
|
"homepage": "https://atomgit.com/HeNanMU-EEA/ModelRouter"
|
||||||
}
|
}
|
||||||
|
|||||||
33
cmd/gui/plugin-sync.js
Normal file
33
cmd/gui/plugin-sync.js
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
// Plugin-tree maintenance for the desktop profile (no Electron imports, so this
|
||||||
|
// is unit-testable with plain node).
|
||||||
|
//
|
||||||
|
// The desktop may configure an operator-selected plugin directory, but it never
|
||||||
|
// owns the files in that directory. In particular, upgrades must not seed,
|
||||||
|
// replace, rename, or delete plugins. Plugin installation is an explicit
|
||||||
|
// operator action (or an external package manager action).
|
||||||
|
const fs = require("fs");
|
||||||
|
|
||||||
|
// ensurePluginDirConfigured back-fills `plugin_dir` into an existing config.
|
||||||
|
// Appending a top-level YAML key is safe for the shapes this file can have
|
||||||
|
// (flat mapping + a keys list). Returns "added" | "present" | "absent" | "error".
|
||||||
|
function ensurePluginDirConfigured(configFile, pluginsDir, log) {
|
||||||
|
try {
|
||||||
|
if (!fs.existsSync(configFile)) return "absent";
|
||||||
|
const cur = fs.readFileSync(configFile, "utf-8");
|
||||||
|
if (/^\s*plugin_dir\s*:/m.test(cur)) return "present";
|
||||||
|
fs.writeFileSync(
|
||||||
|
configFile,
|
||||||
|
cur.replace(/\s*$/, "") +
|
||||||
|
"\n# added by ModelRouter Desktop: enables the operator plugin directory\n" +
|
||||||
|
"plugin_dir: " + JSON.stringify(pluginsDir) + "\n",
|
||||||
|
"utf-8",
|
||||||
|
);
|
||||||
|
if (log) log("[gui] plugin_dir back-filled into " + configFile);
|
||||||
|
return "added";
|
||||||
|
} catch (e) {
|
||||||
|
if (log) log("[gui] ensurePluginDirConfigured failed: " + e.message);
|
||||||
|
return "error";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { ensurePluginDirConfigured };
|
||||||
48
cmd/gui/plugin-sync.test.js
Normal file
48
cmd/gui/plugin-sync.test.js
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
// Unit tests for desktop plugin-directory configuration (plain node).
|
||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert");
|
||||||
|
const fs = require("fs");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
const { ensurePluginDirConfigured } = require("./plugin-sync");
|
||||||
|
|
||||||
|
const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), "plugin-sync-"));
|
||||||
|
|
||||||
|
test("back-fills plugin_dir into an existing profile without it", () => {
|
||||||
|
const dir = tmp();
|
||||||
|
const cfg = path.join(dir, "config.yaml");
|
||||||
|
fs.writeFileSync(cfg, "listen: 127.0.0.1:8787\ndefault_model: AUTO\nkeys:\n - key: k\n role: admin\n");
|
||||||
|
const plugins = path.join(dir, "plugins");
|
||||||
|
assert.equal(ensurePluginDirConfigured(cfg, plugins, null), "added");
|
||||||
|
const out = fs.readFileSync(cfg, "utf-8");
|
||||||
|
assert.match(out, /^plugin_dir: "/m);
|
||||||
|
assert.match(out, /^listen: 127.0.0.1:8787$/m);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("leaves a profile that already sets plugin_dir alone", () => {
|
||||||
|
const dir = tmp();
|
||||||
|
const cfg = path.join(dir, "config.yaml");
|
||||||
|
const orig = 'plugin_dir: "/custom/plugins"\nlisten: ":1"\n';
|
||||||
|
fs.writeFileSync(cfg, orig);
|
||||||
|
assert.equal(ensurePluginDirConfigured(cfg, "/whatever", null), "present");
|
||||||
|
assert.equal(fs.readFileSync(cfg, "utf-8"), orig);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("missing config is not created by the back-fill", () => {
|
||||||
|
const dir = tmp();
|
||||||
|
const cfg = path.join(dir, "config.yaml");
|
||||||
|
assert.equal(ensurePluginDirConfigured(cfg, path.join(dir, "p"), null), "absent");
|
||||||
|
assert.equal(fs.existsSync(cfg), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("desktop upgrade never modifies operator plugin files", () => {
|
||||||
|
const dir = tmp();
|
||||||
|
const plugins = path.join(dir, "plugins");
|
||||||
|
fs.mkdirSync(plugins, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(plugins, "channels.lua"), "OPERATOR FILE");
|
||||||
|
fs.writeFileSync(path.join(plugins, "my-own.lua"), "MINE");
|
||||||
|
// There is deliberately no version stamp and no upgrade synchronizer.
|
||||||
|
assert.equal(fs.readFileSync(path.join(plugins, "channels.lua"), "utf-8"), "OPERATOR FILE");
|
||||||
|
assert.equal(fs.readFileSync(path.join(plugins, "my-own.lua"), "utf-8"), "MINE");
|
||||||
|
assert.equal(fs.readdirSync(dir).includes(".plugin-seed-version"), false);
|
||||||
|
});
|
||||||
@ -1,5 +1,5 @@
|
|||||||
// ModelRouter Desktop GUI — renderer
|
// ModelRouter Desktop GUI — renderer
|
||||||
// Clash-Verge-style: the content area is a full-screen iframe loading the
|
// Desktop shell: the content area is a full-screen iframe loading the
|
||||||
// real WebUI 1:1 (status / chat / keys / priority / sources / adapters).
|
// real WebUI 1:1 (status / chat / keys / priority / sources / adapters).
|
||||||
// The shell only adds the titlebar, settings overlay and tray integration.
|
// The shell only adds the titlebar, settings overlay and tray integration.
|
||||||
const state = {
|
const state = {
|
||||||
|
|||||||
@ -83,7 +83,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- side rail: autostart / silent toggles (Clash-Verge style) -->
|
<!-- side rail: autostart / silent toggles (desktop-shell style) -->
|
||||||
<aside class="rail" id="rail">
|
<aside class="rail" id="rail">
|
||||||
<div class="rail-gap"></div>
|
<div class="rail-gap"></div>
|
||||||
<button
|
<button
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
/* =====================================================================
|
/* =====================================================================
|
||||||
ModelRouter Desktop GUI — Clash-Verge-style shell
|
ModelRouter Desktop GUI — desktop-shell-style shell
|
||||||
Content area = full-screen iframe embedding the real WebUI 1:1.
|
Content area = full-screen iframe embedding the real WebUI 1:1.
|
||||||
Palette mirrors the WebUI (sakura/frost/glassmorphism).
|
Palette mirrors the WebUI (sakura/frost/glassmorphism).
|
||||||
===================================================================== */
|
===================================================================== */
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# One-shot Windows release build for the GUI — ENTIRELY inside docker.
|
# One-shot Windows release build for the GUI — ENTIRELY inside docker.
|
||||||
#
|
#
|
||||||
# 1) cross-compile the embedded core in docker (llmsproxy.exe + lua51.dll)
|
# 1) cross-compile the embedded core in docker (modelrouter.exe + lua51.dll)
|
||||||
# 2) electron-builder NSIS packaging ALSO inside docker (wine for the
|
# 2) electron-builder NSIS packaging ALSO inside docker (wine for the
|
||||||
# uninstaller step lives in the image — see Dockerfile). This is the fix
|
# uninstaller step lives in the image — see Dockerfile). This is the fix
|
||||||
# for the 1.3.0/1.4.0 broken installers: the host had amd64-only wine
|
# for the 1.3.0/1.4.0 broken installers: the host had amd64-only wine
|
||||||
@ -37,13 +37,15 @@ docker run --rm \
|
|||||||
-v "$ROOT":/workspace \
|
-v "$ROOT":/workspace \
|
||||||
-v "$CACHE_VOL":/root/.cache \
|
-v "$CACHE_VOL":/root/.cache \
|
||||||
-e GOPROXY="$PROXY" \
|
-e GOPROXY="$PROXY" \
|
||||||
|
-e ELECTRON_MIRROR="https://npmmirror.com/mirrors/electron/" \
|
||||||
|
-e ELECTRON_BUILDER_BINARIES_MIRROR="https://npmmirror.com/mirrors/electron-builder-binaries/" \
|
||||||
-w /workspace \
|
-w /workspace \
|
||||||
"$IMAGE" bash -c '
|
"$IMAGE" bash -c '
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cp /opt/luajit2/src/lua51.dll /workspace/cmd/gui/bin/lua51.dll
|
cp /opt/luajit2/src/lua51.dll /workspace/cmd/gui/bin/lua51.dll
|
||||||
GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \
|
GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \
|
||||||
go build -buildvcs=false -tags luajit -trimpath \
|
go build -buildvcs=false -tags luajit -trimpath -ldflags "-s -w" \
|
||||||
-o /workspace/cmd/gui/bin/llmsproxy.exe ./cmd/llmsproxy
|
-o /workspace/cmd/gui/bin/modelrouter.exe ./cmd/modelrouter
|
||||||
echo " -> core built"
|
echo " -> core built"
|
||||||
# wine needs one prefix initialization so its wow64 runtime (syswow64/ntdll)
|
# wine needs one prefix initialization so its wow64 runtime (syswow64/ntdll)
|
||||||
# exists — the exact thing that was missing on the broken host. Use a shared
|
# exists — the exact thing that was missing on the broken host. Use a shared
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
// Build the embedded llmsproxy core(s) for the GUI package.
|
// Build the embedded modelrouter core(s) for the GUI package.
|
||||||
// node prepare-core.js -> linux (native, luajit) + win (cross, best effort)
|
// node prepare-core.js -> linux (native, luajit) + win (cross, best effort)
|
||||||
// node prepare-core.js linux -> linux only
|
// node prepare-core.js linux -> linux only
|
||||||
// node prepare-core.js win -> win only
|
// node prepare-core.js win -> win only
|
||||||
@ -12,21 +12,24 @@ const binDir = path.join(guiDir, "bin");
|
|||||||
const projectRoot = path.join(guiDir, "..", "..");
|
const projectRoot = path.join(guiDir, "..", "..");
|
||||||
|
|
||||||
function buildLinux() {
|
function buildLinux() {
|
||||||
const out = path.join(binDir, "llmsproxy");
|
const out = path.join(binDir, "modelrouter");
|
||||||
console.log("[prepare-core] building llmsproxy (linux/luajit)...");
|
console.log("[prepare-core] building modelrouter (linux/luajit)...");
|
||||||
fs.mkdirSync(binDir, { recursive: true });
|
fs.mkdirSync(binDir, { recursive: true });
|
||||||
execFileSync(
|
execFileSync(
|
||||||
"go",
|
"go",
|
||||||
["build", "-tags", "luajit", "-trimpath", "-o", out, "./cmd/llmsproxy"],
|
// -s -w: same stripping as packaging/core-dist.sh. Without it the bundled
|
||||||
|
// core carries debug info (~14.4 MB vs ~10.3 MB) and every installer pays
|
||||||
|
// for it, for a binary the user never debugs.
|
||||||
|
["build", "-tags", "luajit", "-trimpath", "-ldflags", "-s -w", "-o", out, "./cmd/modelrouter"],
|
||||||
{ cwd: projectRoot, stdio: "inherit" },
|
{ cwd: projectRoot, stdio: "inherit" },
|
||||||
);
|
);
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildWin() {
|
function buildWin() {
|
||||||
const out = path.join(binDir, "llmsproxy.exe");
|
const out = path.join(binDir, "modelrouter.exe");
|
||||||
console.log(
|
console.log(
|
||||||
"[prepare-core] cross-building llmsproxy.exe (windows/amd64, luajit)...",
|
"[prepare-core] cross-building modelrouter.exe (windows/amd64, luajit)...",
|
||||||
);
|
);
|
||||||
fs.mkdirSync(binDir, { recursive: true });
|
fs.mkdirSync(binDir, { recursive: true });
|
||||||
const cc = process.env.MINGW_CC || "x86_64-w64-mingw32-gcc";
|
const cc = process.env.MINGW_CC || "x86_64-w64-mingw32-gcc";
|
||||||
@ -43,7 +46,7 @@ function buildWin() {
|
|||||||
}
|
}
|
||||||
execFileSync(
|
execFileSync(
|
||||||
"go",
|
"go",
|
||||||
["build", "-tags", "luajit", "-trimpath", "-o", out, "./cmd/llmsproxy"],
|
["build", "-tags", "luajit", "-trimpath", "-ldflags", "-s -w", "-o", out, "./cmd/modelrouter"],
|
||||||
{
|
{
|
||||||
cwd: projectRoot,
|
cwd: projectRoot,
|
||||||
stdio: "inherit",
|
stdio: "inherit",
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
// Command llmsproxy is a standalone gateway that exposes multiple upstream LLM
|
// Command modelrouter is a standalone gateway that exposes multiple upstream LLM
|
||||||
// sources behind a unified OpenAI-compatible HTTP API. Protocol differences are
|
// sources behind a unified OpenAI-compatible HTTP API. Protocol differences are
|
||||||
// handled by Lua adapters (per source), optionally signing outgoing requests.
|
// handled by Lua adapters (per source), optionally signing outgoing requests.
|
||||||
// It supports explicit model selection or AUTO routing by priority, image
|
// It supports explicit model selection or AUTO routing by priority, image
|
||||||
@ -16,9 +16,9 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
"llmsproxy/internal/gateway"
|
"modelrouter/internal/gateway"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@ -33,12 +33,12 @@ func main() {
|
|||||||
// error here instead of being silently created.
|
// error here instead of being silently created.
|
||||||
if *checkOnly {
|
if *checkOnly {
|
||||||
if _, err := os.Stat(*cfgPath); err != nil {
|
if _, err := os.Stat(*cfgPath); err != nil {
|
||||||
log.Fatalf("[llmsproxy] check: %v", err)
|
log.Fatalf("[modelrouter] check: %v", err)
|
||||||
}
|
}
|
||||||
if _, err := config.Load(*cfgPath); err != nil {
|
if _, err := config.Load(*cfgPath); err != nil {
|
||||||
log.Fatalf("[llmsproxy] check: %v", err)
|
log.Fatalf("[modelrouter] check: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("[llmsproxy] check: %s is valid", *cfgPath)
|
log.Printf("[modelrouter] check: %s is valid", *cfgPath)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -48,34 +48,34 @@ func main() {
|
|||||||
// program prints a credential, and this path never logs to a file.
|
// program prints a credential, and this path never logs to a file.
|
||||||
if *showSecrets {
|
if *showSecrets {
|
||||||
if _, err := os.Stat(*cfgPath); err != nil {
|
if _, err := os.Stat(*cfgPath); err != nil {
|
||||||
log.Fatalf("[llmsproxy] show-secrets: %v", err)
|
log.Fatalf("[modelrouter] show-secrets: %v", err)
|
||||||
}
|
}
|
||||||
if err := config.PrintSecrets(*cfgPath); err != nil {
|
if err := config.PrintSecrets(*cfgPath); err != nil {
|
||||||
log.Fatalf("[llmsproxy] show-secrets: %v", err)
|
log.Fatalf("[modelrouter] show-secrets: %v", err)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
created, err := config.EnsureDefault(*cfgPath)
|
created, err := config.EnsureDefault(*cfgPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("[llmsproxy] config: %v", err)
|
log.Fatalf("[modelrouter] config: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c, err := core.New(*cfgPath)
|
c, err := core.New(*cfgPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("[llmsproxy] core: %v", err)
|
log.Fatalf("[modelrouter] core: %v", err)
|
||||||
}
|
}
|
||||||
defer c.Close()
|
defer c.Close()
|
||||||
|
|
||||||
if created {
|
if created {
|
||||||
for _, k := range c.GatewayKeys() {
|
for _, k := range c.GatewayKeys() {
|
||||||
log.Printf("[llmsproxy] generated default config at %s — admin key: %s (rotate it in the WebUI after first login)", *cfgPath, k)
|
log.Printf("[modelrouter] generated default config at %s — admin key: %s (rotate it in the WebUI after first login)", *cfgPath, k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
gw, err := gateway.New(c)
|
gw, err := gateway.New(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("[llmsproxy] gateway: %v", err)
|
log.Fatalf("[modelrouter] gateway: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ops hygiene: surface the two most common footguns instead of silently
|
// Ops hygiene: surface the two most common footguns instead of silently
|
||||||
@ -88,16 +88,16 @@ func main() {
|
|||||||
// (e.g. after rotating the starter key away) it reported "every request is
|
// (e.g. after rotating the starter key away) it reported "every request is
|
||||||
// rejected" while seven working keys, one of them admin, were in service.
|
// rejected" while seven working keys, one of them admin, were in service.
|
||||||
if keys := c.ListKeys(); len(keys) == 0 {
|
if keys := c.ListKeys(); len(keys) == 0 {
|
||||||
log.Printf("[llmsproxy] WARNING: no gateway key configured — every request will be rejected. Add one in the WebUI or set gateway_keys")
|
log.Printf("[modelrouter] WARNING: no gateway key configured — every request will be rejected. Add one in the WebUI or set gateway_keys")
|
||||||
} else {
|
} else {
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
if k.Key == "sk-gw-local-0001" || k.Key == "sk-local-0001" {
|
if k.Key == "sk-gw-local-0001" || k.Key == "sk-local-0001" {
|
||||||
log.Printf("[llmsproxy] WARNING: gateway key %q looks like the starter/example key — rotate it before exposing the gateway", k.Key)
|
log.Printf("[modelrouter] WARNING: gateway key %q looks like the starter/example key — rotate it before exposing the gateway", k.Key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if l := c.Listen(); strings.HasPrefix(l, "0.0.0.0:") || strings.HasPrefix(l, "::") {
|
if l := c.Listen(); strings.HasPrefix(l, "0.0.0.0:") || strings.HasPrefix(l, "::") {
|
||||||
log.Printf("[llmsproxy] WARNING: listen=%s binds ALL interfaces — bind an internal address in production", l)
|
log.Printf("[modelrouter] WARNING: listen=%s binds ALL interfaces — bind an internal address in production", l)
|
||||||
}
|
}
|
||||||
|
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
@ -108,22 +108,22 @@ func main() {
|
|||||||
go func() {
|
go func() {
|
||||||
cert, key := c.TLS()
|
cert, key := c.TLS()
|
||||||
if cert != "" && key != "" {
|
if cert != "" && key != "" {
|
||||||
log.Printf("[llmsproxy] listening HTTPS on %s (cert=%q) (default_model=%s, models=%v, adapters=%d)",
|
log.Printf("[modelrouter] listening HTTPS on %s (cert=%q) (default_model=%s, models=%v, adapters=%d)",
|
||||||
c.Listen(), cert, c.DefaultModel(), c.Registry().ModelList(), len(c.ListAdapters()))
|
c.Listen(), cert, c.DefaultModel(), c.Registry().ModelList(), len(c.ListAdapters()))
|
||||||
if err := srv.ListenAndServeTLS(cert, key); err != nil && err != http.ErrServerClosed {
|
if err := srv.ListenAndServeTLS(cert, key); err != nil && err != http.ErrServerClosed {
|
||||||
log.Fatalf("[llmsproxy] server: %v", err)
|
log.Fatalf("[modelrouter] server: %v", err)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
log.Printf("[llmsproxy] listening HTTP on %s (default_model=%s, models=%v, adapters=%d)",
|
log.Printf("[modelrouter] listening HTTP on %s (default_model=%s, models=%v, adapters=%d)",
|
||||||
c.Listen(), c.DefaultModel(), c.Registry().ModelList(), len(c.ListAdapters()))
|
c.Listen(), c.DefaultModel(), c.Registry().ModelList(), len(c.ListAdapters()))
|
||||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
log.Fatalf("[llmsproxy] server: %v", err)
|
log.Fatalf("[modelrouter] server: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
stop := make(chan os.Signal, 1)
|
stop := make(chan os.Signal, 1)
|
||||||
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
|
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
|
||||||
<-stop
|
<-stop
|
||||||
log.Printf("[llmsproxy] shutting down")
|
log.Printf("[modelrouter] shutting down")
|
||||||
}
|
}
|
||||||
185
deploy.sh
185
deploy.sh
@ -1,10 +1,10 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# llmsproxy 原子化滚动部署脚本
|
# modelrouter 原子化滚动部署脚本
|
||||||
#
|
#
|
||||||
# 策略:
|
# 策略:
|
||||||
# 1. /tmp (tmpfs) 构建,/usr/local/bin (ext4) 部署
|
# 1. /tmp (tmpfs) 构建,/usr/local/bin (ext4) 部署
|
||||||
# 2. 构建后先 copy 到目标文件系统(.llmsproxy.tmp),
|
# 2. 构建后先 copy 到目标文件系统(.modelrouter.tmp),
|
||||||
# 再 rename(2) 做原子替换(同一文件系统内 mv 是原子的)
|
# 再 rename(2) 做原子替换(同一文件系统内 mv 是原子的)
|
||||||
# 3. runtime.json 不替换(保留运行时密钥、审计状态)
|
# 3. runtime.json 不替换(保留运行时密钥、审计状态)
|
||||||
# 4. systemctl restart → 服务中断仅在 Go 进程重启瞬间 (<500ms)
|
# 4. systemctl restart → 服务中断仅在 Go 进程重启瞬间 (<500ms)
|
||||||
@ -28,15 +28,24 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# ---------- 配置 ----------
|
# ---------- 配置 ----------
|
||||||
readonly TARGET_BIN="/usr/local/bin/llmsproxy"
|
readonly TARGET_BIN="/usr/local/bin/modelrouter"
|
||||||
readonly TARGET_CONFIG="/etc/llmsproxy/config.yaml"
|
readonly TARGET_CONFIG="/etc/modelrouter/config.yaml"
|
||||||
readonly TARGET_ADAPTERS="/etc/llmsproxy/adapters"
|
readonly TARGET_ADAPTERS="/etc/modelrouter/adapters"
|
||||||
readonly REPO_DIR="/home/program/llmsproxy"
|
readonly TARGET_PLUGINS="/etc/modelrouter/plugins"
|
||||||
|
# 仓库所在目录不等于产品名:这里保持真实路径,除非用 MODELROUTER_REPO_DIR 覆盖。
|
||||||
|
readonly REPO_DIR="${MODELROUTER_REPO_DIR:-/home/program/llmsproxy}"
|
||||||
readonly HEALTH_URL="http://127.0.0.1:8081/v1/models"
|
readonly HEALTH_URL="http://127.0.0.1:8081/v1/models"
|
||||||
readonly BIN_NAME="llmsproxy"
|
readonly BIN_NAME="modelrouter"
|
||||||
|
|
||||||
# 构建目录在 /tmp(tmpfs 更快)
|
# 构建目录。默认落在仓库下的构建工作区,而不是 /tmp:本机 /tmp 是 8.8G
|
||||||
readonly TMP_DIR="/tmp/llmsproxy-deploy"
|
# tmpfs,实测被 Go 构建缓存占满后会让 go test/build 在链接阶段直接失败。
|
||||||
|
# 可用 MODELROUTER_BUILD_DIR 覆盖;同一目录也被测试用作 GOTMPDIR。
|
||||||
|
readonly BUILD_SUBDIR=".build-work"
|
||||||
|
if [[ -n "${MODELROUTER_BUILD_DIR:-}" ]]; then
|
||||||
|
readonly TMP_DIR="${MODELROUTER_BUILD_DIR}"
|
||||||
|
else
|
||||||
|
readonly TMP_DIR="${REPO_DIR}/${BUILD_SUBDIR}"
|
||||||
|
fi
|
||||||
# 同文件系统暂存文件(用于原子 rename)
|
# 同文件系统暂存文件(用于原子 rename)
|
||||||
readonly STAGING_BIN="${TARGET_BIN}.staging"
|
readonly STAGING_BIN="${TARGET_BIN}.staging"
|
||||||
readonly STAGING_CONFIG="${TARGET_CONFIG}.staging"
|
readonly STAGING_CONFIG="${TARGET_CONFIG}.staging"
|
||||||
@ -51,7 +60,7 @@ usage() {
|
|||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
用法: ./deploy.sh [--config <file>]
|
用法: ./deploy.sh [--config <file>]
|
||||||
|
|
||||||
--config <file> 投放这份配置到 /etc/llmsproxy/config.yaml(原子替换)。
|
--config <file> 投放这份配置到 /etc/modelrouter/config.yaml(原子替换)。
|
||||||
省略时不改动线上配置,只部署二进制与适配器。
|
省略时不改动线上配置,只部署二进制与适配器。
|
||||||
无论是否指定,重启前都会用新二进制 -check 预检线上配置。
|
无论是否指定,重启前都会用新二进制 -check 预检线上配置。
|
||||||
|
|
||||||
@ -95,6 +104,34 @@ precheck() {
|
|||||||
command -v systemctl &>/dev/null || fail "未找到 systemctl"
|
command -v systemctl &>/dev/null || fail "未找到 systemctl"
|
||||||
[[ $(id -u) -eq 0 ]] || fail "需要 root 权限"
|
[[ $(id -u) -eq 0 ]] || fail "需要 root 权限"
|
||||||
|
|
||||||
|
# 改名保护:本脚本现在部署到 /etc/modelrouter。若机器上仍是改名前的
|
||||||
|
# /etc/llmsproxy 且新路径尚未建立,直接继续会创建一份**空配置**并让新
|
||||||
|
# 服务在没有 key/源的情况下起来 —— 看起来成功,实际网关拒掉所有请求。
|
||||||
|
if [[ -d /etc/llmsproxy && ! -e "$TARGET_CONFIG" ]]; then
|
||||||
|
fail "检测到改名前的部署 /etc/llmsproxy,但 $TARGET_CONFIG 不存在。
|
||||||
|
先迁移(dry-run 默认,确认后加 --apply):
|
||||||
|
sudo ./packaging/migrate-to-modelrouter.sh
|
||||||
|
sudo ./packaging/migrate-to-modelrouter.sh --apply
|
||||||
|
如需全新部署,先把旧目录移开:mv /etc/llmsproxy /etc/llmsproxy.bak"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# unit 与二进制路径必须一致。包安装把二进制放 /usr/bin,源码部署放
|
||||||
|
# /usr/local/bin;unit 里写错一行,systemctl restart 只会给 203/EXEC,
|
||||||
|
# 看起来像"部署成功但服务起不来"。此处提前拦住。
|
||||||
|
local unit="/etc/systemd/system/modelrouter.service"
|
||||||
|
if [[ -f "$unit" ]]; then
|
||||||
|
local unit_exe
|
||||||
|
unit_exe=$(awk -F'=' '/^ExecStart=/{print $2; exit}' "$unit" | awk '{print $1}')
|
||||||
|
if [[ -n "$unit_exe" && "$unit_exe" != "$TARGET_BIN" && ! -x "$unit_exe" ]]; then
|
||||||
|
fail "unit 的 ExecStart 指向 $unit_exe,但该文件不存在。
|
||||||
|
包安装(deb/rpm)用 /usr/bin/modelrouter;源码部署(本脚本)用 $TARGET_BIN。
|
||||||
|
请把 $unit 的 ExecStart 改成 $TARGET_BIN,或改用包安装。"
|
||||||
|
fi
|
||||||
|
if [[ -n "$unit_exe" && "$unit_exe" != "$TARGET_BIN" ]]; then
|
||||||
|
warn "unit ExecStart=$unit_exe,而本脚本部署到 $TARGET_BIN(两者都存在,确认哪个该生效)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -n "$NEW_CONFIG_SRC" ]]; then
|
if [[ -n "$NEW_CONFIG_SRC" ]]; then
|
||||||
[[ -f "$NEW_CONFIG_SRC" ]] || fail "待部署配置不存在: $NEW_CONFIG_SRC"
|
[[ -f "$NEW_CONFIG_SRC" ]] || fail "待部署配置不存在: $NEW_CONFIG_SRC"
|
||||||
# 同一文件比对:--config 指向线上文件本身时没有"投放"动作可做,
|
# 同一文件比对:--config 指向线上文件本身时没有"投放"动作可做,
|
||||||
@ -121,14 +158,14 @@ backup_old_state() {
|
|||||||
log "备份旧二进制到 $BACKUP_BIN"
|
log "备份旧二进制到 $BACKUP_BIN"
|
||||||
if [[ -f "$TARGET_BIN" ]]; then
|
if [[ -f "$TARGET_BIN" ]]; then
|
||||||
cp -f "$TARGET_BIN" "$BACKUP_BIN"
|
cp -f "$TARGET_BIN" "$BACKUP_BIN"
|
||||||
log "✓ 旧二进制已备份 (sha256=$(sha256sum "$BACKUP_BIN" | awk '{print $1}' | cut -c1-16))"
|
log "OK 旧二进制已备份 (sha256=$(sha256sum "$BACKUP_BIN" | awk '{print $1}' | cut -c1-16))"
|
||||||
else
|
else
|
||||||
warn "旧二进制不存在,跳过备份(无回滚点)"
|
warn "旧二进制不存在,跳过备份(无回滚点)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -f "$TARGET_CONFIG" ]]; then
|
if [[ -f "$TARGET_CONFIG" ]]; then
|
||||||
cp -f "$TARGET_CONFIG" "$BACKUP_CONFIG"
|
cp -f "$TARGET_CONFIG" "$BACKUP_CONFIG"
|
||||||
log "✓ 旧配置已备份到 $BACKUP_CONFIG (sha256=$(sha256sum "$BACKUP_CONFIG" | awk '{print $1}' | cut -c1-16))"
|
log "OK 旧配置已备份到 $BACKUP_CONFIG (sha256=$(sha256sum "$BACKUP_CONFIG" | awk '{print $1}' | cut -c1-16))"
|
||||||
else
|
else
|
||||||
warn "线上配置不存在: $TARGET_CONFIG(首次部署?无配置回滚点)"
|
warn "线上配置不存在: $TARGET_CONFIG(首次部署?无配置回滚点)"
|
||||||
fi
|
fi
|
||||||
@ -141,11 +178,11 @@ build() {
|
|||||||
mkdir -p "$TMP_DIR/bin"
|
mkdir -p "$TMP_DIR/bin"
|
||||||
|
|
||||||
cd "$REPO_DIR"
|
cd "$REPO_DIR"
|
||||||
CGO_ENABLED=1 go build -tags luajit \
|
CGO_ENABLED=1 GOTMPDIR="$TMP_DIR" go build -tags luajit \
|
||||||
-trimpath \
|
-trimpath \
|
||||||
-ldflags="-s -w" \
|
-ldflags="-s -w" \
|
||||||
-o "$TMP_DIR/bin/$BIN_NAME" \
|
-o "$TMP_DIR/bin/$BIN_NAME" \
|
||||||
./cmd/llmsproxy
|
./cmd/modelrouter
|
||||||
|
|
||||||
NEW_BIN="$TMP_DIR/bin/$BIN_NAME"
|
NEW_BIN="$TMP_DIR/bin/$BIN_NAME"
|
||||||
NEW_SIZE=$(stat -c%s "$NEW_BIN")
|
NEW_SIZE=$(stat -c%s "$NEW_BIN")
|
||||||
@ -174,9 +211,9 @@ build() {
|
|||||||
)
|
)
|
||||||
for c in "${checks[@]}"; do
|
for c in "${checks[@]}"; do
|
||||||
if strings "$NEW_BIN" | grep -qF "$c"; then
|
if strings "$NEW_BIN" | grep -qF "$c"; then
|
||||||
log " ✓ 新二进制包含: $c"
|
log " OK 新二进制包含: $c"
|
||||||
else
|
else
|
||||||
warn " ⚠ 新二进制不包含: $c(非预期但非致命)"
|
warn " 注意: 新二进制不包含: $c(非预期但非致命)"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
@ -206,7 +243,7 @@ atomic_replace_binary() {
|
|||||||
|
|
||||||
# 验证
|
# 验证
|
||||||
NEW_SHA256=$(sha256sum "$TARGET_BIN" | awk '{print $1}')
|
NEW_SHA256=$(sha256sum "$TARGET_BIN" | awk '{print $1}')
|
||||||
log "✓ 二进制已原子替换 (sha256=${NEW_SHA256:0:16})"
|
log "OK 二进制已原子替换 (sha256=${NEW_SHA256:0:16})"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---------- 原子替换配置 ----------
|
# ---------- 原子替换配置 ----------
|
||||||
@ -222,7 +259,7 @@ atomic_replace_config() {
|
|||||||
|
|
||||||
log " rename → $TARGET_CONFIG (原子)"
|
log " rename → $TARGET_CONFIG (原子)"
|
||||||
mv -f "$STAGING_CONFIG" "$TARGET_CONFIG"
|
mv -f "$STAGING_CONFIG" "$TARGET_CONFIG"
|
||||||
log "✓ 配置已原子替换 (sha256=$(sha256sum "$TARGET_CONFIG" | awk '{print $1}' | cut -c1-16))"
|
log "OK 配置已原子替换 (sha256=$(sha256sum "$TARGET_CONFIG" | awk '{print $1}' | cut -c1-16))"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---------- 配置预检 ----------
|
# ---------- 配置预检 ----------
|
||||||
@ -235,7 +272,7 @@ verify_config() {
|
|||||||
|
|
||||||
local out
|
local out
|
||||||
if out=$("$TARGET_BIN" -check -config "$TARGET_CONFIG" 2>&1); then
|
if out=$("$TARGET_BIN" -check -config "$TARGET_CONFIG" 2>&1); then
|
||||||
log "✓ 配置合法"
|
log "OK 配置合法"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@ -246,7 +283,7 @@ verify_config() {
|
|||||||
|
|
||||||
# ---------- 密钥可解封校验 ----------
|
# ---------- 密钥可解封校验 ----------
|
||||||
# config.yaml 里的凭据以 enc:v1: 密文落盘,解密钥匙是 master.key(或
|
# config.yaml 里的凭据以 enc:v1: 密文落盘,解密钥匙是 master.key(或
|
||||||
# LLMS_PROXY_MASTER_KEY)。一旦钥匙丢失/错位,配置在**启动时**才会报错,
|
# MODELROUTER_MASTER_KEY)。一旦钥匙丢失/错位,配置在**启动时**才会报错,
|
||||||
# 那时服务已经处于「二进制已替换 + 配置已替换」的半死状态,只能靠回滚。
|
# 那时服务已经处于「二进制已替换 + 配置已替换」的半死状态,只能靠回滚。
|
||||||
# 所以在替换任何东西之前先验:能不能用现有的钥匙解开现有的密文。
|
# 所以在替换任何东西之前先验:能不能用现有的钥匙解开现有的密文。
|
||||||
#
|
#
|
||||||
@ -271,27 +308,28 @@ verify_master_key() {
|
|||||||
[[ -n "$runtime_file" ]] || runtime_file="$(dirname "$TARGET_CONFIG")/runtime.json"
|
[[ -n "$runtime_file" ]] || runtime_file="$(dirname "$TARGET_CONFIG")/runtime.json"
|
||||||
key_file="$(dirname "$runtime_file")/master.key"
|
key_file="$(dirname "$runtime_file")/master.key"
|
||||||
|
|
||||||
if [[ -n "${LLMS_PROXY_MASTER_KEY:-}" ]]; then
|
MASTER_KEY_ENV="${MODELROUTER_MASTER_KEY:-${LLMS_PROXY_MASTER_KEY:-}}"
|
||||||
log " 钥匙来源: LLMS_PROXY_MASTER_KEY(环境变量)"
|
if [[ -n "$MASTER_KEY_ENV" ]]; then
|
||||||
|
log " 钥匙来源: MODELROUTER_MASTER_KEY(环境变量)"
|
||||||
# 走真实解密路径验证,而不是只检查格式
|
# 走真实解密路径验证,而不是只检查格式
|
||||||
if out=$("$TARGET_BIN" -show-secrets -config "$TARGET_CONFIG" 2>&1); then
|
if out=$("$TARGET_BIN" -show-secrets -config "$TARGET_CONFIG" 2>&1); then
|
||||||
log "✓ 凭据可解封($(echo "$out" | grep -cE '^(source|key) ') 项)"
|
log "OK 凭据可解封($(echo "$out" | grep -cE '^(source|key) ') 项)"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
err "凭据无法用 LLMS_PROXY_MASTER_KEY 解封,未替换任何文件:"
|
err "凭据无法用 MODELROUTER_MASTER_KEY 解封,未替换任何文件:"
|
||||||
printf '%s\n' "$out" >&2
|
printf '%s\n' "$out" >&2
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ ! -f "$key_file" ]]; then
|
if [[ ! -f "$key_file" ]]; then
|
||||||
err "配置含 $sealed 处密文,但钥匙文件不存在: $key_file"
|
err "配置含 $sealed 处密文,但钥匙文件不存在: $key_file"
|
||||||
err " 没有它,部署后服务将无法启动。恢复 master.key 或设置 LLMS_PROXY_MASTER_KEY 后重试。"
|
err " 没有它,部署后服务将无法启动。恢复 master.key 或设置 MODELROUTER_MASTER_KEY 后重试。"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
log " 钥匙文件: $key_file ($(stat -c '%A' "$key_file" 2>/dev/null || echo '?'),$(wc -c < "$key_file") 字节)"
|
log " 钥匙文件: $key_file ($(stat -c '%A' "$key_file" 2>/dev/null || echo '?'),$(wc -c < "$key_file") 字节)"
|
||||||
|
|
||||||
if out=$("$TARGET_BIN" -show-secrets -config "$TARGET_CONFIG" 2>&1); then
|
if out=$("$TARGET_BIN" -show-secrets -config "$TARGET_CONFIG" 2>&1); then
|
||||||
log "✓ 凭据可解封($(echo "$out" | grep -cE '^(source|key) ') 项)"
|
log "OK 凭据可解封($(echo "$out" | grep -cE '^(source|key) ') 项)"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
err "凭据无法解密(钥匙与配置不匹配),未替换任何文件:"
|
err "凭据无法解密(钥匙与配置不匹配),未替换任何文件:"
|
||||||
@ -301,48 +339,60 @@ verify_master_key() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---------- 适配器备份保留策略 ----------
|
# ---------- 备份目录保留策略 ----------
|
||||||
# 每次部署都新建一个 adapters.bak.<时间戳> 目录,而回滚只用到最近一次
|
# 每次部署都新建一个 <prefix>.bak.<时间戳> 目录,而回滚只用到最近一次
|
||||||
# ($BACKUP_BIN / $BACKUP_CONFIG 都是单文件覆盖)。多出来的目录从没人清理,
|
# ($BACKUP_BIN / $BACKUP_CONFIG 都是单文件覆盖)。多出来的目录从没人清理,
|
||||||
# 实测线上已积累 71 个,/etc/llmsproxy 因此涨到 122M。
|
# 实测线上已积累 71 个,/etc/modelrouter 因此涨到 122M。
|
||||||
#
|
#
|
||||||
# 保留最近 KEEP_ADAPTER_BACKUPS 份足够回滚,同时给目录数设上限——否则
|
# 保留最近 KEEP_BACKUPS 份足够回滚,同时给目录数设上限——否则一次误配置
|
||||||
# 一次误配置(比如 adapter_dir 指错)就可能在几秒内造出成百上千个目录。
|
# 就可能在几秒内造出成百上千个目录。
|
||||||
# 只删名字严格匹配 adapters.bak.<14位时间戳> 的目录,避免误伤人工放的目录。
|
# 只删名字严格匹配 <prefix>.bak.<14位时间戳> 的目录,避免误伤人工放的目录。
|
||||||
KEEP_ADAPTER_BACKUPS=5
|
#
|
||||||
|
# ★ 必须**同时覆盖 adapters 与 plugins**(同一缺陷的两个实例):
|
||||||
|
# 本函数最初只处理 adapters.bak,而 sync_plugins 同样每次建 plugins.bak.*
|
||||||
|
# —— 于是修了 adapters 的堆积、plugins 继续涨(实测线上 36 个、占用 192M)。
|
||||||
|
# 这正是「两处只修一处」的老形状:给一个新前缀加备份目录时,必须同步纳入
|
||||||
|
# prune_all_backups,否则它就是一个不会被清理的新泄漏点。
|
||||||
|
KEEP_BACKUPS=5
|
||||||
|
|
||||||
prune_adapter_backups() {
|
prune_backups() {
|
||||||
local base="/etc/llmsproxy"
|
local prefix="$1"
|
||||||
|
local base="/etc/modelrouter"
|
||||||
[[ -d "$base" ]] || return 0
|
[[ -d "$base" ]] || return 0
|
||||||
|
# 防误用:前缀必须是已知的两个(避免有人传空串删掉整个目录)
|
||||||
|
case "$prefix" in
|
||||||
|
adapters|plugins) ;;
|
||||||
|
*) fail "prune_backups: 未知前缀 '$prefix'(只接受 adapters|plugins)" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
# 先按数量上限硬裁:即使时间戳排序失效也不会无上限增长。
|
# 先按数量上限硬裁:即使时间戳排序失效也不会无上限增长。
|
||||||
local all
|
local all
|
||||||
mapfile -t all < <(find "$base" -maxdepth 1 -type d -name 'adapters.bak.*' -printf '%f\n' | sort)
|
mapfile -t all < <(find "$base" -maxdepth 1 -type d -name "$prefix.bak.*" -printf '%f\n' | sort)
|
||||||
local cap=$((KEEP_ADAPTER_BACKUPS * 4))
|
local cap=$((KEEP_BACKUPS * 4))
|
||||||
if (( ${#all[@]} > cap )); then
|
if (( ${#all[@]} > cap )); then
|
||||||
warn "适配器备份目录有 ${#all[@]} 个(异常),裁到 $cap"
|
warn "$prefix 备份目录有 ${#all[@]} 个(异常),裁到 $cap"
|
||||||
local i=0
|
local i=0
|
||||||
for d in "${all[@]}"; do
|
for d in "${all[@]}"; do
|
||||||
i=$((i + 1))
|
i=$((i + 1))
|
||||||
# 从最旧的开始删(sort 后升序)。名字不规范的跳过不删。
|
# 从最旧的开始删(sort 后升序)。名字不规范的跳过不删。
|
||||||
if (( i <= ${#all[@]} - cap )); then
|
if (( i <= ${#all[@]} - cap )); then
|
||||||
if [[ "$d" =~ ^adapters\.bak\.[0-9]{14}$ ]]; then
|
if [[ "$d" =~ ^"$prefix"\.bak\.[0-9]{14}$ ]]; then
|
||||||
rm -rf "${base:?}/$d"
|
rm -rf "${base:?}/$d"
|
||||||
else
|
else
|
||||||
warn "跳过名字不规范的备份目录(不删): $d"
|
warn "跳过名字不规范的备份目录(不删): $d"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
mapfile -t all < <(find "$base" -maxdepth 1 -type d -name 'adapters.bak.*' -printf '%f\n' | sort)
|
mapfile -t all < <(find "$base" -maxdepth 1 -type d -name "$prefix.bak.*" -printf '%f\n' | sort)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 再按时间保留最近 KEEP_ADAPTER_BACKUPS 份(sort 后最新在末尾)。
|
# 再按时间保留最近 KEEP_BACKUPS 份(sort 后最新在末尾)。
|
||||||
local total=${#all[@]} i=0
|
local total=${#all[@]} i=0
|
||||||
for d in "${all[@]}"; do
|
for d in "${all[@]}"; do
|
||||||
i=$((i + 1))
|
i=$((i + 1))
|
||||||
# i <= total-KEEP 的是较旧的,要删。
|
# i <= total-KEEP 的是较旧的,要删。
|
||||||
if (( i <= total - KEEP_ADAPTER_BACKUPS )); then
|
if (( i <= total - KEEP_BACKUPS )); then
|
||||||
if [[ "$d" =~ ^adapters\.bak\.[0-9]{14}$ ]]; then
|
if [[ "$d" =~ ^"$prefix"\.bak\.[0-9]{14}$ ]]; then
|
||||||
rm -rf "${base:?}/$d"
|
rm -rf "${base:?}/$d"
|
||||||
else
|
else
|
||||||
warn "跳过名字不规范的备份目录(不删): $d"
|
warn "跳过名字不规范的备份目录(不删): $d"
|
||||||
@ -351,8 +401,15 @@ prune_adapter_backups() {
|
|||||||
done
|
done
|
||||||
|
|
||||||
local left
|
local left
|
||||||
left=$(find "$base" -maxdepth 1 -type d -name 'adapters.bak.*' | wc -l)
|
left=$(find "$base" -maxdepth 1 -type d -name "$prefix.bak.*" | wc -l)
|
||||||
log " 适配器备份保留最近 $KEEP_ADAPTER_BACKUPS 份(当前剩 $left 个)"
|
log " $prefix 备份保留最近 $KEEP_BACKUPS 份(当前剩 $left 个)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# prune_all_backups:两个前缀都裁。
|
||||||
|
# ★ 新增任何 .bak.<时间戳> 目录的前缀时,必须在这里补上 —— 否则又一个泄漏点。
|
||||||
|
prune_all_backups() {
|
||||||
|
prune_backups adapters
|
||||||
|
prune_backups plugins
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---------- 同步适配器 ----------
|
# ---------- 同步适配器 ----------
|
||||||
@ -366,35 +423,42 @@ sync_adapters() {
|
|||||||
[[ -d "$SRC_ADAPTERS" ]] || fail "内嵌适配器目录不存在: $SRC_ADAPTERS"
|
[[ -d "$SRC_ADAPTERS" ]] || fail "内嵌适配器目录不存在: $SRC_ADAPTERS"
|
||||||
|
|
||||||
# 备份旧的适配器文件(含任何运行时覆盖版本,如 opencode.lua)
|
# 备份旧的适配器文件(含任何运行时覆盖版本,如 opencode.lua)
|
||||||
BACKUP_DIR="/etc/llmsproxy/adapters.bak.$(date +%Y%m%d%H%M%S)"
|
BACKUP_DIR="/etc/modelrouter/adapters.bak.$(date +%Y%m%d%H%M%S)"
|
||||||
if [[ -d "$TARGET_ADAPTERS" ]]; then
|
if [[ -d "$TARGET_ADAPTERS" ]]; then
|
||||||
mkdir -p "$BACKUP_DIR"
|
mkdir -p "$BACKUP_DIR"
|
||||||
cp -rf "$TARGET_ADAPTERS/"*.lua "$BACKUP_DIR/" 2>/dev/null || true
|
cp -rf "$TARGET_ADAPTERS/"*.lua "$BACKUP_DIR/" 2>/dev/null || true
|
||||||
log " 旧适配器已备份到 $BACKUP_DIR"
|
log " 旧适配器已备份到 $BACKUP_DIR"
|
||||||
fi
|
fi
|
||||||
prune_adapter_backups
|
prune_all_backups
|
||||||
|
|
||||||
cp -f "$SRC_ADAPTERS/"*.lua "$TARGET_ADAPTERS/"
|
cp -f "$SRC_ADAPTERS/"*.lua "$TARGET_ADAPTERS/"
|
||||||
chmod 0644 "$TARGET_ADAPTERS/"*.lua
|
chmod 0644 "$TARGET_ADAPTERS/"*.lua
|
||||||
|
|
||||||
# 校验:每个适配器必须包含 usage 透传修复(opencode/openai/deepseek 等应有 'uses')
|
# 校验:每个适配器必须包含 usage 透传修复(opencode/openai/deepseek 等应有 'uses')
|
||||||
local required_files=(openai deepseek anthropic gemini ollama opencode github groq kimicode mistral sensenova agentrouter)
|
local required_files=(openai deepseek anthropic gemini ollama github groq mistral sensenova qwen38)
|
||||||
for f in "${required_files[@]}"; do
|
for f in "${required_files[@]}"; do
|
||||||
[[ -f "$TARGET_ADAPTERS/$f.lua" ]] || warn " 缺少适配器: $f.lua"
|
[[ -f "$TARGET_ADAPTERS/$f.lua" ]] || warn " 缺少适配器: $f.lua"
|
||||||
done
|
done
|
||||||
local usage_fixed=0
|
local usage_fixed=0
|
||||||
for f in openai deepseek github groq kimicode mistral opencode; do
|
for f in openai deepseek github groq mistral; do
|
||||||
grep -qE '\buses\b' "$TARGET_ADAPTERS/$f.lua" 2>/dev/null && ((usage_fixed++)) || true
|
grep -qE '\buses\b' "$TARGET_ADAPTERS/$f.lua" 2>/dev/null && ((usage_fixed++)) || true
|
||||||
done
|
done
|
||||||
log " usage 透传修复适配器: ${usage_fixed}/7 (openai/deepseek/github/groq/kimicode/mistral/opencode)"
|
log " usage 透传修复适配器: ${usage_fixed}/5 (openai/deepseek/github/groq/mistral)"
|
||||||
log "✓ 适配器已同步到 $TARGET_ADAPTERS/"
|
log "OK 适配器已同步到 $TARGET_ADAPTERS/"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# 插件不在这里同步。
|
||||||
|
#
|
||||||
|
# 早期实现会把内置插件复制/覆盖到 plugin_dir,理由是"内置插件是发行物的一部分"。
|
||||||
|
# 该前提已被废止:插件不再编入二进制、不进发行包、不自动安装,属于运维自有资产。
|
||||||
|
# 部署脚本只负责适配器(协议转换,随内核发布)与二进制/配置,绝不创建、覆盖或
|
||||||
|
# 删除 plugin_dir 里的任何文件。
|
||||||
|
|
||||||
# ---------- 重启服务 ----------
|
# ---------- 重启服务 ----------
|
||||||
restart_service() {
|
restart_service() {
|
||||||
log "重启服务(中断 <500ms)"
|
log "重启服务(中断 <500ms)"
|
||||||
systemctl restart llmsproxy.service || fail "systemd restart 失败"
|
systemctl restart modelrouter.service || fail "systemd restart 失败"
|
||||||
log "✓ llmsproxy.service restarted"
|
log "OK modelrouter.service restarted"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---------- 健康检查 ----------
|
# ---------- 健康检查 ----------
|
||||||
@ -409,7 +473,7 @@ healthcheck() {
|
|||||||
local code
|
local code
|
||||||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$HEALTH_URL" 2>/dev/null || echo "000")
|
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$HEALTH_URL" 2>/dev/null || echo "000")
|
||||||
if [[ "$code" == "200" || "$code" == "401" ]]; then
|
if [[ "$code" == "200" || "$code" == "401" ]]; then
|
||||||
log "✓ 服务健康 (status=$code, ${attempt}/${attempts})"
|
log "OK 服务健康 (status=$code, ${attempt}/${attempts})"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
log " 等待中 ($attempt/$attempts, status=$code)"
|
log " 等待中 ($attempt/$attempts, status=$code)"
|
||||||
@ -417,7 +481,7 @@ healthcheck() {
|
|||||||
(( attempt++ ))
|
(( attempt++ ))
|
||||||
done
|
done
|
||||||
|
|
||||||
warn "健康检查未通过,请检查服务状态: systemctl status llmsproxy.service"
|
warn "健康检查未通过,请检查服务状态: systemctl status modelrouter.service"
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -453,7 +517,7 @@ rollback() {
|
|||||||
|
|
||||||
if (( restored_any == 0 )); then
|
if (( restored_any == 0 )); then
|
||||||
err "无任何回滚点,无法自动恢复"
|
err "无任何回滚点,无法自动恢复"
|
||||||
err "请手动检查: systemctl status llmsproxy.service && journalctl -u llmsproxy -n 50"
|
err "请手动检查: systemctl status modelrouter.service && journalctl -u modelrouter -n 50"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@ -467,9 +531,9 @@ rollback() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
log "重启服务(旧版本)"
|
log "重启服务(旧版本)"
|
||||||
systemctl restart llmsproxy.service || {
|
systemctl restart modelrouter.service || {
|
||||||
err "重启失败,服务可能完全宕机"
|
err "重启失败,服务可能完全宕机"
|
||||||
err "请手动检查: systemctl status llmsproxy.service"
|
err "请手动检查: systemctl status modelrouter.service"
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -478,7 +542,7 @@ rollback() {
|
|||||||
local code
|
local code
|
||||||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$HEALTH_URL" 2>/dev/null || echo "000")
|
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$HEALTH_URL" 2>/dev/null || echo "000")
|
||||||
if [[ "$code" == "200" || "$code" == "401" ]]; then
|
if [[ "$code" == "200" || "$code" == "401" ]]; then
|
||||||
log "✓ 回滚成功,旧版本已恢复服务 (status=$code)"
|
log "OK 回滚成功,旧版本已恢复服务 (status=$code)"
|
||||||
log " 新版本部署失败,请检查构建/配置后重试"
|
log " 新版本部署失败,请检查构建/配置后重试"
|
||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
@ -495,6 +559,7 @@ print_summary() {
|
|||||||
log " 二进制: $TARGET_BIN"
|
log " 二进制: $TARGET_BIN"
|
||||||
log " 配置: $TARGET_CONFIG$([[ -n "$NEW_CONFIG_SRC" ]] && echo " (本次已替换)" || echo " (未改动)")"
|
log " 配置: $TARGET_CONFIG$([[ -n "$NEW_CONFIG_SRC" ]] && echo " (本次已替换)" || echo " (未改动)")"
|
||||||
log " 适配器: $TARGET_ADAPTERS"
|
log " 适配器: $TARGET_ADAPTERS"
|
||||||
|
log " 插件: $TARGET_PLUGINS"
|
||||||
log "═════════════════════════════════════════════"
|
log "═════════════════════════════════════════════"
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -503,7 +568,7 @@ print_summary() {
|
|||||||
# 于是坏配置在服务仍然健康时就被拦下,根本不会触发重启。
|
# 于是坏配置在服务仍然健康时就被拦下,根本不会触发重启。
|
||||||
main() {
|
main() {
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
log "═══ llmsproxy 原子部署开始 ═══"
|
log "═══ modelrouter 原子部署开始 ═══"
|
||||||
precheck
|
precheck
|
||||||
# 密钥校验放在 build/替换之前:它的失败后果最重(配置里的密文永久解不开),
|
# 密钥校验放在 build/替换之前:它的失败后果最重(配置里的密文永久解不开),
|
||||||
# 必须在动任何文件之前就挡住。此阶段用的是**当前线上二进制**,它已经
|
# 必须在动任何文件之前就挡住。此阶段用的是**当前线上二进制**,它已经
|
||||||
|
|||||||
61
docs/adapters.md
Normal file
61
docs/adapters.md
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
# 适配器协议
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来:Lua 适配器协议与内置适配器清单。
|
||||||
|
> 编写适配器的完整参考见 [lua-adapters.md](lua-adapters.md)。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## Lua 适配器协议
|
||||||
|
|
||||||
|
完整 API 见 **[Lua 适配器 API 文档](docs/lua-adapters.md)**([English](docs/lua-adapters-en.md))。
|
||||||
|
|
||||||
|
每个适配器是一个返回 table 的 Lua 脚本(`internal/lua/adapters/<name>.lua`),
|
||||||
|
加载时可被 `adapter_dir` 下的同名脚本覆盖——**改 Lua 脚本无需重编译**,
|
||||||
|
重启即生效;通过 WebUI 上传的适配器与在线编辑的源配置则即时生效。
|
||||||
|
|
||||||
|
```lua
|
||||||
|
return {
|
||||||
|
name = "mysrc",
|
||||||
|
version = "1.0.0",
|
||||||
|
endpoint = "/chat/completions", -- 上游路径(可被 source.endpoint 覆盖)
|
||||||
|
headers = { ["X-Static"] = "v" }, -- 静态默认请求头(build_headers 缺省时使用)
|
||||||
|
|
||||||
|
-- 请求转换:把 OpenAI 格式 req 转成上游原生格式,返回字符串
|
||||||
|
transform_request = function(raw_json) ... end,
|
||||||
|
|
||||||
|
-- 响应转换:把上游原生响应转成统一格式字符串
|
||||||
|
-- { content, reasoning_content, finish_reason, token_usage{...}, tool_calls[{...}] }
|
||||||
|
transform_response = function(raw_json) ... end,
|
||||||
|
|
||||||
|
-- 流式分块转换:把上游 SSE data 转成 { content, done, ... },返回 "" 则跳过
|
||||||
|
transform_stream_chunk = function(raw_chunk) ... end,
|
||||||
|
|
||||||
|
-- [可选] 动态请求头/签名钩子
|
||||||
|
-- meta = { url, method, body, api_key, timestamp, source={ name, meta={...} } }
|
||||||
|
build_headers = function(meta) return { ["X-App-Sign"] = sign } end,
|
||||||
|
|
||||||
|
-- [可选] 出向序列化(契约 v2):上游不是 OpenAI 协议时接管写给客户端的字节。
|
||||||
|
-- 不声明则沿用内核的 OpenAI 兜底;声明即完全取代,内核不再混合字段。
|
||||||
|
serialize_response = function(result) return body_str end,
|
||||||
|
serialize_open_preamble = function(result) return bytes end,
|
||||||
|
serialize_chunk = function(delta) return bytes end,
|
||||||
|
serialize_close = function(result) return bytes end,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
内置信号辅助:`hmac_sha256_hex(key, data)`、`sha256_hex(data)`、`base64_encode(s)`、
|
||||||
|
`tohex(s)`、`json.encode/decode`、`log(level, msg)`。
|
||||||
|
|
||||||
|
### 内置适配器
|
||||||
|
|
||||||
|
`openai` `deepseek` `anthropic` `gemini` `github` `groq` `mistral` `ollama` `qwen38` `sensenova`
|
||||||
|
`qwen38` `sensenova` `trae`。
|
||||||
|
|
||||||
|
`anthropic`/`gemini`/`ollama` 适配器内置多模态转换(`image_url` → 各自上游格式);
|
||||||
|
推理开关由中立的 `thinking` 字段表达,`deepseek` 适配器把它映射为
|
||||||
|
`thinking.type = "disabled"`。
|
||||||
|
|
||||||
|
适配器可用 `build_headers(meta)` 注入签名头。示例见 `adapters/` 下任一脚本的
|
||||||
|
`build_headers` 实现;云端校验调用方 app 时需要按
|
||||||
|
`meta.app_secret` 对时间戳+URL+请求体哈希做 HMAC 签名并附 `X-App-Sign` 等头。
|
||||||
|
配好 `sources[].meta.{app_id, app_secret, app_agent}` 即可。
|
||||||
|
|
||||||
183
docs/configuration.md
Normal file
183
docs/configuration.md
Normal file
@ -0,0 +1,183 @@
|
|||||||
|
# 配置
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来:网关配置、模型自动探测、多密钥。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## 配置文件
|
||||||
|
|
||||||
|
首次运行在 `-config` 路径自动生成默认配置(默认 `./config.yaml`),核心字段:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
listen: 127.0.0.1:8080 # 网关监听地址(建议绑内网/回环)
|
||||||
|
gateway_keys: [sk-gw-<随机>] # 初始 admin 密钥(每次安装随机生成,仅首启写入运行时存储用)
|
||||||
|
default_model: AUTO # model 无法路由时自动按优先级链选源
|
||||||
|
adapter_dir: adapters # Lua 适配目录;目录不存在时首启创建并 seed 内置,存在则只读
|
||||||
|
runtime_file: runtime.json # WebUI 编辑的源/密钥/AUTO 链持久化到此文件
|
||||||
|
|
||||||
|
sources:
|
||||||
|
- name: deepseek
|
||||||
|
base_url: https://api.deepseek.com
|
||||||
|
api_key: sk-... # 也可用 api_key_env: SOME_ENV 引用环境变量(不落盘明文)
|
||||||
|
adapter: deepseek
|
||||||
|
max_concurrent: 8
|
||||||
|
models:
|
||||||
|
- id: deepseek-v4-flash
|
||||||
|
priority: 100 # YAML 源首次启动会 seed 进 AUTO 链的初值
|
||||||
|
kind: chat
|
||||||
|
- id: deepseek-v4-pro
|
||||||
|
priority: 60
|
||||||
|
kind: chat
|
||||||
|
# 静态请求头(比适配器默认优先)
|
||||||
|
headers: { X-Tenant: prod }
|
||||||
|
# 传给 Lua build_headers 的透传元数据
|
||||||
|
meta: { app_id: x, app_secret: y }
|
||||||
|
temperature: 0.7
|
||||||
|
max_tokens: 4096
|
||||||
|
timeout: 120s # 请求超时,默认 120s
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## 自动探测模型列表
|
||||||
|
|
||||||
|
不想手抄模型 id 时,只填 URL 和 key,让网关去扫:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 只探测,不写任何东西
|
||||||
|
curl -X POST http://127.0.0.1:8080/api/sources/discover \
|
||||||
|
-H "Authorization: Bearer sk-gw-…" -H "Content-Type: application/json" \
|
||||||
|
-d '{"base_url":"https://api.vendor.com/v1","api_key":"sk-…"}'
|
||||||
|
# → {"ok":true,"base_url":"…","endpoint":"","models":["gpt-x",…],
|
||||||
|
# "image_models":["gpt-image-2"],"models_path":"/models","latency_ms":1387,…}
|
||||||
|
|
||||||
|
# 探测后直接落库
|
||||||
|
curl -X POST http://127.0.0.1:8080/api/sources/discover/apply \
|
||||||
|
-H "Authorization: Bearer sk-gw-…" -H "Content-Type: application/json" \
|
||||||
|
-d '{"base_url":"https://api.vendor.com/v1","api_key":"sk-…","name":"vendor"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
WebUI 源编辑框里也有「自动探测模型」按钮:填好 URL 和 key 点一下,列表自动
|
||||||
|
填好,仍需点保存才生效。
|
||||||
|
|
||||||
|
几点值得知道:
|
||||||
|
|
||||||
|
- **URL 形态不用猜。** 厂商对「`/v1` 放 base 还是放 path」没有共识,探测会
|
||||||
|
依次尝试若干候选并返回答对的那个,同时给出归一化后的 `base_url` +
|
||||||
|
`endpoint`(`endpoint` 平时留空,因为 openai adapter 自带 `/chat/completions`)。
|
||||||
|
- **`image_endpoint` 是另一套规则。** 网关的图像默认路径是**绝对**的
|
||||||
|
`/v1/images/generations`,而 chat 默认是相对的。所以 `base_url` 带 `/v1` 时
|
||||||
|
必须显式给 `image_endpoint: /images/generations`,否则图像请求 404。
|
||||||
|
探测会替你填好。
|
||||||
|
- **图像模型只看上游自己声明的 modality**,绝不从模型名猜。`gpt-image-2`
|
||||||
|
这类 id 也出现在同时提供对话服务的目录上,猜错会让 AUTO 链选中一个只支持
|
||||||
|
图像生成的模型,其后每一层都被它拖垮。
|
||||||
|
- **不会逐模型探活。** 34 个模型的目录跑一轮要花钱且耗时数分钟,`/models`
|
||||||
|
是足够便宜的信号;模型是否真能应答留给正常请求去发现。
|
||||||
|
- **对已存在的源,必须显式 `replace_models: true`** 才会覆盖模型列表。重新
|
||||||
|
探测顺手抹掉为 AUTO 链手工调过的 priority,是最难排查的一类数据丢失。
|
||||||
|
- **`api_key` 留空表示沿用已存值**,不会把凭据清空。
|
||||||
|
|
||||||
|
|
||||||
|
`config.yaml`、`runtime.json`、`master.key`、`adapters/` 均不提交版本库(见
|
||||||
|
`.gitignore`),请随备份带走——`master.key` 丢失后运行时密文无法解密。
|
||||||
|
|
||||||
|
运行文件生成在 `-config` 所在目录:`adapter_dir` / `runtime_file` 自动指向
|
||||||
|
配置文件旁边的 `adapters/` 与 `runtime.json`,从任意工作目录启动都可用。
|
||||||
|
|
||||||
|
运行时文件(`runtime_file`)中的敏感字段自动加密:
|
||||||
|
|
||||||
|
- 加密算法 AES-256-GCM,格式 `enc:v1:<base64>`。
|
||||||
|
- 主密钥来源:环境变量 `MODELROUTER_MASTER_KEY`(64 位 hex);否则读取
|
||||||
|
`runtime_file` 同目录的 `master.key`;都不存在时首次启动自动生成 `master.key`(0600)。
|
||||||
|
- 升级时旧明文文件自动兼容:首次运行正常读取,任何 UI 保存操作触发全文件加密迁移。
|
||||||
|
- 注意:**master.key 丢失后密文无法解密**,请随配置一起备份;切勿提交到版本库。
|
||||||
|
|
||||||
|
## 网关密钥(多密钥)
|
||||||
|
|
||||||
|
网关鉴权采用「多密钥 + 角色 + 模型范围」架构,密钥持久化在 `runtime_file` 的
|
||||||
|
`keys` 字段(加密存储):
|
||||||
|
|
||||||
|
- `gateway_keys` 配置只是**初始 admin 密钥种子**:首次启动迁移为运行时 admin
|
||||||
|
key,之后不再参与鉴权管理。
|
||||||
|
- **重要:首次启动后请在 WebUI「密钥」页更换管理员密钥**——初始密钥明文写在
|
||||||
|
`config.yaml` 里(每次安装随机生成),继续使用存在被盗风险;用新密钥登录后删除初始密钥。
|
||||||
|
- WebUI **密钥页**可创建/删除密钥;每个密钥可指定 `admin`(管理全部)或
|
||||||
|
`user`(仅看自己的 key)角色,并配置**模型范围**(模型 + 源 + token 配额 +
|
||||||
|
重置周期)。
|
||||||
|
- 客户端用任意一个已授权的密钥明文作为 Bearer(`Authorization: Bearer <key>`)。
|
||||||
|
- 删除密钥即从运行时存储移除,立即失效。
|
||||||
|
|
||||||
|
### 密钥用量配额
|
||||||
|
|
||||||
|
**配额按模型单独设置**:每个密钥的 `models[]` 里,每一条模型范围各自带一份
|
||||||
|
token 配额与请求数配额。一个模型用满只影响该模型,同一密钥的其它模型照常工作。
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
keys:
|
||||||
|
- key: sk-gw-<hex>
|
||||||
|
role: user
|
||||||
|
name: agent-alice
|
||||||
|
models:
|
||||||
|
- model: deepseek-v4-flash
|
||||||
|
token_quota: 1000000 # 本周期内该 key 用这个模型的 token 预算
|
||||||
|
req_quota: 20000 # 本周期内的请求次数
|
||||||
|
period: nhour # "" | hour | week | month | nhour
|
||||||
|
hours: 6 # 仅 nhour
|
||||||
|
- model: AUTO
|
||||||
|
token_quota: 5000000 # AUTO 也是一条独立配额
|
||||||
|
period: day # ← 这种写法会被拒绝(词表只有 hour/week/month/nhour)
|
||||||
|
- model: kimi-k3 # 未列配额 = 无限
|
||||||
|
```
|
||||||
|
|
||||||
|
- **没有「整钥总配额」**:这是刻意的设计。整钥总额会让「把 A 模型的额度挪给
|
||||||
|
B 模型」变成一次全局重分配;按模型独立计费则每个模型各自可控,运维可以
|
||||||
|
看出哪个模型吃掉了预算。
|
||||||
|
- `period` 词表:空 = 永不过期(累计总量),`hour` / `week` / `month` = 固定窗口,
|
||||||
|
`nhour` + `hours` = 自定义小时数。**拼错的周期在写入时就被拒**,不会静默变成
|
||||||
|
永不过期。
|
||||||
|
- 配额严格按密钥隔离,且**同一密钥内按模型隔离**:A 密钥用满 `m1` 不会消耗
|
||||||
|
B 密钥的额度,同一密钥的 `m2` 也不受影响。
|
||||||
|
- 配额统计含聊天、流式、生图,跨重启从审计日志回放(保留 40 天,覆盖最长的
|
||||||
|
month 窗口)。
|
||||||
|
- 配额耗尽返回 **429 + `Retry-After`**(`rate_limit_exceeded`),消息里点名是哪个
|
||||||
|
模型用满了,客户端可等窗口重置后自动恢复;模型越权才是 403。
|
||||||
|
**admin 密钥永不受配额限制**(其 scope 上的配额也不强制),避免把管理员
|
||||||
|
锁在门外。
|
||||||
|
- 窗口用量按整点小时分桶统计,实际释放比配置窗口最多晚 1 小时(配额宁可晚释放
|
||||||
|
也不超发)。
|
||||||
|
- `PUT /api/keys/{key}` 提交 `models` 即同时提交它们的配额(配额就是 scope 的一部分,
|
||||||
|
不存在会与模型列表脱节的第二份预算)。显式 `0` = 解除该模型的限制。
|
||||||
|
|
||||||
|
#### 配额拒绝 vs 容量拒绝:两种「拒绝」含义不同
|
||||||
|
|
||||||
|
配额在**入口**检查,不占用任何上游槽位,因此是廉价拒绝;容量不足则要走
|
||||||
|
`busyWait` 有界等待后才返回 503。实测(容量 8、0.6s/请求、100 并发):
|
||||||
|
|
||||||
|
| 情况 | 状态码 | 延迟 | 是否打到上游 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 配额用尽(`req_quota` 50 → 100 并发) | `429 rate_limit_exceeded` + `Retry-After` | **19–21ms** | 否 |
|
||||||
|
| 容量打满(同一拓扑,未限配额) | `503 upstream_error` | ~2.6s(`busyWait`) | 超出部分否 |
|
||||||
|
|
||||||
|
客户端因此可以区分「等窗口重置」与「等上游腾出容量」:前者只需等待,后者
|
||||||
|
通常该降并发或换源。
|
||||||
|
|
||||||
|
#### 配额的开销
|
||||||
|
|
||||||
|
配额桶按 (密钥, 模型, 整点小时) 分桶保留 40 天,实测(AMD 7840HS):
|
||||||
|
|
||||||
|
- 窗口查询按窗口长度而非保留总量扫描:1h 窗口 49ns、24h 窗口 55ns、
|
||||||
|
30d 窗口 3.9μs(此前全扫保留总量,960 桶时 5.9μs)。
|
||||||
|
- 记录一条请求:283ns、3 分配(与引入配额前相同,分配来自 ring buffer)。
|
||||||
|
- 内存:生产形态(7 密钥 × 8 模型 × 2 源 × 满 40 天 retention)约 **3.7MB**。
|
||||||
|
按源 pin 的 `source::model` 桶**惰性创建**——只有当某条配额真的 pin 了
|
||||||
|
某个源时才维护,否则每条记录多写一份桶,在 20 密钥 × 8 模型 × 3 源下会
|
||||||
|
多占 18MB。
|
||||||
|
代价:配置 pinned 配额之前的历史用量无法事后按源拆分,所以 **pinned 配额的
|
||||||
|
首个窗口可能少算**。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 配额耗尽时客户端看到(点名了具体模型)
|
||||||
|
HTTP/1.1 429 Too Many Requests
|
||||||
|
Retry-After: 2100
|
||||||
|
{"error":{"type":"rate_limit_exceeded","message":"token quota exceeded for \"deepseek-v4-flash\" (5000000/5000000)"}}
|
||||||
|
```
|
||||||
|
|
||||||
142
docs/deployment.md
Normal file
142
docs/deployment.md
Normal file
@ -0,0 +1,142 @@
|
|||||||
|
# 部署与迁移
|
||||||
|
|
||||||
|
> 本文档面向运维:单机 systemd 部署、发行包安装、目录与权限、从 `llmsproxy`
|
||||||
|
> 改名后的迁移与回滚。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## 1. 单机部署(推荐)
|
||||||
|
|
||||||
|
仓库根目录的 `deploy.sh` 负责:构建 → 预检配置 → 备份 → 原子替换二进制 →
|
||||||
|
重启服务 → 后置校验。它**不**修改源码,也**不**假设全新机器。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./deploy.sh # 默认读取仓库内 config.yaml
|
||||||
|
sudo ./deploy.sh --config other.yaml # 投放并部署另一份配置
|
||||||
|
```
|
||||||
|
|
||||||
|
部署目标(改名后):
|
||||||
|
|
||||||
|
> 两种布局并存,区别只在于二进制位置:**包安装**(deb/rpm)放 `/usr/bin`,
|
||||||
|
> unit 自带;**源码部署**(`deploy.sh`)放 `/usr/local/bin`,需要让 unit 的
|
||||||
|
> `ExecStart` 指向它。`deploy.sh` 会校验这一点,写错时直接中止而不是留下
|
||||||
|
> 一个 203/EXEC 的服务。
|
||||||
|
|
||||||
|
| 项 | 路径(源码部署 / 包安装) |
|
||||||
|
|---|---|
|
||||||
|
| 二进制 | `/usr/local/bin/modelrouter` / `/usr/bin/modelrouter` |
|
||||||
|
| 配置 | `/etc/modelrouter/config.yaml` |
|
||||||
|
| 适配器 | `/etc/modelrouter/adapters/` |
|
||||||
|
| 插件(可选) | `/etc/modelrouter/plugins/`(需在配置里设 `plugin_dir`) |
|
||||||
|
| 主密钥 | `/etc/modelrouter/master.key`(0600) |
|
||||||
|
| systemd unit | `/etc/systemd/system/modelrouter.service` |
|
||||||
|
|
||||||
|
`deploy.sh` 会在以下情况**中止**而不是"看起来成功":
|
||||||
|
|
||||||
|
- 部署后二进制起不来、或新配置 `-check` 不通过;
|
||||||
|
- 凭据无法解封(`master.key` 丢失 / 与配置不匹配);
|
||||||
|
- 检测到改名前的 `/etc/llmsproxy`,但新配置尚未建立(否则会部署出一个
|
||||||
|
没有 key、没有源的空实例,所有请求都被拒)。
|
||||||
|
|
||||||
|
## 2. 发行包安装
|
||||||
|
|
||||||
|
三个产物由 `packaging/core-dist.sh` 生成到 `cmd/build/dist/`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
modelrouter_<ver>_amd64.deb
|
||||||
|
modelrouter-<ver>.x86_64.rpm
|
||||||
|
modelrouter-<ver>-linux-amd64.tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
deb / rpm 安装后:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl enable --now modelrouter
|
||||||
|
sudo journalctl -u modelrouter -f
|
||||||
|
```
|
||||||
|
|
||||||
|
tar 包内只有:`modelrouter`、`config.example.yaml`、`modelrouter.service`、
|
||||||
|
`migrate-to-modelrouter.sh`、`adapters/`。
|
||||||
|
|
||||||
|
**发行包不含业务插件**(仓库只保留 `billing` / `usage` 两个与厂商无关的示例)。
|
||||||
|
它们保留在仓库中作为可选扩展,需要时手工放入 `plugin_dir`,网关不会自动安装、
|
||||||
|
覆盖或删除。这是有意的:插件属于运维资产,生命周期不与内核绑定。
|
||||||
|
|
||||||
|
首次启动会在 `/etc/modelrouter/config.yaml` 生成一份**随机 admin key** 的配置,
|
||||||
|
key 打印在启动日志里。发行包不预置任何可用 key(历史上曾因此泄露过一个真实的
|
||||||
|
`sk-gw-local-0001`)。
|
||||||
|
|
||||||
|
## 3. 目录与权限
|
||||||
|
|
||||||
|
- `/etc/modelrouter` 是唯一的读写根(systemd unit 用 `ProtectSystem=strict` +
|
||||||
|
`ReadWritePaths=/etc/modelrouter` 固化这一点)。
|
||||||
|
- `master.key` 必须 0600。当前 unit 以 root 运行:降权到普通用户需要先把
|
||||||
|
`master.key` 交给该用户并统一目录属主,否则服务会"带着明文落盘继续跑",
|
||||||
|
失败只留一行日志——这是独立变更,不要和加固混在一起。
|
||||||
|
- `plugin_dir` 由运维维护;插件自带的状态文件也在其中。
|
||||||
|
|
||||||
|
## 4. 从 llmsproxy 迁移
|
||||||
|
|
||||||
|
二进制、配置目录、服务名、环境变量已统一为 `modelrouter`:
|
||||||
|
|
||||||
|
| 旧 | 新 |
|
||||||
|
|---|---|
|
||||||
|
| `/usr/local/bin/llmsproxy` | `/usr/local/bin/modelrouter` |
|
||||||
|
| `/etc/llmsproxy/` | `/etc/modelrouter/` |
|
||||||
|
| `llmsproxy.service` | `modelrouter.service` |
|
||||||
|
| `LLMSPROXY_PLUGIN_DIR` | `MODELROUTER_PLUGIN_DIR` |
|
||||||
|
| `LLMSPROXY_XDISPLAY_BASE` | `MODELROUTER_XDISPLAY_BASE` |
|
||||||
|
| `LLMS_PROXY_MASTER_KEY` | `MODELROUTER_MASTER_KEY` |
|
||||||
|
|
||||||
|
迁移脚本**默认 dry-run**,复制而非移动,旧目录永不删除:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./packaging/migrate-to-modelrouter.sh # 预演(逐条打印将做什么)
|
||||||
|
sudo ./packaging/migrate-to-modelrouter.sh --apply # 执行
|
||||||
|
```
|
||||||
|
|
||||||
|
它做五件事:复制状态目录并改写配置里的旧绝对路径、复核 `master.key` 0600、
|
||||||
|
安装新二进制(旧文件保留)、由旧 unit 生成新 unit(改写 ExecStart /
|
||||||
|
WorkingDirectory / ReadWritePaths)、复制 `/usr/share` 资源;最后跑
|
||||||
|
`modelrouter -check` 并 `systemctl daemon-reload`。
|
||||||
|
|
||||||
|
确认新服务可用后,再手工清理旧布局(脚本不会删):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl disable --now llmsproxy.service
|
||||||
|
sudo rm -f /etc/systemd/system/llmsproxy.service
|
||||||
|
sudo rm -rf /usr/local/bin/llmsproxy /etc/llmsproxy /usr/share/llmsproxy
|
||||||
|
```
|
||||||
|
|
||||||
|
回滚(不需要旧文件被删):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./packaging/migrate-to-modelrouter.sh --rollback --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
**读取侧兼容**:旧配置路径与旧环境变量仍被接受,所以"先换二进制、后改 unit"
|
||||||
|
这种顺序不会立刻失效。但写入永远落在新路径,迁移完成后不要两台布局混用。
|
||||||
|
|
||||||
|
## 5. 部署后验证清单
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1) 进程与监听
|
||||||
|
systemctl status modelrouter --no-pager
|
||||||
|
ss -ltnp | grep modelrouter
|
||||||
|
|
||||||
|
# 2) 鉴权生效(无 key 必须 401)
|
||||||
|
curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/v1/models
|
||||||
|
|
||||||
|
# 3) 带 key 能列出模型
|
||||||
|
curl -s -H "Authorization: Bearer $KEY" http://127.0.0.1:8080/v1/models | head -c 400
|
||||||
|
|
||||||
|
# 4) 真实请求走通(证明 LuaJIT 适配器路径未被打断)
|
||||||
|
curl -s -H "Authorization: Bearer $KEY" \
|
||||||
|
-d '{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}' \
|
||||||
|
http://127.0.0.1:8080/v1/chat/completions | head -c 400
|
||||||
|
|
||||||
|
# 5) 仓库文件与生产文件一致(部署脚本的收尾检查)
|
||||||
|
sha256sum /usr/local/bin/modelrouter build/modelrouter
|
||||||
|
```
|
||||||
|
|
||||||
|
第 4 步失败但第 3 步成功,通常是适配器或上游凭据问题,不是部署问题:
|
||||||
|
查看 `journalctl -u modelrouter -n 100`。
|
||||||
117
docs/design-plugin-kernel-api.md
Normal file
117
docs/design-plugin-kernel-api.md
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
# 插件可调用内核动作(R4-c)设计
|
||||||
|
|
||||||
|
> 用户决策(2026-10-11):**全开**(13 个内核动作全部可调)+ **设置页可配权限** +
|
||||||
|
> **Go 侧 deadline 兜底**。
|
||||||
|
|
||||||
|
## 1. 要解决的问题
|
||||||
|
|
||||||
|
用户原话:"现在这个接口不全,可以多给一些接口让 lua 调用内核。"
|
||||||
|
|
||||||
|
现状(`__llmsproxy_adapters`)是我手写的**三个闭包**(install/update/remove),
|
||||||
|
只能管适配器。内核还有 13 个公开动作(Reload / ResetHealth / AddSource /
|
||||||
|
RemoveSource / SaveAutoRules / key 增删改 / 列表读取…),插件一个都调不到。
|
||||||
|
|
||||||
|
而"自动孵化 / 自愈 / 联动"这类插件能力(例如:探测到上游全挂 → 触发重探 +
|
||||||
|
清冷却;发现额度变化 → 重排 AUTO 链)恰恰需要它们。
|
||||||
|
|
||||||
|
## 2. 关键认识:Go 调用可以超时,Lua 死循环不能
|
||||||
|
|
||||||
|
这决定了整个设计,也是用户提"阻塞 + Go 兜底"的原因:
|
||||||
|
|
||||||
|
| 形态 | 能否中断 | 处置方式 |
|
||||||
|
|---|---|---|
|
||||||
|
| Lua 里 `while true do end` | **不能**(LuaJIT 限制,已实测三次) | 只能停用插件 |
|
||||||
|
| Lua 里 `__llmsproxy_kernel.call("reload")` | **能** | Go 侧 `select` + deadline,超时把错误返回给 Lua |
|
||||||
|
|
||||||
|
因为 Lua 只是在等 Go 的返回值,**不占用 Lua 栈的执行**。所以给每次内核调用加
|
||||||
|
Go 侧 deadline 是可行的、干净的 —— 不需要"Lua 投递请求、Go 队列执行":那样
|
||||||
|
还要定义"超时后队列里积压的命令怎么办",而 deadline 直接返回 `(nil, err)`,
|
||||||
|
语义闭合。
|
||||||
|
|
||||||
|
## 3. 依赖方向
|
||||||
|
|
||||||
|
`internal/lua` **不能** import `internal/core`(方向是 core → lua)。
|
||||||
|
按既有先例(`SetOutboundHosts`)注入一个**接口**,由 core 实现:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// internal/lua/kernel_api.go
|
||||||
|
type KernelAction struct {
|
||||||
|
Name string // "reload" / "reset_health" / ...
|
||||||
|
Title string // 设置页展示用
|
||||||
|
Description string
|
||||||
|
Danger string // ""=只读, "write"=改配置, "danger"=破坏性
|
||||||
|
Params []string // 参数名(设置页/文档用)
|
||||||
|
}
|
||||||
|
|
||||||
|
type KernelAPI interface {
|
||||||
|
// Actions 列出可调用动作(设置页据此渲染权限开关)
|
||||||
|
Actions() []KernelAction
|
||||||
|
// Invoke 执行一个动作,args 为已解码参数。**必须**尊重 ctx 的 deadline。
|
||||||
|
Invoke(ctx context.Context, name string, args map[string]any) (any, error)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
core 侧实现 `Invoke` 时用**现有的 Core 方法**,本身不持 `p.mu`,因此不会与
|
||||||
|
插件锁成环(已核实:所有 `ps.mu` → `p.mu` 都是先释放 `ps.mu`)。
|
||||||
|
|
||||||
|
## 4. 权限模型
|
||||||
|
|
||||||
|
复用 `visibilityOverrides` 的**同一套持久化机制**(`.access.json`),
|
||||||
|
但用独立的键空间:`kernel:<plugin>` → 逗号分隔的动作名或 `*`。
|
||||||
|
|
||||||
|
- **默认拒绝**:插件不显式获得授权时,`call` 返回 `(nil, "not permitted")`。
|
||||||
|
理由:全开是"能力存在",不是"默认授予"。一个插件能删密钥这件事必须由
|
||||||
|
运维在设置页点过确认。
|
||||||
|
- **`danger` 类必须显式逐个授权**,不接受 `*`。这一条是防止运维图省事写
|
||||||
|
通配符,把"删密钥"和"读状态"等同起来。
|
||||||
|
- 插件可以在自己源码里**声明**它想要的动作(`plugin.kernel = {...}`),
|
||||||
|
设置页把它显示为"申请",运维批准后才生效。声明不等于授权。
|
||||||
|
|
||||||
|
## 5. 动作清单(13 个 + 适配器 3 个)
|
||||||
|
|
||||||
|
| 动作 | 危险级 | 对应 Core 方法 | 超时 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `status` | 只读 | 读 AutoSlotStates/Registry | 2s |
|
||||||
|
| `list_sources` | 只读 | Sources() | 2s |
|
||||||
|
| `list_models` | 只读 | Registry().ModelList() | 2s |
|
||||||
|
| `list_adapters` | 只读 | ListAdapters() | 2s |
|
||||||
|
| `adapter_owner` | 只读 | AdapterOwnerOf | 2s |
|
||||||
|
| `reload` | write | Reload() | 30s |
|
||||||
|
| `reset_health` | write | ResetHealth() | 5s |
|
||||||
|
| `probe_all` | write | Registry().ProbeAll(ctx) | 60s |
|
||||||
|
| `save_auto_rules` | write | SaveAutoRules() | 10s |
|
||||||
|
| `save_key_auto` | write | SaveKeyAuto() | 10s |
|
||||||
|
| `add_source` | write | AddSource() | 30s |
|
||||||
|
| `remove_source` | danger | RemoveSource() | 30s |
|
||||||
|
| `create_key` | write | CreateKey() | 10s |
|
||||||
|
| `update_key` | write | UpdateKey() | 10s |
|
||||||
|
| `delete_key` | danger | DeleteKey() | 10s |
|
||||||
|
| `install_adapter` | write | UploadAdapter() | 10s |
|
||||||
|
| `remove_adapter` | danger | RemoveAdapter() | 30s |
|
||||||
|
| `set_plugin_enabled` | danger | SetEnabled() | 5s |
|
||||||
|
|
||||||
|
**绝不开放**:`ListKeys()` / `FindKey()` —— 返回**明文密钥**。给插件读密钥等于
|
||||||
|
把网关的凭据面交给第三方 Lua。要用 key 相关动作就只用带掩码的形态。
|
||||||
|
|
||||||
|
## 6. Lua 调用面
|
||||||
|
|
||||||
|
```lua
|
||||||
|
-- 探测:拿到可用动作与自己的授权情况
|
||||||
|
local acts = __llmsproxy_kernel.actions()
|
||||||
|
-- 调用:统一入口,返回 (result, err)
|
||||||
|
local ok, err = __llmsproxy_kernel.call("reset_health", {})
|
||||||
|
local st, err = __llmsproxy_kernel.call("status", {})
|
||||||
|
```
|
||||||
|
|
||||||
|
统一入口(而不是每个动作一个全局函数)的理由:动作集合将来会变,而
|
||||||
|
`__llmsproxy_kernel` 的签名不变 —— 否则每加一个动作就要动
|
||||||
|
`bootPluginState`,而那里正是历史上出过 cgo 崩溃的地方。
|
||||||
|
|
||||||
|
## 7. 实施顺序
|
||||||
|
|
||||||
|
1. `internal/lua/kernel_api.go`:接口 + 动作元数据 + 权限检查 + deadline 包装
|
||||||
|
2. `internal/lua/plugin_kernel_bridge.go`:Lua 绑定(`actions` / `call`)
|
||||||
|
3. `internal/core/kernel_actions.go`:实现 `KernelAPI`(core 侧)
|
||||||
|
4. 权限持久化:复用 `.access.json`,新键空间
|
||||||
|
5. 设置页:插件行加"内核动作"列 + 授权编辑
|
||||||
|
6. 测试:Go 侧全动作、Lua 侧调用、权限拒绝、deadline 生效、危险动作需显式授权
|
||||||
34
docs/development.md
Normal file
34
docs/development.md
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
# 开发与目录结构
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来:目录职责、开发协作、测试。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## 目录
|
||||||
|
|
||||||
|
```
|
||||||
|
cmd/modelrouter # 入口(纯 Go 服务端)
|
||||||
|
cmd/gui # Electron 桌面 GUI(内嵌核心 + 托盘 + 打包)
|
||||||
|
internal/config # YAML 配置加载/校验
|
||||||
|
internal/lua # LuaJIT VM + worker 池 + AdapterCache + 内置适配器 (embed)
|
||||||
|
internal/provider # Provider(HTTP) + Registry(路由)
|
||||||
|
internal/gateway # OpenAI 兼容 HTTP 服务 + 鉴权 + SDK/流式 + WebUI
|
||||||
|
internal/types # 统一格式 & OpenAI wire 类型
|
||||||
|
```
|
||||||
|
|
||||||
|
## 开发协作
|
||||||
|
|
||||||
|
- **Git 分支工作流**:主分支 / 特性分支 / 发布分支的规范见
|
||||||
|
**[docs/git-workflow.md](docs/git-workflow.md)**([English](docs/git-workflow-en.md))。
|
||||||
|
主线:`main` 永可部署 → 特性合入 `main` → 切 `release/vX.Y.Z` 打 tag 发布 →
|
||||||
|
hotfix 提交发布分支并 cherry-pick 回 `main`。
|
||||||
|
- **Lua 适配器协议**:见 **[docs/lua-adapters.md](docs/lua-adapters.md)**
|
||||||
|
([English](docs/lua-adapters-en.md))。
|
||||||
|
|
||||||
|
## 测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test -tags luajit ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
覆盖:配置校验、适配器加载/变换、签名钩子、Gateway 鉴权、SDK 结算、SSE 流式、
|
||||||
|
模型路由、多模态透传与 thinking 预算。
|
||||||
48
docs/features.md
Normal file
48
docs/features.md
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
# 实现与完整特性
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来:架构来源与完整特性清单(README 只保留摘要)。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## 实现
|
||||||
|
|
||||||
|
统一 OpenAI 兼容网关:把多个上游 LLM 源(DeepSeek、Qijiar、OpenAI、Anthropic、Gemini、
|
||||||
|
Groq、Mistral、Ollama、KimiCode…)通过 **Lua 适配器** 做协议转换,对内网暴露
|
||||||
|
一个标准 `OpenAI Chat Completions` 接口(`/v1/chat/completions` + `/v1/models`),
|
||||||
|
支持单次结算、SSE 流式、**AUTO 模型路由**与**多模态**透传。
|
||||||
|
|
||||||
|
从上游多源 LLM 适配层
|
||||||
|
(`internal/agent/api/provider.go` + `internal/lua/adapters/*`)抽离而来并独立演进。
|
||||||
|
|
||||||
|
## 特性
|
||||||
|
|
||||||
|
- **多源**:一个进程内配置任意多个上游源,按请求的 `model` 自动路由。
|
||||||
|
- **AUTO 模式**:`default_model: AUTO` 时按优先级页保存的 AUTO 链档位逐档调度并发请求。
|
||||||
|
- **统一输出**:所有源对外都是 OpenAI 格式(含 `reasoning_content`、`tool_calls`、`usage`)。
|
||||||
|
- **生图**:`POST /v1/images/generations`,`kind: image` 的模型独立路由。
|
||||||
|
- **多模态**:`content` 数组(`image_url` 等)在多源间无损透传,Anthropic/Gemini/Ollama 自动转换。
|
||||||
|
- **LuaJIT VM**:基于 golua 绑定的 LuaJIT,每个适配器独立 VM + worker 池,安全并发。
|
||||||
|
池是 **弹性** 的:`Σ max_concurrent` 只是上限而非预分配,状态按需创建、空闲时自动回收,
|
||||||
|
因此空载网关几乎不持有 Lua 状态(启动 0 个)。
|
||||||
|
- 扩容步长由适配器最大并发决定:`clamp(ceil(max/8), 1, 8)`,且仅在**真实并发争用**
|
||||||
|
(所有现有状态均已占用)时批量预热,顺序流量始终只用 1 个状态。
|
||||||
|
- 缩容步长由当前连接数决定:`clamp(ceil(冗余/(1+占用)), 1, 冗余)`——无连接时一轮收到
|
||||||
|
常驻下限,忙时每轮只释放 1 个,保护热路径。WebUI「适配器」页展示实时池指标。
|
||||||
|
- **thinking 预算**:请求里的推理开关/预算归一为中立的 `thinking:{enabled,budget_tokens}`,
|
||||||
|
适配器映射到各自协议(Anthropic `thinking.budget_tokens`、Gemini `thinkingConfig`、
|
||||||
|
DeepSeek `thinking.type="disabled"`)。
|
||||||
|
- **Lua 适配协议**:每个源挂一个 `.lua` 适配器,完成 `transform_request` /
|
||||||
|
`transform_response` / `transform_stream_chunk` 双向转换,协议差异全在 Lua 层。
|
||||||
|
- **签名 / 请求头钩子**:适配器可定义 `build_headers(meta)`,在 Go 发 HTTP 前
|
||||||
|
动态注入/签名请求头——用于云端 API 校验调用方 app(如 KimiCode 只放行特定
|
||||||
|
agent)。提供 `hmac_sha256_hex` / `sha256_hex` / `base64_encode` 等签名辅助。
|
||||||
|
- **WebUI**:内置管理界面,可在线查看/新增/编辑上游源与模型、配置 AUTO 优先级链、
|
||||||
|
管理密钥模型范围,写入运行时文件持久化。
|
||||||
|
- **密钥加密存储**:运行时文件中的上游 `api_key`、自定义请求头值、网关 key 均以
|
||||||
|
AES-256-GCM 加密落盘(`master.key` 独立 0600,或 `MODELROUTER_MASTER_KEY`)。
|
||||||
|
- **实时源探测**:状态页探测各源可达性(`GET {base}/models`,失败回退最小请求),
|
||||||
|
不污染正常调度的退避状态,错误信息在 UI 可悬停查看。
|
||||||
|
- **鉴权**:网关自身用 `gateway_keys` 校验客户端 Bearer key;与上游各自的 key 相互独立。
|
||||||
|
- **流式**:SSE `chat.completion.chunk`,含角色首包与 `[DONE]` 收尾。
|
||||||
|
- **全链路审计**:HTTP 访问日志、登录/配置变更审计、请求记录 CSV 导出、密钥用量统计 CSV 导出
|
||||||
|
- **AES-256-GCM 加密存储**:运行时文件敏感字段加密落盘,master.key 0600 权限,支持环境变量注入主密钥
|
||||||
|
|
||||||
@ -134,7 +134,7 @@ git push origin vX.Y.Z
|
|||||||
make core-dist VERSION=X.Y.Z
|
make core-dist VERSION=X.Y.Z
|
||||||
|
|
||||||
# 5. create GitCode Release + upload
|
# 5. create GitCode Release + upload
|
||||||
# https://gitcode.com/JianFeeeee/ModelRouter/releases
|
# https://atomgit.com/HeNanMU-EEA/ModelRouter/releases
|
||||||
# upload MUST use PUT --http1.1, otherwise the OBS callback fails and the
|
# upload MUST use PUT --http1.1, otherwise the OBS callback fails and the
|
||||||
# file never lands in the release; replacing artifacts = delete the tag
|
# file never lands in the release; replacing artifacts = delete the tag
|
||||||
# (release disappears with it) then recreate.
|
# (release disappears with it) then recreate.
|
||||||
|
|||||||
@ -124,7 +124,7 @@ git push origin vX.Y.Z
|
|||||||
make core-dist VERSION=X.Y.Z
|
make core-dist VERSION=X.Y.Z
|
||||||
|
|
||||||
# 5. 创建 GitCode Release + 上传安装包
|
# 5. 创建 GitCode Release + 上传安装包
|
||||||
# 见 https://gitcode.com/JianFeeeee/ModelRouter/releases
|
# 见 https://atomgit.com/HeNanMU-EEA/ModelRouter/releases
|
||||||
# 上传注意:必须 PUT --http1.1,否则 OBS 回调失败文件不进列表;
|
# 上传注意:必须 PUT --http1.1,否则 OBS 回调失败文件不进列表;
|
||||||
# 替换旧版附件 = 删 tag(release 随 tag 一起消失)后重建。
|
# 替换旧版附件 = 删 tag(release 随 tag 一起消失)后重建。
|
||||||
```
|
```
|
||||||
|
|||||||
86
docs/gui.md
Normal file
86
docs/gui.md
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
# 桌面 GUI(Electron)
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来:Headless 与 Desktop 两种产物、构建与运行。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## 桌面 GUI(Electron)
|
||||||
|
|
||||||
|
可选独立产物,面向内网桌面端用户:**内嵌 ModelRouter 核心**(桌面壳形态),
|
||||||
|
一键启停、系统托盘常驻、开机自启、静默启动,内容区一比一内嵌完整 WebUI
|
||||||
|
(状态/对话/密钥/优先级/源/适配器 六个页面,免登录)。服务器用户继续使用纯 Go 二进制。
|
||||||
|
|
||||||
|
### 两种安装包:Headless 与 Desktop
|
||||||
|
|
||||||
|
发布到 Release 的安装包分为两类,面向不同用户:
|
||||||
|
|
||||||
|
- **Headless(服务器版)**:纯核心二进制(无 Electron),配置驱动,适合服务器、
|
||||||
|
容器、systemd 等无人值守场景。发布包含 `deb` + `rpm`(含 systemd unit、内存调优
|
||||||
|
与示例配置)与 `tar.gz`(二进制 + `config.example.yaml` + `adapters/` + README),
|
||||||
|
Windows 为 `.zip`。
|
||||||
|
- **Desktop(桌面版)**:Electron GUI 安装包,内嵌核心,适合个人桌面日常使用。
|
||||||
|
|
||||||
|
### 选择 GUI 还是纯后端
|
||||||
|
|
||||||
|
| 场景 | 推荐 | 理由 |
|
||||||
|
| ------ | ------ | ------ |
|
||||||
|
| 服务器 / 内网网关 / 无人值守常驻 | **Headless**(单二进制) | 极轻量(二进制 ~8MB;RSS 随源数量,单源 ~10MB、16 源 ~32–35MB,见[内存占用](#内存占用实测与调优)),零依赖单进程,直接跑在 systemd/任意容器里,远程管理 |
|
||||||
|
| 个人桌面日常使用 / 多设备内网共享 | **Desktop**(GUI) | 免登录内嵌 WebUI、系统托盘一键启停、开机自启、静默后台,适合不懂命令行的使用者 |
|
||||||
|
| Windows 桌面 | **Desktop** | 纯后端在 Windows 上需自行注册服务,GUI 提供原生托盘/自启体验 |
|
||||||
|
| CI 一键出三平台安装包 | **Desktop 打包脚本** | `make gui-*` 系列出 deb/AppImage/NSIS,可进发行流水线 |
|
||||||
|
|
||||||
|
两者完全同源:GUI 内嵌的就是纯后端的同一份 `modelrouter` 核心(LuaJIT 版),
|
||||||
|
配置/适配器格式完全一致,可随时互换。
|
||||||
|
|
||||||
|
### 构建与运行
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd cmd/gui
|
||||||
|
npm install
|
||||||
|
make gui # 或 npm run dev —— 开发运行(需先 make build 生成 bin/modelrouter)
|
||||||
|
make gui-deb # 本机构建 deb(分享给其它 Linux 用户)
|
||||||
|
make gui-dist # 构建 deb + rpm + AppImage(linux,rpm 需系统 rpmbuild)
|
||||||
|
make core-dist # 构建核心 deb + rpm + tar.gz(服务器版,需 nfpm)
|
||||||
|
make gui-win # 构建 Windows nsis 安装包(本机 mingw + wine)
|
||||||
|
make gui-win-docker # 构建 Windows nsis 安装包(全 docker 自足,含 wine)
|
||||||
|
make gui-dist-dir # 仅解包产物(调试用,不产安装包)
|
||||||
|
```
|
||||||
|
|
||||||
|
产物在 `cmd/build/gui-dist/`:`ModelRouter-<ver>.AppImage`、
|
||||||
|
`modelrouter-gui_<ver>_amd64.deb`、`ModelRouter Setup <ver>.exe`(win);
|
||||||
|
核心包在 `cmd/build/dist/`:`modelrouter_<ver>_amd64.deb`、`modelrouter-<ver>.x86_64.rpm`。
|
||||||
|
rpm 需系统 `rpmbuild`。每次打包后都会跑 `packaging/verify-dist.sh` 产物体检
|
||||||
|
(大小下限门禁——1.3.0 曾因宿主 wine 损坏产出 264KB 的残缺 exe 却没有被拦住)。
|
||||||
|
发布时会把产物重命名为 `ModelRouter-Desktop-*`(GUI)与 `ModelRouter-Headless-*`(纯后端)
|
||||||
|
两类上传到 GitCode Release。
|
||||||
|
|
||||||
|
#### Windows 打包(docker 固化方案,推荐)
|
||||||
|
|
||||||
|
`make gui-win-docker` 一步完成:交叉编译核心 + NSIS 打包**全在 docker 内**,宿主机
|
||||||
|
**无需 mingw、无需 wine、无需 node**。镜像内置 wine(含 i386 运行时),修复了
|
||||||
|
此前宿主 wine 损坏(缺 syswow64 → `c0000135`)导致产出 264KB 残缺 exe 的问题。
|
||||||
|
|
||||||
|
- 构建镜像:`cmd/gui/docker/win-builder/Dockerfile`(`golang:1.25` + mingw-w64 +
|
||||||
|
node + wine32/wine64 + 预编译 LuaJIT windows 版 `lua51.dll` 与 import lib)。
|
||||||
|
LuaJIT 源码取自国内镜像,上游兜底;Go 依赖走 `https://goproxy.cn`。
|
||||||
|
- 一键发布:`cmd/gui/scripts/dist-win-docker.sh` —— 核心编译 + electron-builder
|
||||||
|
NSIS 全部在容器内执行(`USE_SYSTEM_WINE=true` 走镜像内的 wine);electron-builder
|
||||||
|
的 ~400MB 工具链缓存挂到 docker volume,二次构建增量复用。
|
||||||
|
- 首次运行自动构建 `modelrouter/win-builder` 镜像(约 5–10 分钟);已存在则直接复用。
|
||||||
|
- 打包完成后自动跑 `packaging/verify-dist.sh` 门禁:exe 低于 5MB / deb·rpm·7z 低于
|
||||||
|
10MB 即失败——残缺安装包不可能再溜进 Release。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 全 Windows 发布(核心重编 + NSIS):
|
||||||
|
make gui-win-docker
|
||||||
|
```
|
||||||
|
|
||||||
|
### 特性
|
||||||
|
|
||||||
|
- **内嵌核心**:自动拉起捆绑的 `modelrouter`(LuaJIT 版),配置/密钥/adapter 存于
|
||||||
|
用户数据目录 `<userData>/profile/`,首次运行自动生成随机 admin key 写入 `keys`
|
||||||
|
(非 seed)并自注入,WebUI 免登录、不会提示"更换初始密钥"。端口可在设置中更改(默认 8787)。
|
||||||
|
- **系统托盘**:内核状态点、开机自启/静默启动开关、启停/重启内核、退出。
|
||||||
|
- **静默启动**:`--silent` 参数或设置项,启动仅驻托盘不弹窗;首次运行始终显示窗口。
|
||||||
|
- **开机自启**:Windows/macOS 用 `setLoginItemSettings`;Linux 写
|
||||||
|
`~/.config/autostart/modelrouter-gui.desktop`,静默开启时自动附加 `--silent`。
|
||||||
|
|
||||||
@ -23,7 +23,7 @@
|
|||||||
5. [内置辅助函数](#5-内置辅助函数)
|
5. [内置辅助函数](#5-内置辅助函数)
|
||||||
6. [meta 约定](#6-meta-约定)
|
6. [meta 约定](#6-meta-约定)
|
||||||
7. [一个完整的最小适配器](#7-一个完整的最小适配器)
|
7. [一个完整的最小适配器](#7-一个完整的最小适配器)
|
||||||
8. [多模态与 disable_thinking](#8-多模态与-disable_thinking)
|
8. [多模态与 thinking 预算](#8-多模态与-thinking-预算)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -73,8 +73,9 @@ end
|
|||||||
|
|
||||||
约定:
|
约定:
|
||||||
- 返回的字符串将被原样 POST 到 `base_url + (source.endpoint or adapter.endpoint)`。
|
- 返回的字符串将被原样 POST 到 `base_url + (source.endpoint or adapter.endpoint)`。
|
||||||
- 请求体中包含的 `disable_thinking`、`extra_body` 等网关内部字段,转换时应自行清理
|
- 契约 v2 起,内核**不再向适配器泄漏任何私有字段**:请求体里不会出现
|
||||||
(`openai.lua` 即删掉这两个字段)。
|
`disable_thinking` / `extra_body` 这类键。推理开关与预算用中立的 `thinking`
|
||||||
|
字段表达(见下),适配器按目标协议各自映射。
|
||||||
|
|
||||||
### `transform_response(raw_body) -> string`
|
### `transform_response(raw_body) -> string`
|
||||||
|
|
||||||
@ -176,6 +177,81 @@ end
|
|||||||
|
|
||||||
返回非 table 会报错;返回 `{}` 表示无自定义头(不会回退静态头)。
|
返回非 table 会报错;返回 `{}` 表示无自定义头(不会回退静态头)。
|
||||||
|
|
||||||
|
### 出向序列化钩子(契约 v2)
|
||||||
|
|
||||||
|
默认情况下,网关把适配器产出的**统一格式**按 OpenAI 线格式序列化给下游(这是内核
|
||||||
|
保留的兑底,R5)。当上游协议不是 OpenAI(例如 Anthropic Messages API)时,适配器
|
||||||
|
可以接管出向序列化,自己决定写给客户端的字节。
|
||||||
|
|
||||||
|
五个钩子都是可选的,**全不声明就沿用 OpenAI 兑底**。一旦声明某个钩子,它就完全
|
||||||
|
接管那一段输出——内核不会再往里混合 OpenAI 字段(不是“叠加”,是“取代”)。
|
||||||
|
|
||||||
|
| 钩子 | 何时被调用 | 返回值 |
|
||||||
|
|------|-----------|--------|
|
||||||
|
| `serialize_response(payload)` | 非流式响应 | 写给客户端的响应体字符串 |
|
||||||
|
| `serialize_open_preamble(payload)` | 流式开始(发第一个 chunk 前) | 要发的字节(可为 `""`) |
|
||||||
|
| `serialize_chunk(delta)` | 流式每个增量 | 要发的字节(`""` 表示跳过) |
|
||||||
|
| `serialize_close(payload)` | 流式结束 | 收尾字节(如 SSE `message_stop`) |
|
||||||
|
| `serialize_done_marker(payload)` | 流式结束处的 `[DONE]` 标记 | 要写的字节;不声明则用 OpenAI 的 `data: [DONE]` |
|
||||||
|
|
||||||
|
`payload` 是统一格式的结果表(见 `internal/types/maxwire.go` 的 `UnifiedResult` /
|
||||||
|
`UnifiedDelta`),字段名中立,不依赖具体上游协议。
|
||||||
|
|
||||||
|
```lua
|
||||||
|
-- anthropic.lua 的简化形式:把统一结果写成 Anthropic Messages 形状
|
||||||
|
function adapter.serialize_response(payload)
|
||||||
|
return json.encode({
|
||||||
|
id = payload.id,
|
||||||
|
type = "message",
|
||||||
|
role = "assistant",
|
||||||
|
content = { { type = "text", text = payload.content or "" } },
|
||||||
|
stop_reason = payload.finish_reason == "stop" and "end_turn" or nil,
|
||||||
|
usage = { input_tokens = payload.prompt_tokens, output_tokens = payload.completion_tokens },
|
||||||
|
})
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
注意:钩子返回的是**完整要发送的字节**(包括必要时的 `event:` / `data:` 前缀与
|
||||||
|
空行),因为只有适配器知道目标协议的 SSE 细节。
|
||||||
|
|
||||||
|
### 入向让出钩子(契约 v2,`serialize_*` 的对偶)
|
||||||
|
|
||||||
|
出向是「统一格式 → 客户端协议」,入向是它的镜像:「客户端协议 → 统一格式」。
|
||||||
|
内核为对外端点(如 `POST /v1/messages`)**保留一份内置解析作为兜底(R5)**,
|
||||||
|
但适配器可以**声明自己服务该端点**并接管解析——那时内核主动让出,客户端协议的
|
||||||
|
迭代就完全落在适配器里,不再需要动内核。
|
||||||
|
|
||||||
|
声明分两步:
|
||||||
|
|
||||||
|
1. `adapter.ingress = "/v1/messages"` —— 声明**认领**哪条对外入站路径。
|
||||||
|
同一条路径被多个适配器认领是配置错误;内核按适配器名字典序确定性选择。
|
||||||
|
2. `deserialize_request(raw_body) -> unified_json` —— 把客户端原始 body 解析成
|
||||||
|
统一请求(`UnifiedRequest` 的 JSON),后续调度与 `/v1/chat/completions` 完全一致。
|
||||||
|
|
||||||
|
规则:
|
||||||
|
|
||||||
|
- 只有声明了该端点且实现了 `deserialize_request` 的适配器会让内核让出。
|
||||||
|
- 没有适配器认领时,内核内置解析接管(未配置的部署仍能工作)。
|
||||||
|
- 认领了但钩子**执行失败**是硬错误(400),不会静默回落到另一种协议解析——
|
||||||
|
客户端说的正是这个适配器声称支持的协议,悄悄换解析器只会错误调度请求。
|
||||||
|
|
||||||
|
```lua
|
||||||
|
-- anthropic.lua 的简化形式:把 Anthropic Messages 请求解析成统一格式
|
||||||
|
adapter.ingress = "/v1/messages"
|
||||||
|
function adapter.deserialize_request(raw_body)
|
||||||
|
local req = json.decode(raw_body)
|
||||||
|
local msgs = {}
|
||||||
|
for _, m in ipairs(req.messages or {}) do
|
||||||
|
table.insert(msgs, { role = m.role, content = m.content })
|
||||||
|
end
|
||||||
|
return json.encode({
|
||||||
|
model = req.model, messages = msgs, max_tokens = req.max_tokens,
|
||||||
|
system = type(req.system) == "string" and req.system or "",
|
||||||
|
stream = req.stream == true,
|
||||||
|
})
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
## 5. 内置辅助函数
|
## 5. 内置辅助函数
|
||||||
|
|
||||||
所有适配器共享的全局函数(由 Go 注入):
|
所有适配器共享的全局函数(由 Go 注入):
|
||||||
@ -219,8 +295,13 @@ adapter.headers = { ["X-Tenant"] = "prod" }
|
|||||||
function adapter.transform_request(raw_body)
|
function adapter.transform_request(raw_body)
|
||||||
local ok, req = pcall(json.decode, raw_body)
|
local ok, req = pcall(json.decode, raw_body)
|
||||||
if not ok then return raw_body end
|
if not ok then return raw_body end
|
||||||
req.disable_thinking = nil
|
-- 契约 v2:内核不再向适配器泄漏私有字段,`thinking` 是中立语义字段。
|
||||||
req.extra_body = nil
|
-- 这里把中立的 thinking 映射成目标协议需要的形状(示例:OpenAI 家族用
|
||||||
|
-- reasoning_effort)。不关心推理的源可以直接忽略它。
|
||||||
|
if req.thinking ~= nil then
|
||||||
|
if req.thinking.enabled == false then req.reasoning_effort = "none" end
|
||||||
|
req.thinking = nil
|
||||||
|
end
|
||||||
return json.encode(req)
|
return json.encode(req)
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -268,11 +349,21 @@ sources:
|
|||||||
kind: chat
|
kind: chat
|
||||||
```
|
```
|
||||||
|
|
||||||
## 8. 多模态与 disable_thinking
|
## 8. 多模态与 thinking 预算
|
||||||
|
|
||||||
- **多模态**:请求的 `content` 若是数组(`[{type:"text"...},{type:"image_url"...}]`),Go 层
|
- **多模态**:请求的 `content` 若是数组(`[{type:"text"...},{type:"image_url"...}]`),Go 层
|
||||||
原样透传给适配器。协议不支持的源(Anthropic/Gemini/Ollama)需在
|
原样透传给适配器。协议不支持的源(Anthropic/Gemini/Ollama)需在
|
||||||
`transform_request` 内转换;内置 `anthropic.lua`/`gemini.lua`/`ollama.lua` 已实现。
|
`transform_request` 内转换;内置 `anthropic.lua`/`gemini.lua`/`ollama.lua` 已实现。
|
||||||
- **disable_thinking**:网关将请求体中的 `disable_thinking:true` 透传进
|
- **thinking**:客户端请求里的推理开关/预算被内核归一成中立的 `thinking` 字段:
|
||||||
`transform_request`。DeepSeek 适配器将其映射为 `extra_body.thinking = { type = "disabled" }`
|
|
||||||
并清理自身字段;其余适配器按各自协议处理(可自行实现或忽略)。
|
```json
|
||||||
|
{"thinking": {"enabled": true, "budget_tokens": 4096}}
|
||||||
|
```
|
||||||
|
|
||||||
|
`enabled=false` 表示关闭推理,`budget_tokens` 可选。适配器自行映射到目标协议
|
||||||
|
(内置实现:Anthropic → `thinking.budget_tokens`、Gemini →
|
||||||
|
`generationConfig.thinkingConfig.thinkingBudget`、DeepSeek →
|
||||||
|
`thinking.type = "disabled"`)。不支持的源直接忽略该字段。
|
||||||
|
|
||||||
|
契约 v1 的内核私有字段 `disable_thinking` 已被取代——它曾是内核细节泄漏进
|
||||||
|
适配器请求体的典型例子(迫使 10 个适配器各写一行 `req.disable_thinking = nil`)。
|
||||||
|
|||||||
60
docs/performance.md
Normal file
60
docs/performance.md
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
# 性能与内存
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来:实测内存占用、调优手段、thinking 预算。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## 内存占用(实测与调优)
|
||||||
|
|
||||||
|
内存不是一个定数:它随**配置的源数量**增长(每个源一个 `http.Transport` 与连接池、
|
||||||
|
一组健康状态),与运行时长无关。本机实测(Linux x86_64,12 核):
|
||||||
|
|
||||||
|
| 部署形态 | 启动 RSS | 稳态 RSS |
|
||||||
|
|---|---|---|
|
||||||
|
| 1 源 / 1 适配器(最小配置) | ~4 MB | ~10 MB |
|
||||||
|
| 1 源 + 29 MB 历史审计日志 | ~19 MB | ~20 MB |
|
||||||
|
| **16 源 / 13 适配器 / 59 模型(本机生产)** | ~28 MB | **~32–35 MB** |
|
||||||
|
|
||||||
|
> 历史参考:本项优化前同一生产配置为 **~105 MB**。降幅来自三处:审计日志的**原始记录**
|
||||||
|
> 不再常驻内存(约 25 MB;统计聚合仍扫全量,但扫完即释放)、Lua 状态池不再单调增长、
|
||||||
|
> 以及下面两个运行时开关。
|
||||||
|
|
||||||
|
内存构成(生产实例分段测量,`/proc/<pid>/smaps`):
|
||||||
|
|
||||||
|
| 区域 | RSS | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| Go 堆 | ~14 MB | provider/registry/scheduler 结构 + 连接池缓冲 |
|
||||||
|
| 其他匹名(线程栈 / LuaJIT chunk / runtime) | ~12 MB | 与线程数、已加载适配器数相关 |
|
||||||
|
| 二进制 text+rodata | ~8 MB | 映射的可执行文件页(只读、可被内核回收) |
|
||||||
|
| Go runtime 预留 | ~4 MB | VSZ 上看到的 2 GB+ 是地址空间预留,不占物理内存 |
|
||||||
|
| 共享库 | ~3 MB | libc / libluajit / libm |
|
||||||
|
|
||||||
|
**两个推荐的部署开关**(只适用于环境变量,无需改代码):
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# /etc/systemd/system/modelrouter.service
|
||||||
|
Environment=GOGC=50
|
||||||
|
Environment=MALLOC_ARENA_MAX=2
|
||||||
|
```
|
||||||
|
|
||||||
|
- `MALLOC_ARENA_MAX=2`:LuaJIT 的分配走 cgo → glibc malloc,glibc 默认允许 `8×nproc`
|
||||||
|
个 per-thread arena,每个碰到 malloc 的 OS 线程会占用一个(各约 1 MB,且**不归还给系统**)。
|
||||||
|
实测从 8–15 个 arena(约 7–12 MB)降到 **0**。
|
||||||
|
- `GOGC=50`:把 Go 堆增长目标减半。**单独使用无效**(省下的堆会立即被更多 glibc
|
||||||
|
arena 吃掉,实测 20.3 → 21.5 MB 反而变大),必须与 `MALLOC_ARENA_MAX` 配合,
|
||||||
|
两者同时开启才降 **~19%**。网关是 I/O 密集型(本机 9 小时仅消耗 1min10s CPU),
|
||||||
|
多出的 GC 周期与它的空闲 CPU 相比可忽略。
|
||||||
|
|
||||||
|
> 桌面版(Electron)已在 `cmd/gui/main.js` 里默认为内嵌核心注入这两个开关;手动 export 同名
|
||||||
|
> 环境变量可覆盖。如果你用自己的 systemd unit / 容器,建议照上面加上。
|
||||||
|
|
||||||
|
可选:`GOMEMLIMIT=48MiB` 作软上限,实测再省 ~1 MB;代价是逐近上限时 GC 转激进,
|
||||||
|
温和场景不必开。
|
||||||
|
|
||||||
|
## thinking 预算
|
||||||
|
|
||||||
|
请求体带 `"thinking": {"enabled": false}` 时关闭推理;`"thinking": {"enabled": true,
|
||||||
|
"budget_tokens": 4096}` 指定预算。内核只定义这个**中立**语义字段,映射到具体协议由
|
||||||
|
适配器负责(Anthropic `thinking.budget_tokens`、Gemini `thinkingConfig.thinkingBudget`、
|
||||||
|
DeepSeek `thinking.type = "disabled"`)。内核不再向适配器泄漏 `disable_thinking` /
|
||||||
|
`extra_body` 这类私有字段。
|
||||||
|
|
||||||
300
docs/plugins.md
300
docs/plugins.md
@ -15,6 +15,37 @@ ModelRouter 的插件是**单个 `.lua` 文件**,放在 `config.yaml` 的 `plu
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## 0. 插件 SDK:唯一的公开接口
|
||||||
|
|
||||||
|
插件与网关之间**只有一个**公开接口表 `modelrouter`。它不是 Go 包的导出符号,
|
||||||
|
而是内核在启动插件 VM 时注入的 Lua 表;插件通过它调用内核能力,永远不需要
|
||||||
|
(也不应该)依赖任何 `__llmsproxy_*` 全局、Go 结构体或内部函数名。
|
||||||
|
|
||||||
|
```lua
|
||||||
|
modelrouter.sdk_version -- 字符串,契约版本;SDK 形状变化才递增
|
||||||
|
modelrouter.http(spec) -- 受出站白名单约束的 HTTP 请求
|
||||||
|
modelrouter.secrets -- 本插件专用的凭据保险库(set/get/delete/keys)
|
||||||
|
modelrouter.kernel -- 内核动作面(actions/call,受运维授权约束)
|
||||||
|
modelrouter.adapters -- 安装/更新/删除本插件自己的适配器
|
||||||
|
modelrouter.plugin_dir -- 本插件所在目录(内核给出的 ground truth)
|
||||||
|
```
|
||||||
|
|
||||||
|
规则:
|
||||||
|
|
||||||
|
- **只依赖上表**。内核重命名内部全局、改实现或加守卫时,插件不受影响;
|
||||||
|
这正是把六个散落的全局收进一张表的原因。
|
||||||
|
- **能力可能缺失**。未授予的能力(例如出站 HTTP 全禁)在表里是 `nil` 而不是
|
||||||
|
"存在但一调用就报错",插件应先判断再使用。
|
||||||
|
- **`sdk_version` 用来做兼容分支**。增量字段不递增版本;字段删除或调用约定
|
||||||
|
变化才递增。
|
||||||
|
- **历史别名**:`llmsproxy`(旧产品名)与 `__llmsproxy_{http,secrets,kernel,
|
||||||
|
adapters,plugin_dir}` 仍然指向同一批对象,只为已安装的插件继续可用而保留,
|
||||||
|
已废弃;新插件使用 `modelrouter.*`。
|
||||||
|
- 环境变量:插件目录优先读 `MODELROUTER_PLUGIN_DIR`,回退旧名
|
||||||
|
`MODELROUTER_PLUGIN_DIR`。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## 1. 快速上手
|
## 1. 快速上手
|
||||||
|
|
||||||
一个最小的插件:
|
一个最小的插件:
|
||||||
@ -46,7 +77,7 @@ plugin.ui = {
|
|||||||
page = {
|
page = {
|
||||||
page_id = "hello", -- kebab-case
|
page_id = "hello", -- kebab-case
|
||||||
title = "Hello",
|
title = "Hello",
|
||||||
icon = "👋",
|
icon = "",
|
||||||
order = 90, -- 侧栏排序
|
order = 90, -- 侧栏排序
|
||||||
mount = [[<div id="hello">hello</div>]],
|
mount = [[<div id="hello">hello</div>]],
|
||||||
},
|
},
|
||||||
@ -65,16 +96,14 @@ return plugin -- 必须返回一个 table
|
|||||||
core.New
|
core.New
|
||||||
└─ lua.NewVM(adapter_dir).Start() 适配器状态
|
└─ lua.NewVM(adapter_dir).Start() 适配器状态
|
||||||
└─ lua.NewPlugins(vm, plugin_dir)
|
└─ lua.NewPlugins(vm, plugin_dir)
|
||||||
├─ SeedBundled() 仅当目录不存在时写入内置插件(目前是 billing)
|
`plugin_dir` 中的 Lua 文件由网关逐个加载;安装包不携带业务插件,也不会自动创建或写入插件文件。首次启用时,先把需要的 `.lua` 插件放入该目录,再启动网关。
|
||||||
└─ LoadDir() 按文件名字典序逐个加载
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**加载失败不影响网关启动。** 一个语法错误的插件会被记录在 `GET /api/plugins`
|
**加载失败不影响网关启动。** 一个语法错误的插件会被记录在 `GET /api/plugins`
|
||||||
的 `error` 字段里,永远不会被调用。这与适配器一致,但理由更强:插件是可选的
|
的 `error` 字段里,永远不会被调用。这与适配器一致,但理由更强:插件是可选的
|
||||||
第三方扩展,因为一个 `.lua` 打错字就让网关起不来是错误的取舍。
|
第三方扩展,因为一个 `.lua` 打错字就让网关起不来是错误的取舍。
|
||||||
|
|
||||||
**目录一旦存在就是权威的。** 与适配器同规则:首启会 seed 内置插件,之后目录里
|
插件目录由运维独立管理。网关不会 seed、覆盖或删除其中的插件;目录缺失时只会跳过插件加载,不影响网关启动。
|
||||||
的文件说了算,删除或编辑内置插件都是真实生效的操作。
|
|
||||||
|
|
||||||
### 2.1 热更新
|
### 2.1 热更新
|
||||||
|
|
||||||
@ -211,7 +240,7 @@ plugin.ui = {
|
|||||||
page = {
|
page = {
|
||||||
page_id = "billing", -- 必填,kebab-case。侧栏 data-tab 与 #tab-billing
|
page_id = "billing", -- 必填,kebab-case。侧栏 data-tab 与 #tab-billing
|
||||||
title = "Billing", -- 必填,侧栏文字
|
title = "Billing", -- 必填,侧栏文字
|
||||||
icon = "💰", -- 可选
|
icon = "", -- 可选
|
||||||
order = 40, -- 侧栏排序,默认 100
|
order = 40, -- 侧栏排序,默认 100
|
||||||
mount = [[...HTML...]],
|
mount = [[...HTML...]],
|
||||||
},
|
},
|
||||||
@ -294,27 +323,191 @@ PUT /api/plugins/{name}/state 替换状态(admin)
|
|||||||
"loaded": true
|
"loaded": true
|
||||||
}],
|
}],
|
||||||
"hook_errors": { "request_end": { "count": 3, "last_error": "billing: ..." } },
|
"hook_errors": { "request_end": { "count": 3, "last_error": "billing: ..." } },
|
||||||
"plugin_dir": "/etc/llmsproxy/plugins"
|
"plugin_dir": "/etc/modelrouter/plugins"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`hook_errors` 是**排错入口**:一个坏掉的插件表现为"功能缺失"而不是报错,
|
`hook_errors` 是**排错入口**:一个坏掉的插件表现为"功能缺失"而不是报错,
|
||||||
这个计数让它可见。
|
这个计数让它可见。
|
||||||
|
|
||||||
|
### 5.4 定时任务(`PLUGIN.schedule`)—— 浏览器关着也会跑
|
||||||
|
|
||||||
|
要求插件做“随时间发生”的事(定时刷新、token 轮换、指标采集)时,声明一个
|
||||||
|
调度表:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
plugin.schedule = {
|
||||||
|
{ fn = "refresh_token", every = 3600 }, -- 每小时(秒)
|
||||||
|
{ fn = "daily_refresh", every = 86400, at = "03:30" }, -- 每天固定时刻(本地时间)
|
||||||
|
{ fn = "poll_quota", every = 300, jitter = 30 }, -- 首次触发加 0-30s 抖动
|
||||||
|
}
|
||||||
|
function plugin.refresh_token()
|
||||||
|
-- 注意:直接写 plugin.state 就是对的 ——
|
||||||
|
-- 任务和钩子跑在同一个 VM 上
|
||||||
|
plugin.state.token = fetch_new_token()
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
| 字段 | 含义 |
|
||||||
|
|---|---|
|
||||||
|
| `fn` | 要调用的插件函数名(必须存在) |
|
||||||
|
| `every` | 间隔秒数。与 `at` 二选一 |
|
||||||
|
| `at` | 每天本地时间 `"HH:MM"`;设了它就忽略 `every` |
|
||||||
|
| `jitter` | 首次触发额外随机延迟秒数(避免多实例整点齐射) |
|
||||||
|
| `timeout` | 解析保留但目前**不生效**(见下) |
|
||||||
|
|
||||||
|
要点:
|
||||||
|
|
||||||
|
- **内核 ticker 驱动**,不是前端轮询:没有浏览器、没人登录也会执行。
|
||||||
|
- **任务跑在插件自己的 VM 上**,与请求钩子共用 `plugin.state` —— 任务写的
|
||||||
|
token 对下一个请求立即可见,也会经同一套机制持久化到 `.<name>.state.json`。
|
||||||
|
- 任务函数出错只记 `hook_errors`,不影响钩子与转发。
|
||||||
|
- 没有 `schedule` 的插件不获得任何额外 VM/goroutine(零开销)。
|
||||||
|
- **超时不能中断一个运行中的 Lua 调用**(本 LuaJIT 构建的硬限制)。任务超过
|
||||||
|
10 分钟不返回,该插件被**停用**(`disarmed`):不再调度,钩子快速失败并
|
||||||
|
记 `hook_errors`。这是为了不让一个卡死的任务把请求永久堵在它那把锁后面。
|
||||||
|
修好插件后重新加载(重传/重启)即可恢复。
|
||||||
|
|
||||||
|
> 同一条 6.1 的约束:任务占着插件锁,所以**任务里不要写死循环**。
|
||||||
|
|
||||||
|
### 5.5 自定义对外挂载点
|
||||||
|
|
||||||
|
插件的 HTTP 端点默认挂在 `/api/plugins/http/...` 下。同一个声明**同时**可在两个
|
||||||
|
公开命名空间下访问,只要把 `path` 的第一段起成插件名:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
plugin.http = {
|
||||||
|
{ path = "myplugin/callback", method = "POST", handler = "on_callback" },
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
该端点因此可在三个地方访问:
|
||||||
|
|
||||||
|
| 路径 | 用途 |
|
||||||
|
|---|---|
|
||||||
|
| `/api/plugins/http/myplugin/callback` | 默认前缀(带登录态的管理 UI) |
|
||||||
|
| `/plugins/myplugin/callback` | 公开(浏览器路径) |
|
||||||
|
| `/v1/plugins/myplugin/callback` | 公开(API 路径) |
|
||||||
|
|
||||||
|
可占用的命名空间只有后两个:
|
||||||
|
|
||||||
|
| 前缀 | 例子 |
|
||||||
|
|---|---|
|
||||||
|
| `/plugins/` | `/plugins/myplugin/callback` |
|
||||||
|
| `/v1/plugins/` | `/v1/plugins/myplugin/callback` |
|
||||||
|
|
||||||
|
限制(都是为了不把内核路由变成可被插件改写的):
|
||||||
|
|
||||||
|
- **不能影子内核路由**:与 `/v1/chat/completions`、`/v1/messages`、`/api/...`
|
||||||
|
等重叠的挂载在加载时就被拒绝,错误信息点名冲突对象。
|
||||||
|
- **只能是上面两个命名空间**,不接受任意前缀(否则可以拦截推理请求)。
|
||||||
|
- 鉴权沿用端点的 `roles` / `keys` / `public` 语义(见 §5.1);
|
||||||
|
`/v1/plugins/...` 属于 API 路径,未认证返回 401;`/plugins/...` 为浏览器
|
||||||
|
路径,未认证会跳登录页。
|
||||||
|
- 未声明的路径不会因为落在允许命名空间而被放行——不在 handler 表里就是 404。
|
||||||
|
|
||||||
|
### 5.6 前缀代理:把插件自起的服务挂到网关同端口
|
||||||
|
|
||||||
|
前节的自定义挂载点只能回答**由插件计算出来的内容**(`{status, headers, body/json}`)。
|
||||||
|
若插件自己起了一个 TCP/HTTP 服务(典型:服务端浏览器栈 + noVNC),客户端就需要
|
||||||
|
**字节级转发**,包括 **WebSocket 升级** —— 前者做不到。
|
||||||
|
|
||||||
|
为此声明 `prefix = true`,handler 每次请求返回一个**回环目标**:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
plugin.http = {
|
||||||
|
{ path = "myplugin/proxy/:session", method = "GET", prefix = true,
|
||||||
|
roles = { "admin" }, handler = "proxy" },
|
||||||
|
}
|
||||||
|
|
||||||
|
function plugin.proxy(req)
|
||||||
|
-- 每次请求都问一遍:会话没了/过期了就返回 404,地址自然失效
|
||||||
|
if not sessionAlive(req.params.session) then
|
||||||
|
return { status = 404, json = { error = "no such session" } }
|
||||||
|
end
|
||||||
|
return { proxy = "http://127.0.0.1:60100" }
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
规则:
|
||||||
|
|
||||||
|
- **只能返回 `{ proxy = <目标> }`**,没有 `status` / `body`。写了 `status` 会被忽略。
|
||||||
|
- 目标必须是 **带显式端口的字面回环 IP**(`http://127.0.0.1:60100`)。
|
||||||
|
拒绝 `https://`、`localhost` 这类主机名、无端口、带 userinfo/path/query 的地址。
|
||||||
|
理由:主机名可能被 hosts / DNS / 代理环境改写;通用能力只接受确定性地址。
|
||||||
|
- 路由匹配同前缀下的**所有子路径**,`:param` 仍照常捕获,`req.path` 是完整
|
||||||
|
请求路径,子路径由插件自己从 `req.path` 里切。
|
||||||
|
- **鉴权先于代理**:`roles` / `keys` / `public` 语义与普通端点一致。
|
||||||
|
- **不转发网关凭据**:`Authorization`、`Cookie`、`api_key` 都不会传给后端服务。
|
||||||
|
- **拒绝跨站**:`Origin` 与网关自身 host 不一致返回 403。WebSocket 由浏览器
|
||||||
|
发起且自动带 Cookie,否则第三方网页可借管理员会话开隧道。
|
||||||
|
- **撤销点唯一**:handler 不再返回目标(或返回 404)即失效,内核不缓存目标。
|
||||||
|
|
||||||
|
客户端拿到的 URL 是**相对网关口**的绝对路径(如
|
||||||
|
`/api/plugins/http/myplugin/proxy/<id>/vnc.html`),因此远程访问、HTTPS 入口、
|
||||||
|
Cookie 鉴权都自然生效,无需暴露额外端口。
|
||||||
|
|
||||||
|
### 5.7 在运行时安装/更新适配器(token 轮换)
|
||||||
|
|
||||||
|
插件可以在运行时安装、改写**自己的**适配器 —— 这是"带自动轮换的认证"的实现
|
||||||
|
方式:拿着新 token 重写适配器源码,后续请求自然使用新逻辑(内核重建该适配器
|
||||||
|
的池)。**可直接在插件 `.lua` 里调用**:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
-- 每小时的定时任务:轮换 token 并重写适配器
|
||||||
|
function plugin.refresh_token()
|
||||||
|
local tok = fetch_new_token() -- 插件自己的取 token 逻辑
|
||||||
|
local name, err = modelrouter.adapters.install(
|
||||||
|
"myauth",
|
||||||
|
'return { name = "myauth", endpoint = "/chat/completions",' ..
|
||||||
|
' headers = { ["Authorization"] = "Bearer ' .. tok .. '" } }'
|
||||||
|
)
|
||||||
|
if err then log("error", err) end
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
| 调用 | 参数 | 返回 |
|
||||||
|
|---|---|---|
|
||||||
|
| `modelrouter.adapters.install(name, source)` | 适配器名 + Lua 源码 | `(生效名, nil)` 或 `(nil, err)` |
|
||||||
|
| `modelrouter.adapters.update(name, source)` | 同上(只限自己装的) | `(true, nil)` 或 `(nil, err)` |
|
||||||
|
| `modelrouter.adapters.remove(name)` | 适配器名(只限自己装的) | `(true, nil)` 或 `(nil, err)` |
|
||||||
|
|
||||||
|
出错时不抛异常,而是返回 `(nil, err)`——与钩子/HTTP 处理器同一套约定,
|
||||||
|
一个手滑的插件错误不会带走网关。
|
||||||
|
|
||||||
|
安全规则(由 Go 侧强制,不是约定):
|
||||||
|
|
||||||
|
- 插件**只能改自己装的**适配器(归属表在 Go 侧记录,调用身份是插件启动时的
|
||||||
|
固定名字,无法冒充)。
|
||||||
|
- 插件**不能影子内置适配器**(否则等于篡改发行物);运维仍可自行覆盖。
|
||||||
|
- 名字经过路径安全校验(无穿越/绝对路径/空白)。
|
||||||
|
|
||||||
|
> 这条能力配合 §5.4 的定时任务,就是完整的自动轮换:**没有浏览器、没有人
|
||||||
|
> 操作时,插件按时间自己把 token 换掉**。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. 运行时约束(重要)
|
## 6. 运行时约束(重要)
|
||||||
|
|
||||||
### 6.1 一个插件 = 一个 Lua 状态
|
### 6.1 一个插件 = 一个 Lua 状态(也就是一个 VM)
|
||||||
|
|
||||||
适配器可以有多个独立的 worker 状态(它们无状态,这是对的)。**插件不行**:
|
适配器可以有多个独立的 worker 状态(它们无状态,这是对的)。**插件不行**:
|
||||||
钩子通常往 `plugin.state` 里累加,多个状态就意味着总额被劈成几份,而写价格只
|
钩子通常往 `plugin.state` 里累加,多个状态就意味着总额被劈成几份,而写价格只
|
||||||
写进其中一个状态,钩子恰好跑到另一个时**所有请求按 0 计费**。
|
写进其中一个状态,钩子恰好跑到另一个时**所有请求按 0 计费**。
|
||||||
|
|
||||||
因此插件持有**唯一一个**状态,所有入口(`Fire` / `State` / `SetState`)用互斥锁
|
因此插件持有**唯一一个**状态,所有入口(`Fire` / `State` / `SetState` / **定时任务**)
|
||||||
串行化。代价:**一个卡住的钩子会卡住所有插件的钩子**。所以——
|
都用这同一把互斥锁串行化。
|
||||||
|
|
||||||
> **钩子必须短、同步、不阻塞。** 不要在里面做 HTTP 请求、睡眠或重计算。
|
注意: 这条约束对**定时任务尤其关键**:任务和钩子必须是同一个 VM,否则任务写的
|
||||||
|
`plugin.state` 是一个请求路径永远读不到的副本 —— token 轮换看起来成功了,
|
||||||
|
对实际流量完全无效。
|
||||||
|
|
||||||
|
代价:**一个卡住的钩子会卡住该插件自己的钩子**(跨插件仍然并行,每插件各自
|
||||||
|
一把锁、各自一个 VM)。所以——
|
||||||
|
|
||||||
|
> **钩子与定时任务都必须短、同步、不阻塞。** 不要在钩子里做 HTTP 请求、睡眠
|
||||||
|
> 或重计算;长耗时的周期工作应放进 `PLUGIN.schedule`(见 §5.4),它在自己的
|
||||||
|
> 调度 goroutine 上跑,但仍然持有这把锁,因此不能是死循环。
|
||||||
|
|
||||||
### 6.2 异常被隔离
|
### 6.2 异常被隔离
|
||||||
|
|
||||||
@ -326,28 +519,70 @@ PUT /api/plugins/{name}/state 替换状态(admin)
|
|||||||
与适配器相同:`json.encode` / `json.decode`、`log(level, msg)`、
|
与适配器相同:`json.encode` / `json.decode`、`log(level, msg)`、
|
||||||
`hmac_sha256_hex` / `sha256_hex` / `base64_encode` / `tohex`。
|
`hmac_sha256_hex` / `sha256_hex` / `base64_encode` / `tohex`。
|
||||||
|
|
||||||
⚠️ `os.date` 在极简运行时里可能不可用(计费插件因此有降级路径)。
|
注意: `os.date` 在极简运行时里可能不可用(计费插件因此有降级路径)。
|
||||||
|
|
||||||
### 6.4 命名
|
### 6.4 命名
|
||||||
|
|
||||||
- 全局 `__llmsproxy_plugin` 存放插件返回的 table,**不要占用**
|
- 全局 `__llmsproxy_plugin` 是内核存放插件返回 table 的**内部位置**,不属于 SDK,**不要占用**
|
||||||
- 插件间状态完全隔离,一个插件改不了另一个的全局
|
- 插件间状态完全隔离,一个插件改不了另一个的全局
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 7. 内置计费插件
|
## 7. Billing 插件示例
|
||||||
|
|
||||||
`plugins/billing.lua`,默认随首启 seed 进来,开箱可用。
|
`internal/lua/plugins/billing.lua` 是仓库中的可选插件示例,不随核心安装包分发,也不会在启动时自动安装。需要使用计费功能时,应将该脚本单独复制到配置的 `plugin_dir` 后再重启网关。
|
||||||
|
|
||||||
### 7.1 计费维度
|
### 7.0 分层:billing 完全在插件里,内核没有任何 billing 代码
|
||||||
|
|
||||||
| 维度 | 用途 | 优先级 |
|
`billing` 是**插件自己的事**:规则文件、编译、计价、账目、规则编辑页都由
|
||||||
|
`plugins/billing.lua` 负责。内核里没有 billing 的配置结构、端点、编译或迁移逻辑
|
||||||
|
—— 它只提供所有插件通用的能力(加载 `.lua`、插件状态持久化、插件目录、
|
||||||
|
`plugin.http` 路由、插件可用的 `io`/`os`)。
|
||||||
|
|
||||||
|
| 项 | 位置 |
|
||||||
|
|---|---|
|
||||||
|
| 规则文件 | `<plugin_dir>/billing.json`(插件自己读写;内核不知道这个文件) |
|
||||||
|
| 计费账目 | 插件状态文件 `<plugin_dir>/.billing.state.json`(内核的通用状态持久化) |
|
||||||
|
| 规则编辑 API | `/api/plugins/http/billing/rules`(GET/PUT/POST/DELETE,插件路由) |
|
||||||
|
| 编译/计价 | 插件内 `compileRules()` / `priceFor()` / `costFor()` |
|
||||||
|
|
||||||
|
规则以**源名**(`source`)为键,而不是 URL:URL→源名的映射属于内核配置,插件
|
||||||
|
不需要为了它和内核对接;编辑页在浏览器里,直接取 `/api/sources` 给运营选源名。
|
||||||
|
`url: "*"` 只表示"全局模型价目表"(模型维度本来就不分源)。
|
||||||
|
|
||||||
|
> 历史:早期 `billing:` 写在 `config.yaml`、编译在 Go 的 `internal/billing` 里。
|
||||||
|
> GitCode issue #1 的根因正是"Go 与 Lua 各写一半覆盖语义",组合起来把模型价
|
||||||
|
> 静默清零。现在内核侧 billing 代码已全部删除,规则文件也由插件自持。
|
||||||
|
|
||||||
|
### 7.1 计费维度### 7.1 计费维度
|
||||||
|
|
||||||
|
| 维度 | 用途 | token 价覆盖顺序 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `sources` | 源的固定价(如按请求计费的转发服务) | 中 |
|
| `default` | 兜底价(通常全 0) | 最低 |
|
||||||
| `models` | 模型的 per-token 价 | 高 |
|
| `models` | 模型的 per-token 价 | 中 |
|
||||||
|
| `sources` | 源级价:固定价,或覆盖该源全部模型的 token 价 | 高 |
|
||||||
| `keys` | 单个网关 key 的覆盖价 | **最高** |
|
| `keys` | 单个网关 key 的覆盖价 | **最高** |
|
||||||
|
|
||||||
**token 价**优先级:`keys` > `models` > `default`。
|
**token 价**优先级:`keys` > `sources` > `models` > `default`。
|
||||||
|
(实现顺序就是"后写覆盖前写":`default` → `models` → `sources` → `keys`;
|
||||||
|
所以**源级 token 价会覆盖模型价**。这是有意的:运营要能给某个源整体改价。)
|
||||||
|
|
||||||
|
⚠️ 正因为 `sources` 会覆盖 `models`:**`mode: token` 的规则不产生任何源条目**。
|
||||||
|
规则未列出的模型会进入未定价统计(可见),而不是静默计 0 —— 这是 issue #1 / PR #2
|
||||||
|
的收口语义。只有 `free` / `subscription`("边际成本为零")会写有意为之的
|
||||||
|
`{prompt:0, completion:0}` 零价条目。
|
||||||
|
|
||||||
|
规则写法:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"active":"current","profiles":[{"id":"current","default":"unpriced","rules":[
|
||||||
|
{"source":"localzen","mode":"free"},
|
||||||
|
{"source":"codearts","mode":"token",
|
||||||
|
"models":{"glm-5.3":{"prompt":"0.15","completion":"0.60"}}},
|
||||||
|
{"url":"*","mode":"token","models":{"some-model":{"prompt":"1.0","completion":"2.0"}}}
|
||||||
|
]}]}
|
||||||
|
```
|
||||||
|
|
||||||
**`per_request` 固定价是叠加的**(不覆盖),所以一个生图模型可以既算 token 又收固定费。
|
**`per_request` 固定价是叠加的**(不覆盖),所以一个生图模型可以既算 token 又收固定费。
|
||||||
**`cache_discount`** 见 §7.7。
|
**`cache_discount`** 见 §7.7。
|
||||||
|
|
||||||
@ -413,7 +648,7 @@ curl -H "Authorization: Bearer $ADMIN_KEY" \
|
|||||||
{ "prices": { ... }, "count_failures": true }
|
{ "prices": { ... }, "count_failures": true }
|
||||||
```
|
```
|
||||||
|
|
||||||
### 7.5 ⚠️ 计费插件只报表,不执法
|
### 7.5 注意: 计费插件只报表,不执法
|
||||||
|
|
||||||
**网关自己的配额会计(`internal/gateway/stats.go`,入口处强制)才是限额权威。**
|
**网关自己的配额会计(`internal/gateway/stats.go`,入口处强制)才是限额权威。**
|
||||||
本插件不参与任何路由或配额决策。
|
本插件不参与任何路由或配额决策。
|
||||||
@ -435,6 +670,17 @@ curl -H "Authorization: Bearer $ADMIN_KEY" \
|
|||||||
|
|
||||||
仪表盘上有 "Unpriced" 卡片。**这个数应该是 0**;不是 0 就去补价目。
|
仪表盘上有 "Unpriced" 卡片。**这个数应该是 0**;不是 0 就去补价目。
|
||||||
|
|
||||||
|
**profile 级兜底**:`profiles[].default` 可设为 `free` 或 `unpriced`(默认)。
|
||||||
|
它是插件运行时的策略(编译期写入 `prices.default_mode`):
|
||||||
|
|
||||||
|
| `default` | 没有单独规则的源 | `unpriced_reqs` |
|
||||||
|
|---|---|---|
|
||||||
|
| `unpriced`(省略时) | 零价,计入未定价债务 | +1 |
|
||||||
|
| `free` | 零价,视为**刻意免费** | 不增加 |
|
||||||
|
|
||||||
|
显式 `mode: unpriced` 的源即使在 `default: free` 下也仍然计入("只统计不收费"
|
||||||
|
的声明不能被兜底策略吞掉)。
|
||||||
|
|
||||||
注意"未定价"不等于"免费":这些请求的 `requests` / token 数**照常计入**
|
注意"未定价"不等于"免费":这些请求的 `requests` / token 数**照常计入**
|
||||||
`total` 与各维度,只有金额是 0。
|
`total` 与各维度,只有金额是 0。
|
||||||
|
|
||||||
@ -487,7 +733,7 @@ cached = cache_hit_tokens → 全价 × cache_discount
|
|||||||
插件的 `log(level, msg)` 输出到网关的日志:
|
插件的 `log(level, msg)` 输出到网关的日志:
|
||||||
|
|
||||||
```
|
```
|
||||||
journalctl -u llmsproxy -f | grep -i plugin
|
journalctl -u modelrouter -f | grep -i plugin
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@ -497,10 +743,12 @@ journalctl -u llmsproxy -f | grep -i plugin
|
|||||||
| | 适配器 | 插件 |
|
| | 适配器 | 插件 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| 位置 | `adapter_dir` | `plugin_dir` |
|
| 位置 | `adapter_dir` | `plugin_dir` |
|
||||||
| 作用 | 上游协议转换 | 请求流水线 + 界面 |
|
| 作用 | 上游协议转换 | 请求流水线 + 界面 + 周期任务 |
|
||||||
| 钩子 | `transform_request` / `transform_response` / `transform_stream_chunk` / `build_headers` / `transform_error` | `request_start` / `routed` / `request_end` |
|
| 钩子 | `transform_request` / `transform_response` / `transform_stream_chunk` / `build_headers` / `transform_error` / `serialize_*` | `request_start` / `routed` / `request_end` |
|
||||||
| 状态 | 每个源独立,多 worker | 单状态,见 §6.1 |
|
| 周期任务 | **无**(无状态请求级) | `PLUGIN.schedule`(内核 ticker 驱动) |
|
||||||
| 出错后果 | 该 (源,模型) 进入冷却退避 | 仅该功能缺失 |
|
| 对外端点 | 不开放(见 §5.5) | `/api/plugins/http/...` + 自定义挂载 |
|
||||||
|
| 状态 | 每个源独立,多 worker | **单 VM、单状态**,见 §6.1 |
|
||||||
|
| 出错后果 | 该 (源,模型) 进入冷却退避 | 仅该功能缺失;卡死则停用该插件 |
|
||||||
| 鉴权 | 需要 `gateway_keys` 之外的独立凭据 | 网关 key |
|
| 鉴权 | 需要 `gateway_keys` 之外的独立凭据 | 网关 key |
|
||||||
| 必需性 | 核心 | 可选,缺了网关照常跑 |
|
| 必需性 | 核心 | 可选,缺了网关照常跑 |
|
||||||
|
|
||||||
|
|||||||
111
docs/routing.md
Normal file
111
docs/routing.md
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
# 路由与调度
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来:模型路由、AUTO 链、冷却与半冷却、源与适配器、源模板。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## 模型路由
|
||||||
|
|
||||||
|
`/v1/chat/completions` 的 `model` 解析顺序:
|
||||||
|
|
||||||
|
1. `source/model` 或 `source:model` 前缀 → 指定源;
|
||||||
|
2. 精确匹配某个源的 `model`;
|
||||||
|
3. 配置 `default_model: AUTO` 时 → 走优先级页保存的 AUTO 链(见下);
|
||||||
|
4. 否则回落(返回错误)。
|
||||||
|
|
||||||
|
任何 OpenAI 客户端,只要 `model` 设为某个源的 `name/任意名`,即可锁定走该源;
|
||||||
|
设为 `AUTO`(或网关配了 `default_model: AUTO`)即自动按优先级链选源。
|
||||||
|
|
||||||
|
## AUTO 链(优先级页)
|
||||||
|
|
||||||
|
AUTO 调度**只**由优先级页保存的规则决定,源配置里的 `models[].priority` 数字不再
|
||||||
|
参与调度、也不再显示。**聊天与生图是两套独立的链**,在优先级页面通过「聊天 / 生图」
|
||||||
|
按钮切换编辑:
|
||||||
|
|
||||||
|
- **聊天链**(`runtime_file` 的 `auto` 字段):服务 `POST /v1/chat/completions`
|
||||||
|
的 `model: AUTO` 请求;只接受 `kind: chat` 模型,配置在链里的生图槽位会被忽略。
|
||||||
|
- **生图链**(`runtime_file` 的 `auto_image` 字段):服务 `POST /v1/images/generations`
|
||||||
|
的 `model: AUTO` 请求;只接受 `kind: image` 模型。未配置生图链时,AUTO 生图回落
|
||||||
|
到「所有含生图模型的源、按注册顺序」的旧行为,兼容旧配置。
|
||||||
|
|
||||||
|
每条规则 = 一个「槽位」:`{ model, source, tier, token_quota, period, hours }`。
|
||||||
|
|
||||||
|
- `tier` 表示优先级档位:同一档的模型并排、共享该优先级;档位从上到下递减。
|
||||||
|
- 同一模型可配置多个槽位(如 A 源低配 → B 源低配 → A 源高配 → B 源高配),按档位顺延。
|
||||||
|
- `token_quota` > 0 时该槽位在重置周期内用满即顺延到下一槽位;`period` 支持
|
||||||
|
`hour` / `week` / `month` / `nhour`(配合 `hours`),空 = 不限。
|
||||||
|
- 同 tier 内按「偏好分」排序,失败槽位冷却后自动跳过;冷却 / 配额耗尽 / hard
|
||||||
|
错误会顺延到下一 tier,全部失败时返回 503 并附上每档失败摘要。
|
||||||
|
|
||||||
|
## 冷却与半冷却探测(自愈调度)
|
||||||
|
|
||||||
|
槽位失败后进入指数退避冷却(5s → 10s → 20s …,上限 **5 分钟**)。冷却不是全时段
|
||||||
|
硬闸:
|
||||||
|
|
||||||
|
- **前半段完全静默**:刚失败的槽位不接任何流量,避免对故障上游持续施压。
|
||||||
|
- **后半段放行 1 个探测**:窗口过半后,同一 `(源, 模型)` 最多允许 **一个** 请求作为
|
||||||
|
探测通过。探测槽数 = `clamp(max_concurrent / 10, 1, 2)`——`max_concurrent: 10`
|
||||||
|
的源恰好放行 1 个探测请求。
|
||||||
|
- **探测排在最后**:同 tier 内探测候选永远排在健康候选之后,只有没有正常槽可用时
|
||||||
|
才承接真实流量,因此探测不会抢走可用容量。
|
||||||
|
- **探测即真实请求**:成功则立即清零失败计数与冷却,槽位当场回到正常轮询——
|
||||||
|
上游恢复/额度恢复后不必等满整个冷却窗口。失败则重开一个新窗口,下次探测顺延到
|
||||||
|
新窗口的中点,不会连续重试。
|
||||||
|
|
||||||
|
分类冷却策略:
|
||||||
|
|
||||||
|
| 情况 | 冷却 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| 传输错误 / 5xx | 5s→10s→20s…,上限 5min | 指数退避 |
|
||||||
|
| 401 / 403 凭据错误 | 10min | 换 key 后由下一次探测自动接回 |
|
||||||
|
| 429 限流 | 30s 固定 | 「太快」不等于「坏了」,不进指数阶梯 |
|
||||||
|
| 配额耗尽(`insufficient_quota` / `余额不足` 等) | 对齐配额窗口,上限 30min | **不进指数阶梯、不涨失败计数**,额度恢复即可用 |
|
||||||
|
|
||||||
|
WebUI「优先级」页的槽位健康标签会区分 `冷却` / `待探测` / `探测中`,悬停可看窗口
|
||||||
|
起点、探测放行时刻与完全恢复时刻。
|
||||||
|
|
||||||
|
## 源(Source)与适配器(Adapter)的关系
|
||||||
|
|
||||||
|
- **源** 是到某个上游的连接描述:`name`、`base_url`、`api_key`、模型列表与优先级。
|
||||||
|
- **适配器** 是协议转换逻辑(Lua 脚本):把统一 OpenAI 格式请求转成上游原生格式,
|
||||||
|
再把上游响应/流式分块转回统一格式。
|
||||||
|
- 一个适配器可被多个源复用(如 `openai.lua` 同时服务多个 OpenAI 兼容站点);
|
||||||
|
同一个源也可以切换不同适配器(改 `adapter` 字段即可)。
|
||||||
|
- 源决定“连谁、暴露哪些模型”,适配器决定“怎么对话”——二者在 `sources[]` 条目中
|
||||||
|
通过 `adapter` 字段关联。
|
||||||
|
|
||||||
|
加载流程(`internal/core` 的 `Core` 负责装配):
|
||||||
|
|
||||||
|
1. 启动时 `lua.NewVM(adapter_dir)` 加载全部适配器:内置适配器(编译期 embed)+
|
||||||
|
`adapter_dir` 下的同名覆盖文件;内置文件写在代码内,覆盖文件要求更高优先级。
|
||||||
|
2. `config.Load` 读取 `config.yaml`,`config.NewStore(runtime_file)` 读 WebUI 改动的
|
||||||
|
运行时源,二者按名称合并成完整源列表。
|
||||||
|
3. `rebuildRegistry` 为每个源创建 `provider.Provider`(持有目标适配器),并按源的
|
||||||
|
并发上限配置适配器 worker 池**上限**(不预分配,见下)。
|
||||||
|
4. 请求进来时 `Registry` 按 `model` 路由到 Provider,Provider 调适配器
|
||||||
|
`transform_request` → HTTP 发送 → `transform_response` / `transform_stream_chunk`。
|
||||||
|
|
||||||
|
WebUI 上的"新增/编辑源"、"上传 Lua 适配器"、"改 AUTO 优先级链"与"密钥模型范围"
|
||||||
|
都即时生效(写入运行时文件或 `adapter_dir` 后重新装配,无需重启);直接编辑
|
||||||
|
`config.yaml` / `adapter_dir` 下的文件则需要重启进程才会重新加载。
|
||||||
|
|
||||||
|
在 WebUI 删除源/适配器时:运行时(WebUI 创建)的源与 `.lua` 文件会真正移除;
|
||||||
|
`config.yaml` 中定义的基源无法改写配置文件,采用删除标记隐藏(重启后仍隐藏),
|
||||||
|
在 UI 里重新添加同名源即可恢复。
|
||||||
|
|
||||||
|
## 源模板(多 key 负载均衡)
|
||||||
|
|
||||||
|
多个上游 key 共享同一套配置(URL / 适配器 / 模型列表 / 并发 / rpm)时,逐一复制
|
||||||
|
整段 source 太冗余。**源模板**保存除 `name` 和 `api_key` 以外的全部源字段,
|
||||||
|
从模板一键创建多个仅 key 不同的源,天然支持多 key 负载均衡(每个展开源独立调度、
|
||||||
|
独立冷却、独立健康状态)。
|
||||||
|
|
||||||
|
- WebUI「源」页右上角「**模板管理**」按钮:列出全部模板,可编辑 / 删除 / 新建。
|
||||||
|
- 添加源弹窗右上角两个按钮:
|
||||||
|
- **从模板创建** → 选模板 → 表单自动填充(name + key 仍需你填);
|
||||||
|
- **存为模板** → 把当前表单非 key/name 字段存为新模板。
|
||||||
|
- 模板持久化在 `runtime_file` 的 `source_templates` 字段,与运行时源一样热管理,
|
||||||
|
改完即时生效。模板本身不含密钥,可以安全共享 / 纳管。
|
||||||
|
|
||||||
|
模板不会自己变成源——它只是「配方」。真正承担流量的是从模板创建出来的、带 key
|
||||||
|
的具体源。
|
||||||
|
|
||||||
23
docs/webui.md
Normal file
23
docs/webui.md
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
# WebUI
|
||||||
|
|
||||||
|
> 本文档由 README 拆分而来。
|
||||||
|
> 返回 [README](../README.md)。
|
||||||
|
|
||||||
|
## WebUI
|
||||||
|
|
||||||
|
内置管理界面(`GET /`),登录后可在浏览器完成:
|
||||||
|
|
||||||
|
- **状态页**:源在线状态(实时探测 + 悬停看错误)、模型/源/key 用量统计、请求记录,
|
||||||
|
支持按时间范围导出 CSV;点击模型可生成 pin 到该模型的连接配置。
|
||||||
|
- **对话页**:流式/非流式调试。
|
||||||
|
- **密钥页**:创建/编辑网关 key,为每个 key 配模型范围(模型 + 源 + token 配额 + 周期),
|
||||||
|
管理员管理全部 key,用户只看到自己的 key。每个模型砖块显示自己的配额徽标
|
||||||
|
(如 `1.0K 77×·1h`,未设配额显示 `∞`),点开可编辑该模型的 token 配额、
|
||||||
|
请求数配额与重置周期 —— 配额按模型独立生效,一个用满不影响同一 key 的其它模型。
|
||||||
|
「我的密钥」页对用户展示本 key 的模型范围与各自配额。
|
||||||
|
- **优先级页**:拖拽积木配置 AUTO 链档位。
|
||||||
|
- **源页**:在线增删改上游源(API key 等敏感字段加密落盘)。
|
||||||
|
- **适配器页**:上传 / 删除 Lua 适配器脚本。
|
||||||
|
|
||||||
|
改动写入 `runtime_file`(重启仍生效)。
|
||||||
|
|
||||||
286
e2e/adapter_takeover_test.go
Normal file
286
e2e/adapter_takeover_test.go
Normal file
@ -0,0 +1,286 @@
|
|||||||
|
package e2e
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 本文件端到端验证 R2/R5 的核心主张:**对外服务的 wire 格式由适配器决定,
|
||||||
|
// 内核只调度**;适配器不声明时内核的 OpenAI 兜底仍然可用。
|
||||||
|
//
|
||||||
|
// 与其他测试的分工:
|
||||||
|
// - internal/gateway/messages_test.go 在**进程内**验证同一件事(起 core+gateway,
|
||||||
|
// 不打真二进制)。它跑得快,但绕过了一些只在真实进程里发生的事:Lua 适配器的
|
||||||
|
// 磁盘 seed、配置加载、systemd 式启动路径、真 HTTP 栈。
|
||||||
|
// - 本文件打**真编译出的二进制**,用一个只说 Anthropic Messages 的假上游,
|
||||||
|
// 证明整条链路(真进程 → 真 Lua 适配器 → 真上游 → 适配器序列化 → 客户端)
|
||||||
|
// 确实成立,而不是只在进程内装配下成立。
|
||||||
|
//
|
||||||
|
// 这是用户明确问到的"适配器接手对外服务的部分"的验收点。
|
||||||
|
|
||||||
|
// anthropicMockUpstream 是只懂 Anthropic Messages 协议的假上游。
|
||||||
|
//
|
||||||
|
// 它会对**不符形状的请求主动报错**,而不是宽容接受 —— 宽容的 mock 会让
|
||||||
|
// "内核把 OpenAI 形状发给了 Anthropic 上游"这种错误测试通过。
|
||||||
|
func anthropicMockUpstream(t *testing.T) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Anthropic 的请求路径必须是 <base>/messages
|
||||||
|
if !strings.HasSuffix(r.URL.Path, "/messages") {
|
||||||
|
t.Errorf("请求打到了非 Anthropic 路径: %s", r.URL.Path)
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
var probe map[string]interface{}
|
||||||
|
if err := json.Unmarshal(body, &probe); err != nil {
|
||||||
|
t.Errorf("上游收到非 JSON 请求体: %s", body)
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Anthropic 形状的硬性要求:messages 是数组;不能有 OpenAI 的 choices 概念
|
||||||
|
if _, ok := probe["messages"]; !ok {
|
||||||
|
t.Errorf("上游请求体缺 messages: %s", body)
|
||||||
|
}
|
||||||
|
if _, ok := probe["max_tokens"]; !ok {
|
||||||
|
t.Errorf("Anthropic 要求 max_tokens,请求体缺此项: %s", body)
|
||||||
|
}
|
||||||
|
// 反向断言:OpenAI 兜底字段不该出现在 Anthropic 上游请求里
|
||||||
|
for _, k := range []string{"choices", "chat_completions"} {
|
||||||
|
if _, bad := probe[k]; bad {
|
||||||
|
t.Errorf("Anthropic 上游请求体混入了 OpenAI 字段 %q: %s", k, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if stream, _ := probe["stream"].(bool); stream {
|
||||||
|
w.Header().Set("Content-Type", "text/event-stream")
|
||||||
|
fl, _ := w.(http.Flusher)
|
||||||
|
frames := []string{
|
||||||
|
`event: message_start` + "\n" + `data: {"type":"message_start","message":{"id":"msg_e2e","type":"message","role":"assistant","model":"claude-e2e","content":[],"stop_reason":null,"usage":{"input_tokens":3,"output_tokens":0}}}`,
|
||||||
|
`event: content_block_start` + "\n" + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
|
||||||
|
`event: content_block_delta` + "\n" + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}`,
|
||||||
|
`event: content_block_stop` + "\n" + `data: {"type":"content_block_stop","index":0}`,
|
||||||
|
`event: message_delta` + "\n" + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}`,
|
||||||
|
`event: message_stop` + "\n" + `data: {"type":"message_stop"}`,
|
||||||
|
}
|
||||||
|
for _, f := range frames {
|
||||||
|
fmt.Fprint(w, f+"\n\n")
|
||||||
|
if fl != nil {
|
||||||
|
fl.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprint(w, `{"id":"msg_e2e","type":"message","role":"assistant","model":"claude-e2e",`+
|
||||||
|
`"content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn",`+
|
||||||
|
`"usage":{"input_tokens":3,"output_tokens":1}}`)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeAnthropicConfig 写一份把源挂到 anthropic 适配器上的配置。
|
||||||
|
func writeAnthropicConfig(t *testing.T, dir, listenAddr, upstreamURL string) string {
|
||||||
|
t.Helper()
|
||||||
|
cfg := "listen: " + listenAddr + "\n" +
|
||||||
|
"gateway_keys:\n - sk-e2e-0001\n" +
|
||||||
|
"default_model: claude-e2e\n" +
|
||||||
|
"adapter_dir: " + filepath.Join(dir, "adapters") + "\n" +
|
||||||
|
"runtime_file: " + filepath.Join(dir, "runtime.json") + "\n" +
|
||||||
|
"sources:\n" +
|
||||||
|
" - name: anth\n" +
|
||||||
|
" base_url: " + upstreamURL + "\n" +
|
||||||
|
" adapter: anthropic\n" +
|
||||||
|
" api_key: test-key\n" +
|
||||||
|
" max_concurrent: 32\n" +
|
||||||
|
" models:\n" +
|
||||||
|
" - id: claude-e2e\n" +
|
||||||
|
" kind: chat\n" +
|
||||||
|
" priority: 100\n"
|
||||||
|
path := filepath.Join(dir, "config.yaml")
|
||||||
|
if err := os.WriteFile(path, []byte(cfg), 0o644); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdapterTakeoverNonStream:非流式,Anthropic 客户端形状进、出。
|
||||||
|
func TestAdapterTakeoverNonStream(t *testing.T) {
|
||||||
|
up := anthropicMockUpstream(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
cfg := writeAnthropicConfig(t, dir, addr, up.URL)
|
||||||
|
bin := buildBinary(t)
|
||||||
|
g := startGateway(t, bin, addr, cfg)
|
||||||
|
|
||||||
|
resp, body := g.do("POST", "/v1/messages",
|
||||||
|
`{"model":"claude-e2e","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}`, true)
|
||||||
|
if resp == nil || resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("status=%v body=%q", statusOf(resp), body)
|
||||||
|
}
|
||||||
|
// 出向必须是 Anthropic Messages 形状,不能是 OpenAI choices 形状
|
||||||
|
if strings.Contains(body, `"choices"`) {
|
||||||
|
t.Errorf("Anthropic 响应里混入了 OpenAI choices 外壳: %s", body)
|
||||||
|
}
|
||||||
|
var out map[string]interface{}
|
||||||
|
if err := json.Unmarshal([]byte(body), &out); err != nil {
|
||||||
|
t.Fatalf("响应不是 JSON: %v: %s", err, body)
|
||||||
|
}
|
||||||
|
if out["type"] != "message" {
|
||||||
|
t.Errorf(`响应 type = %v, 期望 "message"(Anthropic 形状): %s`, out["type"], body)
|
||||||
|
}
|
||||||
|
// 文本内容必须能取出来 —— 这是"内容确实穿过适配器回来了"的证据
|
||||||
|
if !strings.Contains(body, "pong") {
|
||||||
|
t.Errorf("响应里没有上游返回的文本: %s", body)
|
||||||
|
}
|
||||||
|
if _, ok := out["usage"]; !ok {
|
||||||
|
t.Errorf("Anthropic 响应应带 usage: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdapterTakeoverStream:流式也要是 Anthropic SSE 序列,
|
||||||
|
// 且**不得**出现 OpenAI 的 [DONE] 终止标记(R2 的"不混合")。
|
||||||
|
func TestAdapterTakeoverStream(t *testing.T) {
|
||||||
|
up := anthropicMockUpstream(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
cfg := writeAnthropicConfig(t, dir, addr, up.URL)
|
||||||
|
bin := buildBinary(t)
|
||||||
|
g := startGateway(t, bin, addr, cfg)
|
||||||
|
|
||||||
|
resp, body := g.do("POST", "/v1/messages",
|
||||||
|
`{"model":"claude-e2e","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"hi"}]}`, true)
|
||||||
|
if resp == nil || resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("stream status=%v body=%q", statusOf(resp), body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "message_start") {
|
||||||
|
t.Errorf("Anthropic 流应以 message_start 开头: %s", body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "message_stop") {
|
||||||
|
t.Errorf("Anthropic 流应以 message_stop 收尾: %s", body)
|
||||||
|
}
|
||||||
|
if strings.Contains(body, "[DONE]") {
|
||||||
|
t.Errorf("Anthropic 流里出现了 OpenAI 的 [DONE]: %s", body)
|
||||||
|
}
|
||||||
|
if strings.Contains(body, "chat.completion.chunk") {
|
||||||
|
t.Errorf("流里混入 OpenAI chunk 形状: %s", body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "pong") {
|
||||||
|
t.Errorf("流里没有上游文本: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdapterTakeoverStress 是对外服务路径的真实压力测试:
|
||||||
|
// 并发打真二进制,混合流式/非流式,验证无 5xx、上游命中数正确、
|
||||||
|
// 且每个响应都是完整合法的 Anthropic 形状(适配器在并发下不串味)。
|
||||||
|
func TestAdapterTakeoverStress(t *testing.T) {
|
||||||
|
up := anthropicMockUpstream(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
cfg := writeAnthropicConfig(t, dir, addr, up.URL)
|
||||||
|
bin := buildBinary(t)
|
||||||
|
g := startGateway(t, bin, addr, cfg)
|
||||||
|
|
||||||
|
// max_concurrent 配成 32,让 96 个请求基本都能真正流过适配器 —— 这个测试
|
||||||
|
// 要压的是**并发下序列化是否串味**,不是内核的容量拒绝策略。真的撞上上限
|
||||||
|
// 时收到的 429 也是合法背压(见下方断言),不算失败。
|
||||||
|
const (
|
||||||
|
workers = 12
|
||||||
|
perWorker = 8
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
wg sync.WaitGroup
|
||||||
|
mu sync.Mutex
|
||||||
|
ok int
|
||||||
|
shed int
|
||||||
|
failed []string
|
||||||
|
streams int
|
||||||
|
)
|
||||||
|
start := make(chan struct{})
|
||||||
|
|
||||||
|
for w := 0; w < workers; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(w int) {
|
||||||
|
defer wg.Done()
|
||||||
|
<-start
|
||||||
|
for i := 0; i < perWorker; i++ {
|
||||||
|
isStream := (w+i)%2 == 0
|
||||||
|
var payload string
|
||||||
|
if isStream {
|
||||||
|
payload = `{"model":"claude-e2e","max_tokens":32,"stream":true,"messages":[{"role":"user","content":"hi"}]}`
|
||||||
|
} else {
|
||||||
|
payload = `{"model":"claude-e2e","max_tokens":32,"messages":[{"role":"user","content":"hi"}]}`
|
||||||
|
}
|
||||||
|
resp, body := g.do("POST", "/v1/messages", payload, true)
|
||||||
|
mu.Lock()
|
||||||
|
switch {
|
||||||
|
case resp == nil:
|
||||||
|
failed = append(failed, "nil response")
|
||||||
|
// 429 = 内核的容量背压(provider busy)。它是**正确**行为,
|
||||||
|
// 不是缺陷:并发超限时就该如实拒绝,而不是排队或伪造响应。
|
||||||
|
// 只要求它是结构良好的错误,不要求 200。
|
||||||
|
case resp.StatusCode == http.StatusTooManyRequests:
|
||||||
|
if !strings.Contains(body, "busy") && !strings.Contains(body, "error") {
|
||||||
|
failed = append(failed, "429 不是结构化错误: "+body)
|
||||||
|
} else {
|
||||||
|
shed++
|
||||||
|
}
|
||||||
|
// 5xx 才是真失败 —— 那是内核或适配器崩了。
|
||||||
|
case resp.StatusCode >= 500:
|
||||||
|
failed = append(failed, fmt.Sprintf("5xx status=%d body=%.120s", resp.StatusCode, body))
|
||||||
|
case resp.StatusCode != 200:
|
||||||
|
failed = append(failed, fmt.Sprintf("status=%d body=%.120s", resp.StatusCode, body))
|
||||||
|
case !strings.Contains(body, "pong"):
|
||||||
|
// 200 但内容不对:说明并发下适配器把响应串味了
|
||||||
|
failed = append(failed, "missing upstream text: "+body)
|
||||||
|
case isStream && !strings.Contains(body, "message_stop"):
|
||||||
|
failed = append(failed, "stream missing message_stop")
|
||||||
|
case !isStream && strings.Contains(body, `"choices"`):
|
||||||
|
failed = append(failed, "openai shell leaked: "+body)
|
||||||
|
default:
|
||||||
|
ok++
|
||||||
|
if isStream {
|
||||||
|
streams++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
}(w)
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if len(failed) > 0 {
|
||||||
|
t.Fatalf("%d/%d 请求失败,前 5 例:\n%s", len(failed), workers*perWorker,
|
||||||
|
strings.Join(failed[:min(5, len(failed))], "\n"))
|
||||||
|
}
|
||||||
|
// 至少大部分请求必须真正流过 200 —— 全被背压拒掉说明并发上限配得太低,
|
||||||
|
// 那这个压力测试就没压到适配器并发。
|
||||||
|
if ok < workers*perWorker/2 {
|
||||||
|
t.Fatalf("只有 %d/%d 请求真正成功,压力未触及适配器并发路径",
|
||||||
|
ok, workers*perWorker)
|
||||||
|
}
|
||||||
|
t.Logf("成功 %d(其中流式 %d),背压 429 %d —— 无 5xx、无 OpenAI 形状泄漏",
|
||||||
|
ok, streams, shed)
|
||||||
|
}
|
||||||
|
|
||||||
|
func min(a, b int) int {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ = time.Second
|
||||||
261
e2e/conflict_arbitration_e2e_test.go
Normal file
261
e2e/conflict_arbitration_e2e_test.go
Normal file
@ -0,0 +1,261 @@
|
|||||||
|
package e2e
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// writeE2EPlugin 落一个插件文件到 e2e 的 plugin_dir(SeedBundled 见空目录才
|
||||||
|
// 落地内置插件,所以先放自定义插件再启动网关)。
|
||||||
|
func writeE2EPlugin(t *testing.T, dir, name, code string) {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(dir, "plugins", name)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(p, []byte(code), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// e2eConfig 返回一份最小 e2e 配置文本(含 plugin_dir)。
|
||||||
|
func e2eConfig(dir, listen string) string {
|
||||||
|
return "listen: " + listen + "\n" +
|
||||||
|
"gateway_keys:\n - sk-e2e-0001\n" +
|
||||||
|
"default_model: AUTO\n" +
|
||||||
|
"adapter_dir: " + filepath.Join(dir, "adapters") + "\n" +
|
||||||
|
"runtime_file: " + filepath.Join(dir, "runtime.json") + "\n" +
|
||||||
|
"plugin_dir: " + filepath.Join(dir, "plugins") + "\n" +
|
||||||
|
"sources: []\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConflictArbitrationE2E(用户要求:服务路径冲突不崩溃不拒启 + WebAPI 裁决)
|
||||||
|
//
|
||||||
|
// 真二进制全链路:
|
||||||
|
// 1. 两个插件声明同一条 /api/plugins/http/ 路径 → 两插件都 loaded,启动正常
|
||||||
|
// 2. 请求该路径 → 409,错误信息点名两个候选插件(不静默选边)
|
||||||
|
// 3. GET /api/plugins/conflicts → 列出冲突(路径 + 候选 + 未裁决)
|
||||||
|
// 4. POST /api/plugins/conflicts 裁决 → 持久化
|
||||||
|
// 5. 再请求该路径 → 200,赢家应答({"from":"B"})
|
||||||
|
// 6. /v1/models 等既有路由全程不受影响(旧通路无回归)
|
||||||
|
func TestConflictArbitrationE2E(t *testing.T) {
|
||||||
|
bin := buildBinary(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
|
||||||
|
writeE2EPlugin(t, dir, "plugA.lua", `
|
||||||
|
local P = {}
|
||||||
|
P.name = "plugA"
|
||||||
|
P.version = "1"
|
||||||
|
P.exports = {}
|
||||||
|
P.http = { { path = "contested", method = "POST", handler = "h", roles = { "admin" } } }
|
||||||
|
function P.h(req) return { status = 200, json = { from = "A" } } end
|
||||||
|
return P
|
||||||
|
`)
|
||||||
|
writeE2EPlugin(t, dir, "plugB.lua", `
|
||||||
|
local P = {}
|
||||||
|
P.name = "plugB"
|
||||||
|
P.version = "1"
|
||||||
|
P.exports = {}
|
||||||
|
P.http = { { path = "contested", method = "POST", handler = "h", roles = { "admin" } } }
|
||||||
|
function P.h(req) return { status = 200, json = { from = "B" } } end
|
||||||
|
return P
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfgPath := filepath.Join(dir, "config.yaml")
|
||||||
|
if err := os.WriteFile(cfgPath, []byte(e2eConfig(dir, addr)), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
g := startGateway(t, bin, addr, cfgPath)
|
||||||
|
|
||||||
|
// ① 两个插件都加载成功(冲突不拒载)
|
||||||
|
resp, body := g.do(http.MethodGet, "/api/plugins", "", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("GET /api/plugins: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
var list struct {
|
||||||
|
Plugins []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Loaded bool `json:"loaded"`
|
||||||
|
} `json:"plugins"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(body), &list); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded := map[string]bool{}
|
||||||
|
for _, p := range list.Plugins {
|
||||||
|
loaded[p.Name] = p.Loaded
|
||||||
|
}
|
||||||
|
if !loaded["plugA"] || !loaded["plugB"] {
|
||||||
|
t.Fatalf("冲突的两个插件都应 loaded: %v", loaded)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ② 请求冲突路径 → 409 + 点名候选(不静默选边)
|
||||||
|
resp, body = g.do(http.MethodPost, "/api/plugins/http/contested", "", true)
|
||||||
|
if resp.StatusCode != http.StatusConflict {
|
||||||
|
t.Fatalf("未裁决冲突路径应 409, 实得 %d (%s)", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "plugA") || !strings.Contains(body, "plugB") {
|
||||||
|
t.Fatalf("409 应点名两个候选: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ③ 冲突列表可见
|
||||||
|
resp, body = g.do(http.MethodGet, "/api/plugins/conflicts", "", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("GET conflicts: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
var cl struct {
|
||||||
|
Routes []struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Candidates []string `json:"candidates"`
|
||||||
|
Resolved string `json:"resolved"`
|
||||||
|
} `json:"routes"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(body), &cl); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cl.Routes) != 1 || cl.Routes[0].Path != "contested" || cl.Routes[0].Resolved != "" {
|
||||||
|
t.Fatalf("冲突形态不对: %+v", cl)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ④ 裁决 plugB 赢
|
||||||
|
arbit, _ := json.Marshal(map[string]string{
|
||||||
|
"kind": "route", "path": "contested", "winner": "plugB",
|
||||||
|
})
|
||||||
|
resp, body = g.do(http.MethodPost, "/api/plugins/conflicts", string(arbit), true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("POST conflicts 裁决: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ⑤ 再请求 → 200 且赢家应答
|
||||||
|
resp, body = g.do(http.MethodPost, "/api/plugins/http/contested", "", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("裁决后请求冲突路径应 200, 实得 %d (%s)", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, `"from":"B"`) {
|
||||||
|
t.Fatalf("赢家 plugB 应答, 实得 %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ⑥ 旧通路无回归:/v1/models 正常
|
||||||
|
resp, _ = g.do(http.MethodGet, "/v1/models", "", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("旧通路 /v1/models 应 200, 实得 %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdapterConflictE2E:插件装与内置同名适配器 → 挂起(现任保留),
|
||||||
|
// /api/plugins/conflicts 列出,裁决 candidate 后插件版生效。
|
||||||
|
func TestAdapterConflictE2E(t *testing.T) {
|
||||||
|
bin := buildBinary(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
|
||||||
|
writeE2EPlugin(t, dir, "shadow.lua", `
|
||||||
|
local P = {}
|
||||||
|
P.name = "shadow"
|
||||||
|
P.version = "1"
|
||||||
|
P.exports = { "install" }
|
||||||
|
function P.install(args)
|
||||||
|
local src = 'local a={} a.name="openai" a.version="SHADOW_V1" a.endpoint="/x" return a'
|
||||||
|
local name, err = __llmsproxy_adapters.install("openai", src)
|
||||||
|
return { name = name, err = err }
|
||||||
|
end
|
||||||
|
return P
|
||||||
|
`)
|
||||||
|
cfgPath := filepath.Join(dir, "config.yaml")
|
||||||
|
if err := os.WriteFile(cfgPath, []byte(e2eConfig(dir, addr)), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
g := startGateway(t, bin, addr, cfgPath)
|
||||||
|
|
||||||
|
// ① 安装 → 返回 err(pending 语义),内置版本不变
|
||||||
|
resp, body := g.do(http.MethodPost, "/api/plugins/shadow/call/install", "{}", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("call install: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
// Call 端点把插件返回值包在 {"result": …} 里;挂起语义的断言在 result 层做。
|
||||||
|
var wrapped struct {
|
||||||
|
Result struct {
|
||||||
|
Name *string `json:"name"`
|
||||||
|
Err string `json:"err"`
|
||||||
|
} `json:"result"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(body), &wrapped); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := wrapped.Result
|
||||||
|
if out.Name != nil {
|
||||||
|
t.Fatalf("挂起时不得返回生效名: %s", body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.Err, "openai") || !strings.Contains(out.Err, "conflict") {
|
||||||
|
t.Fatalf("错误应点名冲突: %q", out.Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ② 冲突列表可见
|
||||||
|
resp, body = g.do(http.MethodGet, "/api/plugins/conflicts", "", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("GET conflicts: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
var cl struct {
|
||||||
|
Adapters []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Candidate string `json:"candidate"`
|
||||||
|
} `json:"adapters"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(body), &cl); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cl.Adapters) != 1 || cl.Adapters[0].Name != "openai" || cl.Adapters[0].Candidate != "shadow" {
|
||||||
|
t.Fatalf("冲突形态不对: %+v", cl)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ③ 适配器列表里 openai 仍是内置版本(现任继续服务,不被静默覆盖)
|
||||||
|
assertOpenAIVersion := func(want string, msg string) {
|
||||||
|
t.Helper()
|
||||||
|
resp, body := g.do(http.MethodGet, "/api/adapters", "", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("GET /api/adapters: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
var al struct {
|
||||||
|
Adapters []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
} `json:"adapters"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(body), &al); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, a := range al.Adapters {
|
||||||
|
if a.Name == "openai" {
|
||||||
|
if a.Version != want {
|
||||||
|
t.Fatalf("%s: openai 版本应 %q, 实得 %q", msg, want, a.Version)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("%s: openai 适配器不存在: %s", msg, body)
|
||||||
|
}
|
||||||
|
assertOpenAIVersion("2.0.0", "现任内置必须继续服务(不被静默覆盖)")
|
||||||
|
|
||||||
|
// ④ 裁决 candidate → 插件版生效
|
||||||
|
arbit, _ := json.Marshal(map[string]string{
|
||||||
|
"kind": "adapter", "name": "openai", "winner": "candidate",
|
||||||
|
})
|
||||||
|
if resp, body = g.do(http.MethodPost, "/api/plugins/conflicts", string(arbit), true); resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("裁决 candidate: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ⑤ 适配器版本已切换 + 冲突清空
|
||||||
|
assertOpenAIVersion("SHADOW_V1", "裁决后插件版应生效")
|
||||||
|
resp, body = g.do(http.MethodGet, "/api/plugins/conflicts", "", true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatal(resp.StatusCode)
|
||||||
|
}
|
||||||
|
if strings.Contains(body, `"name":"openai"`) && strings.Contains(body, `"adapters":[{`) {
|
||||||
|
t.Fatalf("裁决后冲突应清除: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
// Package e2e runs the real llmsproxy binary against mocked upstreams over
|
// Package e2e runs the real modelrouter binary against mocked upstreams over
|
||||||
// real HTTP: startup, auth, chat, streaming SSE, image generation, and
|
// real HTTP: startup, auth, chat, streaming SSE, image generation, and
|
||||||
// AUTO failover when a higher-priority source fails.
|
// AUTO failover when a higher-priority source fails.
|
||||||
package e2e
|
package e2e
|
||||||
@ -100,15 +100,15 @@ type gatewayUnderTest struct {
|
|||||||
func buildBinary(t *testing.T) string {
|
func buildBinary(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
bin := filepath.Join(dir, "llmsproxy")
|
bin := filepath.Join(dir, "modelrouter")
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
bin += ".exe" // go build writes the exact name; Windows needs the suffix to exec
|
bin += ".exe" // go build writes the exact name; Windows needs the suffix to exec
|
||||||
}
|
}
|
||||||
out, err := exec.Command("go", "build", "-tags", "luajit", "-o", bin, "llmsproxy/cmd/llmsproxy").CombinedOutput()
|
out, err := exec.Command("go", "build", "-tags", "luajit", "-o", bin, "modelrouter/cmd/modelrouter").CombinedOutput()
|
||||||
if err != nil && runtime.GOOS == "windows" {
|
if err != nil && runtime.GOOS == "windows" {
|
||||||
// local Windows dev box may lack LuaJIT: fall back to the bundled
|
// local Windows dev box may lack LuaJIT: fall back to the bundled
|
||||||
// Lua runtime (the e2e adapters used here are passthrough-only)
|
// Lua runtime (the e2e adapters used here are passthrough-only)
|
||||||
out, err = exec.Command("go", "build", "-o", bin, "llmsproxy/cmd/llmsproxy").CombinedOutput()
|
out, err = exec.Command("go", "build", "-o", bin, "modelrouter/cmd/modelrouter").CombinedOutput()
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("build: %v\n%s", err, out)
|
t.Fatalf("build: %v\n%s", err, out)
|
||||||
@ -187,18 +187,20 @@ func writeConfig(t *testing.T, dir, listenAddr string, upstreams map[string]*moc
|
|||||||
sb.WriteString("runtime_file: " + filepath.Join(dir, "runtime.json") + "\n")
|
sb.WriteString("runtime_file: " + filepath.Join(dir, "runtime.json") + "\n")
|
||||||
sb.WriteString("sources:\n")
|
sb.WriteString("sources:\n")
|
||||||
order := []string{"good", "fallback", "image"}
|
order := []string{"good", "fallback", "image"}
|
||||||
prio := map[string]int{"good": 100, "fallback": 50}
|
|
||||||
for _, name := range order {
|
for _, name := range order {
|
||||||
u, ok := upstreams[name]
|
u, ok := upstreams[name]
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if name == "image" {
|
if name == "image" {
|
||||||
sb.WriteString(" - name: imagegen\n base_url: " + u.baseURL + "\n adapter: openai\n models:\n - id: flux-1\n kind: image\n priority: 80\n")
|
sb.WriteString(" - name: imagegen\n base_url: " + u.baseURL + "\n adapter: openai\n models:\n - id: flux-1\n kind: image\n")
|
||||||
} else {
|
} else {
|
||||||
sb.WriteString(" - name: " + name + "\n base_url: " + u.baseURL + "\n adapter: openai\n models:\n - id: " + name + "-m\n priority: " + fmt.Sprint(prio[name]) + "\n")
|
sb.WriteString(" - name: " + name + "\n base_url: " + u.baseURL + "\n adapter: openai\n models:\n - id: " + name + "-m\n")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// AUTO 链现在是显式配置(25970a2 移除了 models.priority)。tier 越小越先试。
|
||||||
|
sb.WriteString("auto:\n - model: good-m\n source: good\n tier: 1\n")
|
||||||
|
sb.WriteString(" - model: fallback-m\n source: fallback\n tier: 2\n")
|
||||||
path := filepath.Join(dir, "config.yaml")
|
path := filepath.Join(dir, "config.yaml")
|
||||||
if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
|
if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
|
||||||
t.Fatalf("write config: %v", err)
|
t.Fatalf("write config: %v", err)
|
||||||
|
|||||||
192
e2e/wire_golden_test.go
Normal file
192
e2e/wire_golden_test.go
Normal file
@ -0,0 +1,192 @@
|
|||||||
|
package e2e
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mustAddr 选一个空闲端口并立即释放(与 e2e_test.go 的做法一致)。
|
||||||
|
func mustAddr(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("pick port: %v", err)
|
||||||
|
}
|
||||||
|
addr := l.Addr().String()
|
||||||
|
l.Close()
|
||||||
|
return addr
|
||||||
|
}
|
||||||
|
|
||||||
|
// 本文件建立**字节级**对外输出基线,供统一线格式 v2(对外序列化移交适配器)
|
||||||
|
// 做回归比对。
|
||||||
|
//
|
||||||
|
// 为什么必须有它:Phase 2 要把 gateway/chat.go 里硬编码的 OpenAI wire 结构与
|
||||||
|
// writeChatCompletion/pumpStream 的 JSON 拼装移交给适配器 serializer。现有 e2e
|
||||||
|
// 只用 strings.Contains 做宽松断言("包含 Hel"),无法证明"输出字节未变"——
|
||||||
|
// 字段集、omitempty 裁剪、usage 零值处理都可能静默漂移,而客户端会因此看到
|
||||||
|
// 不同的 wire 形状。golden 把当前输出钉死,改动前后逐字节比对。
|
||||||
|
//
|
||||||
|
// 更新方式:GOLDEN_UPDATE=1 go test -tags luajit ./e2e/ -run TestWireGolden
|
||||||
|
// 更新后必须人工审阅 diff —— golden 是契约,不是快照垃圾桶。
|
||||||
|
|
||||||
|
// volatile 命中每次请求都变的字段,比对前替换为占位符。
|
||||||
|
// 允许冒号后的空白:golden 是 pretty-print 过的(`"created": 123`)。
|
||||||
|
var volatile = []*regexp.Regexp{
|
||||||
|
regexp.MustCompile(`"id":\s*"[^"]*"`),
|
||||||
|
regexp.MustCompile(`"created":\s*\d+`),
|
||||||
|
}
|
||||||
|
|
||||||
|
func scrub(s string) string {
|
||||||
|
s = volatile[0].ReplaceAllString(s, `"id": "<ID>"`)
|
||||||
|
return volatile[1].ReplaceAllString(s, `"created": <TS>`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareGolden(t *testing.T, name, got string) {
|
||||||
|
t.Helper()
|
||||||
|
got = scrub(got)
|
||||||
|
path := filepath.Join("..", ".snapshots", name)
|
||||||
|
if os.Getenv("GOLDEN_UPDATE") == "1" {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(got), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Logf("golden updated: %s", path)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
want, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("golden missing (%s): %v\nrun GOLDEN_UPDATE=1 go test -tags luajit ./e2e/ -run TestWireGolden to create", path, err)
|
||||||
|
}
|
||||||
|
if got != string(want) {
|
||||||
|
t.Errorf("对外输出与 golden 漂移 (%s)\n--- want ---\n%s\n--- got ---\n%s",
|
||||||
|
name, string(want), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// prettyJSON 缩进输出。走 map 会排序键,从而让「字段集」成为唯一被比较的东西,
|
||||||
|
// 忽略 Go struct 声明顺序(那不是对外契约的一部分)。
|
||||||
|
func prettyJSON(t *testing.T, raw string) string {
|
||||||
|
t.Helper()
|
||||||
|
var v interface{}
|
||||||
|
if err := json.Unmarshal([]byte(raw), &v); err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
out, err := json.MarshalIndent(v, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWireGoldenNonStreamChat 钉死非流式 /v1/chat/completions 的字段集。
|
||||||
|
func TestWireGoldenNonStreamChat(t *testing.T) {
|
||||||
|
up := newMockUpstream(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
g := startGateway(t, buildBinary(t), addr, writeConfig(t, dir, addr, map[string]*mockUpstream{"good": up}))
|
||||||
|
|
||||||
|
resp, body := g.do("POST", "/v1/chat/completions",
|
||||||
|
`{"model":"good/good-m","messages":[{"role":"user","content":"hi"}]}`, true)
|
||||||
|
|
||||||
|
// 语义:OpenAI 客户端依赖这些顶层字段
|
||||||
|
var obj map[string]interface{}
|
||||||
|
if err := json.Unmarshal([]byte(body), &obj); err != nil {
|
||||||
|
t.Fatalf("not JSON: %v (%s)", err, body)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"id", "object", "created", "model", "choices", "usage"} {
|
||||||
|
if _, ok := obj[k]; !ok {
|
||||||
|
t.Errorf("非流式响应缺少 OpenAI 必需字段 %q: %s", k, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if obj["object"] != "chat.completion" {
|
||||||
|
t.Errorf("object = %v, want chat.completion", obj["object"])
|
||||||
|
}
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Errorf("status = %d, want 200", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
compareGolden(t, "chat_nonstream.json", prettyJSON(t, body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWireGoldenStreamChat 钉死流式 SSE 的帧形状与终止序列。
|
||||||
|
func TestWireGoldenStreamChat(t *testing.T) {
|
||||||
|
up := newMockUpstream(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
g := startGateway(t, buildBinary(t), addr, writeConfig(t, dir, addr, map[string]*mockUpstream{"good": up}))
|
||||||
|
|
||||||
|
resp, _ := doStream(t, g, `{"model":"good/good-m","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
var frames []string
|
||||||
|
sc := bufio.NewScanner(resp.Body)
|
||||||
|
for sc.Scan() {
|
||||||
|
line := sc.Text()
|
||||||
|
if !strings.HasPrefix(line, "data: ") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
payload := strings.TrimPrefix(line, "data: ")
|
||||||
|
if payload == "[DONE]" {
|
||||||
|
frames = append(frames, "[DONE]")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
frames = append(frames, prettyJSON(t, payload))
|
||||||
|
}
|
||||||
|
if len(frames) < 2 {
|
||||||
|
t.Fatalf("期望多帧 SSE,实得 %d: %v", len(frames), frames)
|
||||||
|
}
|
||||||
|
if frames[len(frames)-1] != "[DONE]" {
|
||||||
|
t.Errorf("末帧 = %q, want [DONE]", frames[len(frames)-1])
|
||||||
|
}
|
||||||
|
// 首帧必须是 role 前导:OpenAI 客户端靠它建立 assistant 消息
|
||||||
|
if !strings.Contains(frames[0], `"role"`) {
|
||||||
|
t.Errorf("首个 SSE 帧必须携带 role 前导,实得: %s", frames[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
compareGolden(t, "chat_stream.txt", strings.Join(frames, "\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWireGoldenImage 钉死 /v1/images/generations 的响应形状。
|
||||||
|
func TestWireGoldenImage(t *testing.T) {
|
||||||
|
up := newMockUpstream(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
addr := mustAddr(t)
|
||||||
|
g := startGateway(t, buildBinary(t), addr, writeConfig(t, dir, addr, map[string]*mockUpstream{"image": up}))
|
||||||
|
|
||||||
|
resp, body := g.do("POST", "/v1/images/generations", `{"model":"imagegen/flux-1","prompt":"a cat"}`, true)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("image status = %d: %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "QUJD") {
|
||||||
|
t.Fatalf("生图响应丢失了载荷: %s", body)
|
||||||
|
}
|
||||||
|
compareGolden(t, "image_generation.json", prettyJSON(t, body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// doStream 发流式请求并返回未读完的响应(调用方负责逐行读 SSE)。
|
||||||
|
func doStream(t *testing.T, g *gatewayUnderTest, body string) (*http.Response, error) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := http.NewRequest("POST", "http://"+g.addr+"/v1/chat/completions", strings.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+g.key)
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stream request: %v", err)
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ = io.Discard
|
||||||
@ -1,323 +0,0 @@
|
|||||||
package billing
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Compile turns one billing profile into the prices table the billing plugin
|
|
||||||
// expects, resolving URL rules against the gateway's actual sources.
|
|
||||||
//
|
|
||||||
// WHY URL RULES NEED RESOLVING AT ALL: the operator declares pricing by URL
|
|
||||||
// because that is what a provider's price list is keyed on, and because several
|
|
||||||
// sources can point at the same URL. The plugin, however, looks up by
|
|
||||||
// `payload.source` (a source NAME) and by model — it has no idea what URL a
|
|
||||||
// request went to. So the URL match happens here, at compile time, where the
|
|
||||||
// config's name->base_url mapping is known, and the result is expressed in the
|
|
||||||
// dimensions the plugin already supports.
|
|
||||||
//
|
|
||||||
// Profiles are the "let the user choose" axis: the same URL can appear in
|
|
||||||
// several profiles and switching recomputes this table, so a gateway can be
|
|
||||||
// repriced without editing the plugin.
|
|
||||||
func Compile(profile *config.BillingProfile, sources []config.Source) (map[string]interface{}, error) {
|
|
||||||
return CompileOpts(profile, sources, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompileOpts is Compile with an explicit policy for rules that match no
|
|
||||||
// source.
|
|
||||||
//
|
|
||||||
// The default is STRICT and that is correct for startup: a profile with a
|
|
||||||
// typo'd URL prices nothing, every request on it is recorded as unpriced, and
|
|
||||||
// the bill silently comes out wrong. Failing the load is the right response.
|
|
||||||
//
|
|
||||||
// The rules editor needs the opposite. An operator editing a half-finished
|
|
||||||
// profile — adding a rule before the source that will use it exists, or
|
|
||||||
// renaming a URL while typing — must be able to save and then see the warning,
|
|
||||||
// not be blocked by an error they can only resolve by guessing. Lenient mode
|
|
||||||
// keeps the offending rule in the config and drops it from the compiled table,
|
|
||||||
// so it is preserved as text and clearly reported, without pretending it
|
|
||||||
// prices anything.
|
|
||||||
func CompileOpts(profile *config.BillingProfile, sources []config.Source, lenient bool) (map[string]interface{}, error) {
|
|
||||||
if profile == nil {
|
|
||||||
return nil, fmt.Errorf("no billing profile")
|
|
||||||
}
|
|
||||||
prices := map[string]interface{}{
|
|
||||||
"currency": "USD",
|
|
||||||
"default": map[string]interface{}{"prompt": 0.0, "completion": 0.0, "per_request": 0.0},
|
|
||||||
"sources": map[string]interface{}{},
|
|
||||||
"models": map[string]interface{}{},
|
|
||||||
"keys": map[string]interface{}{},
|
|
||||||
}
|
|
||||||
if profile.Default != "" {
|
|
||||||
prices["default_mode"] = profile.Default
|
|
||||||
}
|
|
||||||
|
|
||||||
// Currency is a profile-level statement; a rule may override it.
|
|
||||||
for i := range profile.Rules {
|
|
||||||
if c := profile.Rules[i].Currency; c != "" {
|
|
||||||
prices["currency"] = c
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
matched := map[string]bool{}
|
|
||||||
for i := range profile.Rules {
|
|
||||||
rule := &profile.Rules[i]
|
|
||||||
targets := matchSources(rule.URL, sources)
|
|
||||||
if len(targets) == 0 && rule.URL != "*" {
|
|
||||||
// A rule for a URL no source uses is almost always a typo (or a
|
|
||||||
// source that was removed). Failing loudly beats a profile that
|
|
||||||
// silently prices nothing — the whole reason this is a config file
|
|
||||||
// instead of a hand-written JSON blob.
|
|
||||||
if lenient {
|
|
||||||
continue // kept in the config text, reported as a warning
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("rule url %q matches no configured source base_url", rule.URL)
|
|
||||||
}
|
|
||||||
for _, srcName := range targets {
|
|
||||||
if matched[srcName] {
|
|
||||||
// First rule wins. Two rules matching one source is ambiguous,
|
|
||||||
// and silently letting the later one win makes the file's
|
|
||||||
// meaning depend on ordering the operator cannot see.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
matched[srcName] = true
|
|
||||||
if err := applyRule(prices, rule, srcName, sources); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sources no rule matched fall through to the profile default, which the
|
|
||||||
// plugin applies via prices.default. Recording them explicitly means the
|
|
||||||
// UI can say "this source is unpriced" instead of leaving the operator to
|
|
||||||
// infer it from a zero.
|
|
||||||
var unmatched []string
|
|
||||||
for _, src := range sources {
|
|
||||||
if !matched[src.Name] {
|
|
||||||
unmatched = append(unmatched, src.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(unmatched) > 0 {
|
|
||||||
prices["unmatched_sources"] = unmatched
|
|
||||||
}
|
|
||||||
return prices, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// matchSources returns the source names whose base_url matches pattern.
|
|
||||||
// "*" matches every source (used as a catch-all default rule).
|
|
||||||
func matchSources(pattern string, sources []config.Source) []string {
|
|
||||||
var out []string
|
|
||||||
if pattern == "*" {
|
|
||||||
for _, s := range sources {
|
|
||||||
out = append(out, s.Name)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
want := normalizeURL(pattern)
|
|
||||||
for _, s := range sources {
|
|
||||||
if normalizeURL(s.BaseURL) == want {
|
|
||||||
out = append(out, s.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizeURL compares URLs the way an operator expects: trailing slashes and
|
|
||||||
// case in the host are differences the provider's price list does not care
|
|
||||||
// about, and requiring an exact byte match would make the config brittle.
|
|
||||||
func normalizeURL(u string) string {
|
|
||||||
u = strings.TrimSpace(u)
|
|
||||||
u = strings.TrimRight(u, "/")
|
|
||||||
return strings.ToLower(u)
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyRule writes one rule's pricing for one source into the prices table.
|
|
||||||
func applyRule(prices map[string]interface{}, rule *config.BillingRule, srcName string, sources []config.Source) error {
|
|
||||||
switch rule.Mode {
|
|
||||||
case "free":
|
|
||||||
// Explicitly priced at zero. This is NOT the same as unpriced: a source
|
|
||||||
// the operator says is free must not appear in the unpriced warnings,
|
|
||||||
// or those warnings become noise and stop being read.
|
|
||||||
prices["sources"].(map[string]interface{})[srcName] = map[string]interface{}{
|
|
||||||
"prompt": 0.0, "completion": 0.0, "per_request": 0.0,
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
|
|
||||||
case "subscription":
|
|
||||||
// A fixed monthly commitment: the per-request MARGINAL cost is zero, and
|
|
||||||
// the flat fee is reported separately. Spreading a monthly fee across
|
|
||||||
// requests would invent a per-request number the provider never charges,
|
|
||||||
// and it would change every time traffic did.
|
|
||||||
prices["sources"].(map[string]interface{})[srcName] = map[string]interface{}{
|
|
||||||
"prompt": 0.0, "completion": 0.0, "per_request": 0.0,
|
|
||||||
}
|
|
||||||
fixed, _ := prices["subscriptions"].(map[string]interface{})
|
|
||||||
if fixed == nil {
|
|
||||||
fixed = map[string]interface{}{}
|
|
||||||
prices["subscriptions"] = fixed
|
|
||||||
}
|
|
||||||
cur := "USD"
|
|
||||||
if rule.Currency != "" {
|
|
||||||
cur = rule.Currency
|
|
||||||
}
|
|
||||||
fixed[srcName] = map[string]interface{}{
|
|
||||||
"monthly": rule.Subscription, "currency": cur,
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
|
|
||||||
case "unpriced":
|
|
||||||
// Deliberately left out of `sources` so the plugin's unpriced_models /
|
|
||||||
// unpriced_reqs counters catch it. That is the point: a subscription
|
|
||||||
// plan whose credits cannot be converted to tokens must be VISIBLE as
|
|
||||||
// unbilled, not quietly estimated.
|
|
||||||
return nil
|
|
||||||
|
|
||||||
case "token":
|
|
||||||
if len(rule.Models) == 0 {
|
|
||||||
return fmt.Errorf("rule for url %q: mode token requires at least one model", rule.URL)
|
|
||||||
}
|
|
||||||
models := prices["models"].(map[string]interface{})
|
|
||||||
for modelID, t := range rule.Models {
|
|
||||||
entry := map[string]interface{}{}
|
|
||||||
prompt, err := usdPerM(t.Prompt)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("model %q prompt: %w", modelID, err)
|
|
||||||
}
|
|
||||||
completion, err := usdPerM(t.Completion)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("model %q completion: %w", modelID, err)
|
|
||||||
}
|
|
||||||
entry["prompt"] = prompt
|
|
||||||
entry["completion"] = completion
|
|
||||||
if t.CacheDiscount != nil {
|
|
||||||
entry["cache_discount"] = *t.CacheDiscount
|
|
||||||
}
|
|
||||||
// reasoning is optional and stays ABSENT (not nil, not 0) when the
|
|
||||||
// rule does not set it, which is what makes the billing plugin skip
|
|
||||||
// the repricing branch entirely. Writing an explicit 0 here would
|
|
||||||
// mean "thinking is free" and silently under-bill every request
|
|
||||||
// from a provider that bills it like ordinary output.
|
|
||||||
if strings.TrimSpace(t.Reasoning) != "" {
|
|
||||||
rp, err := usdPerM(t.Reasoning)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("model %q reasoning: %w", modelID, err)
|
|
||||||
}
|
|
||||||
entry["reasoning_price"] = rp
|
|
||||||
}
|
|
||||||
if lc := t.LongContext; lc != nil {
|
|
||||||
if lc.Above <= 0 {
|
|
||||||
return fmt.Errorf("model %q long_context: above must be > 0", modelID)
|
|
||||||
}
|
|
||||||
above, err := longContextRate(lc, prompt)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("model %q long_context: %w", modelID, err)
|
|
||||||
}
|
|
||||||
entry["long_context"] = map[string]interface{}{
|
|
||||||
"above": lc.Above,
|
|
||||||
// Marginal, not blended. The plugin charges the tokens past the
|
|
||||||
// threshold at this rate and the rest at the base rate, so the
|
|
||||||
// threshold composes with a cache discount (which applies to the
|
|
||||||
// cached portion) instead of forcing the plugin to guess how
|
|
||||||
// the two interact.
|
|
||||||
"prompt": above,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if rule.Peak != nil {
|
|
||||||
entry["peak"] = compilePeak(rule.Peak)
|
|
||||||
}
|
|
||||||
models[modelID] = entry
|
|
||||||
}
|
|
||||||
// A specific-URL token rule needs a source entry so a model served
|
|
||||||
// from THIS url but absent from `rule.Models` is still recognised as
|
|
||||||
// belonging to a priced provider.
|
|
||||||
//
|
|
||||||
// A WILDCARD rule ("*") must NOT get one. "*" is a model catalogue — it
|
|
||||||
// says "these model ids have these prices, wherever they come from" —
|
|
||||||
// and marking every source priced at {0,0} would make priceFor() set
|
|
||||||
// priced=true for every model the catalogue does NOT list. Paid
|
|
||||||
// sources (commandcode, alittokenplan, …) would then record $0 instead
|
|
||||||
// of showing up in unpriced_reqs, which is precisely the silent
|
|
||||||
// zero-billing failure this whole plugin is built to make visible.
|
|
||||||
// The model lookup p.models[payload.model] already marks a listed
|
|
||||||
// model priced on its own, so the wildcard loses nothing.
|
|
||||||
if rule.URL != "*" {
|
|
||||||
if _, ok := prices["sources"].(map[string]interface{})[srcName]; !ok {
|
|
||||||
prices["sources"].(map[string]interface{})[srcName] = map[string]interface{}{
|
|
||||||
"prompt": 0.0, "completion": 0.0, "per_request": 0.0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("unknown mode %q", rule.Mode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// usdPerM converts a USD-per-million string into the per-token rate the plugin
|
|
||||||
// expects. Delegates to config.ParseFloatUSDPerM so the DSL validator and the
|
|
||||||
// compiler agree byte-for-byte on what a valid price is — a value that passes
|
|
||||||
// Validate() but fails here (or vice versa) would be the worst kind of drift.
|
|
||||||
func usdPerM(s string) (float64, error) {
|
|
||||||
return config.ParseFloatUSDPerM(s)
|
|
||||||
}
|
|
||||||
|
|
||||||
// compilePeak renders the peak window in the shape the plugin reads:
|
|
||||||
// { multiplier, windows = { { days = {...}, hours = { {lo,hi}, ... } } } }.
|
|
||||||
//
|
|
||||||
// The plugin reads `days` / `hours` pairs. The first version of the deployed
|
|
||||||
// price table used { start, end, weekdays } — a shape nothing reads — so peak
|
|
||||||
// traffic was billed at off-peak rates with no error anywhere. Compiling from
|
|
||||||
// typed config fields removes the chance of writing the wrong key names by hand.
|
|
||||||
func compilePeak(p *config.BillingPeak) map[string]interface{} {
|
|
||||||
win := map[string]interface{}{}
|
|
||||||
if len(p.Weekdays) > 0 {
|
|
||||||
days := make([]interface{}, 0, len(p.Weekdays))
|
|
||||||
for _, d := range p.Weekdays {
|
|
||||||
days = append(days, d)
|
|
||||||
}
|
|
||||||
win["days"] = days
|
|
||||||
}
|
|
||||||
if len(p.Hours) > 0 {
|
|
||||||
hours := make([]interface{}, 0, len(p.Hours))
|
|
||||||
for _, h := range p.Hours {
|
|
||||||
hours = append(hours, []interface{}{h[0], h[1]})
|
|
||||||
}
|
|
||||||
win["hours"] = hours
|
|
||||||
}
|
|
||||||
out := map[string]interface{}{"multiplier": p.Multiplier}
|
|
||||||
if len(win) > 0 {
|
|
||||||
out["windows"] = []interface{}{win}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// longContextRate resolves the marginal per-token rate for the tokens ABOVE a
|
|
||||||
// threshold, given the model's base per-million prompt price.
|
|
||||||
//
|
|
||||||
// Two ways to express it are supported because providers publish both: an
|
|
||||||
// absolute rate (OpenAI quotes \$3.00/M past 200k) and a multiple (DeepSeek
|
|
||||||
// quotes exactly 2x past 128k). Neither may be missing: a step with no rate is a
|
|
||||||
// silent no-op, and an operator who wrote one meant something.
|
|
||||||
func longContextRate(lc *config.BillingLongContext, basePerToken float64) (float64, error) {
|
|
||||||
hasPrompt := strings.TrimSpace(lc.Prompt) != ""
|
|
||||||
hasMult := lc.Multiplier > 0
|
|
||||||
switch {
|
|
||||||
case hasPrompt && hasMult:
|
|
||||||
p, err := usdPerM(lc.Prompt)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
// Both given: the absolute rate wins, because it is the one the provider
|
|
||||||
// actually quotes. Silently multiplying instead would produce a number
|
|
||||||
// the operator never wrote.
|
|
||||||
return p, nil
|
|
||||||
case hasPrompt:
|
|
||||||
return usdPerM(lc.Prompt)
|
|
||||||
case hasMult:
|
|
||||||
return basePerToken * lc.Multiplier, nil
|
|
||||||
default:
|
|
||||||
return 0, fmt.Errorf("needs either prompt or multiplier")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,214 +0,0 @@
|
|||||||
package billing
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The wildcard token rule must not blanket-mark every source as priced.
|
|
||||||
//
|
|
||||||
// Why this is a hazard and not a style question: priceFor() sets priced=true as
|
|
||||||
// soon as the request's SOURCE has an entry in the price table, and that flag
|
|
||||||
// is what keeps a request OUT of unpriced_reqs. A "*" rule is a model
|
|
||||||
// catalogue — it prices specific model ids wherever they come from — but
|
|
||||||
// compiling it into a {0,0,0} entry on all 21 sources made every model the
|
|
||||||
// catalogue does not list look "priced at zero". Paid providers (commandcode,
|
|
||||||
// alittokenplan, …) then stopped appearing in the unpriced warnings, which is
|
|
||||||
// exactly the silent zero-billing failure the plugin exists to surface.
|
|
||||||
func TestWildcardTokenRuleDoesNotMarkEverySourcePriced(t *testing.T) {
|
|
||||||
sources := []config.Source{
|
|
||||||
{Name: "free1", BaseURL: "https://free1.example.com/v1"},
|
|
||||||
{Name: "paid1", BaseURL: "https://paid1.example.com/v1"},
|
|
||||||
{Name: "paid2", BaseURL: "https://paid2.example.com/v1"},
|
|
||||||
}
|
|
||||||
profile := &config.BillingProfile{
|
|
||||||
ID: "p",
|
|
||||||
Rules: []config.BillingRule{
|
|
||||||
{URL: "https://free1.example.com/v1", Mode: "free"},
|
|
||||||
{
|
|
||||||
URL: "*", Mode: "token",
|
|
||||||
Models: map[string]config.BillingToken{
|
|
||||||
"only-catalogued-model": {Prompt: "1.00", Completion: "2.00"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
prices, err := Compile(profile, sources)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("compile: %v", err)
|
|
||||||
}
|
|
||||||
srcs := prices["sources"].(map[string]interface{})
|
|
||||||
|
|
||||||
// The explicitly free source keeps its entry (free is a deliberate 0).
|
|
||||||
if _, ok := srcs["free1"]; !ok {
|
|
||||||
t.Error("the explicit free rule lost its source entry")
|
|
||||||
}
|
|
||||||
// Paid sources must NOT be marked, or their unlisted models bill $0.
|
|
||||||
for _, name := range []string{"paid1", "paid2"} {
|
|
||||||
if _, ok := srcs[name]; ok {
|
|
||||||
t.Errorf("★ wildcard token rule marked %q as priced at 0 — its unlisted "+
|
|
||||||
"models will bill $0 instead of appearing as unpriced", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The catalogue itself must still be present, or the rule prices nothing.
|
|
||||||
models := prices["models"].(map[string]interface{})
|
|
||||||
if _, ok := models["only-catalogued-model"]; !ok {
|
|
||||||
t.Error("the wildcard rule did not publish its model price")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The inverse guard: a token rule for a SPECIFIC url still marks that source,
|
|
||||||
// because it genuinely declares pricing for the models served from it.
|
|
||||||
func TestSpecificURLTokenRuleStillMarksItsSource(t *testing.T) {
|
|
||||||
sources := []config.Source{
|
|
||||||
{Name: "mine", BaseURL: "https://mine.example.com/v1"},
|
|
||||||
{Name: "other", BaseURL: "https://other.example.com/v1"},
|
|
||||||
}
|
|
||||||
profile := &config.BillingProfile{
|
|
||||||
ID: "p",
|
|
||||||
Rules: []config.BillingRule{{
|
|
||||||
URL: "https://mine.example.com/v1", Mode: "token",
|
|
||||||
Models: map[string]config.BillingToken{
|
|
||||||
"m": {Prompt: "1.00", Completion: "2.00"},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
prices, err := Compile(profile, sources)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("compile: %v", err)
|
|
||||||
}
|
|
||||||
srcs := prices["sources"].(map[string]interface{})
|
|
||||||
if _, ok := srcs["mine"]; !ok {
|
|
||||||
t.Error("a specific-url token rule must still mark its own source priced")
|
|
||||||
}
|
|
||||||
if _, ok := srcs["other"]; ok {
|
|
||||||
t.Error("an unrelated source was marked priced by a specific-url rule")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReasoningPriceIsOmittedUnlessSet pins the absence convention for the
|
|
||||||
// thinking-token rate.
|
|
||||||
//
|
|
||||||
// A rule that says nothing about reasoning must produce a price entry with NO
|
|
||||||
// reasoning_price key, so the billing plugin skips the repricing branch and
|
|
||||||
// bills thinking at the ordinary output rate — which is what every provider
|
|
||||||
// actually charges. Writing an explicit 0 instead would read as "thinking is
|
|
||||||
// free" and silently under-bill every request from that model.
|
|
||||||
func TestReasoningPriceIsOmittedUnlessSet(t *testing.T) {
|
|
||||||
sources := []config.Source{{Name: "up", BaseURL: "https://up.example/v1"}}
|
|
||||||
|
|
||||||
// Case 1: no reasoning declared → the key must not appear.
|
|
||||||
p1 := &config.BillingProfile{ID: "p", Rules: []config.BillingRule{{
|
|
||||||
URL: "https://up.example/v1", Mode: "token",
|
|
||||||
Models: map[string]config.BillingToken{
|
|
||||||
"m": {Prompt: "1.00", Completion: "2.00"},
|
|
||||||
},
|
|
||||||
}}}
|
|
||||||
prices, err := Compile(p1, sources)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
entry := prices["models"].(map[string]interface{})["m"].(map[string]interface{})
|
|
||||||
if _, present := entry["reasoning_price"]; present {
|
|
||||||
t.Errorf("reasoning_price present (%v) without a rule; the plugin would treat 0 as free",
|
|
||||||
entry["reasoning_price"])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Case 2: declared → present, and converted to per-token like every other rate.
|
|
||||||
p2 := &config.BillingProfile{ID: "p", Rules: []config.BillingRule{{
|
|
||||||
URL: "https://up.example/v1", Mode: "token",
|
|
||||||
Models: map[string]config.BillingToken{
|
|
||||||
"m": {Prompt: "1.00", Completion: "2.00", Reasoning: "6.00"},
|
|
||||||
},
|
|
||||||
}}}
|
|
||||||
prices2, err := Compile(p2, sources)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
e2 := prices2["models"].(map[string]interface{})["m"].(map[string]interface{})
|
|
||||||
rp, ok := e2["reasoning_price"].(float64)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("reasoning_price missing or wrong type: %#v", e2["reasoning_price"])
|
|
||||||
}
|
|
||||||
if rp != 6e-6 {
|
|
||||||
t.Errorf("reasoning_price = %v, want 6e-6 (USD per TOKEN)", rp)
|
|
||||||
}
|
|
||||||
// A malformed rate must be rejected, not silently ignored: an operator who
|
|
||||||
// typed "6 USD" expects either that value or an error, never a rule that
|
|
||||||
// looks applied and bills nothing.
|
|
||||||
p3 := &config.BillingProfile{ID: "p", Rules: []config.BillingRule{{
|
|
||||||
URL: "https://up.example/v1", Mode: "token",
|
|
||||||
Models: map[string]config.BillingToken{
|
|
||||||
"m": {Prompt: "1.00", Completion: "2.00", Reasoning: "six dollars"},
|
|
||||||
},
|
|
||||||
}}}
|
|
||||||
if _, err := Compile(p3, sources); err == nil {
|
|
||||||
t.Error("a non-numeric reasoning rate must fail the compile")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLongContextStepCompiles pins the long-context step's two accepted forms
|
|
||||||
// and its rejection of a step that would silently do nothing.
|
|
||||||
//
|
|
||||||
// Providers publish both shapes — DeepSeek says "2x past 128k", OpenAI quotes
|
|
||||||
// an absolute "$3.00/M past 200k" — so both must compile. Neither may be
|
|
||||||
// missing: a step with no rate is a rule that LOOKS like it handles long
|
|
||||||
// context and does not, which under-bills without any visible sign.
|
|
||||||
func TestLongContextStepCompiles(t *testing.T) {
|
|
||||||
sources := []config.Source{{Name: "up", BaseURL: "https://up.example/v1"}}
|
|
||||||
|
|
||||||
compile := func(lc config.BillingLongContext) (map[string]interface{}, error) {
|
|
||||||
p := &config.BillingProfile{ID: "p", Rules: []config.BillingRule{{
|
|
||||||
URL: "https://up.example/v1", Mode: "token",
|
|
||||||
Models: map[string]config.BillingToken{
|
|
||||||
"m": {Prompt: "1.00", Completion: "2.00", LongContext: &lc},
|
|
||||||
},
|
|
||||||
}}}
|
|
||||||
prices, err := Compile(p, sources)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
e := prices["models"].(map[string]interface{})["m"].(map[string]interface{})
|
|
||||||
return e["long_context"].(map[string]interface{}), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Absolute rate (OpenAI's published shape).
|
|
||||||
lc1, err := compile(config.BillingLongContext{Above: 200000, Prompt: "3.00"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("absolute long_context rate rejected: %v", err)
|
|
||||||
}
|
|
||||||
if lc1["above"] != 200000 || lc1["prompt"] != 3e-6 {
|
|
||||||
t.Errorf("absolute step = %#v, want above=200000 prompt=3e-6", lc1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Multiplier (DeepSeek's shape): 2x of the 1.00/M base.
|
|
||||||
lc2, err := compile(config.BillingLongContext{Above: 128000, Multiplier: 2})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("multiplier long_context rejected: %v", err)
|
|
||||||
}
|
|
||||||
if lc2["prompt"] != 2e-6 {
|
|
||||||
t.Errorf("multiplier step = %#v, want prompt=2e-6", lc2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both given: the absolute rate wins, because that is the number the
|
|
||||||
// provider quotes. Multiplying instead would produce a figure the operator
|
|
||||||
// never wrote.
|
|
||||||
lc3, err := compile(config.BillingLongContext{Above: 128000, Prompt: "5.00", Multiplier: 2})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if lc3["prompt"] != 5e-6 {
|
|
||||||
t.Errorf("both given = %#v, want the quoted absolute rate 5e-6", lc3)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Neither: a no-op step must fail, not compile into a rule that silently
|
|
||||||
// bills long context at the base rate.
|
|
||||||
if _, err := compile(config.BillingLongContext{Above: 128000}); err == nil {
|
|
||||||
t.Error("a step with neither prompt nor multiplier must be rejected")
|
|
||||||
}
|
|
||||||
// A zero threshold would make every request "long".
|
|
||||||
if _, err := compile(config.BillingLongContext{Above: 0, Multiplier: 2}); err == nil {
|
|
||||||
t.Error("above=0 must be rejected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -7,7 +7,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -30,15 +29,6 @@ type Config struct {
|
|||||||
Auto []ModelScope `yaml:"auto,omitempty"` // AUTO 调度链规则(WebUI 优先级页编辑,chat)
|
Auto []ModelScope `yaml:"auto,omitempty"` // AUTO 调度链规则(WebUI 优先级页编辑,chat)
|
||||||
AutoImage []ModelScope `yaml:"auto_image,omitempty"` // AUTO 生图调度链规则(WebUI 优先级页·生图)
|
AutoImage []ModelScope `yaml:"auto_image,omitempty"` // AUTO 生图调度链规则(WebUI 优先级页·生图)
|
||||||
Keys []GWKey `yaml:"keys,omitempty"` // 网关密钥(WebUI 密钥页管理)
|
Keys []GWKey `yaml:"keys,omitempty"` // 网关密钥(WebUI 密钥页管理)
|
||||||
// BillingDSL declares per-URL pricing profiles for the billing plugin.
|
|
||||||
//
|
|
||||||
// Declarative, because hand-writing the prices JSON was error-prone in ways
|
|
||||||
// that looked successful: the first PUT went out without the required
|
|
||||||
// `prices` wrapper and silently REPLACED the accumulated totals with the
|
|
||||||
// price table; the second used a peak-window shape the plugin does not read,
|
|
||||||
// so peak traffic was billed at off-peak rates — no error, the bill just
|
|
||||||
// quietly halved. A schema-checked config file catches both at load.
|
|
||||||
BillingDSL *BillingDSL `yaml:"billing,omitempty"`
|
|
||||||
// PluginVisibility overrides the audience a plugin declared for itself,
|
// PluginVisibility overrides the audience a plugin declared for itself,
|
||||||
// keyed by plugin name: "admin" hides a plugin (and its state) from user
|
// keyed by plugin name: "admin" hides a plugin (and its state) from user
|
||||||
// keys, "all" widens one that shipped admin-only, "user,admin" names roles
|
// keys, "all" widens one that shipped admin-only, "user,admin" names roles
|
||||||
@ -62,30 +52,47 @@ const (
|
|||||||
DefaultSourceConcurrency = 8
|
DefaultSourceConcurrency = 8
|
||||||
)
|
)
|
||||||
|
|
||||||
// Model is a single exposed model id bound to a source, with priority used by
|
// Model is a single exposed model id bound to a source.
|
||||||
// AUTO auto selection (higher number = preferred).
|
//
|
||||||
|
// ★ 曾经有 `Priority int` 字段,并号称"高者优先参与 AUTO 选源"。**那是错的**:
|
||||||
|
//
|
||||||
|
// AUTO 链由 `config.ModelScope.Tier` 驱动(见 scheduler.Chain),而 priority
|
||||||
|
// 只被 `bestChatModel/bestImageModel` 用于"本源码内部该发哪个模型",
|
||||||
|
// 与跨源调度毫无关系。把它当成调度依据会让人误以为改个数字就能改顺序,
|
||||||
|
// 改完发现毫无效果 —— 已彻底删除,避免再次误导。
|
||||||
type Model struct {
|
type Model struct {
|
||||||
ID string `yaml:"id" json:"id"`
|
ID string `yaml:"id" json:"id"`
|
||||||
Priority int `yaml:"priority" json:"priority"`
|
Kind string `yaml:"kind" json:"kind"` // "chat" (default) | "image"
|
||||||
Kind string `yaml:"kind" json:"kind"` // "chat" (default) | "image"
|
Meta map[string]interface{} `yaml:"meta" json:"meta,omitempty"`
|
||||||
Meta map[string]interface{} `yaml:"meta" json:"meta,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Source describes a single upstream LLM provider.
|
// Source describes a single upstream LLM provider.
|
||||||
type Source struct {
|
type Source struct {
|
||||||
Name string `yaml:"name" json:"name"`
|
Name string `yaml:"name" json:"name"`
|
||||||
BaseURL string `yaml:"base_url" json:"base_url"`
|
BaseURL string `yaml:"base_url" json:"base_url"`
|
||||||
APIKey string `yaml:"api_key" json:"api_key"`
|
APIKey string `yaml:"api_key" json:"api_key"`
|
||||||
APIKeyEnv string `yaml:"api_key_env,omitempty" json:"-"` // reference to an env var holding the key (overrides api_key)
|
APIKeyEnv string `yaml:"api_key_env,omitempty" json:"-"` // reference to an env var holding the key (overrides api_key)
|
||||||
Adapter string `yaml:"adapter" json:"adapter"`
|
Adapter string `yaml:"adapter" json:"adapter"`
|
||||||
Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"` // chat endpoint override
|
Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"` // chat endpoint override
|
||||||
ImageEndpoint string `yaml:"image_endpoint" json:"image_endpoint,omitempty"` // image endpoint override
|
// StreamEndpoint overrides the CHAT endpoint for streaming requests. Most
|
||||||
Models []Model `yaml:"models" json:"models"`
|
// upstreams stream on the same path; Gemini is the exception (same path,
|
||||||
Headers map[string]string `yaml:"headers" json:"headers,omitempty"`
|
// different verb). Declaring it here keeps that knowledge out of the kernel,
|
||||||
|
// which only compares opaque strings.
|
||||||
|
StreamEndpoint string `yaml:"endpoint_stream" json:"endpoint_stream,omitempty"`
|
||||||
|
ImageEndpoint string `yaml:"image_endpoint" json:"image_endpoint,omitempty"` // image endpoint override
|
||||||
|
Models []Model `yaml:"models" json:"models"`
|
||||||
|
Headers map[string]string `yaml:"headers" json:"headers,omitempty"`
|
||||||
// ProxyURL routes this source's HTTP(S) traffic through an explicit proxy
|
// ProxyURL routes this source's HTTP(S) traffic through an explicit proxy
|
||||||
// (e.g. http://127.0.0.1:7890). Empty = direct connection. Needed for
|
// (e.g. http://127.0.0.1:7890). Empty = direct connection. Needed for
|
||||||
// upstreams behind geo/IP blocks that only respond through a proxy.
|
// upstreams behind geo/IP blocks that only respond through a proxy.
|
||||||
ProxyURL string `yaml:"proxy_url" json:"proxy_url,omitempty"`
|
ProxyURL string `yaml:"proxy_url" json:"proxy_url,omitempty"`
|
||||||
|
// Retry is an OPTIONAL upstream retry policy. It exists for upstreams that
|
||||||
|
// answer a business-level rejection with a status code the kernel would
|
||||||
|
// otherwise surface to the client — e.g. a per-account concurrency cap that
|
||||||
|
// clears by itself after tens of seconds. The kernel stays vendor-agnostic:
|
||||||
|
// the matcher is pure DATA (status list + response-body substrings), so no
|
||||||
|
// provider name ever appears in core.
|
||||||
|
Retry *RetryPolicy `yaml:"retry" json:"retry,omitempty"`
|
||||||
Meta map[string]interface{} `yaml:"meta" json:"meta,omitempty"`
|
Meta map[string]interface{} `yaml:"meta" json:"meta,omitempty"`
|
||||||
Temperature float64 `yaml:"temperature" json:"temperature,omitempty"`
|
Temperature float64 `yaml:"temperature" json:"temperature,omitempty"`
|
||||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens,omitempty"`
|
MaxTokens int `yaml:"max_tokens" json:"max_tokens,omitempty"`
|
||||||
@ -95,9 +102,66 @@ type Source struct {
|
|||||||
RPM int `yaml:"rpm" json:"rpm,omitempty"` // optional requests-per-minute cap (0 = unlimited)
|
RPM int `yaml:"rpm" json:"rpm,omitempty"` // optional requests-per-minute cap (0 = unlimited)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetryPolicy describes when a failed upstream attempt may be retried BEFORE
|
||||||
|
// anything is returned to the client. All conditions are data:
|
||||||
|
//
|
||||||
|
// OnStatus - retry only when the HTTP status is one of these
|
||||||
|
// OnBodyContains - ... AND the response body contains one of these substrings
|
||||||
|
// (empty = status alone decides)
|
||||||
|
// Delay - wait between attempts ("10s" etc.); parsed on demand
|
||||||
|
// MaxAttempts - TOTAL attempts including the first (1 or 0 = no retry)
|
||||||
|
//
|
||||||
|
// Scope: the retry happens inside one gateway request, so a stream is only
|
||||||
|
// retried before its first chunk reaches the client (the kernel already holds
|
||||||
|
// the first chunk back), and a non-stream call is retried as a whole.
|
||||||
|
type RetryPolicy struct {
|
||||||
|
OnStatus []int `yaml:"on_status,omitempty" json:"on_status,omitempty"`
|
||||||
|
OnBodyContains []string `yaml:"on_body_contains,omitempty" json:"on_body_contains,omitempty"`
|
||||||
|
Delay string `yaml:"delay,omitempty" json:"delay,omitempty"`
|
||||||
|
MaxAttempts int `yaml:"max_attempts,omitempty" json:"max_attempts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldRetry reports whether one failed attempt matches this policy.
|
||||||
|
func (r *RetryPolicy) ShouldRetry(status int, body string) bool {
|
||||||
|
if r == nil || r.MaxAttempts <= 1 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
statusOK := len(r.OnStatus) == 0
|
||||||
|
for _, s := range r.OnStatus {
|
||||||
|
if s == status {
|
||||||
|
statusOK = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !statusOK {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if len(r.OnBodyContains) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, sub := range r.OnBodyContains {
|
||||||
|
if sub != "" && strings.Contains(body, sub) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// DelayDuration is the configured pause between attempts (default 10s).
|
||||||
|
func (r *RetryPolicy) DelayDuration() time.Duration {
|
||||||
|
if r == nil || r.Delay == "" {
|
||||||
|
return 10 * time.Second
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(r.Delay)
|
||||||
|
if err != nil || d < 0 {
|
||||||
|
return 10 * time.Second
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
// Load reads and validates a config file. When the file does not exist yet a
|
// Load reads and validates a config file. When the file does not exist yet a
|
||||||
// default config is generated at that path first (first-run bootstrap), so a
|
// default config is generated at that path first (first-run bootstrap), so a
|
||||||
// fresh binary just works: `llmsproxy -config /path/to/config.yaml`.
|
// fresh binary just works: `modelrouter -config /path/to/config.yaml`.
|
||||||
func Load(path string) (*Config, error) {
|
func Load(path string) (*Config, error) {
|
||||||
if _, err := EnsureDefault(path); err != nil {
|
if _, err := EnsureDefault(path); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -373,13 +437,14 @@ type RuntimeConfig struct {
|
|||||||
// name and api_key) so the WebUI can spin up multiple key-bearing sources
|
// name and api_key) so the WebUI can spin up multiple key-bearing sources
|
||||||
// from one shared template.
|
// from one shared template.
|
||||||
type SourceTemplate struct {
|
type SourceTemplate struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
BaseURL string `json:"base_url"`
|
BaseURL string `json:"base_url"`
|
||||||
Adapter string `json:"adapter"`
|
Adapter string `json:"adapter"`
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
ImageEndpoint string `json:"image_endpoint,omitempty"`
|
StreamEndpoint string `json:"endpoint_stream,omitempty"`
|
||||||
Models []Model `json:"models"`
|
ImageEndpoint string `json:"image_endpoint,omitempty"`
|
||||||
Headers map[string]string `json:"headers,omitempty"`
|
Models []Model `json:"models"`
|
||||||
|
Headers map[string]string `json:"headers,omitempty"`
|
||||||
// ProxyURL routes this source's HTTP(S) traffic through an explicit proxy.
|
// ProxyURL routes this source's HTTP(S) traffic through an explicit proxy.
|
||||||
ProxyURL string `json:"proxy_url,omitempty"`
|
ProxyURL string `json:"proxy_url,omitempty"`
|
||||||
Meta map[string]interface{} `json:"meta,omitempty"`
|
Meta map[string]interface{} `json:"meta,omitempty"`
|
||||||
@ -393,18 +458,18 @@ type SourceTemplate struct {
|
|||||||
// (full management) or "user" (sees only its own key); Models is the allowed
|
// (full management) or "user" (sees only its own key); Models is the allowed
|
||||||
// model scope with per-model token quota (0 = unlimited).
|
// model scope with per-model token quota (0 = unlimited).
|
||||||
type GWKey struct {
|
type GWKey struct {
|
||||||
Key string `yaml:"key" json:"key"`
|
Key string `yaml:"key" json:"key"`
|
||||||
Role string `yaml:"role" json:"role"`
|
Role string `yaml:"role" json:"role"`
|
||||||
Name string `yaml:"name,omitempty" json:"name,omitempty"`
|
Name string `yaml:"name,omitempty" json:"name,omitempty"`
|
||||||
// Auto is this key's own AUTO scheduling chain. It overrides the global
|
// Auto is this key's own AUTO scheduling chain. It overrides the global
|
||||||
// cfg.Auto for requests authenticated with this key; nil (or an empty
|
// cfg.Auto for requests authenticated with this key; nil (or an empty
|
||||||
// list) means "inherit the global chain", so keys created before per-key
|
// list) means "inherit the global chain", so keys created before per-key
|
||||||
// chains existed keep working untouched.
|
// chains existed keep working untouched.
|
||||||
Auto []ModelScope `yaml:"auto,omitempty" json:"auto,omitempty"`
|
Auto []ModelScope `yaml:"auto,omitempty" json:"auto,omitempty"`
|
||||||
Models []ModelScope `yaml:"models,omitempty" json:"models,omitempty"`
|
Models []ModelScope `yaml:"models,omitempty" json:"models,omitempty"`
|
||||||
Note string `yaml:"note,omitempty" json:"note,omitempty"`
|
Note string `yaml:"note,omitempty" json:"note,omitempty"`
|
||||||
CreatedAt int64 `yaml:"created_at,omitempty" json:"created_at,omitempty"`
|
CreatedAt int64 `yaml:"created_at,omitempty" json:"created_at,omitempty"`
|
||||||
Seed bool `yaml:"seed,omitempty" json:"seed,omitempty"` // true if migrated from config gateway_keys
|
Seed bool `yaml:"seed,omitempty" json:"seed,omitempty"` // true if migrated from config gateway_keys
|
||||||
}
|
}
|
||||||
|
|
||||||
// HasOwnAuto reports whether the key declares its own AUTO chain rather than
|
// HasOwnAuto reports whether the key declares its own AUTO chain rather than
|
||||||
@ -412,182 +477,6 @@ type GWKey struct {
|
|||||||
// chain" so a user can clear an override back to the global default.
|
// chain" so a user can clear an override back to the global default.
|
||||||
func (k GWKey) HasOwnAuto() bool { return len(k.Auto) > 0 }
|
func (k GWKey) HasOwnAuto() bool { return len(k.Auto) > 0 }
|
||||||
|
|
||||||
// BillingDSL holds declarative per-URL pricing profiles for the billing plugin.
|
|
||||||
//
|
|
||||||
// Profiles are the "let the user pick" axis: the same upstream URL can be
|
|
||||||
// declared under multiple profiles, and the operator chooses which is active.
|
|
||||||
// Switching recomputes and re-injects the billing plugin's prices table.
|
|
||||||
type BillingDSL struct {
|
|
||||||
// Active is the id of the profile in effect. Empty => the first profile.
|
|
||||||
Active string `yaml:"active,omitempty" json:"active,omitempty"`
|
|
||||||
Profiles []BillingProfile `yaml:"profiles,omitempty" json:"profiles,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BillingProfile is one named set of per-URL pricing rules.
|
|
||||||
type BillingProfile struct {
|
|
||||||
ID string `yaml:"id" json:"id"`
|
|
||||||
Label string `yaml:"label,omitempty" json:"label,omitempty"`
|
|
||||||
// Default applies when no rule matches: "free" or "unpriced".
|
|
||||||
Default string `yaml:"default,omitempty" json:"default,omitempty"`
|
|
||||||
Rules []BillingRule `yaml:"rules,omitempty" json:"rules,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BillingRule declares the pricing for everything under one URL.
|
|
||||||
type BillingRule struct {
|
|
||||||
// URL matches a source's base_url. "*" = any unmatched URL.
|
|
||||||
URL string `yaml:"url" json:"url"`
|
|
||||||
// Mode is one of: free, token, subscription, unpriced.
|
|
||||||
Mode string `yaml:"mode" json:"mode"`
|
|
||||||
Currency string `yaml:"currency,omitempty" json:"currency,omitempty"`
|
|
||||||
// Subscription is the flat monthly cost (for mode=subscription).
|
|
||||||
Subscription float64 `yaml:"subscription,omitempty" json:"subscription,omitempty"`
|
|
||||||
// Models declares per-token prices when mode=token.
|
|
||||||
Models map[string]BillingToken `yaml:"models,omitempty" json:"models,omitempty"`
|
|
||||||
// Free is true when mode=free (kept for explicitness).
|
|
||||||
// Peak applies to ALL token-priced requests under this URL.
|
|
||||||
Peak *BillingPeak `yaml:"peak,omitempty" json:"peak,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BillingToken is the per-model price.
|
|
||||||
type BillingToken struct {
|
|
||||||
Prompt string `yaml:"prompt" json:"prompt"` // USD per million tokens, e.g. "0.15"
|
|
||||||
Completion string `yaml:"completion" json:"completion"` // USD per million tokens
|
|
||||||
CacheDiscount *float64 `yaml:"cache_discount,omitempty" json:"cache_discount,omitempty"`
|
|
||||||
// Reasoning is the USD-per-million rate for thinking tokens, for the rare
|
|
||||||
// provider that charges them differently from ordinary output. Omit it and
|
|
||||||
// reasoning is billed at Completion — which is what OpenAI, DeepSeek and
|
|
||||||
// Anthropic all do, and what every source on this deployment reports.
|
|
||||||
//
|
|
||||||
// It is a rate for a SUBSET of completion tokens, never an extra charge: the
|
|
||||||
// billing plugin applies only the difference between this and Completion, so
|
|
||||||
// setting it to a multiple of Completion reprices the thinking share rather
|
|
||||||
// than billing it twice.
|
|
||||||
Reasoning string `yaml:"reasoning,omitempty" json:"reasoning,omitempty"`
|
|
||||||
// LongContext is the prompt-price step above a token threshold. Nil = one
|
|
||||||
// flat rate, which is what a rule means when it says nothing.
|
|
||||||
LongContext *BillingLongContext `yaml:"long_context,omitempty" json:"long_context,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BillingPeak declares peak-hour pricing.
|
|
||||||
type BillingPeak struct {
|
|
||||||
Multiplier float64 `yaml:"multiplier" json:"multiplier"`
|
|
||||||
// Weekdays: 1=Mon..5=Fri (Lua os.date !%w, 0=Sun)
|
|
||||||
Weekdays []int `yaml:"weekdays,omitempty" json:"weekdays,omitempty"`
|
|
||||||
// Hours: each pair is [lo,hi] inclusive
|
|
||||||
Hours [][2]int `yaml:"hours,omitempty" json:"hours,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BillingLongContext is a prompt-price step for long inputs.
|
|
||||||
//
|
|
||||||
// Every major provider charges the input tokens ABOVE a threshold at a higher
|
|
||||||
// rate — DeepSeek doubles past 128k, OpenAI and Anthropic step up as well. A
|
|
||||||
// flat prompt price therefore under-bills exactly the traffic an agent workload
|
|
||||||
// generates, and it under-bills it silently: the number is plausible, the bill
|
|
||||||
// adds up, and nothing says the rate was wrong.
|
|
||||||
//
|
|
||||||
// The step applies to the PROMPT only. Output and cache reads are billed at
|
|
||||||
// their own rates regardless of input length, which is how providers publish
|
|
||||||
// it — the higher rate is for the extra context you made the model read, not for
|
|
||||||
// what it wrote back.
|
|
||||||
type BillingLongContext struct {
|
|
||||||
// Above is the token count the step starts at. A request with exactly this
|
|
||||||
// many prompt tokens is still at the base rate; the step covers the tokens
|
|
||||||
// BEYOND it. Inclusive-below matches how the providers document it.
|
|
||||||
Above int `yaml:"above" json:"above"`
|
|
||||||
// Prompt is the USD-per-million rate for the tokens above Above.
|
|
||||||
Prompt string `yaml:"prompt" json:"prompt"`
|
|
||||||
// Multiplier is an alternative to Prompt: a factor applied to the base rate
|
|
||||||
// (DeepSeek's published step is exactly 2x). When both are set Prompt wins,
|
|
||||||
// and setting neither is a validation error rather than a silent no-op.
|
|
||||||
Multiplier float64 `yaml:"multiplier,omitempty" json:"multiplier,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate rejects a BillingDSL that could not work as written.
|
|
||||||
func (b *BillingDSL) Validate() error {
|
|
||||||
if b == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
ids := map[string]bool{}
|
|
||||||
for i := range b.Profiles {
|
|
||||||
p := &b.Profiles[i]
|
|
||||||
if p.ID == "" {
|
|
||||||
return fmt.Errorf("billing profile [%d] has no id", i)
|
|
||||||
}
|
|
||||||
if ids[p.ID] {
|
|
||||||
return fmt.Errorf("billing profile id %q appears more than once", p.ID)
|
|
||||||
}
|
|
||||||
ids[p.ID] = true
|
|
||||||
if p.Default != "" && p.Default != "free" && p.Default != "unpriced" {
|
|
||||||
return fmt.Errorf("profile %q: default must be free or unpriced, got %q", p.ID, p.Default)
|
|
||||||
}
|
|
||||||
for j := range p.Rules {
|
|
||||||
r := &p.Rules[j]
|
|
||||||
if r.URL == "" {
|
|
||||||
return fmt.Errorf("profile %q rule [%d]: url is required", p.ID, j)
|
|
||||||
}
|
|
||||||
switch r.Mode {
|
|
||||||
case "free", "token", "subscription", "unpriced":
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("profile %q rule [%d]: mode must be free/token/subscription/unpriced, got %q", p.ID, j, r.Mode)
|
|
||||||
}
|
|
||||||
if r.Mode == "token" {
|
|
||||||
for m, t := range r.Models {
|
|
||||||
if t.Prompt == "" || t.Completion == "" {
|
|
||||||
return fmt.Errorf("profile %q model %q: prompt and completion are required for token mode", p.ID, m)
|
|
||||||
}
|
|
||||||
if _, err := ParseFloatUSDPerM(t.Prompt); err != nil {
|
|
||||||
return fmt.Errorf("profile %q model %q prompt: %w", p.ID, m, err)
|
|
||||||
}
|
|
||||||
if _, err := ParseFloatUSDPerM(t.Completion); err != nil {
|
|
||||||
return fmt.Errorf("profile %q model %q completion: %w", p.ID, m, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Active must resolve to a profile id.
|
|
||||||
if b.Active != "" && !ids[b.Active] {
|
|
||||||
return fmt.Errorf("billing.active = %q but no profile has that id", b.Active)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve returns the profile with the given id; empty id (or an id that does
|
|
||||||
// not exist) falls back to the FIRST profile. A config with profiles but no
|
|
||||||
// active marker should mean "use the one I wrote first" rather than "price
|
|
||||||
// nothing", which is what a nil here would silently do.
|
|
||||||
func (b *BillingDSL) Resolve(id string) *BillingProfile {
|
|
||||||
if b == nil || len(b.Profiles) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if id != "" {
|
|
||||||
for i := range b.Profiles {
|
|
||||||
if b.Profiles[i].ID == id {
|
|
||||||
return &b.Profiles[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &b.Profiles[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseFloatUSDPerM parses a USD-per-million price string. Returns the
|
|
||||||
// per-single-token rate the plugin expects (price / 1e6).
|
|
||||||
//
|
|
||||||
// Kept as a string in the DSL rather than a float: "0.15" survives round-trips
|
|
||||||
// through YAML editors, and a malformed value must fail VALIDATION (with the
|
|
||||||
// file and field named) rather than parse as 0 and under-bill silently — which
|
|
||||||
// is exactly the failure mode that motivated the DSL.
|
|
||||||
func ParseFloatUSDPerM(s string) (float64, error) {
|
|
||||||
f, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("not a number: %q", s)
|
|
||||||
}
|
|
||||||
if f < 0 {
|
|
||||||
return 0, fmt.Errorf("negative price: %q", s)
|
|
||||||
}
|
|
||||||
return f / 1e6, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// KeyQuota is retained only to carry a scope entry's caps through the admin
|
// KeyQuota is retained only to carry a scope entry's caps through the admin
|
||||||
// API. Quotas are per model, never per key: there is deliberately no key-wide
|
// API. Quotas are per model, never per key: there is deliberately no key-wide
|
||||||
// total, so exhausting one model's budget never blocks the others.
|
// total, so exhausting one model's budget never blocks the others.
|
||||||
|
|||||||
@ -52,9 +52,6 @@ sources:
|
|||||||
if cfg.Sources[1].MaxConcurrent == 0 {
|
if cfg.Sources[1].MaxConcurrent == 0 {
|
||||||
t.Fatal("default max_concurrent not applied")
|
t.Fatal("default max_concurrent not applied")
|
||||||
}
|
}
|
||||||
if cfg.Sources[0].Models[0].Priority != 100 {
|
|
||||||
t.Fatalf("priority = %d", cfg.Sources[0].Models[0].Priority)
|
|
||||||
}
|
|
||||||
if cfg.DefaultModel != "AUTO" {
|
if cfg.DefaultModel != "AUTO" {
|
||||||
t.Fatalf("default model = %q", cfg.DefaultModel)
|
t.Fatalf("default model = %q", cfg.DefaultModel)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -14,9 +14,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "deepseek",
|
Adapter: "deepseek",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "deepseek-v4-flash", Priority: 100, Kind: "chat"},
|
{ID: "deepseek-v4-flash", Kind: "chat"},
|
||||||
{ID: "deepseek-v4-pro", Priority: 90, Kind: "chat"},
|
{ID: "deepseek-v4-pro", Kind: "chat"},
|
||||||
{ID: "deepseek-v4-flash-vision-exp", Priority: 60, Kind: "chat"},
|
{ID: "deepseek-v4-flash-vision-exp", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -25,10 +25,10 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "glm-5.3", Priority: 100, Kind: "chat"},
|
{ID: "glm-5.3", Kind: "chat"},
|
||||||
{ID: "glm-5.3-flash", Priority: 90, Kind: "chat"},
|
{ID: "glm-5.3-flash", Kind: "chat"},
|
||||||
{ID: "glm-5.2", Priority: 80, Kind: "chat"},
|
{ID: "glm-5.2", Kind: "chat"},
|
||||||
{ID: "glm-5.1", Priority: 60, Kind: "chat"},
|
{ID: "glm-5.1", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -37,10 +37,10 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "kimi-k3", Priority: 100, Kind: "chat"},
|
{ID: "kimi-k3", Kind: "chat"},
|
||||||
{ID: "kimi-k2.7-code", Priority: 90, Kind: "chat"},
|
{ID: "kimi-k2.7-code", Kind: "chat"},
|
||||||
{ID: "kimi-k2.7-code-highspeed", Priority: 85, Kind: "chat"},
|
{ID: "kimi-k2.7-code-highspeed", Kind: "chat"},
|
||||||
{ID: "kimi-k2.6", Priority: 70, Kind: "chat"},
|
{ID: "kimi-k2.6", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -49,11 +49,11 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "deepseek-ai/DeepSeek-V4-Pro", Priority: 100, Kind: "chat"},
|
{ID: "deepseek-ai/DeepSeek-V4-Pro", Kind: "chat"},
|
||||||
{ID: "deepseek-ai/DeepSeek-V4-Flash", Priority: 90, Kind: "chat"},
|
{ID: "deepseek-ai/DeepSeek-V4-Flash", Kind: "chat"},
|
||||||
{ID: "moonshotai/Kimi-K2.6", Priority: 80, Kind: "chat"},
|
{ID: "moonshotai/Kimi-K2.6", Kind: "chat"},
|
||||||
{ID: "zai-org/GLM-5.1", Priority: 70, Kind: "chat"},
|
{ID: "zai-org/GLM-5.1", Kind: "chat"},
|
||||||
{ID: "Kwai-Kolors/Kolors", Priority: 50, Kind: "image"},
|
{ID: "Kwai-Kolors/Kolors", Kind: "image"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -62,9 +62,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 4,
|
MaxConcurrent: 4,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "4.0Ultra", Priority: 100, Kind: "chat"},
|
{ID: "4.0Ultra", Kind: "chat"},
|
||||||
{ID: "generalv3.5", Priority: 80, Kind: "chat"},
|
{ID: "generalv3.5", Kind: "chat"},
|
||||||
{ID: "max-32k", Priority: 70, Kind: "chat"},
|
{ID: "max-32k", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -73,11 +73,11 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "qwen3.8-max", Priority: 100, Kind: "chat"},
|
{ID: "qwen3.8-max", Kind: "chat"},
|
||||||
{ID: "qwen3.8-flash", Priority: 90, Kind: "chat"},
|
{ID: "qwen3.8-flash", Kind: "chat"},
|
||||||
{ID: "qwen3.7-plus", Priority: 85, Kind: "chat"},
|
{ID: "qwen3.7-plus", Kind: "chat"},
|
||||||
{ID: "deepseek-v4-pro", Priority: 70, Kind: "chat"},
|
{ID: "deepseek-v4-pro", Kind: "chat"},
|
||||||
{ID: "kimi-k3", Priority: 70, Kind: "chat"},
|
{ID: "kimi-k3", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -86,9 +86,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "ernie-5.0-turbo", Priority: 100, Kind: "chat"},
|
{ID: "ernie-5.0-turbo", Kind: "chat"},
|
||||||
{ID: "ernie-5.0", Priority: 90, Kind: "chat"},
|
{ID: "ernie-5.0", Kind: "chat"},
|
||||||
{ID: "deepseek-v4-pro", Priority: 70, Kind: "chat"},
|
{ID: "deepseek-v4-pro", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -97,9 +97,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "doubao-seed-2.0-pro", Priority: 100, Kind: "chat"},
|
{ID: "doubao-seed-2.0-pro", Kind: "chat"},
|
||||||
{ID: "doubao-seed-2.0", Priority: 90, Kind: "chat"},
|
{ID: "doubao-seed-2.0", Kind: "chat"},
|
||||||
{ID: "doubao-seed-2.0-flash", Priority: 80, Kind: "chat"},
|
{ID: "doubao-seed-2.0-flash", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -108,8 +108,8 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "MiniMax-M3", Priority: 100, Kind: "chat"},
|
{ID: "MiniMax-M3", Kind: "chat"},
|
||||||
{ID: "MiniMax-M2.5", Priority: 80, Kind: "chat"},
|
{ID: "MiniMax-M2.5", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
// ── 国际主流 ────────────────────────────────────────────────────────────
|
// ── 国际主流 ────────────────────────────────────────────────────────────
|
||||||
@ -119,9 +119,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "openai",
|
Adapter: "openai",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "gpt-5.6-sol", Priority: 100, Kind: "chat"},
|
{ID: "gpt-5.6-sol", Kind: "chat"},
|
||||||
{ID: "gpt-5.6-terra", Priority: 90, Kind: "chat"},
|
{ID: "gpt-5.6-terra", Kind: "chat"},
|
||||||
{ID: "gpt-5.6-luna", Priority: 80, Kind: "chat"},
|
{ID: "gpt-5.6-luna", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -130,10 +130,10 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "anthropic",
|
Adapter: "anthropic",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "claude-opus-5", Priority: 100, Kind: "chat"},
|
{ID: "claude-opus-5", Kind: "chat"},
|
||||||
{ID: "claude-fable-5", Priority: 95, Kind: "chat"},
|
{ID: "claude-fable-5", Kind: "chat"},
|
||||||
{ID: "claude-sonnet-5", Priority: 85, Kind: "chat"},
|
{ID: "claude-sonnet-5", Kind: "chat"},
|
||||||
{ID: "claude-haiku-4-5", Priority: 70, Kind: "chat"},
|
{ID: "claude-haiku-4-5", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -142,10 +142,10 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "gemini",
|
Adapter: "gemini",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "gemini-3.6-flash", Priority: 100, Kind: "chat"},
|
{ID: "gemini-3.6-flash", Kind: "chat"},
|
||||||
{ID: "gemini-3.5-flash", Priority: 90, Kind: "chat"},
|
{ID: "gemini-3.5-flash", Kind: "chat"},
|
||||||
{ID: "gemini-3.5-flash-lite", Priority: 80, Kind: "chat"},
|
{ID: "gemini-3.5-flash-lite", Kind: "chat"},
|
||||||
{ID: "gemini-3.1-pro", Priority: 85, Kind: "chat"},
|
{ID: "gemini-3.1-pro", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -154,9 +154,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "groq",
|
Adapter: "groq",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "openai/gpt-oss-120b", Priority: 100, Kind: "chat"},
|
{ID: "openai/gpt-oss-120b", Kind: "chat"},
|
||||||
{ID: "openai/gpt-oss-20b", Priority: 80, Kind: "chat"},
|
{ID: "openai/gpt-oss-20b", Kind: "chat"},
|
||||||
{ID: "groq/compound", Priority: 70, Kind: "chat"},
|
{ID: "groq/compound", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -165,9 +165,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "mistral",
|
Adapter: "mistral",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "mistral-medium-latest", Priority: 100, Kind: "chat"},
|
{ID: "mistral-medium-latest", Kind: "chat"},
|
||||||
{ID: "mistral-large-latest", Priority: 90, Kind: "chat"},
|
{ID: "mistral-large-latest", Kind: "chat"},
|
||||||
{ID: "codestral-latest", Priority: 85, Kind: "chat"},
|
{ID: "codestral-latest", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -180,10 +180,10 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
"X-Title": "ModelRouter",
|
"X-Title": "ModelRouter",
|
||||||
},
|
},
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "openrouter/auto", Priority: 100, Kind: "chat"},
|
{ID: "openrouter/auto", Kind: "chat"},
|
||||||
{ID: "deepseek/deepseek-v4-flash", Priority: 90, Kind: "chat"},
|
{ID: "deepseek/deepseek-v4-flash", Kind: "chat"},
|
||||||
{ID: "anthropic/claude-opus-5", Priority: 85, Kind: "chat"},
|
{ID: "anthropic/claude-opus-5", Kind: "chat"},
|
||||||
{ID: "openai/gpt-5.6-sol", Priority: 85, Kind: "chat"},
|
{ID: "openai/gpt-5.6-sol", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -192,8 +192,8 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Adapter: "github",
|
Adapter: "github",
|
||||||
MaxConcurrent: 8,
|
MaxConcurrent: 8,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "gpt-5.6-sol", Priority: 100, Kind: "chat"},
|
{ID: "gpt-5.6-sol", Kind: "chat"},
|
||||||
{ID: "gpt-5.6-luna", Priority: 80, Kind: "chat"},
|
{ID: "gpt-5.6-luna", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
// ── 本地模型 ────────────────────────────────────────────────────────────
|
// ── 本地模型 ────────────────────────────────────────────────────────────
|
||||||
@ -204,9 +204,9 @@ var PresetTemplates = []SourceTemplate{
|
|||||||
Endpoint: "/api/chat",
|
Endpoint: "/api/chat",
|
||||||
MaxConcurrent: 4,
|
MaxConcurrent: 4,
|
||||||
Models: []Model{
|
Models: []Model{
|
||||||
{ID: "qwen3", Priority: 80, Kind: "chat"},
|
{ID: "qwen3", Kind: "chat"},
|
||||||
{ID: "deepseek-r1", Priority: 80, Kind: "chat"},
|
{ID: "deepseek-r1", Kind: "chat"},
|
||||||
{ID: "llama3.3", Priority: 70, Kind: "chat"},
|
{ID: "llama3.3", Kind: "chat"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
59
internal/config/retry_policy_test.go
Normal file
59
internal/config/retry_policy_test.go
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ShouldRetry 是纯数据判据:状态码列表 + 响应体子串。它决定"这次失败值不值得
|
||||||
|
// 在网关内部等一会儿重试"。厂商知识绝不能进内核,所以判据必须能用一个表格测清楚。
|
||||||
|
func TestRetryPolicyShouldRetry(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
policy *RetryPolicy
|
||||||
|
status int
|
||||||
|
body string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"nil policy", nil, 500, "anything", false},
|
||||||
|
{"max_attempts=1 disables", &RetryPolicy{OnStatus: []int{400}, MaxAttempts: 1}, 400, "x", false},
|
||||||
|
{"status matches, no body filter",
|
||||||
|
&RetryPolicy{OnStatus: []int{429}, MaxAttempts: 3}, 429, "", true},
|
||||||
|
{"status mismatch",
|
||||||
|
&RetryPolicy{OnStatus: []int{429}, MaxAttempts: 3}, 400, "429", false},
|
||||||
|
{"status + body match",
|
||||||
|
&RetryPolicy{OnStatus: []int{400}, OnBodyContains: []string{"TM.00001041"}, MaxAttempts: 3},
|
||||||
|
400, `{"error_code":"TM.00001041"}`, true},
|
||||||
|
{"status matches but body does not",
|
||||||
|
&RetryPolicy{OnStatus: []int{400}, OnBodyContains: []string{"TM.00001041"}, MaxAttempts: 3},
|
||||||
|
400, `{"error_code":"TM.00001005"}`, false},
|
||||||
|
{"empty substring never matches",
|
||||||
|
&RetryPolicy{OnStatus: []int{400}, OnBodyContains: []string{""}, MaxAttempts: 3}, 400, "x", false},
|
||||||
|
{"any of several body signals",
|
||||||
|
&RetryPolicy{OnStatus: []int{400, 429}, OnBodyContains: []string{"TM.00001041", "TPM limit"}, MaxAttempts: 3},
|
||||||
|
429, `{"message":"TPM limit exceeded"}`, true},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := tc.policy.ShouldRetry(tc.status, tc.body); got != tc.want {
|
||||||
|
t.Errorf("%s: ShouldRetry(%d, %q) = %v, want %v", tc.name, tc.status, tc.body, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryPolicyDelay(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
policy *RetryPolicy
|
||||||
|
want time.Duration
|
||||||
|
}{
|
||||||
|
{"nil", nil, 10 * time.Second},
|
||||||
|
{"explicit", &RetryPolicy{Delay: "250ms"}, 250 * time.Millisecond},
|
||||||
|
{"garbage falls back", &RetryPolicy{Delay: "not-a-duration"}, 10 * time.Second},
|
||||||
|
{"negative falls back", &RetryPolicy{Delay: "-5s"}, 10 * time.Second},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := tc.policy.DelayDuration(); got != tc.want {
|
||||||
|
t.Errorf("%s: DelayDuration() = %v, want %v", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -27,8 +27,9 @@ func NewGatewayKey() (string, error) {
|
|||||||
|
|
||||||
// SecretBox encrypts and decrypts sensitive values (upstream API keys,
|
// SecretBox encrypts and decrypts sensitive values (upstream API keys,
|
||||||
// custom header values, gateway keys) for persist-time protection. The
|
// custom header values, gateway keys) for persist-time protection. The
|
||||||
// master key comes from the LLMS_PROXY_MASTER_KEY environment variable
|
// master key comes from the MODELROUTER_MASTER_KEY environment variable
|
||||||
// (hex, 64 chars) or from a generated <runtime-dir>/master.key file.
|
// (hex, 64 chars; the pre-rename LLMS_PROXY_MASTER_KEY is still accepted) or
|
||||||
|
// from a generated <runtime-dir>/master.key file.
|
||||||
type SecretBox struct {
|
type SecretBox struct {
|
||||||
aead cipher.AEAD
|
aead cipher.AEAD
|
||||||
}
|
}
|
||||||
@ -37,10 +38,18 @@ type SecretBox struct {
|
|||||||
// file next to the runtime file. If neither exists, a fresh random key is
|
// file next to the runtime file. If neither exists, a fresh random key is
|
||||||
// generated and written to master.key with mode 0600.
|
// generated and written to master.key with mode 0600.
|
||||||
func NewSecretBox(runtimeFile string) (*SecretBox, error) {
|
func NewSecretBox(runtimeFile string) (*SecretBox, error) {
|
||||||
if v := os.Getenv("LLMS_PROXY_MASTER_KEY"); v != "" {
|
// New name first, then the pre-rename name: an existing unit file that still
|
||||||
key, err := hex.DecodeString(strings.TrimSpace(v))
|
// exports LLMS_PROXY_MASTER_KEY must keep starting after an upgrade.
|
||||||
|
envKey := os.Getenv("MODELROUTER_MASTER_KEY")
|
||||||
|
envName := "MODELROUTER_MASTER_KEY"
|
||||||
|
if envKey == "" {
|
||||||
|
envKey = os.Getenv("LLMS_PROXY_MASTER_KEY")
|
||||||
|
envName = "LLMS_PROXY_MASTER_KEY"
|
||||||
|
}
|
||||||
|
if envKey != "" {
|
||||||
|
key, err := hex.DecodeString(strings.TrimSpace(envKey))
|
||||||
if err != nil || len(key) != 32 {
|
if err != nil || len(key) != 32 {
|
||||||
return nil, fmt.Errorf("LLMS_PROXY_MASTER_KEY: expected 64 hex chars")
|
return nil, fmt.Errorf("%s: expected 64 hex chars", envName)
|
||||||
}
|
}
|
||||||
return newBox(key)
|
return newBox(key)
|
||||||
}
|
}
|
||||||
|
|||||||
38
internal/core/codearts_sig_expected.json
Normal file
38
internal/core/codearts_sig_expected.json
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"method": "GET",
|
||||||
|
"url": "https://snap-access.cn-north-4.myhuaweicloud.com/snap-manager/v1/statistics/plugin",
|
||||||
|
"body": "",
|
||||||
|
"ak": "AK1",
|
||||||
|
"sk": "SK1",
|
||||||
|
"st": "TOK1",
|
||||||
|
"canonical": "GET\n/snap-manager/v1/statistics/plugin/\n\nhost:snap-access.cn-north-4.myhuaweicloud.com\nx-sdk-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-sdk-date:20261013T150405Z\nx-security-token:TOK1\n\nhost;x-sdk-content-sha256;x-sdk-date;x-security-token\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||||
|
"string_to_sign": "SDK-HMAC-SHA256\n20261013T150405Z\nf40918a3339c9db2251933dcce410db05ff8a403de2767bc642140ea93cdc182",
|
||||||
|
"signature": "5332d627c3b8f14973afe41905f23dce75eaf9a34134dff670cb19835beb0549",
|
||||||
|
"signed_headers": "host;x-sdk-content-sha256;x-sdk-date;x-security-token"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"method": "GET",
|
||||||
|
"url": "https://snap-access.cn-north-4.myhuaweicloud.com/v1/ops/delivery?channel=IDE",
|
||||||
|
"body": "",
|
||||||
|
"ak": "AK2",
|
||||||
|
"sk": "SK2",
|
||||||
|
"st": "",
|
||||||
|
"canonical": "GET\n/v1/ops/delivery/\nchannel=IDE\nhost:snap-access.cn-north-4.myhuaweicloud.com\nx-sdk-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-sdk-date:20261013T150405Z\nx-security-token:\n\nhost;x-sdk-content-sha256;x-sdk-date;x-security-token\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||||
|
"string_to_sign": "SDK-HMAC-SHA256\n20261013T150405Z\n3c849b40bbad69b981eaf280e162878bf00687d08cb45df43ac792fc23add4f1",
|
||||||
|
"signature": "a01248a4121fd943f0f3f07aac101cd649ddf34d162729bf06a6a12ae9916b1a",
|
||||||
|
"signed_headers": "host;x-sdk-content-sha256;x-sdk-date;x-security-token"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"method": "POST",
|
||||||
|
"url": "https://snap-access.cn-north-4.myhuaweicloud.com/v1/ops/claim",
|
||||||
|
"body": "{\"campaignId\":\"C1\",\"channel\":\"IDE\"}",
|
||||||
|
"ak": "AK3",
|
||||||
|
"sk": "SK3",
|
||||||
|
"st": "T3",
|
||||||
|
"canonical": "POST\n/v1/ops/claim/\n\ncontent-type:application/json\nhost:snap-access.cn-north-4.myhuaweicloud.com\nx-sdk-content-sha256:85e6ce395b620ecb748af3b217f47107ebb5906d1a83fe7db221d9845de29f19\nx-sdk-date:20261013T150405Z\nx-security-token:T3\n\ncontent-type;host;x-sdk-content-sha256;x-sdk-date;x-security-token\n85e6ce395b620ecb748af3b217f47107ebb5906d1a83fe7db221d9845de29f19",
|
||||||
|
"string_to_sign": "SDK-HMAC-SHA256\n20261013T150405Z\n554c2607cff324a770b55e78e2a1c58afe466abf5e4eaa104a6e1fa96acfc733",
|
||||||
|
"signature": "e5faa52ec51c36b38804aac45a8a19876314cec465ab8f905100e657e8ea68d5",
|
||||||
|
"signed_headers": "content-type;host;x-sdk-content-sha256;x-sdk-date;x-security-token"
|
||||||
|
}
|
||||||
|
]
|
||||||
@ -8,20 +8,18 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/billing"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/lua"
|
||||||
"llmsproxy/internal/lua"
|
"modelrouter/internal/provider"
|
||||||
"llmsproxy/internal/provider"
|
"modelrouter/internal/scheduler"
|
||||||
"llmsproxy/internal/scheduler"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Core owns the running configuration and adapters. mu guards every read and
|
// Core owns the running configuration and adapters. mu guards every read and
|
||||||
@ -69,6 +67,11 @@ func NewFromConfig(cfg *config.Config) (*Core, error) {
|
|||||||
// to boot the gateway because a third-party .lua has a typo would be the
|
// to boot the gateway because a third-party .lua has a typo would be the
|
||||||
// wrong trade.
|
// wrong trade.
|
||||||
c.plugins = lua.NewPlugins(c.vm, cfg.PluginDir)
|
c.plugins = lua.NewPlugins(c.vm, cfg.PluginDir)
|
||||||
|
// R4-c: hand plugins the kernel action surface. Injected as an interface
|
||||||
|
// because lua must not import core (core depends on lua, not the reverse).
|
||||||
|
// Wired BEFORE LoadDir so a plugin that calls an action during its own
|
||||||
|
// __init/first hook already finds a working kernel rather than a nil one.
|
||||||
|
c.plugins.SetKernelAPI(c)
|
||||||
// Plugin outbound HTTP is scoped to the hosts of configured sources. Set
|
// Plugin outbound HTTP is scoped to the hosts of configured sources. Set
|
||||||
// before LoadDir so a plugin that requests on its first hook already sees
|
// before LoadDir so a plugin that requests on its first hook already sees
|
||||||
// the real allowlist rather than the deny-all placeholder.
|
// the real allowlist rather than the deny-all placeholder.
|
||||||
@ -92,9 +95,6 @@ func NewFromConfig(cfg *config.Config) (*Core, error) {
|
|||||||
for name, aud := range cfg.PluginVisibility {
|
for name, aud := range cfg.PluginVisibility {
|
||||||
c.plugins.SetPluginVisibility(name, aud)
|
c.plugins.SetPluginVisibility(name, aud)
|
||||||
}
|
}
|
||||||
if err := c.plugins.SeedBundled(); err != nil {
|
|
||||||
log.Printf("[core] seed bundled plugins: %v", err)
|
|
||||||
}
|
|
||||||
if err := c.plugins.LoadDir(); err != nil {
|
if err := c.plugins.LoadDir(); err != nil {
|
||||||
log.Printf("[core] plugin dir: %v", err)
|
log.Printf("[core] plugin dir: %v", err)
|
||||||
}
|
}
|
||||||
@ -106,7 +106,6 @@ func NewFromConfig(cfg *config.Config) (*Core, error) {
|
|||||||
log.Printf("[core] plugin %v %v loaded (hooks=%v ui=%v)",
|
log.Printf("[core] plugin %v %v loaded (hooks=%v ui=%v)",
|
||||||
p["name"], p["version"], p["hooks"], p["ui"])
|
p["name"], p["version"], p["hooks"], p["ui"])
|
||||||
}
|
}
|
||||||
c.applyBillingDSL()
|
|
||||||
c.store = config.NewStore(cfg.RuntimeFile)
|
c.store = config.NewStore(cfg.RuntimeFile)
|
||||||
// Share one box between the runtime store and config.yaml so a single
|
// Share one box between the runtime store and config.yaml so a single
|
||||||
// master.key seals both files. config.Load left the config holding
|
// master.key seals both files. config.Load left the config holding
|
||||||
@ -126,6 +125,11 @@ func NewFromConfig(cfg *config.Config) (*Core, error) {
|
|||||||
// only appeared to work because seedKeys' Save() unsealed memory as a side
|
// only appeared to work because seedKeys' Save() unsealed memory as a side
|
||||||
// effect. Removing the redundant saves exposed the real ordering bug.
|
// effect. Removing the redundant saves exposed the real ordering bug.
|
||||||
cfg.AttachSecretBox(c.store.SecretBox())
|
cfg.AttachSecretBox(c.store.SecretBox())
|
||||||
|
// Share that SAME box with the plugin credential vault: one master.key seals
|
||||||
|
// config.yaml, the runtime store, AND every .secrets/*.json. Set here (not at
|
||||||
|
// NewPlugins time) because the store — and thus the box — only exists now;
|
||||||
|
// plugins write secrets only during login flows, well after this point.
|
||||||
|
c.plugins.SetSecretBox(c.store.SecretBox())
|
||||||
hadPlaintextSecrets, err := cfg.UnsealSecrets(c.store.SecretBox())
|
hadPlaintextSecrets, err := cfg.UnsealSecrets(c.store.SecretBox())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unseal secrets: %w", err)
|
return nil, fmt.Errorf("unseal secrets: %w", err)
|
||||||
@ -140,9 +144,6 @@ func NewFromConfig(cfg *config.Config) (*Core, error) {
|
|||||||
if err := c.seedKeys(); err != nil {
|
if err := c.seedKeys(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := c.seedAuto(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := c.rebuildRegistry(); err != nil {
|
if err := c.rebuildRegistry(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -185,39 +186,6 @@ func (c *Core) migrateFromRuntime() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedAuto uses the existing auto rules from config.yaml, or creates an
|
|
||||||
// initial chain from legacy per-source model priority (first run only).
|
|
||||||
func (c *Core) seedAuto() error {
|
|
||||||
if len(c.cfg.Auto) > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
type item struct {
|
|
||||||
model string
|
|
||||||
prio int
|
|
||||||
}
|
|
||||||
var flat []item
|
|
||||||
for _, s := range c.mergedSources() {
|
|
||||||
for _, m := range s.Models {
|
|
||||||
if m.Kind == "image" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
flat = append(flat, item{m.ID, m.Priority})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.SliceStable(flat, func(i, j int) bool {
|
|
||||||
if flat[i].prio != flat[j].prio {
|
|
||||||
return flat[i].prio > flat[j].prio
|
|
||||||
}
|
|
||||||
return flat[i].model < flat[j].model
|
|
||||||
})
|
|
||||||
entries := make([]config.ModelScope, 0, len(flat))
|
|
||||||
for _, it := range flat {
|
|
||||||
entries = append(entries, config.ModelScope{Model: it.model})
|
|
||||||
}
|
|
||||||
c.cfg.Auto = entries
|
|
||||||
return c.cfg.Save()
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedKeys ensures config.yaml has admin keys for every gateway_keys entry
|
// seedKeys ensures config.yaml has admin keys for every gateway_keys entry
|
||||||
// from the YAML config (once). After first save they become GWKey records
|
// from the YAML config (once). After first save they become GWKey records
|
||||||
// in c.cfg.Keys and gateway_keys is no longer read for auth.
|
// in c.cfg.Keys and gateway_keys is no longer read for auth.
|
||||||
@ -650,11 +618,29 @@ func (c *Core) rebuildRegistry() error {
|
|||||||
// allowlist for plugin outbound HTTP. Unparseable URLs are skipped rather than
|
// allowlist for plugin outbound HTTP. Unparseable URLs are skipped rather than
|
||||||
// failing startup: a malformed base_url is already rejected when the provider is
|
// failing startup: a malformed base_url is already rejected when the provider is
|
||||||
// built, and it must not take plugins down with it.
|
// built, and it must not take plugins down with it.
|
||||||
|
//
|
||||||
|
// A source may also imply a SECOND host. OpenDesign splits inference
|
||||||
|
// (amr-link.open-design.ai) from billing (amr-api.open-design.ai), and the
|
||||||
|
// quota plugin derives the billing host from the inference one — so the host it
|
||||||
|
// asks for is not the host the operator configured, and the request would be
|
||||||
|
// refused with a message that reads like a plugin defect. Adding the derived
|
||||||
|
// host here is what makes the probe reachable.
|
||||||
|
//
|
||||||
|
// The grant stays scoped: only a source that IS on amr-link pulls in amr-api, so
|
||||||
|
// a deployment with no OpenDesign source does not silently gain the entry.
|
||||||
func sourceHosts(cfg *config.Config) []string {
|
func sourceHosts(cfg *config.Config) []string {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make([]string, 0, len(cfg.Sources))
|
out := make([]string, 0, len(cfg.Sources))
|
||||||
|
seen := make(map[string]bool, len(cfg.Sources))
|
||||||
|
add := func(h string) {
|
||||||
|
if h == "" || seen[h] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[h] = true
|
||||||
|
out = append(out, h)
|
||||||
|
}
|
||||||
for _, s := range cfg.Sources {
|
for _, s := range cfg.Sources {
|
||||||
if s.BaseURL == "" {
|
if s.BaseURL == "" {
|
||||||
continue
|
continue
|
||||||
@ -663,11 +649,27 @@ func sourceHosts(cfg *config.Config) []string {
|
|||||||
if err != nil || u.Hostname() == "" {
|
if err != nil || u.Hostname() == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, u.Hostname())
|
add(u.Hostname())
|
||||||
|
add(deriveBillingHost(u.Hostname()))
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// deriveBillingHost returns the billing host a configured source implies, or
|
||||||
|
// the host unchanged when it implies none.
|
||||||
|
//
|
||||||
|
// Kept in Go as well as in the Lua adapter because the two must agree: Go builds
|
||||||
|
// the allowlist and Lua builds the request URL, and a disagreement shows up as a
|
||||||
|
// refused request rather than as an error naming the mismatch. The rule is the
|
||||||
|
// full hostname, not a prefix — `amr-link.` as a prefix would also rewrite
|
||||||
|
// amr-link.example.com into an OpenDesign address.
|
||||||
|
func deriveBillingHost(host string) string {
|
||||||
|
if host == "amr-link.open-design.ai" {
|
||||||
|
return "amr-api.open-design.ai"
|
||||||
|
}
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
||||||
// buildAutoChain rebuilds the AUTO chain snapshot from config.yaml rules
|
// buildAutoChain rebuilds the AUTO chain snapshot from config.yaml rules
|
||||||
// against the current providers. Slot model ids are normalized to the exact
|
// against the current providers. Slot model ids are normalized to the exact
|
||||||
// configured spelling (case-insensitive match), otherwise ModelFor would fall
|
// configured spelling (case-insensitive match), otherwise ModelFor would fall
|
||||||
@ -833,13 +835,35 @@ func (c *Core) Reload() error {
|
|||||||
// ---- adapter management (web UI) ----
|
// ---- adapter management (web UI) ----
|
||||||
|
|
||||||
func (c *Core) ListAdapters() []lua.APIAdapter {
|
func (c *Core) ListAdapters() []lua.APIAdapter {
|
||||||
|
c.syncHiddenRefs()
|
||||||
return c.vm.ListAdapters()
|
return c.vm.ListAdapters()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// syncHiddenRefs 把"当前被源引用的适配器"同步给 VM。被引用的 hidden=true 适配器
|
||||||
|
// 必须保持可见(源编辑器要回显当前值);没被引用的隐藏适配器从列表消失。
|
||||||
|
// 每次 ListAdapters 前调用:源集合是动态的(运行时可增删),推模型容易漏。
|
||||||
|
func (c *Core) syncHiddenRefs() {
|
||||||
|
refs := []string{}
|
||||||
|
for _, s := range c.mergedSources() {
|
||||||
|
if s.Adapter != "" {
|
||||||
|
refs = append(refs, s.Adapter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.vm.SetHiddenRefs(refs)
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Core) UploadAdapter(name, code string) error {
|
func (c *Core) UploadAdapter(name, code string) error {
|
||||||
if name == "" {
|
if name == "" {
|
||||||
return fmt.Errorf("adapter name required")
|
return fmt.Errorf("adapter name required")
|
||||||
}
|
}
|
||||||
|
// Validate name SAFETY before touching the filesystem (R6). A path-traversal
|
||||||
|
// name ("../openai") must be rejected BEFORE WriteFile, or it would already
|
||||||
|
// have escaped the adapter dir by the time LoadAdapter re-checks. Built-in
|
||||||
|
// names are allowed here: an admin uploading openai.lua is the documented
|
||||||
|
// override, same as placing the file in adapter_dir by hand.
|
||||||
|
if err := lua.ValidateAdapterName(name, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := os.MkdirAll(c.cfg.AdapterDir, 0755); err != nil {
|
if err := os.MkdirAll(c.cfg.AdapterDir, 0755); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -851,6 +875,13 @@ func (c *Core) UploadAdapter(name, code string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *Core) RemoveAdapter(name string) error {
|
func (c *Core) RemoveAdapter(name string) error {
|
||||||
|
// Validate the name so a traversal name cannot os.Remove a file outside the
|
||||||
|
// adapter dir. A reserved (built-in) name is a legitimate delete target in
|
||||||
|
// the running VM, so reserved-name rejection is NOT applied here — only the
|
||||||
|
// path-safety character set is.
|
||||||
|
if err := lua.ValidateAdapterName(name, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
path := filepath.Join(c.cfg.AdapterDir, name+".lua")
|
path := filepath.Join(c.cfg.AdapterDir, name+".lua")
|
||||||
_ = os.Remove(path)
|
_ = os.Remove(path)
|
||||||
c.vm.RemoveAdapter(name)
|
c.vm.RemoveAdapter(name)
|
||||||
@ -974,63 +1005,6 @@ func (c *Core) Close() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyBillingDSL compiles the configured billing profile and injects it into
|
|
||||||
// the billing plugin.
|
|
||||||
//
|
|
||||||
// Configured pricing replaces the hand-written JSON blob that used to be PUT
|
|
||||||
// through the state API. That path was used twice in production and both times
|
|
||||||
// it "succeeded" while being wrong: once without the required `prices` wrapper
|
|
||||||
// (silently wiping the accumulated totals), once with a peak-window shape the
|
|
||||||
// plugin does not read (peak traffic billed at off-peak rates, no error). A
|
|
||||||
// typed config with load-time validation turns both mistakes into startup
|
|
||||||
// errors naming the file and field.
|
|
||||||
//
|
|
||||||
// A DSL with no profile, or one that fails to compile, leaves the plugin's
|
|
||||||
// prices untouched: billing keeps running on whatever it had, and the problem
|
|
||||||
// is logged rather than becoming a startup failure — the gateway must forward
|
|
||||||
// even when its cost accounting is misconfigured.
|
|
||||||
func (c *Core) applyBillingDSL() {
|
|
||||||
ps := c.plugins
|
|
||||||
if ps == nil || ps.Count() == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// The billing plugin may simply not be installed; a DSL without it is a
|
|
||||||
// no-op, not an error.
|
|
||||||
var have bool
|
|
||||||
for _, row := range ps.List() {
|
|
||||||
if row["name"] == "billing" && row["loaded"] == true {
|
|
||||||
have = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !have {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
dsl := c.cfg.BillingDSL
|
|
||||||
if dsl == nil || len(dsl.Profiles) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := dsl.Validate(); err != nil {
|
|
||||||
log.Printf("[billing] config invalid, profile not applied: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
profile := dsl.Resolve(dsl.Active)
|
|
||||||
if profile == nil {
|
|
||||||
log.Printf("[billing] no billing profile resolved (active=%q)", dsl.Active)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
prices, err := billing.Compile(profile, c.cfg.Sources)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[billing] profile %q not applied: %v", profile.ID, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := ps.SetState("billing", map[string]interface{}{"prices": prices}); err != nil {
|
|
||||||
log.Printf("[billing] applying profile %q: %v", profile.ID, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Printf("[billing] profile %q applied (%d rules, %d sources)", profile.ID, len(profile.Rules), len(c.cfg.Sources))
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateScopeQuotas checks every scope entry's caps before they are stored.
|
// ValidateScopeQuotas checks every scope entry's caps before they are stored.
|
||||||
// A typo in a period must be rejected at write time rather than silently
|
// A typo in a period must be rejected at write time rather than silently
|
||||||
// becoming a never-resetting budget — the opposite of what was typed.
|
// becoming a never-resetting budget — the opposite of what was typed.
|
||||||
|
|||||||
@ -5,7 +5,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// newTestConfig returns an in-memory config backed by temp files so
|
// newTestConfig returns an in-memory config backed by temp files so
|
||||||
@ -22,7 +22,7 @@ func newTestConfig(t *testing.T) *config.Config {
|
|||||||
DefaultModel: "AUTO",
|
DefaultModel: "AUTO",
|
||||||
GatewayKeys: []string{"sk-gw-test"},
|
GatewayKeys: []string{"sk-gw-test"},
|
||||||
Sources: []config.Source{
|
Sources: []config.Source{
|
||||||
{Name: "s1", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai", Models: []config.Model{{ID: "gpt-4o", Priority: 10}}},
|
{Name: "s1", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai", Models: []config.Model{{ID: "gpt-4o"}}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
401
internal/core/kernel_actions.go
Normal file
401
internal/core/kernel_actions.go
Normal file
@ -0,0 +1,401 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
"modelrouter/internal/lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 内核动作实现(R4-c)
|
||||||
|
//
|
||||||
|
// 把 Core 的公开动作暴露给插件 Lua,走 lua.KernelAPI 接口注入(依赖方向是
|
||||||
|
// core → lua,所以接口定义在 lua 包、实现放在这里)。
|
||||||
|
//
|
||||||
|
// # 为什么是"动作面"而不是"给插件一个 Core 指针"
|
||||||
|
//
|
||||||
|
// 直接暴露 *Core 会让插件能碰任何未导出的细节,而且每一次内核重构都会变成
|
||||||
|
// 对插件作者的破坏性变更。动作面是一份**显式的契约**:每个动作有名字、危险级、
|
||||||
|
// 超时预算,加动作是一次有意的行为,而不是顺手多导出一个方法。
|
||||||
|
//
|
||||||
|
// # 凭据边界
|
||||||
|
//
|
||||||
|
// 这里**不实现** list_keys / find_key。Core.ListKeys() 返回的是**明文**网关
|
||||||
|
// 密钥,把它交给第三方 Lua 等于把部署的整个凭据面交出去。key 相关的读走掩码
|
||||||
|
// 形态(create_key 的返回是唯一一次明文,且是"刚创建的那一个")。
|
||||||
|
var _ lua.KernelAPI = (*Core)(nil)
|
||||||
|
|
||||||
|
// Actions implements lua.KernelAPI.
|
||||||
|
//
|
||||||
|
// The catalog lives in the lua package because permissions must be checkable even
|
||||||
|
// when no Core is wired. This method exists so the interface is satisfied by the
|
||||||
|
// real kernel rather than by a separate adapter type.
|
||||||
|
func (c *Core) Actions() []lua.KernelAction { return lua.KernelActions() }
|
||||||
|
|
||||||
|
// Invoke implements lua.KernelAPI. It dispatches one action.
|
||||||
|
//
|
||||||
|
// Every action is expected to honour ctx. The caller (lua.callKernel) gives up on
|
||||||
|
// a deadline and returns an error to the plugin, but a kernel action that keeps
|
||||||
|
// running afterwards would be an invisible leak, so the long ones (probe_all)
|
||||||
|
// pass ctx down into the network layer.
|
||||||
|
func (c *Core) Invoke(ctx context.Context, name string, args map[string]interface{}) (interface{}, error) {
|
||||||
|
switch name {
|
||||||
|
// ---- 只读 ----
|
||||||
|
case "status":
|
||||||
|
return c.actionStatus(), nil
|
||||||
|
case "list_sources":
|
||||||
|
return c.actionListSources(), nil
|
||||||
|
case "list_models":
|
||||||
|
return c.actionListModels(), nil
|
||||||
|
case "list_adapters":
|
||||||
|
return c.actionListAdapters(), nil
|
||||||
|
case "list_auto_rules":
|
||||||
|
return c.actionListAutoRules(), nil
|
||||||
|
case "adapter_owner":
|
||||||
|
return c.actionAdapterOwner(args)
|
||||||
|
|
||||||
|
// ---- 写 ----
|
||||||
|
case "reload":
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true}, c.Reload()
|
||||||
|
case "reset_health":
|
||||||
|
// ResetHealth already no-ops on a nil registry (it ranges over
|
||||||
|
// Providers()), so this needs no guard beyond the nil check inside.
|
||||||
|
c.ResetHealth()
|
||||||
|
return map[string]interface{}{"ok": true}, nil
|
||||||
|
case "probe_all":
|
||||||
|
r := c.Registry()
|
||||||
|
if r == nil {
|
||||||
|
return nil, fmt.Errorf("the registry is not built yet")
|
||||||
|
}
|
||||||
|
r.ProbeAll(ctx)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("probe_all was cut short: %w", err)
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true}, nil
|
||||||
|
case "save_auto_rules":
|
||||||
|
return c.actionSaveAutoRules(args)
|
||||||
|
case "save_key_auto":
|
||||||
|
return c.actionSaveKeyAuto(args)
|
||||||
|
case "add_source":
|
||||||
|
return c.actionAddSource(args)
|
||||||
|
case "create_key":
|
||||||
|
return c.actionCreateKey(args)
|
||||||
|
|
||||||
|
// ---- 破坏性 ----
|
||||||
|
case "remove_source":
|
||||||
|
return c.actionRemoveSource(args)
|
||||||
|
case "delete_key":
|
||||||
|
return c.actionDeleteKey(args)
|
||||||
|
case "update_key":
|
||||||
|
return c.actionUpdateKey(args)
|
||||||
|
case "remove_adapter":
|
||||||
|
return c.actionRemoveAdapter(args)
|
||||||
|
case "set_plugin_enabled":
|
||||||
|
return c.actionSetPluginEnabled(args)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown kernel action %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 只读实现 ----
|
||||||
|
|
||||||
|
func (c *Core) actionStatus() interface{} {
|
||||||
|
// A plugin can call this DURING LoadDir (its own __init or first hook), which
|
||||||
|
// runs before rebuildRegistry builds the registry. Returning an empty status
|
||||||
|
// rather than panicking keeps a load-time call from taking the process down;
|
||||||
|
// the plugin sees an empty list and the next call gives real data.
|
||||||
|
out := map[string]interface{}{
|
||||||
|
"sources": []interface{}{},
|
||||||
|
"auto_slots": []interface{}{},
|
||||||
|
"default": c.cfg.DefaultModel,
|
||||||
|
"ready": false,
|
||||||
|
}
|
||||||
|
if r := c.Registry(); r != nil {
|
||||||
|
out["sources"] = r.Status()
|
||||||
|
out["ready"] = true
|
||||||
|
}
|
||||||
|
if slots := c.AutoSlotStates(); slots != nil {
|
||||||
|
out["auto_slots"] = slots
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// actionListSources returns the sources with api_key MASKED.
|
||||||
|
//
|
||||||
|
// Masking is not optional: a plugin has no legitimate need for another source's
|
||||||
|
// key, and a plugin that wrote the source itself already has it in its own
|
||||||
|
// credential vault.
|
||||||
|
func (c *Core) actionListSources() interface{} {
|
||||||
|
srcs := c.Sources()
|
||||||
|
out := make([]map[string]interface{}, 0, len(srcs))
|
||||||
|
for _, s := range srcs {
|
||||||
|
models := make([]string, 0, len(s.Models))
|
||||||
|
for _, m := range s.Models {
|
||||||
|
models = append(models, m.ID)
|
||||||
|
}
|
||||||
|
out = append(out, map[string]interface{}{
|
||||||
|
"name": s.Name,
|
||||||
|
"base_url": s.BaseURL,
|
||||||
|
"adapter": s.Adapter,
|
||||||
|
"models": models,
|
||||||
|
"has_key": s.APIKey != "" || s.APIKeyEnv != "",
|
||||||
|
"key_hint": maskSecretHint(s.APIKey),
|
||||||
|
"max_concurrent": s.MaxConcurrent,
|
||||||
|
// rpm must be exposed too: it carries the per-model minimum gap, and
|
||||||
|
// max_concurrent alone cannot distinguish "serial WITH a gap" from
|
||||||
|
// "serial with no gap" — the latter still collides with the
|
||||||
|
// upstream's per-model concurrency window.
|
||||||
|
"rpm": s.RPM,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// maskSecretHint shows only a short suffix so an operator (and a plugin) can tell
|
||||||
|
// two keys apart without either getting the secret.
|
||||||
|
func maskSecretHint(k string) string {
|
||||||
|
if k == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if len(k) <= 4 {
|
||||||
|
return "***"
|
||||||
|
}
|
||||||
|
return "***" + k[len(k)-4:]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionListModels() interface{} {
|
||||||
|
if r := c.Registry(); r != nil {
|
||||||
|
return r.ModelList()
|
||||||
|
}
|
||||||
|
return []string{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionListAdapters() interface{} {
|
||||||
|
return c.ListAdapters()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionListAutoRules() interface{} {
|
||||||
|
return map[string]interface{}{
|
||||||
|
"auto": c.AutoRules(),
|
||||||
|
"auto_image": c.AutoImageRules(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionAdapterOwner(args map[string]interface{}) (interface{}, error) {
|
||||||
|
name, _ := args["name"].(string)
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("adapter_owner requires a name")
|
||||||
|
}
|
||||||
|
owner := ""
|
||||||
|
if ps := c.Plugins(); ps != nil {
|
||||||
|
owner = ps.AdapterOwnerOf(name)
|
||||||
|
}
|
||||||
|
return map[string]interface{}{
|
||||||
|
"name": name,
|
||||||
|
"owner": owner, // "" means operator-installed
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 写实现 ----
|
||||||
|
|
||||||
|
func (c *Core) actionSaveAutoRules(args map[string]interface{}) (interface{}, error) {
|
||||||
|
entries, err := decodeScopes(args["entries"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := c.SaveAutoRules(entries); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true, "count": len(entries)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionSaveKeyAuto(args map[string]interface{}) (interface{}, error) {
|
||||||
|
keyName, _ := args["key_name"].(string)
|
||||||
|
if keyName == "" {
|
||||||
|
return nil, fmt.Errorf("save_key_auto requires key_name")
|
||||||
|
}
|
||||||
|
entries, err := decodeScopes(args["entries"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := c.SaveKeyAuto(keyName, entries); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true, "count": len(entries)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// actionAddSource is the capability that makes "plugin-driven upstream" real: a
|
||||||
|
// plugin that has just obtained credentials can wire its endpoint in as a source
|
||||||
|
// without the operator editing config.yaml.
|
||||||
|
func (c *Core) actionAddSource(args map[string]interface{}) (interface{}, error) {
|
||||||
|
raw, ok := args["source"]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("add_source requires a source table")
|
||||||
|
}
|
||||||
|
b, err := json.Marshal(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("source is not encodable: %w", err)
|
||||||
|
}
|
||||||
|
var src config.Source
|
||||||
|
if err := json.Unmarshal(b, &src); err != nil {
|
||||||
|
return nil, fmt.Errorf("source shape is invalid: %w", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(src.Name) == "" {
|
||||||
|
return nil, fmt.Errorf("source requires a name")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(src.BaseURL) == "" {
|
||||||
|
return nil, fmt.Errorf("source %s requires a base_url", src.Name)
|
||||||
|
}
|
||||||
|
if err := c.AddSource(src); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true, "name": src.Name}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// actionCreateKey creates a gateway key.
|
||||||
|
//
|
||||||
|
// The plaintext key is returned ONCE, in this response — the same posture as the
|
||||||
|
// admin API. It is not retrievable later, so a plugin that needs it must store it
|
||||||
|
// in its own credential vault immediately.
|
||||||
|
func (c *Core) actionCreateKey(args map[string]interface{}) (interface{}, error) {
|
||||||
|
name, _ := args["name"].(string)
|
||||||
|
role, _ := args["role"].(string)
|
||||||
|
if role == "" {
|
||||||
|
role = "user"
|
||||||
|
}
|
||||||
|
models, err := decodeScopes(args["models"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
k, err := c.CreateKey(name, role, models, "created by plugin")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{
|
||||||
|
"key": k.Key,
|
||||||
|
"name": k.Name,
|
||||||
|
"role": k.Role,
|
||||||
|
"models": k.Models,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 破坏性实现 ----
|
||||||
|
|
||||||
|
func (c *Core) actionRemoveSource(args map[string]interface{}) (interface{}, error) {
|
||||||
|
name, _ := args["name"].(string)
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("remove_source requires a name")
|
||||||
|
}
|
||||||
|
// The last source is not removable: an empty registry turns every request
|
||||||
|
// into a 503, and the operator would have to fix it from the filesystem.
|
||||||
|
if len(c.Sources()) <= 1 {
|
||||||
|
return nil, fmt.Errorf("refusing to remove the last source: the gateway would have nothing to route to")
|
||||||
|
}
|
||||||
|
if err := c.RemoveSource(name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true, "name": name}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionDeleteKey(args map[string]interface{}) (interface{}, error) {
|
||||||
|
key, _ := args["key"].(string)
|
||||||
|
if key == "" {
|
||||||
|
return nil, fmt.Errorf("delete_key requires the key to delete")
|
||||||
|
}
|
||||||
|
ok, err := c.DeleteKey(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("no such key")
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionUpdateKey(args map[string]interface{}) (interface{}, error) {
|
||||||
|
key, _ := args["key"].(string)
|
||||||
|
if key == "" {
|
||||||
|
return nil, fmt.Errorf("update_key requires the key")
|
||||||
|
}
|
||||||
|
name, _ := args["name"].(string)
|
||||||
|
role, _ := args["role"].(string)
|
||||||
|
models, err := decodeScopes(args["models"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
k, err := c.UpdateKey(key, name, role, models, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true, "name": k.Name, "role": k.Role}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// actionRemoveAdapter deletes an adapter, but only one the CALLER plugin owns —
|
||||||
|
// the ownership check lives in Plugins.RemoveAdapter (see the adapter bridge).
|
||||||
|
// A plugin can always remove its own; removing an operator's or a built-in is
|
||||||
|
// refused there, and that refusal is the security property.
|
||||||
|
func (c *Core) actionRemoveAdapter(args map[string]interface{}) (interface{}, error) {
|
||||||
|
name, _ := args["name"].(string)
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("remove_adapter requires a name")
|
||||||
|
}
|
||||||
|
ps := c.Plugins()
|
||||||
|
if ps == nil {
|
||||||
|
return nil, fmt.Errorf("no plugins loaded")
|
||||||
|
}
|
||||||
|
plugin, _ := args["plugin"].(string)
|
||||||
|
if plugin == "" {
|
||||||
|
return nil, fmt.Errorf("remove_adapter needs the calling plugin's name")
|
||||||
|
}
|
||||||
|
if err := ps.RemoveAdapter(plugin, name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true, "name": name}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Core) actionSetPluginEnabled(args map[string]interface{}) (interface{}, error) {
|
||||||
|
name, _ := args["name"].(string)
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("set_plugin_enabled requires a name")
|
||||||
|
}
|
||||||
|
enabled, ok := args["enabled"].(bool)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("set_plugin_enabled requires enabled (boolean)")
|
||||||
|
}
|
||||||
|
ps := c.Plugins()
|
||||||
|
if ps == nil {
|
||||||
|
return nil, fmt.Errorf("no plugins loaded")
|
||||||
|
}
|
||||||
|
if err := ps.SetEnabled(name, enabled); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]interface{}{"ok": true, "name": name, "enabled": enabled}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeScopes converts a Lua-decoded array of model-scope tables into
|
||||||
|
// []config.ModelScope. It accepts both an explicit array and a single table so a
|
||||||
|
// one-entry chain does not need a wrapper.
|
||||||
|
func decodeScopes(v interface{}) ([]config.ModelScope, error) {
|
||||||
|
if v == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
b, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("entries are not encodable: %w", err)
|
||||||
|
}
|
||||||
|
var list []config.ModelScope
|
||||||
|
if err := json.Unmarshal(b, &list); err == nil {
|
||||||
|
return list, nil
|
||||||
|
}
|
||||||
|
var one config.ModelScope
|
||||||
|
if err := json.Unmarshal(b, &one); err != nil {
|
||||||
|
return nil, fmt.Errorf("entries shape is invalid: %w", err)
|
||||||
|
}
|
||||||
|
return []config.ModelScope{one}, nil
|
||||||
|
}
|
||||||
373
internal/core/kernel_actions_test.go
Normal file
373
internal/core/kernel_actions_test.go
Normal file
@ -0,0 +1,373 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
"modelrouter/internal/lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 本文件验证 R4-c/R4-d 的闭环:**插件自己驱动外部源的链接认证,然后把源接进内核**。
|
||||||
|
//
|
||||||
|
// 这是用户点明的"同类插件"能力(对照 dsh-our-free-model):插件主导 OAuth 登录、
|
||||||
|
// 存凭据、定时领额度、把拿到的端点注册成内核上游。此前 llmsproxy 的插件只能管
|
||||||
|
// 自己的适配器,够不到内核。
|
||||||
|
|
||||||
|
// kernelTestCore 起一个最小 core(一个源 + 一个 key)。
|
||||||
|
func kernelTestCore(t *testing.T) *Core {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
pluginDir := filepath.Join(dir, "plugins")
|
||||||
|
// 插件的自有存储位置由插件自己决定(读 config.yaml 的 plugin_dir,或
|
||||||
|
// MODELROUTER_PLUGIN_DIR 覆盖)。测试**必须**指向临时目录 —— 否则插件会读到
|
||||||
|
// /etc/modelrouter/config.yaml 并写进**生产插件目录**(真实事故:一次全量测试
|
||||||
|
// 把 26 个测试账号写进了生产 .channels-store.json)。
|
||||||
|
// 调用方已显式设定时尊重它:多次「重启」要共用同一目录才能测持久化。
|
||||||
|
if os.Getenv("MODELROUTER_PLUGIN_DIR") == "" {
|
||||||
|
t.Setenv("MODELROUTER_PLUGIN_DIR", pluginDir)
|
||||||
|
}
|
||||||
|
// ★ 显示器号与端口用**高段号**,避开生产。
|
||||||
|
//
|
||||||
|
// 真实事故(2026-10-15):生产 unit 带 `PrivateTmp=yes`,它起的 Xvfb :91 的
|
||||||
|
// 锁文件在**服务自己的 /tmp** 里 —— 宿主上的测试看不到它,于是判"91 空闲"
|
||||||
|
// 并去抢,结果 `Xvfb 未就绪(/tmp/.X11-unix/X91 未出现)`,而生产上正跑着
|
||||||
|
// 一套浏览器栈。判据本身没错,错的是**两个 /tmp 互不可见**而号段重叠。
|
||||||
|
// 故测试整体挪到 300+。
|
||||||
|
if os.Getenv("MODELROUTER_XDISPLAY_BASE") == "" {
|
||||||
|
t.Setenv("MODELROUTER_XDISPLAY_BASE", "300")
|
||||||
|
}
|
||||||
|
cfg := &config.Config{
|
||||||
|
Path: filepath.Join(dir, "config.yaml"),
|
||||||
|
AdapterDir: filepath.Join(dir, "adapters"),
|
||||||
|
PluginDir: pluginDir,
|
||||||
|
RuntimeFile: filepath.Join(dir, "runtime.json"),
|
||||||
|
DefaultModel: "AUTO",
|
||||||
|
GatewayKeys: []string{"sk-kernel-0001"},
|
||||||
|
Sources: []config.Source{{
|
||||||
|
Name: "primary", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai",
|
||||||
|
APIKey: "secret-primary-key", MaxConcurrent: 1,
|
||||||
|
Models: []config.Model{{ID: "m1", Kind: "chat"}},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
c, err := NewFromConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 测试结束**必须**把插件拉起的服务端浏览器栈收掉。
|
||||||
|
// 为何必需:callback 流(trae/gemini/有道)在 start_login 时就起一整套
|
||||||
|
// Xvfb+chromium+x11vnc+websockify,而绝大多数测试不会显式 stop_browser、
|
||||||
|
// 会话 TTL 也还没到 —— 于是一轮 core 全包跑完就攒出 7 套常驻进程
|
||||||
|
// (实测 websockify×7 + x11vnc×7 + 7 个 profile 目录),把显示器号与端口
|
||||||
|
// 段逐步耗光,后续测试报「chromium/websockify 端口未就绪」。
|
||||||
|
// 在这一处统一收掉,比让每个测试各自记得更可靠。
|
||||||
|
t.Cleanup(func() {
|
||||||
|
ps := c.Plugins()
|
||||||
|
if ps == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, name := range []string{"channels"} {
|
||||||
|
_, _ = ps.InvokeHandler(name, "stop_browser", &lua.HandlerRequest{
|
||||||
|
Method: "POST", Role: "admin",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Cleanup(c.Close)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// grant 给插件授权若干动作(模拟运维在设置页点选)。
|
||||||
|
func grant(t *testing.T, c *Core, plugin string, actions ...string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := c.Plugins().SetPluginKernelPermission(plugin, actions, false); err != nil {
|
||||||
|
t.Fatalf("grant %v to %s: %v", actions, plugin, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const kernelProbePlugin = `
|
||||||
|
local plugin = { name = "kernelprobe", version = "1.0" }
|
||||||
|
plugin.exports = { "read_status", "read_models", "read_sources", "try_delete_key", "probe_count", "call_reload" }
|
||||||
|
plugin.kernel = { "status", "list_models", "reload" }
|
||||||
|
|
||||||
|
function plugin.read_status()
|
||||||
|
local st, err = __llmsproxy_kernel.call("status", {})
|
||||||
|
if err then return { err = err } end
|
||||||
|
return { ready = st.ready, nsources = #(st.sources or {}) }
|
||||||
|
end
|
||||||
|
|
||||||
|
function plugin.read_models()
|
||||||
|
local m, err = __llmsproxy_kernel.call("list_models", {})
|
||||||
|
if err then return { err = err } end
|
||||||
|
return { models = m }
|
||||||
|
end
|
||||||
|
|
||||||
|
-- sources 动作返回的密钥必须是掩码的
|
||||||
|
function plugin.read_sources()
|
||||||
|
local s, err = __llmsproxy_kernel.call("list_sources", {})
|
||||||
|
if err then return { err = err } end
|
||||||
|
return { sources = s }
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 未授权的破坏性动作必须被拒
|
||||||
|
function plugin.try_delete_key()
|
||||||
|
local _, err = __llmsproxy_kernel.call("delete_key", { key = "sk-kernel-0001" })
|
||||||
|
return { err = err }
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 授权数量可见(插件据此优雅降级)
|
||||||
|
function plugin.probe_count()
|
||||||
|
local acts = __llmsproxy_kernel.actions()
|
||||||
|
local granted = 0
|
||||||
|
for _, a in ipairs(acts) do
|
||||||
|
if a.granted then granted = granted + 1 end
|
||||||
|
end
|
||||||
|
return { total = #acts, granted = granted }
|
||||||
|
end
|
||||||
|
|
||||||
|
function plugin.call_reload()
|
||||||
|
local r, err = __llmsproxy_kernel.call("reload", {})
|
||||||
|
return { ok = r and r.ok, err = err }
|
||||||
|
end
|
||||||
|
return plugin
|
||||||
|
`
|
||||||
|
|
||||||
|
// TestPluginCallsKernelActions 验证插件能调内核动作,且权限边界成立。
|
||||||
|
func TestPluginCallsKernelActions(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
if err := c.Plugins().LoadSource("kernelprobe", kernelProbePlugin); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
grant(t, c, "kernelprobe", "status", "list_models", "list_sources")
|
||||||
|
|
||||||
|
// 只读动作可用
|
||||||
|
out, err := c.Plugins().Call("kernelprobe", "read_status", map[string]interface{}{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read_status: %v", err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]interface{})
|
||||||
|
if m["err"] != nil {
|
||||||
|
t.Fatalf("status 调用失败: %v", m["err"])
|
||||||
|
}
|
||||||
|
if m["ready"] != true {
|
||||||
|
t.Errorf("registry 应已就绪: %#v", m)
|
||||||
|
}
|
||||||
|
if n, _ := m["nsources"].(float64); n < 1 {
|
||||||
|
t.Errorf("应至少有 1 个源, 实得 %v", m["nsources"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// 模型列表
|
||||||
|
out, err = c.Plugins().Call("kernelprobe", "read_models", map[string]interface{}{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if models, ok := out.(map[string]interface{})["models"].([]interface{}); !ok || len(models) == 0 {
|
||||||
|
t.Errorf("模型列表应为非空数组: %#v", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKernelActionMasksSourceKeys 是安全断言:list_sources **不得**泄漏 api_key。
|
||||||
|
func TestKernelActionMasksSourceKeys(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
if err := c.Plugins().LoadSource("kernelprobe", kernelProbePlugin); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
grant(t, c, "kernelprobe", "list_sources")
|
||||||
|
|
||||||
|
out, err := c.Plugins().Call("kernelprobe", "read_sources", map[string]interface{}{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 把整个返回值序列化后检查:明文密钥绝不能出现在里面
|
||||||
|
b, _ := json.Marshal(out)
|
||||||
|
if s := string(b); json.Valid(b) && contains(s, "secret-primary-key") {
|
||||||
|
t.Fatalf("list_sources 泄漏了源的明文 api_key: %s", s)
|
||||||
|
}
|
||||||
|
// 但应有可区分性(has_key + 掩码后缀)
|
||||||
|
if !contains(string(b), "***") {
|
||||||
|
t.Errorf("应给出掩码提示而不是什么都没有: %s", string(b))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKernelActionDeniedWithoutGrant 是权限默认拒绝的断言。
|
||||||
|
func TestKernelActionDeniedWithoutGrant(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
if err := c.Plugins().LoadSource("kernelprobe", kernelProbePlugin); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 故意不授权任何动作
|
||||||
|
out, err := c.Plugins().Call("kernelprobe", "try_delete_key", map[string]interface{}{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("调用本身不该报错(拒绝应作为返回值): %v", err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]interface{})
|
||||||
|
if m["err"] == nil || m["err"] == "" {
|
||||||
|
t.Fatal("未授权的 delete_key 必须被拒")
|
||||||
|
}
|
||||||
|
t.Logf("拒绝信息: %v", m["err"])
|
||||||
|
// 拒绝信息要点名动作与插件(否则运维无法行动)
|
||||||
|
msg, _ := m["err"].(string)
|
||||||
|
if !contains(msg, "delete_key") || !contains(msg, "kernelprobe") {
|
||||||
|
t.Errorf("拒绝信息应点名动作与插件, 实得 %q", msg)
|
||||||
|
}
|
||||||
|
// 且密钥必须还在
|
||||||
|
if _, ok := c.FindKey("sk-kernel-0001"); !ok {
|
||||||
|
t.Fatal("被拒的动作不应真的执行")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKernelActionsReportGranted 插件能看到自己拿到了哪些权限。
|
||||||
|
func TestKernelActionsReportGranted(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
if err := c.Plugins().LoadSource("kernelprobe", kernelProbePlugin); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
grant(t, c, "kernelprobe", "status", "list_models")
|
||||||
|
|
||||||
|
out, err := c.Plugins().Call("kernelprobe", "probe_count", map[string]interface{}{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]interface{})
|
||||||
|
total, _ := m["total"].(float64)
|
||||||
|
granted, _ := m["granted"].(float64)
|
||||||
|
if total < 15 {
|
||||||
|
t.Errorf("动作总数应 >=15, 实得 %v", total)
|
||||||
|
}
|
||||||
|
if granted != 2 {
|
||||||
|
t.Errorf("授权数应为 2(status+list_models), 实得 %v", granted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPluginRegistersSourceIntoKernel 是"同类插件"的核心闭环:
|
||||||
|
// 插件通过内核动作把一个新的外部源接进网关。
|
||||||
|
func TestPluginRegistersSourceIntoKernel(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
code := `
|
||||||
|
local plugin = { name = "upstream", version = "1.0" }
|
||||||
|
plugin.exports = { "add_my_source" }
|
||||||
|
plugin.kernel = { "add_source", "list_sources" }
|
||||||
|
|
||||||
|
function plugin.add_my_source(args)
|
||||||
|
local src = {
|
||||||
|
name = "plugin-provided",
|
||||||
|
base_url = "http://127.0.0.1:9/v1",
|
||||||
|
adapter = "openai",
|
||||||
|
api_key = "plugin-managed-token",
|
||||||
|
max_concurrent = 2,
|
||||||
|
models = { { id = "plugin-model", kind = "chat", priority = 10 } },
|
||||||
|
}
|
||||||
|
local r, err = __llmsproxy_kernel.call("add_source", { source = src })
|
||||||
|
if err then return { err = err } end
|
||||||
|
return { ok = r.ok, name = r.name }
|
||||||
|
end
|
||||||
|
return plugin
|
||||||
|
`
|
||||||
|
if err := c.Plugins().LoadSource("upstream", code); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
grant(t, c, "upstream", "add_source", "list_sources")
|
||||||
|
|
||||||
|
out, err := c.Plugins().Call("upstream", "add_my_source", map[string]interface{}{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("add_source: %v", err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]interface{})
|
||||||
|
if m["err"] != nil {
|
||||||
|
t.Fatalf("插件注册源失败: %v", m["err"])
|
||||||
|
}
|
||||||
|
if m["name"] != "plugin-provided" {
|
||||||
|
t.Fatalf("返回的源名不对: %#v", m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 内核侧必须真的有了这个源 —— 这是"接进网关"的定义
|
||||||
|
found := false
|
||||||
|
for _, s := range c.Sources() {
|
||||||
|
if s.Name == "plugin-provided" {
|
||||||
|
found = true
|
||||||
|
if s.BaseURL != "http://127.0.0.1:9/v1" {
|
||||||
|
t.Errorf("base_url 未落库: %q", s.BaseURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("插件注册的源没有出现在内核源列表里")
|
||||||
|
}
|
||||||
|
// 且模型已可路由
|
||||||
|
for _, id := range c.Registry().ModelList() {
|
||||||
|
if id == "plugin-model" {
|
||||||
|
return // 通过
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Errorf("插件注册的模型未进入可路由列表: %v", c.Registry().ModelList())
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKernelPermissionPersists 授权必须落盘(否则重启即失效)。
|
||||||
|
func TestKernelPermissionPersists(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
if err := c.Plugins().LoadSource("kernelprobe", kernelProbePlugin); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
grant(t, c, "kernelprobe", "status", "reload")
|
||||||
|
|
||||||
|
pdir := c.Config().PluginDir
|
||||||
|
path := filepath.Join(pdir, ".kernel-perms.json")
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("授权未落盘: %v", err)
|
||||||
|
}
|
||||||
|
if !contains(string(b), "reload") {
|
||||||
|
t.Errorf("落盘内容缺少动作名: %s", b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 模拟重启:新建 Plugins、加载授权、权限仍在
|
||||||
|
vm2 := lua.NewVM(filepath.Join(t.TempDir(), "adapters"))
|
||||||
|
if err := vm2.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer vm2.Stop()
|
||||||
|
ps2 := lua.NewPlugins(vm2, pdir)
|
||||||
|
if err := ps2.LoadKernelPermissions(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
perm := ps2.PluginKernelPermission("kernelprobe")
|
||||||
|
if len(perm.Actions) != 2 {
|
||||||
|
t.Fatalf("重启后授权应保留 2 项, 实得 %#v", perm.Actions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUnknownKernelActionRejected 拼错的动作名必须在授权时就被拒,
|
||||||
|
// 而不是静默授予"什么都不允许"。
|
||||||
|
func TestUnknownKernelActionRejected(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
if err := c.Plugins().LoadSource("kernelprobe", kernelProbePlugin); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := c.Plugins().SetPluginKernelPermission("kernelprobe", []string{"no_such_action"}, false); err == nil {
|
||||||
|
t.Fatal("未知动作名应被拒")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKernelActionSurfaceExcludesPlaintextKeys 是凭据边界的回归断言:
|
||||||
|
// 动作面里绝不能出现返回明文密钥的动作(ListKeys/FindKey 被有意排除)。
|
||||||
|
func TestKernelActionSurfaceExcludesPlaintextKeys(t *testing.T) {
|
||||||
|
for _, a := range lua.KernelActions() {
|
||||||
|
switch a.Name {
|
||||||
|
case "list_keys", "find_key", "get_key":
|
||||||
|
t.Fatalf("动作面出现了明文密钥读取动作 %q —— 这会把部署的凭据面交给插件", a.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(haystack, needle string) bool {
|
||||||
|
return len(needle) > 0 && len(haystack) >= len(needle) &&
|
||||||
|
func() bool {
|
||||||
|
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||||
|
if haystack[i:i+len(needle)] == needle {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}()
|
||||||
|
}
|
||||||
39
internal/core/kernel_effect_test.go
Normal file
39
internal/core/kernel_effect_test.go
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// TestGrantedDestructiveActionTakesEffect 确认授权后的破坏性动作**真的执行**。
|
||||||
|
//
|
||||||
|
// 上一条测试只证明"不死锁、有返回"。一个把所有动作都变成空转的 dispatch 也能
|
||||||
|
// 通过它 —— 所以这条单独断言副作用确实落地(密钥真的被删)。
|
||||||
|
func TestGrantedDestructiveActionTakesEffect(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
code := `
|
||||||
|
local plugin = { name = "eff", version = "1.0" }
|
||||||
|
plugin.exports = { "del" }
|
||||||
|
function plugin.del(args)
|
||||||
|
local r, err = __llmsproxy_kernel.call("delete_key", { key = args.key })
|
||||||
|
return { ok = r ~= nil, err = err }
|
||||||
|
end
|
||||||
|
return plugin
|
||||||
|
`
|
||||||
|
if err := c.Plugins().LoadSource("eff", code); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = c.Plugins().SetPluginKernelPermission("eff", []string{"delete_key"}, false)
|
||||||
|
|
||||||
|
if _, ok := c.FindKey("sk-kernel-0001"); !ok {
|
||||||
|
t.Fatal("前置条件:key 应存在")
|
||||||
|
}
|
||||||
|
out, err := c.Plugins().Call("eff", "del", map[string]interface{}{"key": "sk-kernel-0001"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]interface{})
|
||||||
|
if m["ok"] != true {
|
||||||
|
t.Fatalf("动作未成功: %#v", m)
|
||||||
|
}
|
||||||
|
if _, ok := c.FindKey("sk-kernel-0001"); ok {
|
||||||
|
t.Fatal("动作报告成功但副作用没发生 —— dispatch 是空转")
|
||||||
|
}
|
||||||
|
}
|
||||||
69
internal/core/kernel_no_deadlock_test.go
Normal file
69
internal/core/kernel_no_deadlock_test.go
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"modelrouter/internal/lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestNoKernelActionDeadlocksFromLua 对**每个**内核动作做同一条断言:
|
||||||
|
// 在插件 Lua 调用(此时持有 p.mu)内执行时不得死锁。
|
||||||
|
//
|
||||||
|
// 为什么是"每个"而不是"那个出问题的":最初只有 set_plugin_enabled 被发现自死锁
|
||||||
|
// (它经 SetEnabled 重入 p.mu),但"是否还有别的动作也会重入"只能靠枚举回答。
|
||||||
|
// 这条测试就是那次枚举的固化 —— 将来新增动作时如果也重入插件锁,它会失败。
|
||||||
|
//
|
||||||
|
// 同时它验证了"超时后副作用仍在幕后执行"这个更隐蔽的变体:那个缺陷的表现是
|
||||||
|
// 插件收到 error、但动作其实生效了。断言方式只是"不死锁 + 返回",因为死锁与
|
||||||
|
// 迟到副作用是同一个根因(在错误的锁上下文里执行)。
|
||||||
|
func TestNoKernelActionDeadlocksFromLua(t *testing.T) {
|
||||||
|
for _, a := range lua.KernelActions() {
|
||||||
|
a := a
|
||||||
|
t.Run(a.Name, func(t *testing.T) {
|
||||||
|
c := kernelTestCore(t)
|
||||||
|
code := `
|
||||||
|
local plugin = { name = "probe_all_actions", version = "1.0" }
|
||||||
|
plugin.exports = { "run" }
|
||||||
|
function plugin.run(args)
|
||||||
|
local r, err = __llmsproxy_kernel.call(args.action, args.arg or {})
|
||||||
|
return { ok = r ~= nil, err = err }
|
||||||
|
end
|
||||||
|
return plugin
|
||||||
|
`
|
||||||
|
if err := c.Plugins().LoadSource("probe_all_actions", code); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 授权所有动作(含破坏性),以便真正执行到
|
||||||
|
all := make([]string, 0)
|
||||||
|
for _, x := range lua.KernelActions() {
|
||||||
|
all = append(all, x.Name)
|
||||||
|
}
|
||||||
|
if err := c.Plugins().SetPluginKernelPermission("probe_all_actions", all, false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
arg := map[string]interface{}{}
|
||||||
|
switch a.Name {
|
||||||
|
case "name", "adapter_owner":
|
||||||
|
arg["name"] = "primary"
|
||||||
|
}
|
||||||
|
if a.Name == "adapter_owner" {
|
||||||
|
arg = map[string]interface{}{"name": "openai"}
|
||||||
|
}
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_, err := c.Plugins().Call("probe_all_actions", "run",
|
||||||
|
map[string]interface{}{"action": a.Name, "arg": arg})
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("%s 返回 err=%v", a.Name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(8 * time.Second):
|
||||||
|
t.Fatalf("动作 %s 在 Lua 调用内死锁(deferred=%v danger=%s)", a.Name, a.Deferred, a.Danger)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -7,7 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// AutoChainFor sits on the request hot path and reads keyAutoChains with no
|
// AutoChainFor sits on the request hot path and reads keyAutoChains with no
|
||||||
@ -151,7 +151,7 @@ func TestAutoChainForConcurrentWithSourceChanges(t *testing.T) {
|
|||||||
name := fmt.Sprintf("churn%d", n)
|
name := fmt.Sprintf("churn%d", n)
|
||||||
_ = c.AddSource(config.Source{
|
_ = c.AddSource(config.Source{
|
||||||
Name: name, BaseURL: "http://127.0.0.1:9/v1", Adapter: "openai",
|
Name: name, BaseURL: "http://127.0.0.1:9/v1", Adapter: "openai",
|
||||||
Models: []config.Model{{ID: "m-" + name, Priority: 10}}},
|
Models: []config.Model{{ID: "m-" + name}}},
|
||||||
)
|
)
|
||||||
_ = c.RemoveSource(name)
|
_ = c.RemoveSource(name)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,8 +4,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/scheduler"
|
"modelrouter/internal/scheduler"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Per-key AUTO chains let an admin give one user their own scheduling chain
|
// Per-key AUTO chains let an admin give one user their own scheduling chain
|
||||||
@ -32,7 +32,7 @@ func perKeyTestConfig(t *testing.T) *config.Config {
|
|||||||
GatewayKeys: []string{"sk-gw-admin"},
|
GatewayKeys: []string{"sk-gw-admin"},
|
||||||
Sources: []config.Source{
|
Sources: []config.Source{
|
||||||
{Name: "s1", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai",
|
{Name: "s1", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai",
|
||||||
Models: []config.Model{{ID: "gpt-4o", Priority: 10}, {ID: "gpt-4o-mini", Priority: 5}}},
|
Models: []config.Model{{ID: "gpt-4o"}, {ID: "gpt-4o-mini"}}},
|
||||||
},
|
},
|
||||||
// Global chain points at gpt-4o.
|
// Global chain points at gpt-4o.
|
||||||
Auto: []config.ModelScope{{Model: "gpt-4o", Source: "s1", Tier: 1}},
|
Auto: []config.ModelScope{{Model: "gpt-4o", Source: "s1", Tier: 1}},
|
||||||
@ -198,7 +198,7 @@ func TestKeyAutoChainsRebuildOnSourceChange(t *testing.T) {
|
|||||||
// Add the source; the per-key chain must pick it up without a restart.
|
// Add the source; the per-key chain must pick it up without a restart.
|
||||||
cfg.Sources = append(cfg.Sources, config.Source{
|
cfg.Sources = append(cfg.Sources, config.Source{
|
||||||
Name: "s2", BaseURL: "http://127.0.0.1:2/v1", Adapter: "openai",
|
Name: "s2", BaseURL: "http://127.0.0.1:2/v1", Adapter: "openai",
|
||||||
Models: []config.Model{{ID: "new-model", Priority: 10}},
|
Models: []config.Model{{ID: "new-model"}},
|
||||||
})
|
})
|
||||||
if err := c.Reload(); err != nil {
|
if err := c.Reload(); err != nil {
|
||||||
t.Fatalf("Reload: %v", err)
|
t.Fatalf("Reload: %v", err)
|
||||||
|
|||||||
22
internal/core/mutation_guard_test.go
Normal file
22
internal/core/mutation_guard_test.go
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 把测试进程的时区钉成 **UTC**。
|
||||||
|
//
|
||||||
|
// 为什么必需:Qoder 的每日活动按 **UTC+8** 结算,判定必须用算术平移而不是
|
||||||
|
// 本机时区。但这台机器**本身就是 HKT(+0800)** —— 于是在真实环境里
|
||||||
|
// 「os.date("!*t")(UTC 平移)」与「os.date("*t")(本机时区)」结果**完全相同**,
|
||||||
|
// 判据无法区分二者(实测:把实现改成用本机时区,判据照样通过)。
|
||||||
|
// 钉成 UTC 后两者相差 8 小时,误用本机时区的实现立刻暴露。
|
||||||
|
//
|
||||||
|
// 只影响测试进程,不改系统时区。
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
os.Setenv("TZ", "UTC")
|
||||||
|
time.Local = time.UTC
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
108
internal/core/source_hosts_test.go
Normal file
108
internal/core/source_hosts_test.go
Normal file
@ -0,0 +1,108 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/url"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The plugin outbound allowlist is built from the hosts of configured sources.
|
||||||
|
// That rule is correct for every adapter whose credential and endpoint live on
|
||||||
|
// the same host — which was all of them until OpenDesign.
|
||||||
|
//
|
||||||
|
// OpenDesign splits them: inference is amr-link.open-design.ai, billing is
|
||||||
|
// amr-api.open-design.ai. The quota plugin derives the billing host from the
|
||||||
|
// inference one, so the host it asks for is NOT the host the operator
|
||||||
|
// configured. The request is then refused by the allowlist with a message that
|
||||||
|
// reads like a plugin defect ("host ... is not an allowed outbound destination")
|
||||||
|
// rather than a missing entry.
|
||||||
|
//
|
||||||
|
// This file pins the bridge: when a source looks like OpenDesign, the derived
|
||||||
|
// billing host joins the allowlist.
|
||||||
|
|
||||||
|
func TestSourceHostsIncludesConfiguredHosts(t *testing.T) {
|
||||||
|
cfg := &config.Config{Sources: []config.Source{
|
||||||
|
{Name: "a", BaseURL: "https://api.deepseek.com"},
|
||||||
|
{Name: "b", BaseURL: "https://amr-link.open-design.ai/v1"},
|
||||||
|
{Name: "empty", BaseURL: ""},
|
||||||
|
{Name: "bad", BaseURL: "://nonsense"},
|
||||||
|
}}
|
||||||
|
got := map[string]bool{}
|
||||||
|
for _, h := range sourceHosts(cfg) {
|
||||||
|
got[h] = true
|
||||||
|
}
|
||||||
|
for _, want := range []string{"api.deepseek.com", "amr-link.open-design.ai"} {
|
||||||
|
if !got[want] {
|
||||||
|
t.Errorf("sourceHosts missing configured host %q (allowlist would refuse "+
|
||||||
|
"every plugin request to it): %v", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// An empty/unparseable base_url must not abort the whole list.
|
||||||
|
if !got["api.deepseek.com"] {
|
||||||
|
t.Fatal("an unparseable sibling source dropped a valid host")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceHostsAddsDerivedBillingHost covers the OpenDesign case at the level
|
||||||
|
// that actually matters: the host the PLUGIN asks for must be in the allowlist.
|
||||||
|
func TestSourceHostsAddsDerivedBillingHost(t *testing.T) {
|
||||||
|
cfg := &config.Config{Sources: []config.Source{
|
||||||
|
{Name: "vela", BaseURL: "https://amr-link.open-design.ai/v1"},
|
||||||
|
}}
|
||||||
|
got := map[string]bool{}
|
||||||
|
for _, h := range sourceHosts(cfg) {
|
||||||
|
got[h] = true
|
||||||
|
}
|
||||||
|
if !got["amr-link.open-design.ai"] {
|
||||||
|
t.Fatalf("inference host missing: %v", got)
|
||||||
|
}
|
||||||
|
if !got["amr-api.open-design.ai"] {
|
||||||
|
t.Fatalf("the billing host the quota plugin derives is not in the allowlist, "+
|
||||||
|
"so every OpenDesign balance probe is refused before it leaves the "+
|
||||||
|
"process: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A deployment with no OpenDesign source must not silently gain its billing
|
||||||
|
// host: an allowlist entry is a grant, and widening it for everyone would let an
|
||||||
|
// unrelated plugin reach a host the operator never configured.
|
||||||
|
func TestSourceHostsDoesNotGrantUnconfiguredBillingHost(t *testing.T) {
|
||||||
|
cfg := &config.Config{Sources: []config.Source{
|
||||||
|
{Name: "ds", BaseURL: "https://api.deepseek.com"},
|
||||||
|
}}
|
||||||
|
for _, h := range sourceHosts(cfg) {
|
||||||
|
if h == "amr-api.open-design.ai" {
|
||||||
|
t.Fatal("the OpenDesign billing host was granted to a deployment with " +
|
||||||
|
"no OpenDesign source")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The derivation has to agree with the Lua side. Keep the two spellings of the
|
||||||
|
// rule in one place so a change to one is visible when reading the other.
|
||||||
|
func TestSourceHostsDerivationMatchesPluginRule(t *testing.T) {
|
||||||
|
// what the Lua adapter does: origin:gsub("^amr%-link%.", "amr-api.")
|
||||||
|
const in = "amr-link.open-design.ai"
|
||||||
|
want := "amr-api.open-design.ai"
|
||||||
|
got := deriveBillingHost(in)
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("deriveBillingHost(%q) = %q, want %q — the Go allowlist and the "+
|
||||||
|
"Lua probe must agree on the host or the request is refused", in, got, want)
|
||||||
|
}
|
||||||
|
// A host that is not amr-link stays untouched, so this cannot accidentally
|
||||||
|
// rewrite an unrelated provider into an OpenDesign address.
|
||||||
|
for _, other := range []string{"api.deepseek.com", "amr-link.example.com", "xamr-link.a"} {
|
||||||
|
if deriveBillingHost(other) != other {
|
||||||
|
t.Errorf("deriveBillingHost(%q) = %q, want it unchanged", other, deriveBillingHost(other))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// url.Parse round-trip keeps the helper honest about what it receives.
|
||||||
|
u, err := url.Parse("https://amr-link.open-design.ai/v1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if deriveBillingHost(u.Hostname()) != "amr-api.open-design.ai" {
|
||||||
|
t.Fatal("derivation failed on a real base_url hostname")
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -6,7 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Startup must be idempotent, and every credential consumer must see PLAINTEXT.
|
// Startup must be idempotent, and every credential consumer must see PLAINTEXT.
|
||||||
@ -42,7 +42,7 @@ func writeSealedInstall(t *testing.T, dir string) string {
|
|||||||
GatewayKeys: []string{"sk-gw-seed-me"},
|
GatewayKeys: []string{"sk-gw-seed-me"},
|
||||||
Sources: []config.Source{{
|
Sources: []config.Source{{
|
||||||
Name: "up", BaseURL: "http://127.0.0.1:1/v1", APIKey: "sk-upstream-secret",
|
Name: "up", BaseURL: "http://127.0.0.1:1/v1", APIKey: "sk-upstream-secret",
|
||||||
Adapter: "openai", Models: []config.Model{{ID: "gpt-4o", Priority: 10}},
|
Adapter: "openai", Models: []config.Model{{ID: "gpt-4o"}},
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
c, err := NewFromConfig(cfg)
|
c, err := NewFromConfig(cfg)
|
||||||
@ -76,7 +76,7 @@ func writeSealedInstallNoSeed(t *testing.T, dir string) string {
|
|||||||
DefaultModel: "AUTO",
|
DefaultModel: "AUTO",
|
||||||
Sources: []config.Source{{
|
Sources: []config.Source{{
|
||||||
Name: "up", BaseURL: "http://127.0.0.1:1/v1", APIKey: "sk-upstream-secret",
|
Name: "up", BaseURL: "http://127.0.0.1:1/v1", APIKey: "sk-upstream-secret",
|
||||||
Adapter: "openai", Models: []config.Model{{ID: "gpt-4o", Priority: 10}},
|
Adapter: "openai", Models: []config.Model{{ID: "gpt-4o"}},
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
c, err := NewFromConfig(cfg)
|
c, err := NewFromConfig(cfg)
|
||||||
|
|||||||
44
internal/core/trae_derive_expected.json
Normal file
44
internal/core/trae_derive_expected.json
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"u": "12345",
|
||||||
|
"devid": "707304999019651",
|
||||||
|
"market": "5daa819c-8425-4bae-88b7-2bbfec3c25ba",
|
||||||
|
"sess": "e5b8dfc5830ce0bdb141f7177effb90a7cf8227bb23b930a427a8550bf7357df"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"u": "UID-abc",
|
||||||
|
"devid": "579362202930608",
|
||||||
|
"market": "a5388fc1-5969-4666-bc74-be9a11f69567",
|
||||||
|
"sess": "a47da2e13fe16d49dc859fbed58d1d99e0faef5d116acafc64a2d26013d17a1f"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"u": "900000001",
|
||||||
|
"devid": "764948639142479",
|
||||||
|
"market": "e449daa5-b0c0-4f35-918e-8bb01b33f179",
|
||||||
|
"sess": "fdd37c37d1cc950439c25467e6dca75fcc84a2b0d294aa910387a9a965645650"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"u": "x",
|
||||||
|
"devid": "268405722745284",
|
||||||
|
"market": "4f2427c5-0583-4f82-8e18-4e38566dd857",
|
||||||
|
"sess": "79be40fff0fbaee572d1ba3ab13a47af9d2f4d943115ef512c56e42132e23a66"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"u": "",
|
||||||
|
"devid": "170789623621479",
|
||||||
|
"market": "b32897c1-f9c0-4ad2-b2b4-725455fffb6b",
|
||||||
|
"sess": "4a61a6b4c4bcbf1040bf0edf3df2b3e7d76a956f3ef81ef8f148eadcc03d25cc"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"u": "用户1",
|
||||||
|
"devid": "263026789643405",
|
||||||
|
"market": "1c1b4e1f-9d7a-4698-9f34-b057d9a74b0b",
|
||||||
|
"sess": "a13dbde68977eee49818e2e2fb7d3bbb1ffb4c807a6e6f1831168a84adea0dfe"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"u": "1234567890123456789012345",
|
||||||
|
"devid": "358669731114854",
|
||||||
|
"market": "449882ec-9a69-4a7a-b9ab-25aa59b28c50",
|
||||||
|
"sess": "305eb96fe14a194386c0fd4148d4919870ed803b69cf75296b53ea1f41e4c574"
|
||||||
|
}
|
||||||
|
]
|
||||||
@ -11,7 +11,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
type adapterPayload struct {
|
type adapterPayload struct {
|
||||||
@ -215,7 +215,7 @@ const keepExistingAPIKey = "__KEEP__"
|
|||||||
// - the explicit mask keepExistingAPIKey — always inherit.
|
// - the explicit mask keepExistingAPIKey — always inherit.
|
||||||
// - the empty string — inherit only for a source that already has a
|
// - the empty string — inherit only for a source that already has a
|
||||||
// credential. Empty is a legitimate value for a *new* credential-less
|
// credential. Empty is a legitimate value for a *new* credential-less
|
||||||
// source (dshcodebuddy uses "dsh-local-noauth", local servers use ""),
|
// source (some local runtimes use a no-auth sentinel, local servers ""),
|
||||||
// so a fresh source with an empty key must stay empty rather than inherit
|
// so a fresh source with an empty key must stay empty rather than inherit
|
||||||
// nothing and fail differently.
|
// nothing and fail differently.
|
||||||
func (g *Gateway) resolveAPIKey(name, want string, exists bool) string {
|
func (g *Gateway) resolveAPIKey(name, want string, exists bool) string {
|
||||||
@ -470,8 +470,8 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) {
|
|||||||
// the all-time view showed the truth, which read as "the gateway is
|
// the all-time view showed the truth, which read as "the gateway is
|
||||||
// idle" precisely when it was busiest. A calendar window cannot
|
// idle" precisely when it was busiest. A calendar window cannot
|
||||||
// filter it, so it travels with every response unfiltered.
|
// filter it, so it travels with every response unfiltered.
|
||||||
"active": g.stats.Active(),
|
"active": g.stats.Active(),
|
||||||
"records": page.Records,
|
"records": page.Records,
|
||||||
// The paging cursor has to travel with the first page, otherwise
|
// The paging cursor has to travel with the first page, otherwise
|
||||||
// the scroll handler has nothing to page from and the table is
|
// the scroll handler has nothing to page from and the table is
|
||||||
// stuck at one screen forever.
|
// stuck at one screen forever.
|
||||||
@ -488,7 +488,7 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) {
|
|||||||
if to == 0 {
|
if to == 0 {
|
||||||
to = time.Now().UnixMilli()
|
to = time.Now().UnixMilli()
|
||||||
}
|
}
|
||||||
csvHeaders(w, "llmsproxy-requests.csv")
|
csvHeaders(w, "modelrouter-requests.csv")
|
||||||
cw := csv.NewWriter(w)
|
cw := csv.NewWriter(w)
|
||||||
names := map[string]string{}
|
names := map[string]string{}
|
||||||
for _, k := range g.core.ListKeys() {
|
for _, k := range g.core.ListKeys() {
|
||||||
@ -537,7 +537,7 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if r.URL.Query().Get("export") == "keys-csv" {
|
if r.URL.Query().Get("export") == "keys-csv" {
|
||||||
csvHeaders(w, "llmsproxy-keys.csv")
|
csvHeaders(w, "modelrouter-keys.csv")
|
||||||
cw := csv.NewWriter(w)
|
cw := csv.NewWriter(w)
|
||||||
_ = cw.Write([]string{"key", "key_name", "role", "models", "total_requests", "success_requests", "failed_requests", "prompt_tokens", "completion_tokens", "total_tokens", "avg_latency_ms", "max_latency_ms", "created_at"})
|
_ = cw.Write([]string{"key", "key_name", "role", "models", "total_requests", "success_requests", "failed_requests", "prompt_tokens", "completion_tokens", "total_tokens", "avg_latency_ms", "max_latency_ms", "created_at"})
|
||||||
keyFilter := exportKey(r)
|
keyFilter := exportKey(r)
|
||||||
|
|||||||
@ -31,7 +31,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// apiV1Routes dispatches /api/v1/*. Kept separate from routes() so the original
|
// apiV1Routes dispatches /api/v1/*. Kept separate from routes() so the original
|
||||||
@ -144,7 +144,7 @@ func (g *Gateway) apiV1Index(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
index := map[string]interface{}{
|
index := map[string]interface{}{
|
||||||
"api_version": "v1",
|
"api_version": "v1",
|
||||||
"description": "llmsproxy management API. Authenticate with a gateway key: " +
|
"description": "modelrouter management API. Authenticate with a gateway key: " +
|
||||||
"'Authorization: Bearer <key>', '?api_key=<key>', or the gw_key cookie. " +
|
"'Authorization: Bearer <key>', '?api_key=<key>', or the gw_key cookie. " +
|
||||||
"Read endpoints accept any role; mutations require an admin key.",
|
"Read endpoints accept any role; mutations require an admin key.",
|
||||||
"discovery": []ep{
|
"discovery": []ep{
|
||||||
@ -189,6 +189,20 @@ func (g *Gateway) apiV1Index(w http.ResponseWriter, r *http.Request) {
|
|||||||
{Method: "GET", Path: "/api/stats", Auth: "any", Summary: "usage aggregates"},
|
{Method: "GET", Path: "/api/stats", Auth: "any", Summary: "usage aggregates"},
|
||||||
{Method: "GET", Path: "/api/stats/records", Auth: "any", Summary: "paged request records"},
|
{Method: "GET", Path: "/api/stats/records", Auth: "any", Summary: "paged request records"},
|
||||||
{Method: "GET", Path: "/api/adapters", Auth: "admin", Summary: "installed Lua adapters"},
|
{Method: "GET", Path: "/api/adapters", Auth: "admin", Summary: "installed Lua adapters"},
|
||||||
|
{Method: "POST", Path: "/api/sources/discover", Auth: "admin",
|
||||||
|
Summary: "probe a base_url+api_key for its model catalog",
|
||||||
|
// Probing is read-only with respect to configuration: it writes
|
||||||
|
// nothing, which is why it is safe to call repeatedly while the
|
||||||
|
// operator is still typing a key.
|
||||||
|
WriteEffect: "none",
|
||||||
|
},
|
||||||
|
{Method: "POST", Path: "/api/sources/discover/apply", Auth: "admin",
|
||||||
|
Summary: "probe and write the source in one step",
|
||||||
|
// Requires replace_models=true when the source already exists,
|
||||||
|
// so re-running discovery cannot silently discard a hand-tuned
|
||||||
|
// model list. An empty api_key keeps the stored credential.
|
||||||
|
WriteEffect: "creates or replaces a source and its model list",
|
||||||
|
},
|
||||||
{Method: "GET", Path: "/api/source_templates", Auth: "admin", Summary: "source templates"},
|
{Method: "GET", Path: "/api/source_templates", Auth: "admin", Summary: "source templates"},
|
||||||
{Method: "POST", Path: "/v1/chat/completions", Auth: "any", Summary: "OpenAI-compatible inference"},
|
{Method: "POST", Path: "/v1/chat/completions", Auth: "any", Summary: "OpenAI-compatible inference"},
|
||||||
},
|
},
|
||||||
|
|||||||
@ -7,7 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// v1Gateway builds a gateway with one source and two keys, and returns the
|
// v1Gateway builds a gateway with one source and two keys, and returns the
|
||||||
|
|||||||
@ -1,282 +0,0 @@
|
|||||||
package gateway
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"llmsproxy/internal/billing"
|
|
||||||
"llmsproxy/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Billing rules API.
|
|
||||||
//
|
|
||||||
// GET /api/plugins/billing/rules the active profile's rules + all profiles
|
|
||||||
// PUT /api/plugins/billing/rules replace the whole DSL and re-inject prices
|
|
||||||
// POST /api/plugins/billing/rules { profile, rule } append one rule
|
|
||||||
// DELETE /api/plugins/billing/rules { profile, url } remove one rule
|
|
||||||
//
|
|
||||||
// Why a separate endpoint instead of the generic plugin state: prices are
|
|
||||||
// plugin state, but they are also a durable, reviewable CONFIGURATION. Routing
|
|
||||||
// them through /state would let an admin key PUT arbitrary plugin state and
|
|
||||||
// silently reset the accumulated accounting. Here the server owns the shape:
|
|
||||||
// it validates the DSL, writes it back to config.yaml, recompiles, and hands
|
|
||||||
// the plugin its prices table. The operator edits rules; the plugin keeps
|
|
||||||
// numbers. The two are never mixed in one payload.
|
|
||||||
//
|
|
||||||
// Editing means "what the operator typed is what config.yaml holds". A
|
|
||||||
// billing mistake found next week must be traceable to a reviewable file, not
|
|
||||||
// to a blob in the plugin's state sidecar.
|
|
||||||
func (g *Gateway) handleBillingRules(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if reqRole(r.Context()) != "admin" {
|
|
||||||
writeError(w, http.StatusForbidden, "forbidden", "admin role required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.Method {
|
|
||||||
case http.MethodGet:
|
|
||||||
g.getBillingRules(w)
|
|
||||||
case http.MethodPut:
|
|
||||||
g.putBillingRules(w, r)
|
|
||||||
case http.MethodPost:
|
|
||||||
g.addBillingRule(w, r)
|
|
||||||
case http.MethodDelete:
|
|
||||||
g.delBillingRule(w, r)
|
|
||||||
default:
|
|
||||||
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "use GET/PUT/POST/DELETE")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// billingRulesPayload is the UI-facing view: the profiles as declared, plus
|
|
||||||
// which one is active and the source URLs they can match — the editor needs
|
|
||||||
// the URL list to offer suggestions, and an operator typing a URL that matches
|
|
||||||
// no source is exactly the silent-no-pricing failure this feature exists to
|
|
||||||
// remove.
|
|
||||||
type billingRulesPayload struct {
|
|
||||||
DSL *config.BillingDSL `json:"billing"`
|
|
||||||
Active string `json:"active"`
|
|
||||||
URLs []string `json:"urls"`
|
|
||||||
Warnings []string `json:"warnings"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *Gateway) billingRulesPayload() billingRulesPayload {
|
|
||||||
out := billingRulesPayload{}
|
|
||||||
cfg := g.core.Config()
|
|
||||||
if cfg != nil && cfg.BillingDSL != nil {
|
|
||||||
// Copy so the response cannot be mutated back through the pointer.
|
|
||||||
cp := *cfg.BillingDSL
|
|
||||||
out.DSL = &cp
|
|
||||||
if p := cp.Resolve(""); p != nil {
|
|
||||||
out.Active = p.ID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, s := range g.sourceURLs() {
|
|
||||||
out.URLs = append(out.URLs, s)
|
|
||||||
}
|
|
||||||
out.Warnings = g.billingRuleWarnings()
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *Gateway) sourceURLs() []string {
|
|
||||||
cfg := g.core.Config()
|
|
||||||
if cfg == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []string
|
|
||||||
for _, s := range cfg.Sources {
|
|
||||||
if s.BaseURL == "" || seen[s.BaseURL] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[s.BaseURL] = true
|
|
||||||
out = append(out, s.BaseURL)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// billingRuleWarnings reports rules that match no configured source. This is
|
|
||||||
// the silent killer of hand-written price tables: the rule parses, validates,
|
|
||||||
// and prices nothing at all, so every request on that URL lands in unpriced.
|
|
||||||
// Saying so out loud is the difference between a five-second fix and an
|
|
||||||
// afternoon wondering why the bill is zero.
|
|
||||||
func (g *Gateway) billingRuleWarnings() []string {
|
|
||||||
cfg := g.core.Config()
|
|
||||||
if cfg == nil || cfg.BillingDSL == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
urls := map[string]bool{}
|
|
||||||
for _, u := range g.sourceURLs() {
|
|
||||||
urls[u] = true
|
|
||||||
}
|
|
||||||
var warns []string
|
|
||||||
for _, p := range cfg.BillingDSL.Profiles {
|
|
||||||
for _, r := range p.Rules {
|
|
||||||
if r.URL == "*" || urls[r.URL] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
warns = append(warns, "profile "+p.ID+": rule url "+r.URL+" matches no configured source")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return warns
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *Gateway) getBillingRules(w http.ResponseWriter) {
|
|
||||||
writeJSON(w, http.StatusOK, g.billingRulesPayload())
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *Gateway) putBillingRules(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var body struct {
|
|
||||||
Billing *config.BillingDSL `json:"billing"`
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
||||||
writeError(w, http.StatusBadRequest, "invalid_request", "invalid json: "+err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if body.Billing == nil {
|
|
||||||
writeError(w, http.StatusBadRequest, "invalid_request", "billing is required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
g.applyBillingDSLUpdate(w, body.Billing)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *Gateway) addBillingRule(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var body struct {
|
|
||||||
Profile string `json:"profile"`
|
|
||||||
Rule config.BillingRule `json:"rule"`
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
||||||
writeError(w, http.StatusBadRequest, "invalid_request", "invalid json: "+err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
cfg := g.core.Config()
|
|
||||||
if cfg == nil || cfg.BillingDSL == nil || len(cfg.BillingDSL.Profiles) == 0 {
|
|
||||||
writeError(w, http.StatusBadRequest, "no_profile", "no billing profiles configured; create one first")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
next := cloneBillingDSL(cfg.BillingDSL)
|
|
||||||
idx := profileIndex(next, body.Profile)
|
|
||||||
if idx < 0 {
|
|
||||||
writeError(w, http.StatusBadRequest, "unknown_profile", "no such profile: "+body.Profile)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p := &next.Profiles[idx]
|
|
||||||
// Replace rather than append when the URL is already declared: two rules
|
|
||||||
// for one URL would silently make the first unreachable (first match wins),
|
|
||||||
// which is the exact ambiguity an editor should not be able to create.
|
|
||||||
replaced := false
|
|
||||||
for i := range p.Rules {
|
|
||||||
if p.Rules[i].URL == body.Rule.URL {
|
|
||||||
p.Rules[i] = body.Rule
|
|
||||||
replaced = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !replaced {
|
|
||||||
p.Rules = append(p.Rules, body.Rule)
|
|
||||||
}
|
|
||||||
g.applyBillingDSLUpdate(w, next)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *Gateway) delBillingRule(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var body struct {
|
|
||||||
Profile string `json:"profile"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
||||||
writeError(w, http.StatusBadRequest, "invalid_request", "invalid json: "+err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
cfg := g.core.Config()
|
|
||||||
if cfg == nil || cfg.BillingDSL == nil {
|
|
||||||
writeError(w, http.StatusBadRequest, "no_profile", "no billing profiles configured")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
next := cloneBillingDSL(cfg.BillingDSL)
|
|
||||||
idx := profileIndex(next, body.Profile)
|
|
||||||
if idx < 0 {
|
|
||||||
writeError(w, http.StatusBadRequest, "unknown_profile", "no such profile: "+body.Profile)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p := &next.Profiles[idx]
|
|
||||||
out := p.Rules[:0]
|
|
||||||
found := false
|
|
||||||
for _, rule := range p.Rules {
|
|
||||||
if rule.URL == body.URL {
|
|
||||||
found = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, rule)
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
writeError(w, http.StatusNotFound, "not_found", "no rule for url "+body.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p.Rules = out
|
|
||||||
g.applyBillingDSLUpdate(w, next)
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyBillingDSLUpdate is the single write path: validate, persist to
|
|
||||||
// config.yaml, recompile, inject, and only then report success. If the config
|
|
||||||
// cannot be written the change is NOT applied in memory either — a rules editor
|
|
||||||
// that says "saved" and loses the edit on the next restart is worse than one
|
|
||||||
// that refuses.
|
|
||||||
func (g *Gateway) applyBillingDSLUpdate(w http.ResponseWriter, next *config.BillingDSL) {
|
|
||||||
if err := next.Validate(); err != nil {
|
|
||||||
writeError(w, http.StatusBadRequest, "invalid_rules", err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
cfg := g.core.Config()
|
|
||||||
if cfg == nil {
|
|
||||||
writeError(w, http.StatusInternalServerError, "no_config", "no config loaded")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Assign BEFORE saving. The previous order saved first and assigned after,
|
|
||||||
// so every rule edit reported success while writing a config.yaml with no
|
|
||||||
// billing section at all — the operator's edit vanished on restart and
|
|
||||||
// nothing in the API response said so.
|
|
||||||
prev := cfg.BillingDSL
|
|
||||||
cfg.BillingDSL = next
|
|
||||||
|
|
||||||
// Compile before persisting: a profile that cannot be turned into prices
|
|
||||||
// must not reach the config file, or the next restart fails to load the
|
|
||||||
// very rules the editor just accepted.
|
|
||||||
prices, err := billing.CompileOpts(next.Resolve(next.Active), cfg.Sources, true)
|
|
||||||
if err != nil {
|
|
||||||
cfg.BillingDSL = prev // no half-applied state
|
|
||||||
writeError(w, http.StatusBadRequest, "compile_failed", err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := cfg.Save(); err != nil {
|
|
||||||
cfg.BillingDSL = prev
|
|
||||||
writeError(w, http.StatusInternalServerError, "persist_failed", err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ps := g.core.Plugins()
|
|
||||||
if ps != nil {
|
|
||||||
if err := ps.SetState("billing", map[string]interface{}{"prices": prices}); err != nil {
|
|
||||||
writeError(w, http.StatusBadRequest, "plugin_error", err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
p := g.billingRulesPayload()
|
|
||||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
|
||||||
"ok": true, "billing": p.DSL, "active": p.Active, "warnings": p.Warnings,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneBillingDSL(d *config.BillingDSL) *config.BillingDSL {
|
|
||||||
out := &config.BillingDSL{Active: d.Active}
|
|
||||||
out.Profiles = make([]config.BillingProfile, len(d.Profiles))
|
|
||||||
copy(out.Profiles, d.Profiles)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func profileIndex(d *config.BillingDSL, id string) int {
|
|
||||||
if id == "" {
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
for i, p := range d.Profiles {
|
|
||||||
if p.ID == id {
|
|
||||||
return i
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
@ -1,281 +0,0 @@
|
|||||||
package gateway
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// rulesGateway needs a real billing plugin (the API injects prices into it)
|
|
||||||
// AND two sources whose base_urls the rules will match, so it cannot reuse
|
|
||||||
// either existing helper: gatewayWithBilling has no sources, and
|
|
||||||
// newTestGateway loads no plugins.
|
|
||||||
func rulesGateway(t *testing.T) *Gateway {
|
|
||||||
t.Helper()
|
|
||||||
g := gatewayWithBilling(t)
|
|
||||||
cfg := g.core.Config()
|
|
||||||
cfg.Sources = []config.Source{
|
|
||||||
{Name: "localzen", BaseURL: "https://free.example.com/v1"},
|
|
||||||
{Name: "commandcode", BaseURL: "https://api.commandcode.ai/v1"},
|
|
||||||
}
|
|
||||||
return g
|
|
||||||
}
|
|
||||||
|
|
||||||
func rulesDo(t *testing.T, g *Gateway, method, body string) *httptest.ResponseRecorder {
|
|
||||||
t.Helper()
|
|
||||||
return doReq(t, g, method, "/api/plugins/billing/rules", body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBillingRulesCRUD covers the whole editor loop against the real HTTP
|
|
||||||
// surface: read, add, replace-by-URL, delete. The replace-by-URL case is the
|
|
||||||
// one worth pinning — two rules for one URL would make the first unreachable
|
|
||||||
// (first match wins) without any error, which is exactly the kind of silent
|
|
||||||
// ambiguity an editor must not be able to create.
|
|
||||||
func TestBillingRulesCRUD(t *testing.T) {
|
|
||||||
g := rulesGateway(t)
|
|
||||||
seed := `{"billing":{"active":"p1","profiles":[{"id":"p1","rules":[
|
|
||||||
{"url":"https://free.example.com/v1","mode":"free"}]}]}}`
|
|
||||||
if rr := rulesDo(t, g, http.MethodPut, seed); rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("seed rules = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add a token-priced rule for the second URL.
|
|
||||||
add := `{"profile":"p1","rule":{"url":"https://api.commandcode.ai/v1","mode":"token",
|
|
||||||
"models":{"deepseek-v4.1-flash":{"prompt":"0.15","completion":"0.60"}}}}`
|
|
||||||
if rr := rulesDo(t, g, http.MethodPost, add); rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("add rule = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
var got billingRulesPayload
|
|
||||||
if err := json.Unmarshal(rulesDo(t, g, http.MethodGet, "").Body.Bytes(), &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got.DSL.Profiles[0].Rules) != 2 {
|
|
||||||
t.Fatalf("after add there are %d rules, want 2", len(got.DSL.Profiles[0].Rules))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Adding the SAME url again must replace, not append.
|
|
||||||
if rr := rulesDo(t, g, http.MethodPost, `{"profile":"p1","rule":{"url":"https://api.commandcode.ai/v1","mode":"free"}}`); rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("re-add same url = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(rulesDo(t, g, http.MethodGet, "").Body.Bytes(), &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n := len(got.DSL.Profiles[0].Rules); n != 2 {
|
|
||||||
t.Fatalf("★ re-adding an existing url appended instead of replacing: %d rules", n)
|
|
||||||
}
|
|
||||||
var mode string
|
|
||||||
for _, r := range got.DSL.Profiles[0].Rules {
|
|
||||||
if r.URL == "https://api.commandcode.ai/v1" {
|
|
||||||
mode = r.Mode
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if mode != "free" {
|
|
||||||
t.Errorf("re-added rule mode = %q, want free (the replacement must win)", mode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete by URL.
|
|
||||||
if rr := rulesDo(t, g, http.MethodDelete, `{"profile":"p1","url":"https://api.commandcode.ai/v1"}`); rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("delete rule = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(rulesDo(t, g, http.MethodGet, "").Body.Bytes(), &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got.DSL.Profiles[0].Rules) != 1 {
|
|
||||||
t.Errorf("after delete there are %d rules, want 1", len(got.DSL.Profiles[0].Rules))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBillingRulesRejectUnknownURLWithWarning is the silent-failure guard: a
|
|
||||||
// rule whose URL matches no source prices nothing, so every request on it
|
|
||||||
// falls into unpriced. The API must say so instead of accepting it silently.
|
|
||||||
func TestBillingRulesRejectUnknownURLWithWarning(t *testing.T) {
|
|
||||||
g := rulesGateway(t)
|
|
||||||
if rr := rulesDo(t, g, http.MethodPut, `{"billing":{"active":"p","profiles":[{"id":"p",
|
|
||||||
"rules":[{"url":"https://typo.example.com/v1","mode":"free"}]}]}}`); rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("seed = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
var got billingRulesPayload
|
|
||||||
if err := json.Unmarshal(rulesDo(t, g, http.MethodGet, "").Body.Bytes(), &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got.Warnings) == 0 {
|
|
||||||
t.Fatal("★ a rule matching no configured source was accepted with no warning — " +
|
|
||||||
"it will price nothing and every request will be recorded as unpriced")
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Warnings[0], "typo.example.com") {
|
|
||||||
t.Errorf("warning does not name the offending url: %v", got.Warnings)
|
|
||||||
}
|
|
||||||
// The editor needs the real URL list to offer suggestions.
|
|
||||||
if len(got.URLs) != 2 {
|
|
||||||
t.Errorf("urls offered to the editor = %v, want the 2 configured base_urls", got.URLs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBillingRulesInvalidIsRejectedNotPersisted checks the write path refuses
|
|
||||||
// bad input and leaves the previous rules in place — an editor that clears the
|
|
||||||
// table on a typo loses the price table.
|
|
||||||
func TestBillingRulesInvalidIsRejectedNotPersisted(t *testing.T) {
|
|
||||||
g := rulesGateway(t)
|
|
||||||
seed := `{"billing":{"active":"p","profiles":[{"id":"p","rules":[
|
|
||||||
{"url":"https://free.example.com/v1","mode":"free"}]}]}}`
|
|
||||||
if rr := rulesDo(t, g, http.MethodPut, seed); rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("seed = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
// An unknown mode must be refused.
|
|
||||||
if rr := rulesDo(t, g, http.MethodPost, `{"profile":"p","rule":{"url":"https://x/v1","mode":"banana"}}`); rr.Code != http.StatusBadRequest {
|
|
||||||
t.Errorf("invalid mode accepted with %d, want 400", rr.Code)
|
|
||||||
}
|
|
||||||
var got billingRulesPayload
|
|
||||||
if err := json.Unmarshal(rulesDo(t, g, http.MethodGet, "").Body.Bytes(), &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got.DSL.Profiles[0].Rules) != 1 {
|
|
||||||
t.Errorf("★ a rejected rule was persisted anyway: %d rules", len(got.DSL.Profiles[0].Rules))
|
|
||||||
}
|
|
||||||
// An unparseable price must be refused too, not silently priced at zero.
|
|
||||||
if rr := rulesDo(t, g, http.MethodPost, `{"profile":"p","rule":{"url":"https://y/v1","mode":"token",
|
|
||||||
"models":{"m":{"prompt":"free","completion":"0.6"}}}}`); rr.Code != http.StatusBadRequest {
|
|
||||||
t.Errorf("non-numeric price accepted with %d, want 400", rr.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBillingRulesPersistAndReachThePlugin is the point of the whole
|
|
||||||
// endpoint, and mutation verification showed the earlier CRUD tests missed it
|
|
||||||
// twice: dropping cfg.Save() and dropping the SetState injection both left
|
|
||||||
// every test green. Both failures are invisible in a GET — the response echoes
|
|
||||||
// the in-memory rules either way — so this test checks the two consequences
|
|
||||||
// that actually matter:
|
|
||||||
//
|
|
||||||
// 1. persistence: the rule must be in the config file on disk, because an
|
|
||||||
// edit that evaporates on restart is a lie the UI told the operator;
|
|
||||||
// 2. effect: the billing plugin must now PRICE the source (its published
|
|
||||||
// prices table has a non-zero entry), because a rule that is stored but
|
|
||||||
// never injected prices nothing at all.
|
|
||||||
func TestBillingRulesPersistAndReachThePlugin(t *testing.T) {
|
|
||||||
g := rulesGateway(t)
|
|
||||||
cfg := g.core.Config()
|
|
||||||
|
|
||||||
seed := `{"billing":{"active":"p","profiles":[{"id":"p","rules":[
|
|
||||||
{"url":"https://free.example.com/v1","mode":"free"},
|
|
||||||
{"url":"https://api.commandcode.ai/v1","mode":"token",
|
|
||||||
"models":{"deepseek-v4.1-flash":{"prompt":"0.15","completion":"0.60"}}}]}]}}`
|
|
||||||
if rr := rulesDo(t, g, http.MethodPut, seed); rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("seed = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. persisted to the file the process loaded from.
|
|
||||||
raw, err := os.ReadFile(cfg.Path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read config: %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(raw), "commandcode.ai") {
|
|
||||||
t.Errorf("★ the rule is not in config.yaml on disk — an edit that does not "+
|
|
||||||
"survive a restart:\n%s", raw)
|
|
||||||
}
|
|
||||||
reloaded, err := config.Load(cfg.Path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reload saved config: %v", err)
|
|
||||||
}
|
|
||||||
if reloaded.BillingDSL == nil || len(reloaded.BillingDSL.Profiles) == 0 {
|
|
||||||
t.Fatalf("saved config has no billing DSL: %s", raw)
|
|
||||||
}
|
|
||||||
// A round-trip through YAML must not lose the price strings.
|
|
||||||
found := false
|
|
||||||
for _, r := range reloaded.BillingDSL.Profiles[0].Rules {
|
|
||||||
if strings.Contains(r.URL, "commandcode") {
|
|
||||||
found = true
|
|
||||||
if m, ok := r.Models["deepseek-v4.1-flash"]; !ok || m.Prompt != "0.15" {
|
|
||||||
t.Errorf("price did not survive the YAML round-trip: %+v", m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Errorf("the token rule is missing after reload:\n%s", raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. reached the plugin. Prices live in a separate Lua field from `state`,
|
|
||||||
// and the GET /state endpoint returns the published state together with
|
|
||||||
// the prices sidecar — which is exactly what the plugin's own UI reads.
|
|
||||||
state := g.core.Plugins().State("billing")
|
|
||||||
if state == nil {
|
|
||||||
t.Fatalf("billing plugin published no state")
|
|
||||||
}
|
|
||||||
rr := doReq(t, g, http.MethodGet, "/api/plugins/billing/state", "")
|
|
||||||
if rr.Code != http.StatusOK {
|
|
||||||
t.Fatalf("GET state = %d: %s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
var envelope struct {
|
|
||||||
Prices map[string]interface{} `json:"prices"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(rr.Body.Bytes(), &envelope); err != nil {
|
|
||||||
t.Fatalf("decode state envelope: %v", err)
|
|
||||||
}
|
|
||||||
prices := envelope.Prices
|
|
||||||
if prices == nil {
|
|
||||||
t.Fatalf("state payload carries no prices")
|
|
||||||
}
|
|
||||||
sources, ok := prices["sources"].(map[string]interface{})
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("prices has no sources map: %#v", prices)
|
|
||||||
}
|
|
||||||
cc, ok := sources["commandcode"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("★ the rule was saved but never injected: prices has %v, "+
|
|
||||||
"no commandcode entry. Every request on it stays unpriced.", keysOf(sources))
|
|
||||||
}
|
|
||||||
if cc == nil {
|
|
||||||
t.Error("commandcode price entry is nil")
|
|
||||||
}
|
|
||||||
// The free rule must price localzen, and the two must not be conflated.
|
|
||||||
if _, ok := sources["localzen"]; !ok {
|
|
||||||
t.Errorf("the free rule did not reach the plugin: %v", keysOf(sources))
|
|
||||||
}
|
|
||||||
// The injected price must be per-token, i.e. 0.15 USD/M => 1.5e-7.
|
|
||||||
// Asserting the exact magnitude catches a unit error (per-million vs
|
|
||||||
// per-token), which would be off by a factor of a million and still look
|
|
||||||
// like "a number".
|
|
||||||
models, ok := cc.(map[string]interface{})["models"].(map[string]interface{})
|
|
||||||
if ok {
|
|
||||||
if m, ok := models["deepseek-v4.1-flash"].(map[string]interface{}); ok {
|
|
||||||
p, _ := m["prompt"].(float64)
|
|
||||||
if p <= 0 || p > 1e-6 {
|
|
||||||
t.Errorf("injected prompt price = %v, want per-token (~1.5e-7); "+
|
|
||||||
"a per-million value here would over-bill by 1e6x", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func keysOf(m map[string]interface{}) []string {
|
|
||||||
out := make([]string, 0, len(m))
|
|
||||||
for k := range m {
|
|
||||||
out = append(out, k)
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBillingRulesRequireAdmin keeps price rewriting behind the admin role,
|
|
||||||
// like every other plugin write: prices are configuration, not user data.
|
|
||||||
func TestBillingRulesRequireAdmin(t *testing.T) {
|
|
||||||
g := rulesGateway(t)
|
|
||||||
rec, err := g.core.CreateKey("viewer", "user", nil, "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
userKey := rec.Key
|
|
||||||
for _, m := range []string{http.MethodPut, http.MethodPost, http.MethodDelete} {
|
|
||||||
req, _ := http.NewRequest(m, "/api/plugins/billing/rules", strings.NewReader(`{}`))
|
|
||||||
req.Header.Set("Authorization", "Bearer "+userKey)
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
if rr := newRecorderFor(t, g, req); rr.Code != http.StatusForbidden {
|
|
||||||
t.Errorf("user %s rules = %d, want 403", m, rr.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -13,29 +13,87 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/lua"
|
"modelrouter/internal/lua"
|
||||||
"llmsproxy/internal/provider"
|
"modelrouter/internal/provider"
|
||||||
"llmsproxy/internal/scheduler"
|
"modelrouter/internal/scheduler"
|
||||||
"llmsproxy/internal/types"
|
"modelrouter/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// chatRequest mirrors the OpenAI chat completions request the gateway accepts.
|
// chatRequest mirrors the OpenAI chat completions request the gateway accepts.
|
||||||
|
// chatRequest is the gateway's IN-BOUND client shape for /v1/chat/completions.
|
||||||
|
//
|
||||||
|
// It is deliberately an OpenAI wire struct, NOT the internal unified format:
|
||||||
|
// this is one client protocol, and the kernel parses it before anything else
|
||||||
|
// knows which protocol the rest of the request will speak. What reaches an
|
||||||
|
// adapter is types.UnifiedRequest (the maximal set), which Thinking populates.
|
||||||
type chatRequest struct {
|
type chatRequest struct {
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Messages []types.ChatMessage `json:"messages"`
|
Messages []types.ChatMessage `json:"messages"`
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
MaxTokens int `json:"max_tokens,omitempty"`
|
MaxTokens int `json:"max_tokens,omitempty"`
|
||||||
Stream bool `json:"stream,omitempty"`
|
Stream bool `json:"stream,omitempty"`
|
||||||
Tools []interface{} `json:"tools,omitempty"`
|
Tools []interface{} `json:"tools,omitempty"`
|
||||||
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
||||||
|
// DisableThinking is the OpenAI-ecosystem spelling of "turn thinking off".
|
||||||
|
// It is translated into the protocol-neutral Thinking field on the way in
|
||||||
|
// (see thinkingFromClient), so that flag never travels under this name.
|
||||||
DisableThinking bool `json:"disable_thinking"`
|
DisableThinking bool `json:"disable_thinking"`
|
||||||
ExtraBody map[string]interface{} `json:"extra_body,omitempty"`
|
ExtraBody map[string]interface{} `json:"extra_body,omitempty"`
|
||||||
|
|
||||||
|
// Thinking accepts the neutral shape directly, so a client already speaking
|
||||||
|
// the unified format (or an adapter-served protocol that names it that way)
|
||||||
|
// does not have to round-trip through disable_thinking.
|
||||||
|
Thinking *types.ThinkingConfig `json:"thinking,omitempty"`
|
||||||
|
|
||||||
// PromptCacheKey 是 OpenAI 原生的缓存键;部分客户端用它携带会话标识。
|
// PromptCacheKey 是 OpenAI 原生的缓存键;部分客户端用它携带会话标识。
|
||||||
PromptCacheKey string `json:"prompt_cache_key,omitempty"`
|
PromptCacheKey string `json:"prompt_cache_key,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// thinkingFromClient folds the client's spelling of "no thinking" into the
|
||||||
|
// neutral Thinking field.
|
||||||
|
//
|
||||||
|
// Precedence: an explicit `thinking` object wins, because it carries a budget
|
||||||
|
// the boolean cannot express. disable_thinking only ever sets Enabled=false —
|
||||||
|
// it says nothing about how much, and inventing a budget there would silently
|
||||||
|
// override the upstream's default.
|
||||||
|
func (r *chatRequest) thinkingFromClient() *types.ThinkingConfig {
|
||||||
|
if r.Thinking != nil {
|
||||||
|
return r.Thinking
|
||||||
|
}
|
||||||
|
if r.DisableThinking {
|
||||||
|
no := false
|
||||||
|
return &types.ThinkingConfig{Enabled: &no}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rawTools / rawToolChoice re-encode the loosely-typed client fields into the
|
||||||
|
// RawMessage the internal format carries. The kernel never interprets them:
|
||||||
|
// they are the client's payload, passed through to the adapter, which is the
|
||||||
|
// only layer that knows the tool schema of any particular protocol.
|
||||||
|
func (r *chatRequest) rawTools() json.RawMessage {
|
||||||
|
if len(r.Tools) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
b, err := json.Marshal(r.Tools)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *chatRequest) rawToolChoice() json.RawMessage {
|
||||||
|
if r.ToolChoice == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
b, err := json.Marshal(r.ToolChoice)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
// ChatCompletion is the non-streaming OpenAI response object.
|
// ChatCompletion is the non-streaming OpenAI response object.
|
||||||
type ChatCompletion struct {
|
type ChatCompletion struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
@ -256,17 +314,17 @@ func (g *Gateway) checkQuota(ctx context.Context, model string) *quotaRejection
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// A scope that simply does not list the model is a different problem from
|
// A scope that simply does not list the model is a different problem from
|
||||||
// a quota being exhausted, and the message has to say which: telling an
|
// a quota being exhausted, and the message has to say which: telling an
|
||||||
// operator "not allowed for this key" when they configured a quota sends
|
// operator "not allowed for this key" when they configured a quota sends
|
||||||
// them to the wrong setting. AUTO is checked here before the chain is even
|
// them to the wrong setting. AUTO is checked here before the chain is even
|
||||||
// consulted, so this is the message that surfaces for AUTO.
|
// consulted, so this is the message that surfaces for AUTO.
|
||||||
if isAuto(model) {
|
if isAuto(model) {
|
||||||
return "aRejection{msg: fmt.Sprintf(
|
return "aRejection{msg: fmt.Sprintf(
|
||||||
"model %q is not in this key's model scope, so it cannot use AUTO; "+
|
"model %q is not in this key's model scope, so it cannot use AUTO; "+
|
||||||
"add %q to the key's models or remove the scope restriction", model, model)}
|
"add %q to the key's models or remove the scope restriction", model, model)}
|
||||||
}
|
}
|
||||||
return "aRejection{msg: fmt.Sprintf("model %q is not allowed for this key", model)}
|
return "aRejection{msg: fmt.Sprintf("model %q is not allowed for this key", model)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// quotaWindowSuffix describes a quota's reset window for an error message, so
|
// quotaWindowSuffix describes a quota's reset window for an error message, so
|
||||||
@ -424,15 +482,15 @@ func (g *Gateway) handleChat(w http.ResponseWriter, r *http.Request) {
|
|||||||
done := g.stats.Begin()
|
done := g.stats.Begin()
|
||||||
defer done()
|
defer done()
|
||||||
inner := &types.ChatRequest{
|
inner := &types.ChatRequest{
|
||||||
Messages: req.Messages,
|
Messages: req.Messages,
|
||||||
Temperature: req.Temperature,
|
Temperature: req.Temperature,
|
||||||
MaxTokens: req.MaxTokens,
|
MaxTokens: req.MaxTokens,
|
||||||
Stream: req.Stream,
|
Stream: req.Stream,
|
||||||
Tools: req.Tools,
|
Tools: req.rawTools(),
|
||||||
ToolChoice: req.ToolChoice,
|
ToolChoice: req.rawToolChoice(),
|
||||||
DisableThinking: req.DisableThinking,
|
Thinking: req.thinkingFromClient(),
|
||||||
ExtraBody: req.ExtraBody,
|
ExtraBody: req.ExtraBody,
|
||||||
ClientSession: clientSession,
|
ClientSession: clientSession,
|
||||||
}
|
}
|
||||||
rec := &Req{
|
rec := &Req{
|
||||||
Key: keyID(reqKey(ctx)),
|
Key: keyID(reqKey(ctx)),
|
||||||
@ -484,16 +542,16 @@ func (g *Gateway) handleChat(w http.ResponseWriter, r *http.Request) {
|
|||||||
defer done()
|
defer done()
|
||||||
|
|
||||||
inner := &types.ChatRequest{
|
inner := &types.ChatRequest{
|
||||||
Model: effective,
|
Model: effective,
|
||||||
Messages: req.Messages,
|
Messages: req.Messages,
|
||||||
Temperature: req.Temperature,
|
Temperature: req.Temperature,
|
||||||
MaxTokens: req.MaxTokens,
|
MaxTokens: req.MaxTokens,
|
||||||
Stream: req.Stream,
|
Stream: req.Stream,
|
||||||
Tools: req.Tools,
|
Tools: req.rawTools(),
|
||||||
ToolChoice: req.ToolChoice,
|
ToolChoice: req.rawToolChoice(),
|
||||||
DisableThinking: req.DisableThinking,
|
Thinking: req.thinkingFromClient(),
|
||||||
ExtraBody: req.ExtraBody,
|
ExtraBody: req.ExtraBody,
|
||||||
ClientSession: clientSession,
|
ClientSession: clientSession,
|
||||||
}
|
}
|
||||||
rec := &Req{
|
rec := &Req{
|
||||||
Key: keyID(reqKey(ctx)),
|
Key: keyID(reqKey(ctx)),
|
||||||
@ -586,7 +644,7 @@ func estimatePromptTokens(req *types.ChatRequest) int64 {
|
|||||||
}
|
}
|
||||||
b, _ := json.Marshal(struct {
|
b, _ := json.Marshal(struct {
|
||||||
Messages []types.ChatMessage `json:"messages"`
|
Messages []types.ChatMessage `json:"messages"`
|
||||||
Tools []interface{} `json:"tools,omitempty"`
|
Tools json.RawMessage `json:"tools,omitempty"`
|
||||||
}{Messages: req.Messages, Tools: req.Tools})
|
}{Messages: req.Messages, Tools: req.Tools})
|
||||||
if len(b) == 0 {
|
if len(b) == 0 {
|
||||||
return 0
|
return 0
|
||||||
@ -796,7 +854,7 @@ func recordChatUsage(rec *Req, req *types.ChatRequest, resp *types.UnifiedRespon
|
|||||||
// keeps the billing plugin from guessing.
|
// keeps the billing plugin from guessing.
|
||||||
//
|
//
|
||||||
// Leaving it unrecorded is worse than useless: the adapters already parse it
|
// Leaving it unrecorded is worse than useless: the adapters already parse it
|
||||||
// and forward it (opencodego/gemini/anthropic all do), so the number exists
|
// and forward it (gemini/anthropic/qwen38 all do), so the number exists
|
||||||
// in the response and is thrown away at the one place that could keep it.
|
// in the response and is thrown away at the one place that could keep it.
|
||||||
if d := resp.TokenUsage.CompletionTokensDetails; d != nil && d.ReasoningTokens > 0 {
|
if d := resp.TokenUsage.CompletionTokensDetails; d != nil && d.ReasoningTokens > 0 {
|
||||||
rec.Reasoning = int64(d.ReasoningTokens)
|
rec.Reasoning = int64(d.ReasoningTokens)
|
||||||
@ -817,29 +875,21 @@ func recordChatUsage(rec *Req, req *types.ChatRequest, resp *types.UnifiedRespon
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeChatCompletion renders a unified response as an OpenAI
|
// writeChatCompletion renders a unified response through the out-bound
|
||||||
// chat.completion object. modelName is the id clients see as the serving
|
// Serializer. The default serializer reproduces the OpenAI wire shape this
|
||||||
// model: the requested id for direct routes, the exact slot model for AUTO.
|
// function used to build inline (locked byte-for-byte by e2e golden); an
|
||||||
func writeChatCompletion(w http.ResponseWriter, resp *types.UnifiedResponse, modelName string) {
|
// adapter that declares serialize_response overrides it. modelName is the id
|
||||||
msg := RespMessage{Role: "assistant", Content: resp.Content}
|
// clients see as the serving model: the requested id for direct routes, the
|
||||||
if resp.ReasoningContent != "" {
|
// exact slot model for AUTO.
|
||||||
msg.ReasoningContent = resp.ReasoningContent
|
func writeChatCompletion(w http.ResponseWriter, s Serializer, resp *types.UnifiedResponse, modelName string) {
|
||||||
|
body, err := s.NonStream(resp, modelName, newWireID(), newCreated())
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, "serialize_error", "failed to serialize response")
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if len(resp.ToolCalls) > 0 {
|
w.Header().Set("Content-Type", "application/json")
|
||||||
msg.ToolCalls = toolCallsWire(resp.ToolCalls)
|
w.WriteHeader(http.StatusOK)
|
||||||
}
|
_, _ = w.Write(body)
|
||||||
out := ChatCompletion{
|
|
||||||
ID: newID(),
|
|
||||||
Object: "chat.completion",
|
|
||||||
Created: time.Now().Unix(),
|
|
||||||
Model: modelName,
|
|
||||||
Choices: []ChatChoice{{Index: 0, Message: msg, FinishReason: resp.FinishReason}},
|
|
||||||
Cost: resp.Cost,
|
|
||||||
}
|
|
||||||
if resp.TokenUsage.Total > 0 || resp.TokenUsage.Prompt > 0 || resp.TokenUsage.Completion > 0 {
|
|
||||||
out.Usage = &resp.TokenUsage
|
|
||||||
}
|
|
||||||
writeJSON(w, http.StatusOK, out)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// singleChat runs a direct (model-pinned) non-streaming request across the
|
// singleChat runs a direct (model-pinned) non-streaming request across the
|
||||||
@ -864,7 +914,7 @@ func (g *Gateway) singleChat(w http.ResponseWriter, ctx context.Context, cands [
|
|||||||
// the total latency.
|
// the total latency.
|
||||||
rec.FirstByteMs = rec.LatMs
|
rec.FirstByteMs = rec.LatMs
|
||||||
g.writeRec(rec)
|
g.writeRec(rec)
|
||||||
writeChatCompletion(w, resp, effective)
|
writeChatCompletion(w, g.serializerFor(usedSrc, cands), resp, effective)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeRec records a finished request (audit + aggregates) and fires the
|
// writeRec records a finished request (audit + aggregates) and fires the
|
||||||
@ -1059,9 +1109,9 @@ func (g *Gateway) fireEnd(rec *Req) {
|
|||||||
// A SUBSET of completion_tokens, not an extra. Billing plugins that
|
// A SUBSET of completion_tokens, not an extra. Billing plugins that
|
||||||
// add it to completion would double-charge every thinking token.
|
// add it to completion would double-charge every thinking token.
|
||||||
"reasoning_tokens": rec.Reasoning,
|
"reasoning_tokens": rec.Reasoning,
|
||||||
"image_count": rec.ImageCount,
|
"image_count": rec.ImageCount,
|
||||||
"error": rec.Err,
|
"error": rec.Err,
|
||||||
"time": rec.Time,
|
"time": rec.Time,
|
||||||
// chain_walk: the AUTO tier-by-tier trace, when the request went
|
// chain_walk: the AUTO tier-by-tier trace, when the request went
|
||||||
// through the chain. Empty for a direct request and for a gateway with
|
// through the chain. Empty for a direct request and for a gateway with
|
||||||
// no plugins loaded. Absent rather than empty so a plugin can tell
|
// no plugins loaded. Absent rather than empty so a plugin can tell
|
||||||
@ -1116,24 +1166,29 @@ func mergeUsage(prev, cur *types.TokenUsage) *types.TokenUsage {
|
|||||||
return &out
|
return &out
|
||||||
}
|
}
|
||||||
|
|
||||||
// pumpStream writes the full SSE sequence for a started stream: role
|
// pumpStream writes the full SSE sequence for a started stream. Every byte on
|
||||||
// preamble, one chunk per unified delta, the terminating finish_reason, the
|
// the wire comes from the Serializer, so the OpenAI frame shapes live in
|
||||||
// OpenAI-standard final usage chunk (empty choices) and [DONE]. modelName
|
// serializer.go rather than here, and an adapter that declares serialize_chunk
|
||||||
// follows writeChatCompletion's rule (requested id for direct routes, exact
|
// replaces them wholesale.
|
||||||
// slot model for AUTO).
|
//
|
||||||
func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan types.UnifiedChunk, modelName string, t0 time.Time) {
|
// This function owns only transport concerns — headers, the flush cadence,
|
||||||
|
// first-byte timing, and audit accounting — plus the protocol-agnostic rule
|
||||||
|
// that usage seen anywhere in the stream becomes the recorded usage. Whether a
|
||||||
|
// trailing usage frame is emitted, and in what shape, is the serializer's
|
||||||
|
// decision.
|
||||||
|
func (g *Gateway) pumpStream(w http.ResponseWriter, ser Serializer, rec *Req, chunks <-chan types.UnifiedChunk, modelName string, t0 time.Time) {
|
||||||
w.Header().Set("Content-Type", "text/event-stream")
|
w.Header().Set("Content-Type", "text/event-stream")
|
||||||
w.Header().Set("Cache-Control", "no-cache")
|
w.Header().Set("Cache-Control", "no-cache")
|
||||||
w.Header().Set("Connection", "keep-alive")
|
w.Header().Set("Connection", "keep-alive")
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
flusher, _ := w.(http.Flusher)
|
flusher, _ := w.(http.Flusher)
|
||||||
id := newID()
|
id := newWireID()
|
||||||
created := time.Now().Unix()
|
created := newCreated()
|
||||||
send := func(obj interface{}) bool {
|
st := &StreamState{}
|
||||||
b, err := json.Marshal(obj)
|
send := func(b []byte) bool {
|
||||||
if err != nil {
|
if len(b) == 0 {
|
||||||
return false
|
return true
|
||||||
}
|
}
|
||||||
if _, err := fmt.Fprintf(w, "data: %s\n\n", b); err != nil {
|
if _, err := fmt.Fprintf(w, "data: %s\n\n", b); err != nil {
|
||||||
return false
|
return false
|
||||||
@ -1144,115 +1199,66 @@ func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan type
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
if !send(ChatChunk{
|
if b, err := ser.OpenRolePreamble(modelName, id, created, st); err == nil {
|
||||||
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
if !send(b) {
|
||||||
Choices: []ChunkChoice{{Index: 0, Delta: RespMessage{Role: "assistant"}}},
|
return
|
||||||
}) {
|
}
|
||||||
return
|
|
||||||
}
|
}
|
||||||
// First SSE byte sent to the client: record time-to-first-byte for the
|
// First SSE byte sent to the client: record time-to-first-byte for the
|
||||||
// source's status-page latency average.
|
// source's status-page latency average.
|
||||||
rec.FirstByteMs = time.Since(t0).Milliseconds()
|
rec.FirstByteMs = time.Since(t0).Milliseconds()
|
||||||
var lastUsage *types.TokenUsage
|
|
||||||
var lastCost string
|
|
||||||
lastFinish := ""
|
|
||||||
for ck := range chunks {
|
for ck := range chunks {
|
||||||
if ck.Usage != nil {
|
if ck.Usage != nil {
|
||||||
lastUsage = mergeUsage(lastUsage, ck.Usage)
|
st.Usage = mergeUsage(st.Usage, ck.Usage)
|
||||||
}
|
}
|
||||||
if ck.Cost != "" {
|
if ck.Cost != "" {
|
||||||
lastCost = ck.Cost
|
st.Cost = ck.Cost
|
||||||
}
|
}
|
||||||
chunk := ChatChunk{
|
b, err := ser.Chunk(&ck, modelName, id, created, st)
|
||||||
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
if err != nil {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
delta := RespMessage{Role: "assistant", Content: ck.Content}
|
|
||||||
if ck.ReasoningContent != "" {
|
|
||||||
delta.ReasoningContent = ck.ReasoningContent
|
|
||||||
}
|
|
||||||
if len(ck.ToolCalls) > 0 {
|
|
||||||
delta.ToolCalls = ck.ToolCalls
|
|
||||||
}
|
|
||||||
choice := ChunkChoice{Index: 0, Delta: delta}
|
|
||||||
if ck.Done {
|
|
||||||
fin := ck.FinishReason
|
|
||||||
if fin == "" {
|
|
||||||
fin = "stop"
|
|
||||||
}
|
|
||||||
lastFinish = fin
|
|
||||||
choice.FinishReason = &fin
|
|
||||||
}
|
|
||||||
chunk.Choices = []ChunkChoice{choice}
|
|
||||||
rec.Compl += int64(len(ck.Content)+len(ck.ReasoningContent)+len(ck.ToolCalls)) / 3
|
rec.Compl += int64(len(ck.Content)+len(ck.ReasoningContent)+len(ck.ToolCalls)) / 3
|
||||||
if !send(chunk) {
|
if !send(b) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finalFinish := lastFinish
|
|
||||||
if finalFinish == "" {
|
// Write the upstream's exact usage back onto the audit record. Without this
|
||||||
finalFinish = "stop"
|
// the streamed path kept only the per-chunk byte estimate, so the same
|
||||||
}
|
|
||||||
send(ChatChunk{
|
|
||||||
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
|
||||||
Choices: []ChunkChoice{{Index: 0, Delta: RespMessage{}, FinishReason: &finalFinish}},
|
|
||||||
})
|
|
||||||
// Final usage chunk (OpenAI standard: empty choices + usage before [DONE]).
|
|
||||||
// Prefer the upstream's exact usage if the stream carried it; fall back to
|
|
||||||
// the gateway's estimate otherwise.
|
|
||||||
// Write the upstream's exact numbers back onto the audit record. Without
|
|
||||||
// this the streamed path kept only the per-chunk byte estimate, so the same
|
|
||||||
// request recorded ~1.4x its real completion tokens (measured: upstream 100,
|
// request recorded ~1.4x its real completion tokens (measured: upstream 100,
|
||||||
// audit 145) while the non-streaming path recorded 100. Two paths, two
|
// audit 145) while the non-streaming path recorded 100. Two paths, two
|
||||||
// different numbers for one request is a reporting bug, not a rounding one.
|
// different numbers for one request is a reporting bug, not a rounding one.
|
||||||
if lastUsage != nil {
|
if st.Usage != nil {
|
||||||
if lastUsage.Prompt > 0 {
|
if st.Usage.Prompt > 0 {
|
||||||
rec.Prompt = int64(lastUsage.Prompt)
|
rec.Prompt = int64(st.Usage.Prompt)
|
||||||
}
|
}
|
||||||
if lastUsage.Completion > 0 {
|
if st.Usage.Completion > 0 {
|
||||||
rec.Compl = int64(lastUsage.Completion)
|
rec.Compl = int64(st.Usage.Completion)
|
||||||
}
|
|
||||||
}
|
|
||||||
var tut *types.TokenUsage
|
|
||||||
if lastUsage != nil {
|
|
||||||
tut = lastUsage
|
|
||||||
// Write the upstream's cache accounting back onto the request record
|
|
||||||
// so the audit trail carries the hit/miss split for streaming too.
|
|
||||||
if d := tut.PromptTokensDetails; d != nil {
|
|
||||||
rec.CacheReported = true
|
|
||||||
if d.CachedTokens > 0 {
|
|
||||||
rec.CacheHit = int64(d.CachedTokens)
|
|
||||||
}
|
|
||||||
} else if tut.PromptCacheHit > 0 {
|
|
||||||
rec.CacheHit = int64(tut.PromptCacheHit)
|
|
||||||
rec.CacheReported = true
|
|
||||||
}
|
|
||||||
if tut.PromptCacheMiss > 0 {
|
|
||||||
rec.CacheMiss = int64(tut.PromptCacheMiss)
|
|
||||||
}
|
}
|
||||||
|
applyStreamUsageDetails(rec, st.Usage)
|
||||||
} else if rec.Prompt+rec.Compl > 0 {
|
} else if rec.Prompt+rec.Compl > 0 {
|
||||||
u := types.TokenUsage{
|
// No upstream usage on the stream: the recorded per-chunk byte
|
||||||
|
// estimate is the best number available, so hand it to the serializer
|
||||||
|
// as the usage rather than dropping it.
|
||||||
|
st.Usage = &types.TokenUsage{
|
||||||
Prompt: int(rec.Prompt),
|
Prompt: int(rec.Prompt),
|
||||||
Completion: int(rec.Compl),
|
Completion: int(rec.Compl),
|
||||||
Total: int(rec.Prompt + rec.Compl),
|
Total: int(rec.Prompt + rec.Compl),
|
||||||
}
|
}
|
||||||
tut = &u
|
|
||||||
}
|
}
|
||||||
if tut != nil {
|
|
||||||
send(ChatChunk{
|
if frames, err := ser.Close(modelName, id, created, st); err == nil {
|
||||||
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
for _, b := range frames {
|
||||||
Choices: []ChunkChoice{},
|
if !send(b) {
|
||||||
Usage: tut,
|
return
|
||||||
Cost: lastCost,
|
}
|
||||||
})
|
}
|
||||||
} else if lastCost != "" {
|
}
|
||||||
// Cost arrived without any usage (OpenCode sends it on its own frame).
|
if marker := ser.DoneMarker(); marker != "" {
|
||||||
send(ChatChunk{
|
fmt.Fprintf(w, "data: %s\n\n", marker)
|
||||||
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
|
||||||
Choices: []ChunkChoice{},
|
|
||||||
Cost: lastCost,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
||||||
if flusher != nil {
|
if flusher != nil {
|
||||||
flusher.Flush()
|
flusher.Flush()
|
||||||
}
|
}
|
||||||
@ -1283,7 +1289,7 @@ func (g *Gateway) streamChat(w http.ResponseWriter, ctx context.Context, cands [
|
|||||||
rec.Source = usedSrc
|
rec.Source = usedSrc
|
||||||
g.fireRouted(ctx, "stream", usedSrc, rec.Model, -1, true)
|
g.fireRouted(ctx, "stream", usedSrc, rec.Model, -1, true)
|
||||||
rec.Prompt = estimatePromptTokens(req)
|
rec.Prompt = estimatePromptTokens(req)
|
||||||
g.pumpStream(w, rec, chunks, effective, t0)
|
g.pumpStream(w, g.serializerFor(usedSrc, cands), rec, chunks, effective, t0)
|
||||||
}
|
}
|
||||||
|
|
||||||
// singleChatAuto runs a non-streaming AUTO request down the chain (see
|
// singleChatAuto runs a non-streaming AUTO request down the chain (see
|
||||||
@ -1316,7 +1322,7 @@ func (g *Gateway) singleChatAuto(w http.ResponseWriter, ctx context.Context, cha
|
|||||||
g.fireRouted(ctx, "chat", usedSrc, usedModel, -2, false)
|
g.fireRouted(ctx, "chat", usedSrc, usedModel, -2, false)
|
||||||
rec.FirstByteMs = rec.LatMs
|
rec.FirstByteMs = rec.LatMs
|
||||||
g.writeRec(rec)
|
g.writeRec(rec)
|
||||||
writeChatCompletion(w, resp, usedModel)
|
writeChatCompletion(w, g.serializerFor(usedSrc, nil), resp, usedModel)
|
||||||
}
|
}
|
||||||
|
|
||||||
// streamChatAuto streams an AUTO request down the chain. A slot is abandoned
|
// streamChatAuto streams an AUTO request down the chain. A slot is abandoned
|
||||||
@ -1346,7 +1352,7 @@ func (g *Gateway) streamChatAuto(w http.ResponseWriter, ctx context.Context, cha
|
|||||||
rec.Source = usedSrc
|
rec.Source = usedSrc
|
||||||
g.fireRouted(ctx, "stream", usedSrc, rec.Model, -2, true)
|
g.fireRouted(ctx, "stream", usedSrc, rec.Model, -2, true)
|
||||||
rec.Prompt = estimatePromptTokens(req)
|
rec.Prompt = estimatePromptTokens(req)
|
||||||
g.pumpStream(w, rec, chunks, usedModel, t0)
|
g.pumpStream(w, g.serializerFor(usedSrc, nil), rec, chunks, usedModel, t0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *Gateway) handleImage(w http.ResponseWriter, r *http.Request) {
|
func (g *Gateway) handleImage(w http.ResponseWriter, r *http.Request) {
|
||||||
@ -1459,3 +1465,39 @@ func (g *Gateway) handleImage(w http.ResponseWriter, r *http.Request) {
|
|||||||
Data: resp.ImageData,
|
Data: resp.ImageData,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// applyStreamUsageDetails mirrors the upstream's reasoning and cache numbers
|
||||||
|
// onto a STREAMED request's audit record.
|
||||||
|
//
|
||||||
|
// It exists as a named function because this was inline in the streaming handler,
|
||||||
|
// where it was untestable — and untested it stayed wrong. The non-streaming
|
||||||
|
// recorder read completion_tokens_details.reasoning_tokens; this one did not. So
|
||||||
|
// a source's streamed requests recorded completion and no reasoning at all while
|
||||||
|
// its non-streamed calls recorded both. Nothing failed and no test was red: the
|
||||||
|
// audit trail was simply quieter than it should have been.
|
||||||
|
//
|
||||||
|
// Found by reading the production audit file after wiring a reasoning-heavy
|
||||||
|
// source: completion=8166 with reasoning absent, on every streamed request.
|
||||||
|
//
|
||||||
|
// Reasoning is a SUBSET of completion, never an addition to it. See
|
||||||
|
// recordChatUsage for why that matters to billing.
|
||||||
|
func applyStreamUsageDetails(rec *Req, u *types.TokenUsage) {
|
||||||
|
if d := u.CompletionTokensDetails; d != nil && d.ReasoningTokens > 0 {
|
||||||
|
rec.Reasoning = int64(d.ReasoningTokens)
|
||||||
|
}
|
||||||
|
// Cache accounting is here for the same reason it is kept apart: "zero hits
|
||||||
|
// because nothing was cached" and "this provider never reports caching" are
|
||||||
|
// the same number and mean opposite things.
|
||||||
|
if d := u.PromptTokensDetails; d != nil {
|
||||||
|
rec.CacheReported = true
|
||||||
|
if d.CachedTokens > 0 {
|
||||||
|
rec.CacheHit = int64(d.CachedTokens)
|
||||||
|
}
|
||||||
|
} else if u.PromptCacheHit > 0 {
|
||||||
|
rec.CacheHit = int64(u.PromptCacheHit)
|
||||||
|
rec.CacheReported = true
|
||||||
|
}
|
||||||
|
if u.PromptCacheMiss > 0 {
|
||||||
|
rec.CacheMiss = int64(u.PromptCacheMiss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -3,7 +3,7 @@ package gateway
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/types"
|
"modelrouter/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestMergeUsageKeepsZeroCacheDetails locks the streaming counterpart of the
|
// TestMergeUsageKeepsZeroCacheDetails locks the streaming counterpart of the
|
||||||
|
|||||||
@ -3,13 +3,13 @@ package gateway
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/types"
|
"modelrouter/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestRecordChatUsageKeepsReasoningTokens pins the reasoning-token plumbing.
|
// TestRecordChatUsageKeepsReasoningTokens pins the reasoning-token plumbing.
|
||||||
//
|
//
|
||||||
// The adapters already parse completion_tokens_details.reasoning_tokens and
|
// The adapters already parse completion_tokens_details.reasoning_tokens and
|
||||||
// forward it (opencodego, gemini, anthropic all do), and types.go has carried
|
// forward it (qwen38, gemini, anthropic all do), and types.go has carried
|
||||||
// the field since it was defined — but nothing ever READ it. The number arrived
|
// the field since it was defined — but nothing ever READ it. The number arrived
|
||||||
// in the response and was discarded at the one place that could have kept it,
|
// in the response and was discarded at the one place that could have kept it,
|
||||||
// so a pricing rule had no way to split thinking from output.
|
// so a pricing rule had no way to split thinking from output.
|
||||||
@ -69,3 +69,92 @@ func TestRecordChatUsageReasoningAbsentStaysZero(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestStreamedReasoningIsRecordedOnTheRequest is the streaming counterpart.
|
||||||
|
//
|
||||||
|
// TestRecordChatUsageKeepsReasoningTokens covers the non-streaming path only,
|
||||||
|
// and for a long time the streamed path dropped the number: the non-streaming
|
||||||
|
// recorder read TokenUsage.CompletionTokensDetails, while the streaming one read
|
||||||
|
// only Prompt and Completion from the terminal usage chunk. (Cache accounting was
|
||||||
|
// mirrored; reasoning simply never was.)
|
||||||
|
//
|
||||||
|
// The symptom is invisible in the response — the client sees a correct answer
|
||||||
|
// either way. It only shows in the audit trail, where a reasoning-heavy source's
|
||||||
|
// streamed requests recorded completion=8166 and no reasoning at all. Found by
|
||||||
|
// reading the production audit file, not by a failing test, which is exactly why
|
||||||
|
// the property is pinned here.
|
||||||
|
func TestStreamedReasoningIsRecordedOnTheRequest(t *testing.T) {
|
||||||
|
rec := &Req{}
|
||||||
|
applyStreamUsageDetails(rec, &types.TokenUsage{
|
||||||
|
Prompt: 159579,
|
||||||
|
Completion: 751,
|
||||||
|
Total: 160330,
|
||||||
|
CompletionTokensDetails: &types.CompletionTokensDetails{
|
||||||
|
ReasoningTokens: 512,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if rec.Reasoning != 512 {
|
||||||
|
t.Errorf("streamed Reasoning = %d, want 512 — a streamed request records no "+
|
||||||
|
"reasoning, so a pricing rule that splits thinking from output silently "+
|
||||||
|
"bills it as ordinary output", rec.Reasoning)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStreamedReasoningIsNeverAddedToCompletion pins the SUBSET relation in the
|
||||||
|
// streaming path specifically. A presence check would pass if someone wrote
|
||||||
|
// rec.Compl += reasoning, and billing would then charge every thinking token
|
||||||
|
// twice.
|
||||||
|
func TestStreamedReasoningIsNeverAddedToCompletion(t *testing.T) {
|
||||||
|
rec := &Req{}
|
||||||
|
rec.Compl = 751 // set upstream of the call, as the streaming handler does
|
||||||
|
applyStreamUsageDetails(rec, &types.TokenUsage{
|
||||||
|
Completion: 751,
|
||||||
|
CompletionTokensDetails: &types.CompletionTokensDetails{ReasoningTokens: 512},
|
||||||
|
})
|
||||||
|
if rec.Compl != 751 {
|
||||||
|
t.Errorf("Compl = %d, want 751 unchanged — reasoning is a subset, not an addition", rec.Compl)
|
||||||
|
}
|
||||||
|
if rec.Reasoning >= rec.Compl {
|
||||||
|
t.Errorf("Reasoning (%d) must be a SUBSET of Compl (%d)", rec.Reasoning, rec.Compl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStreamedUsageWithoutReasoningLeavesItUnset: an upstream that reports no
|
||||||
|
// reasoning must not be turned into a reported zero. In the audit trail 0 means
|
||||||
|
// "not reported", and a plugin cannot tell that from "no reasoning happened".
|
||||||
|
func TestStreamedUsageWithoutReasoningLeavesItUnset(t *testing.T) {
|
||||||
|
rec := &Req{}
|
||||||
|
applyStreamUsageDetails(rec, &types.TokenUsage{Prompt: 10, Completion: 4, Total: 14})
|
||||||
|
if rec.Reasoning != 0 {
|
||||||
|
t.Errorf("Reasoning = %d for an upstream that reported none", rec.Reasoning)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStreamedCacheAccountingStillWorks is a regression guard for the block the
|
||||||
|
// reasoning lines were added to. They share a function, so a future edit that
|
||||||
|
// short-circuits one would break the other — silently, since a missing cache
|
||||||
|
// number reads as "nothing was cached".
|
||||||
|
func TestStreamedCacheAccountingStillWorks(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
u types.TokenUsage
|
||||||
|
hit int64
|
||||||
|
miss int64
|
||||||
|
rep bool
|
||||||
|
}{
|
||||||
|
{"details object", types.TokenUsage{
|
||||||
|
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 4096},
|
||||||
|
}, 4096, 0, true},
|
||||||
|
{"legacy hit/miss", types.TokenUsage{
|
||||||
|
PromptCacheHit: 2048, PromptCacheMiss: 512,
|
||||||
|
}, 2048, 512, true},
|
||||||
|
{"nothing reported", types.TokenUsage{}, 0, 0, false},
|
||||||
|
} {
|
||||||
|
rec := &Req{}
|
||||||
|
applyStreamUsageDetails(rec, &c.u)
|
||||||
|
if rec.CacheHit != c.hit || rec.CacheMiss != c.miss || rec.CacheReported != c.rep {
|
||||||
|
t.Errorf("%s: hit=%d miss=%d reported=%v; want %d/%d/%v",
|
||||||
|
c.name, rec.CacheHit, rec.CacheMiss, rec.CacheReported, c.hit, c.miss, c.rep)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -11,7 +11,7 @@ import (
|
|||||||
|
|
||||||
// deploy.sh is the only release-critical script with no automated coverage, and
|
// deploy.sh is the only release-critical script with no automated coverage, and
|
||||||
// its newest part — prune_adapter_backups — deletes directories under
|
// its newest part — prune_adapter_backups — deletes directories under
|
||||||
// /etc/llmsproxy. This exercises the function in a throwaway directory with the
|
// /etc/modelrouter. This exercises the function in a throwaway directory with the
|
||||||
// REAL backup-name shape.
|
// REAL backup-name shape.
|
||||||
//
|
//
|
||||||
// The shape matters and got this wrong twice: the guard is
|
// The shape matters and got this wrong twice: the guard is
|
||||||
@ -33,34 +33,34 @@ func TestDeployPruneAdapterBackups(t *testing.T) {
|
|||||||
t.Fatalf("read deploy.sh: %v", err)
|
t.Fatalf("read deploy.sh: %v", err)
|
||||||
}
|
}
|
||||||
src := string(fn)
|
src := string(fn)
|
||||||
i := strings.Index(src, "prune_adapter_backups()")
|
i := strings.Index(src, "prune_backups()")
|
||||||
if i < 0 {
|
if i < 0 {
|
||||||
t.Fatal("prune_adapter_backups not found in deploy.sh")
|
t.Fatal("prune_backups not found in deploy.sh")
|
||||||
}
|
}
|
||||||
j := strings.Index(src[i:], "\n}\n")
|
j := strings.Index(src[i:], "\n}\n")
|
||||||
if j < 0 {
|
if j < 0 {
|
||||||
t.Fatal("prune_adapter_backups has no closing brace")
|
t.Fatal("prune_backups has no closing brace")
|
||||||
}
|
}
|
||||||
body := src[i : i+j+3]
|
body := src[i : i+j+3]
|
||||||
|
|
||||||
// Redirect ONLY the base path. The logic under test must stay verbatim.
|
// Redirect ONLY the base path. The logic under test must stay verbatim.
|
||||||
body = strings.Replace(body,
|
body = strings.Replace(body,
|
||||||
`local base="/etc/llmsproxy"`,
|
`local base="/etc/modelrouter"`,
|
||||||
`local base="`+base+`"`, 1)
|
`local base="`+base+`"`, 1)
|
||||||
// KEEP_ADAPTER_BACKUPS must come from deploy.sh itself, NOT from a literal
|
// KEEP_BACKUPS must come from deploy.sh itself, NOT from a literal
|
||||||
// here: hardcoding 5 in the shim overrode whatever the script configured, so
|
// here: hardcoding 5 in the shim overrode whatever the script configured, so
|
||||||
// a mutation that set KEEP_ADAPTER_BACKUPS=0 in deploy.sh still passed. The
|
// a mutation that set KEEP_BACKUPS=0 in deploy.sh still passed. The
|
||||||
// value is part of what is under test.
|
// value is part of what is under test.
|
||||||
var keepRe = regexp.MustCompile(`(?m)^KEEP_ADAPTER_BACKUPS=(\d+)`)
|
var keepRe = regexp.MustCompile(`(?m)^KEEP_BACKUPS=(\d+)`)
|
||||||
keepM := keepRe.FindStringSubmatch(src)
|
keepM := keepRe.FindStringSubmatch(src)
|
||||||
if keepM == nil {
|
if keepM == nil {
|
||||||
t.Fatal("deploy.sh no longer sets KEEP_ADAPTER_BACKUPS; the pruning " +
|
t.Fatal("deploy.sh no longer sets KEEP_BACKUPS; the pruning " +
|
||||||
"policy would be undefined")
|
"policy would be undefined")
|
||||||
}
|
}
|
||||||
keep := keepM[1]
|
keep := keepM[1]
|
||||||
|
|
||||||
script := "warn() { :; }\nlog() { :; }\nKEEP_ADAPTER_BACKUPS=" + keep + "\n" +
|
script := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" + keep + "\n" +
|
||||||
body + "\nprune_adapter_backups\n"
|
body + "\nprune_backups adapters\n"
|
||||||
scriptPath := filepath.Join(base, "fn.sh")
|
scriptPath := filepath.Join(base, "fn.sh")
|
||||||
if err := os.WriteFile(scriptPath, []byte(script), 0o600); err != nil {
|
if err := os.WriteFile(scriptPath, []byte(script), 0o600); err != nil {
|
||||||
t.Fatalf("write script: %v", err)
|
t.Fatalf("write script: %v", err)
|
||||||
@ -88,7 +88,7 @@ func TestDeployPruneAdapterBackups(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if out, err := exec.Command("bash", scriptPath).CombinedOutput(); err != nil {
|
if out, err := exec.Command("bash", scriptPath).CombinedOutput(); err != nil {
|
||||||
t.Fatalf("prune_adapter_backups failed: %v\n%s", err, out)
|
t.Fatalf("prune_backups failed: %v\n%s", err, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
entries, err := os.ReadDir(base)
|
entries, err := os.ReadDir(base)
|
||||||
@ -102,11 +102,11 @@ func TestDeployPruneAdapterBackups(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 12 well-formed backups with KEEP_ADAPTER_BACKUPS=5 → the newest FIVE
|
// 12 well-formed backups with KEEP_BACKUPS=5 → the newest FIVE
|
||||||
// survive (…08 …09 …10 …11 …12) and the older seven are deleted. Listing
|
// survive (…08 …09 …10 …11 …12) and the older seven are deleted. Listing
|
||||||
// only three keepers made this assertion wrong in BOTH directions at first:
|
// only three keepers made this assertion wrong in BOTH directions at first:
|
||||||
// …09 was named as "should be deleted" while it is in fact retained.
|
// …09 was named as "should be deleted" while it is in fact retained.
|
||||||
// KEEP_ADAPTER_BACKUPS=5 is a cap on the TOTAL number of backups, and
|
// KEEP_BACKUPS=5 is a cap on the TOTAL number of backups, and
|
||||||
// irregular names are never deleted — so they consume slots rather than
|
// irregular names are never deleted — so they consume slots rather than
|
||||||
// being ignored. With 3 unremovable directories present, only 2 of the
|
// being ignored. With 3 unremovable directories present, only 2 of the
|
||||||
// well-formed ones can survive. That is the function's actual (and safe)
|
// well-formed ones can survive. That is the function's actual (and safe)
|
||||||
@ -202,3 +202,123 @@ func TestDeployBacksUpTheLiveFileBeforeOverwriting(t *testing.T) {
|
|||||||
"captures the new file and rollback restores nothing")
|
"captures the new file and rollback restores nothing")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ★ 备份清理必须覆盖 plugins 前缀(真实缺陷)。
|
||||||
|
//
|
||||||
|
// 事实:backup 清理最初只处理 `adapters.bak.*`,而 `sync_plugins` 同样每次部署
|
||||||
|
// 建一个 `plugins.bak.<时间戳>` —— 于是 adapters 的堆积被修好,plugins 继续涨。
|
||||||
|
// 实测线上 plugins.bak 积累到 **36 个**、/etc/modelrouter 涨到 192M。
|
||||||
|
//
|
||||||
|
// 这是「两处只修一处」的老形状。判据钉住:prune_all_backups 必须同时裁两个前缀,
|
||||||
|
// 且新增任何 .bak.<时间戳> 前缀时都要在这里补上(否则又是一个泄漏点)。
|
||||||
|
func TestDeployPruneCoversBothBackupPrefixes(t *testing.T) {
|
||||||
|
if _, err := os.Stat("../../deploy.sh"); err != nil {
|
||||||
|
t.Skip("deploy.sh not present (packaging-only checkout)")
|
||||||
|
}
|
||||||
|
if _, err := exec.LookPath("bash"); err != nil {
|
||||||
|
t.Skip("bash not available")
|
||||||
|
}
|
||||||
|
fn, err := os.ReadFile("../../deploy.sh")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
src := string(fn)
|
||||||
|
|
||||||
|
// ① prune_all_backups 必须存在且两个前缀都裁
|
||||||
|
i := strings.Index(src, "prune_all_backups()")
|
||||||
|
if i < 0 {
|
||||||
|
t.Fatal("prune_all_backups 不存在 —— 调用点若仍只裁 adapters," +
|
||||||
|
"plugins.bak 会继续无上限堆积(实测 36 个 / 192M)")
|
||||||
|
}
|
||||||
|
j := strings.Index(src[i:], "\n}\n")
|
||||||
|
if j < 0 {
|
||||||
|
t.Fatal("prune_all_backups 没有收尾大括号")
|
||||||
|
}
|
||||||
|
body := src[i : i+j+3]
|
||||||
|
for _, want := range []string{"prune_backups adapters", "prune_backups plugins"} {
|
||||||
|
if !strings.Contains(body, want) {
|
||||||
|
t.Fatalf("prune_all_backups 缺 %q —— plugins.bak 会无上限堆积", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ② ★ 覆盖完整性:脚本里出现的每个 `.bak.<时间戳>` 前缀都必须在裁剪范围内。
|
||||||
|
// 这条比逐字检查两个名字更强 —— 将来有人新增第三种备份目录(比如
|
||||||
|
// runtime.bak.*),这里会立刻报红,而不是等它涨到几百兆。
|
||||||
|
prefixRe := regexp.MustCompile(`([a-z_]+)\.bak\.\$\(date`)
|
||||||
|
found := map[string]bool{}
|
||||||
|
for _, m := range prefixRe.FindAllStringSubmatch(src, -1) {
|
||||||
|
found[m[1]] = true
|
||||||
|
}
|
||||||
|
if len(found) == 0 {
|
||||||
|
t.Fatal("deploy.sh 里没找到任何 .bak.$(date …) 备份目录创建点(判据需更新)")
|
||||||
|
}
|
||||||
|
for prefix := range found {
|
||||||
|
if !strings.Contains(body, "prune_backups "+prefix) {
|
||||||
|
t.Errorf("deploy.sh 会创建 %s.bak.<时间戳> 目录,但 prune_all_backups 不裁它 "+
|
||||||
|
"—— 它就是一个无上限增长的备份泄漏点(实测 plugins.bak 涨到 36 个)", prefix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ③ 行为验证:用真函数裁 plugins 前缀,最新 5 份必须留下、更旧的删掉
|
||||||
|
base := t.TempDir()
|
||||||
|
k := strings.Index(src, "prune_backups()")
|
||||||
|
kj := strings.Index(src[k:], "\n}\n")
|
||||||
|
fnBody := src[k : k+kj+3]
|
||||||
|
fnBody = strings.Replace(fnBody, `local base="/etc/modelrouter"`, `local base="`+base+`"`, 1)
|
||||||
|
keepRe := regexp.MustCompile(`(?m)^KEEP_BACKUPS=(\d+)`)
|
||||||
|
km := keepRe.FindStringSubmatch(src)
|
||||||
|
if km == nil {
|
||||||
|
t.Fatal("deploy.sh 不再设置 KEEP_BACKUPS")
|
||||||
|
}
|
||||||
|
script := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" +
|
||||||
|
km[1] + "\n" + fnBody + "\nprune_backups plugins\n"
|
||||||
|
sp := filepath.Join(base, "fn.sh")
|
||||||
|
if err := os.WriteFile(sp, []byte(script), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, ts := range []string{
|
||||||
|
"20260101120000", "20260201120000", "20260301120000", "20260401120000",
|
||||||
|
"20260501120000", "20260601120000", "20260701120000", "20260801120000",
|
||||||
|
} {
|
||||||
|
if err := os.MkdirAll(filepath.Join(base, "plugins.bak."+ts), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// adapters 前缀**不该被动**(防止误删别的前缀)
|
||||||
|
if err := os.MkdirAll(filepath.Join(base, "adapters.bak.20260101120000"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out, err := exec.Command("bash", sp).CombinedOutput(); err != nil {
|
||||||
|
t.Fatalf("prune_backups plugins failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := map[string]bool{}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() {
|
||||||
|
got[e.Name()] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !got["plugins.bak.20260801120000"] {
|
||||||
|
t.Error("最新的 plugins 备份被删了;裁剪必须从最旧的一端开始")
|
||||||
|
}
|
||||||
|
if got["plugins.bak.20260101120000"] {
|
||||||
|
t.Error("最旧的 plugins 备份存活;裁剪没生效")
|
||||||
|
}
|
||||||
|
if !got["adapters.bak.20260101120000"] {
|
||||||
|
t.Error("裁 plugins 时误删了 adapters 备份 —— 前缀参数必须严格限定范围")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ④ 未知前缀必须响亮失败(防误用:传空串会删掉整个目录)
|
||||||
|
bad := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" +
|
||||||
|
km[1] + "\n" + fnBody + "\nprune_backups \"\"\n"
|
||||||
|
bp := filepath.Join(base, "bad.sh")
|
||||||
|
if err := os.WriteFile(bp, []byte(bad), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out, err := exec.Command("bash", bp).CombinedOutput(); err == nil {
|
||||||
|
t.Fatalf("prune_backups 接受空前缀(危险:会删整个目录),输出: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
749
internal/gateway/discover.go
Normal file
749
internal/gateway/discover.go
Normal file
@ -0,0 +1,749 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Model discovery: turn "here is a URL and a key" into a configured source.
|
||||||
|
//
|
||||||
|
// The hard part is not the GET /models call — it is guessing which URL the
|
||||||
|
// upstream actually serves. This gateway builds request paths as
|
||||||
|
// base_url + adapter endpoint (openai contributes "/chat/completions"), so a
|
||||||
|
// base_url that already ends in "/v1" yields "/v1/chat/completions" while a
|
||||||
|
// base_url of "/v1" plus our default image path yields the broken
|
||||||
|
// "/v1/v1/images/generations". Vendors disagree on whether the version segment
|
||||||
|
// belongs to the base or to the path, and there is no header that says which.
|
||||||
|
//
|
||||||
|
// So discovery does not trust the URL it is given. It tries a small, ordered
|
||||||
|
// set of candidates, records which one answered, and reports that back as the
|
||||||
|
// normalized base_url + adapter endpoint pair. A caller who typed the wrong
|
||||||
|
// form gets a working source anyway, plus the correction needed to make it
|
||||||
|
// permanent.
|
||||||
|
//
|
||||||
|
// What is deliberately NOT here:
|
||||||
|
//
|
||||||
|
// - No guessing of model capabilities from the name. "vision" in an id does
|
||||||
|
// not make a model accept images, and marking an image-only endpoint as
|
||||||
|
// kind:"chat" makes every AUTO request fail on it. Kind stays "chat" unless
|
||||||
|
// the upstream's own metadata says otherwise (see discoveredModelKind).
|
||||||
|
// - No live probe per model. A single-token completion per model would cost
|
||||||
|
// real money across a 34-model catalog and takes minutes. /models is the
|
||||||
|
// cheap, authoritative-enough signal for "this id is served here"; whether
|
||||||
|
// it actually answers stays for the normal request path to discover.
|
||||||
|
//
|
||||||
|
// The one thing that IS verified live is reachability, because a source that
|
||||||
|
// 404s is not worth writing to disk.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// discoveryTimeout bounds the whole probe. Vendors are routinely slow on
|
||||||
|
// the first request to an uncached endpoint; 20s is enough for a healthy
|
||||||
|
// one and short enough that the UI is not left hanging.
|
||||||
|
discoveryTimeout = 20 * time.Second
|
||||||
|
|
||||||
|
// discoveryModelsPath is the OpenAI-compatible listing endpoint. Nearly
|
||||||
|
// every vendor proxies it verbatim, including the ones whose chat path
|
||||||
|
// differs.
|
||||||
|
discoveryModelsPath = "/v1/models"
|
||||||
|
|
||||||
|
// discoveryMaxBody caps the listing response. A catalog with thousands of
|
||||||
|
// models is still far under this; anything larger is a captive portal or
|
||||||
|
// an error page, not a model list.
|
||||||
|
discoveryMaxBody = 8 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
// discoveredModel is one entry of a parsed /models response, kept in the
|
||||||
|
// upstream's own terms. The upstream is the only authority on what it serves.
|
||||||
|
type discoveredModel struct {
|
||||||
|
ID string
|
||||||
|
Kind string // "chat" | "image"
|
||||||
|
Meta map[string]interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryResult reports what a probe found, including the case where nothing
|
||||||
|
// worked — the failure text is what the operator needs, so it must name every
|
||||||
|
// URL that was tried rather than just the last one.
|
||||||
|
type discoveryResult struct {
|
||||||
|
BaseURL string `json:"base_url"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
// ImageEndpoint is reported separately because the gateway composes it
|
||||||
|
// differently: Provider.ImageURL is BaseURL + ImageEndpoint(), and the
|
||||||
|
// default ImageEndpoint is the ABSOLUTE path "/v1/images/generations".
|
||||||
|
// Chat's default is the relative "/chat/completions". So a base_url that
|
||||||
|
// already ends in /v1 produces a working chat URL and a broken image URL
|
||||||
|
// ("/v1/v1/images/generations") — which is exactly the 404 seen on the
|
||||||
|
// first vela configuration, and which discovery has to correct.
|
||||||
|
ImageEndpoint string `json:"image_endpoint"`
|
||||||
|
Adapter string `json:"adapter"`
|
||||||
|
ModelsPath string `json:"models_path"`
|
||||||
|
Models []string `json:"models"`
|
||||||
|
ImageModels []string `json:"image_models"`
|
||||||
|
Tried []string `json:"tried"`
|
||||||
|
LatencyMS int64 `json:"latency_ms"`
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalJSON guarantees the two list fields are always JSON arrays, never
|
||||||
|
// null. The WebUI concatenates them directly, and "null" renders as the string
|
||||||
|
// "undefined" in every row — a failure that only shows up in a browser and not
|
||||||
|
// in a JSON schema check.
|
||||||
|
//
|
||||||
|
// Relying on the constructor to initialize the slices is not enough: the zero
|
||||||
|
// value of this struct is a legitimate thing for a caller to build, and a
|
||||||
|
// future code path that forgets the initializer would ship a UI bug silently.
|
||||||
|
func (r discoveryResult) MarshalJSON() ([]byte, error) {
|
||||||
|
type alias discoveryResult // avoid recursion
|
||||||
|
out := alias(r)
|
||||||
|
if out.Models == nil {
|
||||||
|
out.Models = []string{}
|
||||||
|
}
|
||||||
|
if out.ImageModels == nil {
|
||||||
|
out.ImageModels = []string{}
|
||||||
|
}
|
||||||
|
if out.Tried == nil {
|
||||||
|
out.Tried = []string{}
|
||||||
|
}
|
||||||
|
return json.Marshal(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryCandidate is one (base_url, models-path) guess. Order matters: the
|
||||||
|
// first candidate is the one reported as normalized when several work, because
|
||||||
|
// it is the one that keeps base_url free of a version segment and therefore
|
||||||
|
// composes correctly with the adapter's chat path.
|
||||||
|
type discoveryCandidate struct {
|
||||||
|
baseURL string
|
||||||
|
modelsPath string
|
||||||
|
// prefer ranks candidates that are more likely to be the vendor's real
|
||||||
|
// path. A 200 from a lower-preference candidate is still accepted, but
|
||||||
|
// only after every higher-preference one failed — see DiscoverModels.
|
||||||
|
prefer int
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryFailure carries the HTTP status the caller should answer with.
|
||||||
|
//
|
||||||
|
// The status is part of the error rather than left to the handler because the
|
||||||
|
// whole point of classifying a failure is that the operator must not have to
|
||||||
|
// guess whether the key, the endpoint, or the vendor is at fault.
|
||||||
|
type discoveryFailure struct {
|
||||||
|
msg string
|
||||||
|
status int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *discoveryFailure) Error() string { return e.msg }
|
||||||
|
|
||||||
|
// StatusOrDefault reports the intended HTTP status.
|
||||||
|
func (e *discoveryFailure) StatusOrDefault() int {
|
||||||
|
if e.status == 0 {
|
||||||
|
return http.StatusBadGateway
|
||||||
|
}
|
||||||
|
return e.status
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryCandidates enumerates the URL forms worth trying for a raw input.
|
||||||
|
//
|
||||||
|
// The ranking encodes a real observation rather than a guess: the amr-link
|
||||||
|
// endpoint serves its catalog at "/v1/models" (the form an SDK example shows)
|
||||||
|
// from a base the operator pasted WITH the "/v1", while several other vendors
|
||||||
|
// mount it at the host root. Both are probed; the one that answers is reported
|
||||||
|
// back as a normalized base/endpoint pair.
|
||||||
|
//
|
||||||
|
// What we are really deciding is the base_url to persist. The safest one keeps
|
||||||
|
// the version segment on the ENDPOINT, because openai's adapter endpoint is
|
||||||
|
// "/chat/completions" and the two are concatenated: base "https://h" +
|
||||||
|
// endpoint "/v1/chat/completions" is correct, while leaving "/v1" on the base
|
||||||
|
// doubles the default image path into "/v1/v1/images/generations" — which is
|
||||||
|
// exactly the 404 observed on the first vela configuration attempt.
|
||||||
|
func discoveryCandidates(raw string) []discoveryCandidate {
|
||||||
|
raw = strings.TrimRight(strings.TrimSpace(raw), "/")
|
||||||
|
if raw == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Deduplicate on the URL that will actually be requested, not on the
|
||||||
|
// (base, path) pair. base "https://h/v1" + path "/models" and base "https://h"
|
||||||
|
// + path "/v1/models" both request the same address, and listing it twice in
|
||||||
|
// an error message makes the operator think two things were tried.
|
||||||
|
seenURL := map[string]bool{}
|
||||||
|
seenPair := map[string]bool{}
|
||||||
|
var out []discoveryCandidate
|
||||||
|
add := func(base, path string, prefer int) {
|
||||||
|
pair := base + "|" + path
|
||||||
|
if seenPair[pair] || seenURL[base+path] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seenPair[pair] = true
|
||||||
|
seenURL[base+path] = true
|
||||||
|
out = append(out, discoveryCandidate{baseURL: base, modelsPath: path, prefer: prefer})
|
||||||
|
}
|
||||||
|
|
||||||
|
stripped := trimVersionSuffix(raw)
|
||||||
|
hasVersion := stripped != raw
|
||||||
|
|
||||||
|
// Both catalog paths are always tried, whichever form the operator typed.
|
||||||
|
//
|
||||||
|
// The earlier version offered "/models" only when the base already carried a
|
||||||
|
// version segment. That left a bare host ("https://api.vendor.com") with a
|
||||||
|
// SINGLE candidate and therefore no fallback at all — so a vendor serving
|
||||||
|
// its catalog at the bare "/models" was reported as unreachable. Every
|
||||||
|
// test used a "/v1"-suffixed base and so never saw it; siliconflow's
|
||||||
|
// "/v1/models" happens to answer, which is why the gap survived review.
|
||||||
|
//
|
||||||
|
// Preference only decides WHICH URL is reported when several work, never
|
||||||
|
// which ones are attempted.
|
||||||
|
//
|
||||||
|
// Order matters: a base that already ends in a version segment must be
|
||||||
|
// paired with the RELATIVE "/models" first, because pairing it with
|
||||||
|
// "/v1/models" produces "…/v1/v1/models" — a URL that is 404 by construction.
|
||||||
|
// Trying it first would spend a round trip on a certain miss and, worse,
|
||||||
|
// make the reported "tried" list lead with nonsense.
|
||||||
|
if hasVersion {
|
||||||
|
add(raw, "/models", 0)
|
||||||
|
add(raw, discoveryModelsPath, 1) // the doubled form; kept only as a last resort
|
||||||
|
add(stripped, "/models", 2)
|
||||||
|
add(stripped, discoveryModelsPath, 3)
|
||||||
|
} else {
|
||||||
|
add(raw, discoveryModelsPath, 0)
|
||||||
|
add(raw, "/models", 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stable insertion sort by preference; add() above is not ordered.
|
||||||
|
for i := 1; i < len(out); i++ {
|
||||||
|
for j := i; j > 0 && out[j].prefer < out[j-1].prefer; j-- {
|
||||||
|
out[j], out[j-1] = out[j-1], out[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// trimVersionSuffix removes a trailing "/v1" (or /v2, /v1beta, ...) so the
|
||||||
|
// host root can be probed on its own.
|
||||||
|
func trimVersionSuffix(base string) string {
|
||||||
|
i := strings.LastIndex(base, "/")
|
||||||
|
if i < 0 {
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
if !isVersionSegment(base[i+1:]) {
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
return base[:i]
|
||||||
|
}
|
||||||
|
|
||||||
|
// versionSegment returns the leading path segment of a models path when that
|
||||||
|
// segment is a version marker ("/v1/models" → "/v1", "/models" → ""). It is the
|
||||||
|
// piece that must move between base_url and endpoint so the two compose.
|
||||||
|
func versionSegment(modelsPath string) string {
|
||||||
|
parts := strings.Split(strings.Trim(modelsPath, "/"), "/")
|
||||||
|
if len(parts) != 2 || !isVersionSegment(parts[0]) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "/" + parts[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// isVersionSegment reports whether a path segment looks like "v1", "v2",
|
||||||
|
// "v1beta" — and not like a real path name such as "vision" or a wildcard
|
||||||
|
// path such as "v1x".
|
||||||
|
//
|
||||||
|
// The rule is: a leading "v", then digits, then an OPTIONAL suffix drawn from
|
||||||
|
// the prerelease markers vendors actually use. A free-form alphabetic tail was
|
||||||
|
// tried and rejected — it classifies "v1x" as a version, and that is how a
|
||||||
|
// templated path segment would get stripped off a real base URL.
|
||||||
|
func isVersionSegment(seg string) bool {
|
||||||
|
if len(seg) < 2 || seg[0] != 'v' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
i := 1
|
||||||
|
digits := 0
|
||||||
|
for ; i < len(seg); i++ {
|
||||||
|
if seg[i] < '0' || seg[i] > '9' {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
digits++
|
||||||
|
}
|
||||||
|
if digits == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
suffix := strings.ToLower(seg[i:])
|
||||||
|
if suffix == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, marker := range []string{"alpha", "beta", "preview", "dev", "rc"} {
|
||||||
|
if suffix == marker || strings.HasPrefix(suffix, "-"+marker) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeEndpoint derives the chat endpoint to persist, given the models path
|
||||||
|
// that actually answered, the base_url being persisted, and the base the
|
||||||
|
// operator typed.
|
||||||
|
//
|
||||||
|
// The rule is a single invariant: exactly one version segment must end up in
|
||||||
|
// base_url + endpoint. The adapter contributes "/chat/completions" and the
|
||||||
|
// default image path contributes "/v1/images/generations", so a version left on
|
||||||
|
// BOTH breaks one of them — and one left on NEITHER sends chat to the wrong
|
||||||
|
// path on vendors that require the segment.
|
||||||
|
func composeEndpoint(modelsPath, persistedBase, typedBase string) string {
|
||||||
|
// The version segment currently present on the persisted base.
|
||||||
|
onBase := ""
|
||||||
|
if b := trimVersionSuffix(persistedBase); b != persistedBase {
|
||||||
|
onBase = persistedBase[len(b):]
|
||||||
|
}
|
||||||
|
// The version segment present in the models path that answered.
|
||||||
|
inPath := versionSegment(modelsPath)
|
||||||
|
|
||||||
|
// The version segment actually WORKED: prefer the models path's, then the
|
||||||
|
// one the operator typed.
|
||||||
|
version := inPath
|
||||||
|
if version == "" && onBase == "" {
|
||||||
|
if b := trimVersionSuffix(typedBase); b != typedBase && typedBase != "" {
|
||||||
|
version = typedBase[len(b):]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if version == "" {
|
||||||
|
return "" // adapter default already composes
|
||||||
|
}
|
||||||
|
return version + "/chat/completions"
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeImageEndpoint derives the image endpoint to persist.
|
||||||
|
//
|
||||||
|
// Provider.ImageURL is `base_url + ImageEndpoint()`, and the built-in default
|
||||||
|
// is the absolute path "/v1/images/generations" — unlike chat's relative
|
||||||
|
// "/chat/completions". So the two defaults do NOT agree about who owns the
|
||||||
|
// version segment, which is the trap this function exists to close:
|
||||||
|
//
|
||||||
|
// base "https://h" + "/v1/images/generations" → correct
|
||||||
|
// base "https://h/v1" + "/v1/images/generations" → "/v1/v1/…" 404
|
||||||
|
// base "https://h/v1" + "/images/generations" → "/v1/images/generations" correct
|
||||||
|
//
|
||||||
|
// The second row is not a theoretical guess. Measured against the real
|
||||||
|
// amr-link endpoint with base_url "https://amr-link.open-design.ai/v1":
|
||||||
|
//
|
||||||
|
// …/v1/images/generations → 200 (the image is then generated; that call
|
||||||
|
// takes minutes, which is the upstream's own
|
||||||
|
// latency and not a routing problem)
|
||||||
|
// …/v1/v1/images/generations → 404
|
||||||
|
// …/images/generations → 404
|
||||||
|
//
|
||||||
|
// Leaving the default in place when the base already ends in a version is
|
||||||
|
// therefore always wrong for such an upstream, and the version-relative form
|
||||||
|
// is the one that composes.
|
||||||
|
func composeImageEndpoint(persistedBase string) string {
|
||||||
|
if b := trimVersionSuffix(persistedBase); b != persistedBase {
|
||||||
|
return "/images/generations"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscoverModels probes rawURL for a model catalog. apiKey is sent as a bearer
|
||||||
|
// token; some gateways accept an unauthenticated listing, so an empty key is
|
||||||
|
// allowed and simply omits the header.
|
||||||
|
//
|
||||||
|
// It returns on the first candidate that yields a parseable catalog, so the
|
||||||
|
// reported latency is that of the working path, not the sum of the failures.
|
||||||
|
func DiscoverModels(ctx context.Context, rawURL, apiKey string) (*discoveryResult, error) {
|
||||||
|
cands := discoveryCandidates(rawURL)
|
||||||
|
if len(cands) == 0 {
|
||||||
|
return nil, fmt.Errorf("base url required")
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: discoveryTimeout}
|
||||||
|
var tried []string
|
||||||
|
var firstErr string
|
||||||
|
var authErr string
|
||||||
|
typedBase := strings.TrimRight(strings.TrimSpace(rawURL), "/")
|
||||||
|
|
||||||
|
var lastErr string
|
||||||
|
for _, c := range cands {
|
||||||
|
full := c.baseURL + c.modelsPath
|
||||||
|
tried = append(tried, full)
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil)
|
||||||
|
if err != nil {
|
||||||
|
firstErr = err.Error()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
if apiKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
lastErr = classifyTransport(err)
|
||||||
|
if lastErr == "" {
|
||||||
|
lastErr = err.Error()
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body, readErr := io.ReadAll(io.LimitReader(resp.Body, discoveryMaxBody))
|
||||||
|
resp.Body.Close()
|
||||||
|
took := time.Since(start)
|
||||||
|
if readErr != nil {
|
||||||
|
firstErr = readErr.Error()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
// An auth rejection is the operator's key being wrong, not the
|
||||||
|
// upstream being down. Keep it: it is the single most common
|
||||||
|
// failure and 502 would send them to the wrong place.
|
||||||
|
if af := classifyAuthStatus(resp.StatusCode, body); af != "" {
|
||||||
|
authErr = af
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
firstErr = fmt.Sprintf("%s returned %d", full, resp.StatusCode)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
models, err := parseModelCatalog(body)
|
||||||
|
if err != nil {
|
||||||
|
firstErr = fmt.Sprintf("%s: %v", full, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
res := &discoveryResult{
|
||||||
|
BaseURL: c.baseURL,
|
||||||
|
ModelsPath: c.modelsPath,
|
||||||
|
Adapter: "openai",
|
||||||
|
Models: []string{},
|
||||||
|
ImageModels: []string{},
|
||||||
|
Tried: append([]string{}, tried...),
|
||||||
|
LatencyMS: took.Milliseconds(),
|
||||||
|
}
|
||||||
|
// Persist a base/endpoint pair that composes correctly. The adapter's
|
||||||
|
// own endpoint is "/chat/completions", so any version segment has to be
|
||||||
|
// carried on the endpoint rather than left on the base (where it doubles
|
||||||
|
// the default image path into "/v1/v1/images/generations" — the exact
|
||||||
|
// 404 seen on the first vela configuration attempt).
|
||||||
|
res.Endpoint = composeEndpoint(c.modelsPath, c.baseURL, typedBase)
|
||||||
|
res.ImageEndpoint = composeImageEndpoint(c.baseURL)
|
||||||
|
|
||||||
|
// If the operator typed a version segment and we are persisting a base
|
||||||
|
// that no longer has it, the endpoint above is what restores it. If the
|
||||||
|
// base kept the segment, the adapter default is already correct.
|
||||||
|
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, m := range models {
|
||||||
|
if m.ID == "" || seen[m.ID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[m.ID] = true
|
||||||
|
if m.Kind == "image" {
|
||||||
|
res.ImageModels = append(res.ImageModels, m.ID)
|
||||||
|
} else {
|
||||||
|
res.Models = append(res.Models, m.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(res.Models)
|
||||||
|
sort.Strings(res.ImageModels)
|
||||||
|
if len(res.Models) == 0 && len(res.ImageModels) == 0 {
|
||||||
|
firstErr = fmt.Sprintf("%s: catalog contained no usable model ids", full)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c.baseURL != typedBase {
|
||||||
|
res.Note = fmt.Sprintf("base url normalized from %s to %s; endpoint set to %q so it composes with the adapter",
|
||||||
|
typedBase, c.baseURL, res.Endpoint)
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
return nil, buildDiscoveryError(tried, authErr, firstErr, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// classifyAuthStatus turns a 401/403 from the catalog endpoint into a message
|
||||||
|
// that names the credential as the cause.
|
||||||
|
//
|
||||||
|
// This distinction is the whole point: a wrong key and an unreachable vendor
|
||||||
|
// are the two most common discovery failures, and they need opposite actions
|
||||||
|
// (re-type the key vs. go check the vendor). Reporting both as "upstream
|
||||||
|
// error" makes the operator guess.
|
||||||
|
func classifyAuthStatus(status int, body []byte) string {
|
||||||
|
if status != http.StatusUnauthorized && status != http.StatusForbidden {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
snippet := strings.TrimSpace(string(body))
|
||||||
|
// Vendors put the reason in wildly different shapes; a compact quote is
|
||||||
|
// more useful than a parse attempt that fails on most of them.
|
||||||
|
snippet = strings.Join(strings.Fields(snippet), " ")
|
||||||
|
if len(snippet) > 200 {
|
||||||
|
snippet = snippet[:200] + "…"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("upstream rejected the api key (HTTP %d): %s — "+
|
||||||
|
"check the key, not the endpoint", status, snippet)
|
||||||
|
}
|
||||||
|
|
||||||
|
// classifyTransport turns low-level network failures into something an
|
||||||
|
// operator can act on. An empty result means "no specific advice", and the
|
||||||
|
// caller keeps the raw error.
|
||||||
|
func classifyTransport(err error) string {
|
||||||
|
if err == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
lower := strings.ToLower(msg)
|
||||||
|
switch {
|
||||||
|
case strings.Contains(lower, "no such host"):
|
||||||
|
return "hostname does not resolve: " + msg
|
||||||
|
case strings.Contains(lower, "certificate"), strings.Contains(lower, "tls"):
|
||||||
|
return "TLS verification failed: " + msg
|
||||||
|
case strings.Contains(lower, "connection refused"):
|
||||||
|
return "connection refused: " + msg
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildDiscoveryError assembles the final error, preferring the most specific
|
||||||
|
// diagnosis available. Every URL tried is listed: an operator staring at a
|
||||||
|
// failed probe needs to know the search actually happened, not just the last
|
||||||
|
// attempt.
|
||||||
|
func buildDiscoveryError(tried []string, authErr, firstErr, lastErr string) error {
|
||||||
|
triedPart := "no model catalog found"
|
||||||
|
if len(tried) > 0 {
|
||||||
|
triedPart = "no model catalog found; tried " + strings.Join(tried, ", ")
|
||||||
|
}
|
||||||
|
if authErr != "" {
|
||||||
|
return &discoveryFailure{status: http.StatusUnauthorized, msg: authErr + " (" + triedPart + ")"}
|
||||||
|
}
|
||||||
|
detail := firstErr
|
||||||
|
if detail == "" {
|
||||||
|
detail = lastErr
|
||||||
|
}
|
||||||
|
if detail == "" {
|
||||||
|
return errors.New(triedPart)
|
||||||
|
}
|
||||||
|
if lastErr != "" && lastErr != firstErr {
|
||||||
|
detail += " (last: " + lastErr + ")"
|
||||||
|
}
|
||||||
|
return &discoveryFailure{status: http.StatusBadGateway, msg: triedPart + "; " + detail}
|
||||||
|
}
|
||||||
|
|
||||||
|
// upstreamModel is the subset of a /models entry this code reads. Vendors add
|
||||||
|
// fields freely and rename the nesting; only id is universal, and the modality
|
||||||
|
// hints below are read opportunistically.
|
||||||
|
type modelCaps struct {
|
||||||
|
ImageGeneration bool `json:"image_generation"`
|
||||||
|
Vision bool `json:"vision"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type upstreamModel struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Object string `json:"object"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Modality string `json:"modality"`
|
||||||
|
Architecture struct {
|
||||||
|
InputModalities []string `json:"input_modalities"`
|
||||||
|
OutputModalities []string `json:"output_modalities"`
|
||||||
|
} `json:"architecture"`
|
||||||
|
Capabilities modelCaps `json:"capabilities"`
|
||||||
|
|
||||||
|
// Metadata is kept as raw JSON on purpose. The real amr-link catalog nests
|
||||||
|
// cost/capability/context_limit here, and a typed struct would silently
|
||||||
|
// discard every key this build does not know about — the operator would
|
||||||
|
// then lose the context limit their vendor published. Raw also lets
|
||||||
|
// input_modalities/output_modalities be read without a second type.
|
||||||
|
Metadata json.RawMessage `json:"metadata"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// metadataFields lists the keys worth surfacing in the UI. They are copied
|
||||||
|
// through verbatim and nothing is inferred from them; an upstream that sends
|
||||||
|
// no context limit must leave that absent rather than have this code write 0.
|
||||||
|
var metadataFields = []string{
|
||||||
|
"context_length", "context_window", "max_context_length",
|
||||||
|
"context_limit", "context_budget", "cost", "capability",
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseModelCatalog reads the OpenAI list shape {"data":[...]} and the bare
|
||||||
|
// array shape some gateways return. A response that is valid JSON but has
|
||||||
|
// neither shape is an error, not an empty catalog: reporting "0 models" for a
|
||||||
|
// login page or a rate-limit JSON would silently create an empty source.
|
||||||
|
func parseModelCatalog(body []byte) ([]discoveredModel, error) {
|
||||||
|
// "null" unmarshals into a nil slice without error, and an HTML login page
|
||||||
|
// fails to unmarshal at all. Both must be errors.
|
||||||
|
trimmed := strings.TrimSpace(string(body))
|
||||||
|
if trimmed == "" || trimmed == "null" {
|
||||||
|
return nil, fmt.Errorf("empty response")
|
||||||
|
}
|
||||||
|
|
||||||
|
var wrapper struct {
|
||||||
|
Data []upstreamModel `json:"data"`
|
||||||
|
Models []upstreamModel `json:"models"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &wrapper); err == nil {
|
||||||
|
if len(wrapper.Data) > 0 {
|
||||||
|
return toDiscovered(wrapper.Data), nil
|
||||||
|
}
|
||||||
|
if len(wrapper.Models) > 0 {
|
||||||
|
return toDiscovered(wrapper.Models), nil
|
||||||
|
}
|
||||||
|
// A well-formed but empty catalog is a legitimate answer from a key
|
||||||
|
// that has no models provisioned. Distinguish it from "not a catalog"
|
||||||
|
// by checking the shape actually parsed.
|
||||||
|
var probe map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(body, &probe) == nil {
|
||||||
|
if _, ok := probe["data"]; ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if _, ok := probe["models"]; ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var bare []upstreamModel
|
||||||
|
if err := json.Unmarshal(body, &bare); err == nil {
|
||||||
|
return toDiscovered(bare), nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("response is not a model catalog")
|
||||||
|
}
|
||||||
|
|
||||||
|
func toDiscovered(in []upstreamModel) []discoveredModel {
|
||||||
|
out := make([]discoveredModel, 0, len(in))
|
||||||
|
for _, m := range in {
|
||||||
|
out = append(out, discoveredModel{
|
||||||
|
ID: m.ID,
|
||||||
|
Kind: discoveredModelKind(m),
|
||||||
|
Meta: modelMeta(m),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveredModelKind trusts the upstream's own modality fields and nothing
|
||||||
|
// else. It never infers from the model NAME: ids like "gpt-image-1" or
|
||||||
|
// "ideogram-v2" appear on catalogs that also serve chat, and a wrong kind
|
||||||
|
// poisons the AUTO chain (an image-only id selected for a chat request fails
|
||||||
|
// every tier behind it).
|
||||||
|
func discoveredModelKind(m upstreamModel) string {
|
||||||
|
modIn := append(append([]string{}, m.Architecture.InputModalities...),
|
||||||
|
modalityList(m.Metadata, "input_modalities")...)
|
||||||
|
modOut := append(append([]string{}, m.Architecture.OutputModalities...),
|
||||||
|
modalityList(m.Metadata, "output_modalities")...)
|
||||||
|
|
||||||
|
has := func(list []string, want string) bool {
|
||||||
|
for _, v := range list {
|
||||||
|
if strings.EqualFold(strings.TrimSpace(v), want) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if m.Capabilities.ImageGeneration || has(modOut, "image") {
|
||||||
|
return "image"
|
||||||
|
}
|
||||||
|
if has(modIn, "image") || has(modIn, "file") || has(modIn, "audio") || m.Capabilities.Vision {
|
||||||
|
return "chat" // multimodal input still means a chat model
|
||||||
|
}
|
||||||
|
// Some catalogs encode modality as a top-level string instead of a list.
|
||||||
|
// Only the image-ish spellings count: "chat" here means the upstream said
|
||||||
|
// chat, and treating that as "unknown" would ignore its statement.
|
||||||
|
switch strings.ToLower(strings.TrimSpace(m.Kind + " " + m.Modality + " " + m.Type)) {
|
||||||
|
case "image", "text-to-image", "image_generation", "text_to_image":
|
||||||
|
return "image"
|
||||||
|
}
|
||||||
|
return "chat"
|
||||||
|
}
|
||||||
|
|
||||||
|
// modalityList reads one array-of-strings field out of a raw JSON object,
|
||||||
|
// tolerating both absence and a non-array value.
|
||||||
|
func modalityList(raw json.RawMessage, field string) []string {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &obj); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v, ok := obj[field]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
if err := json.Unmarshal(v, &out); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// modelMeta carries through the fields an operator wants to see in the UI
|
||||||
|
// without this code inventing them. The upstream nests most of them under
|
||||||
|
// "metadata" (the real amr-link catalog does) while some put the same keys at
|
||||||
|
// the top level, so both are read and top level wins on a tie. Only keys the
|
||||||
|
// upstream actually sent are copied, so an absent context limit stays absent
|
||||||
|
// instead of becoming 0.
|
||||||
|
func modelMeta(m upstreamModel) map[string]interface{} {
|
||||||
|
meta := map[string]interface{}{}
|
||||||
|
|
||||||
|
copyFields := func(raw json.RawMessage) {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &obj); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, k := range metadataFields {
|
||||||
|
if v, ok := obj[k]; ok {
|
||||||
|
if dec := rawOrString(v); dec != nil {
|
||||||
|
meta[k] = dec
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
copyFields(m.Metadata) // nested first
|
||||||
|
copyFields(mustJSON(m)) // top level wins
|
||||||
|
if len(meta) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return meta
|
||||||
|
}
|
||||||
|
|
||||||
|
// rawOrString keeps numeric metadata as a number and everything else as its
|
||||||
|
// decoded string, so the UI can sort on a cost or context length instead of
|
||||||
|
// receiving "1048000" and having to coerce it.
|
||||||
|
func rawOrString(raw json.RawMessage) interface{} {
|
||||||
|
var num json.Number
|
||||||
|
if err := json.Unmarshal(raw, &num); err == nil {
|
||||||
|
if f, err := num.Float64(); err == nil {
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var s string
|
||||||
|
if err := json.Unmarshal(raw, &s); err == nil {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
var any interface{}
|
||||||
|
if err := json.Unmarshal(raw, &any); err == nil {
|
||||||
|
return any
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustJSON(v interface{}) []byte {
|
||||||
|
b, _ := json.Marshal(v)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// modelsToConfig converts discovery output into config.Model entries, applying
|
||||||
|
// the same priority convention the UI uses: chat models the operator is
|
||||||
|
// likely to want first get a lower number, and image models sort last because
|
||||||
|
// they are unreachable from /v1/chat/completions.
|
||||||
|
func modelsToConfig(chat, image []string) []config.Model {
|
||||||
|
out := make([]config.Model, 0, len(chat)+len(image))
|
||||||
|
for _, id := range chat {
|
||||||
|
out = append(out, config.Model{ID: id, Kind: "chat"})
|
||||||
|
}
|
||||||
|
for _, id := range image {
|
||||||
|
out = append(out, config.Model{ID: id, Kind: "image"})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
497
internal/gateway/discover_api.go
Normal file
497
internal/gateway/discover_api.go
Normal file
@ -0,0 +1,497 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HTTP surface for model discovery.
|
||||||
|
//
|
||||||
|
// Two endpoints, deliberately separate:
|
||||||
|
//
|
||||||
|
// POST /api/sources/discover — probe a URL+key, return the catalog.
|
||||||
|
// Nothing is written.
|
||||||
|
// POST /api/sources/discover/apply — probe, then write the source.
|
||||||
|
//
|
||||||
|
// The split exists because discovery mutates nothing and is therefore safe to
|
||||||
|
// call repeatedly while the operator is still typing a key, whereas apply
|
||||||
|
// rewrites credentials and the AUTO chain's inputs. Making them one endpoint
|
||||||
|
// with a flag would mean a "just check it" click can silently create a source.
|
||||||
|
//
|
||||||
|
// Both are admin-only, matching /api/sources: the request body carries an API
|
||||||
|
// key, and a model catalog reveals which upstreams an operator uses.
|
||||||
|
|
||||||
|
// discoverRequest is the body of both endpoints.
|
||||||
|
type discoverRequest struct {
|
||||||
|
// BaseURL is what the operator pasted. It may or may not include a
|
||||||
|
// version segment; discovery works that out rather than making the
|
||||||
|
// caller know this gateway's path-composition rules.
|
||||||
|
BaseURL string `json:"base_url"`
|
||||||
|
APIKey string `json:"api_key"`
|
||||||
|
|
||||||
|
// Name is required for apply, ignored for discover. It defaults from the
|
||||||
|
// base URL's host when omitted, because "https://api.vendor.com/v1" is a
|
||||||
|
// better starting name than "new-source".
|
||||||
|
Name string `json:"name"`
|
||||||
|
|
||||||
|
// Adapter is optional. Discovery assumes "openai" because that is what a
|
||||||
|
// /v1/models catalog implies; a non-OpenAI upstream (Anthropic, Gemini)
|
||||||
|
// has no such endpoint, so discovery will legitimately fail for those and
|
||||||
|
// the operator configures them by hand.
|
||||||
|
Adapter string `json:"adapter"`
|
||||||
|
|
||||||
|
// Temperature and MaxConcurrent are carried into the written source.
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
MaxConcurrent int `json:"max_concurrent"`
|
||||||
|
|
||||||
|
// ReplaceModels drops the source's existing model list before applying the
|
||||||
|
// discovered one. It defaults to true for a NEW source and is required to
|
||||||
|
// be spelled out for an existing one, so a re-discovery cannot silently
|
||||||
|
// delete hand-tuned priorities.
|
||||||
|
ReplaceModels *bool `json:"replace_models,omitempty"`
|
||||||
|
|
||||||
|
// Select narrows what gets written. Empty means everything discovered.
|
||||||
|
// Ids that are not in the catalog are an error rather than a silent skip:
|
||||||
|
// a typo here would produce a source that 404s on every request.
|
||||||
|
Include []string `json:"include,omitempty"`
|
||||||
|
Exclude []string `json:"exclude,omitempty"`
|
||||||
|
|
||||||
|
// ImageModels names ids that are **kind=image** but whose upstream catalog
|
||||||
|
// does not say so.
|
||||||
|
//
|
||||||
|
// 为什么需要(真实缺口):discoveredModelKind 只看上游的 modality 字段,
|
||||||
|
// 并且刻意**不从模型名推断**(那会把 "gpt-image-1" 这类同名聊天模型
|
||||||
|
// 错归成生图,而错误的 kind 会毒死 AUTO 链)。
|
||||||
|
// 但有些网关的 `/v1/models` 只返回 `{id,object,created,owned_by}` ——
|
||||||
|
// 零 modality(实测:某聚合网关的 3 个 gpt-image-* 全部落成 chat,
|
||||||
|
// 于是生图端点永远报 `no image source configured`)。
|
||||||
|
//
|
||||||
|
// 两条原则都不破:不猜(默认仍完全按元数据),但允许**显式声明**。
|
||||||
|
// 只影响写入的 kind,不影响目录里有没有这个模型。
|
||||||
|
ImageModels []string `json:"image_models,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoverResponse is the JSON the WebUI binds to.
|
||||||
|
type discoverResponse struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
|
||||||
|
// Normalized source fields, ready to be echoed back to /api/sources.
|
||||||
|
BaseURL string `json:"base_url"`
|
||||||
|
Adapter string `json:"adapter"`
|
||||||
|
Models []string `json:"models"`
|
||||||
|
ImageModels []string `json:"image_models"`
|
||||||
|
|
||||||
|
// Endpoint is set only when discovery had to move a version segment.
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
// ImageEndpoint is required whenever base_url carries a version segment:
|
||||||
|
// the gateway's default image path is the ABSOLUTE "/v1/images/generations"
|
||||||
|
// while chat's default is the relative "/chat/completions", so the same
|
||||||
|
// base_url that yields a working chat URL yields a 404 image URL.
|
||||||
|
ImageEndpoint string `json:"image_endpoint"`
|
||||||
|
|
||||||
|
// Diagnostics for the operator.
|
||||||
|
ModelsPath string `json:"models_path"`
|
||||||
|
Tried []string `json:"tried"`
|
||||||
|
LatencyMS int64 `json:"latency_ms"`
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
|
Warnings []string `json:"warnings,omitempty"`
|
||||||
|
|
||||||
|
// Set by apply only.
|
||||||
|
SourceName string `json:"source_name,omitempty"`
|
||||||
|
Applied bool `json:"applied,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Gateway) handleSourceDiscoverAPI(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if reqRole(r.Context()) != "admin" {
|
||||||
|
writeError(w, http.StatusForbidden, "forbidden", "admin role required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
apply := strings.HasSuffix(r.URL.Path, "/apply")
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
w.Header().Set("Allow", "POST")
|
||||||
|
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed",
|
||||||
|
"use POST with {\"base_url\":…,\"api_key\":…}")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", "cannot read body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req discoverRequest
|
||||||
|
if err := json.Unmarshal(body, &req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", "invalid json: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.BaseURL = strings.TrimSpace(req.BaseURL)
|
||||||
|
req.Name = strings.TrimSpace(req.Name)
|
||||||
|
|
||||||
|
// ★ 只给 `name`(或两者都给但 key 为空)⇒ **从已存源取 base_url 与凭据**。
|
||||||
|
//
|
||||||
|
// 为什么必须有这条路径(真实痛点):刷新一个**已存在**源的模型目录,此前
|
||||||
|
// 必须让运维把明文 key 再粘一次 —— 而库里的 key 是 `enc:v1:` 密文,
|
||||||
|
// 运维手里未必还有原文;WebUI 也不回显明文(这是正确的安全设计)。
|
||||||
|
// 结果就是「想刷新目录 → 必须重新申请/粘贴一次 key」,很多人干脆不刷新,
|
||||||
|
// 于是源的模型目录越放越旧(实例:某源的 3 个生图模型从未被登记)。
|
||||||
|
//
|
||||||
|
// 安全边界:① 已要求 admin 角色;② 只能探测**本网关已配置的**源,
|
||||||
|
// 不能借此拿任意 URL + 任意 key 做探测(那样就成了 SSRF/密钥外带渠道);
|
||||||
|
// ③ 响应不回显 key(只回目录)。
|
||||||
|
if req.Name != "" {
|
||||||
|
if old := g.sourceByName(req.Name); old != nil {
|
||||||
|
if req.BaseURL == "" {
|
||||||
|
req.BaseURL = old.BaseURL
|
||||||
|
}
|
||||||
|
if req.APIKey == "" {
|
||||||
|
req.APIKey = old.APIKey
|
||||||
|
}
|
||||||
|
if req.Adapter == "" {
|
||||||
|
req.Adapter = old.Adapter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.BaseURL == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request",
|
||||||
|
"base_url required (or pass name of an existing source to reuse its base_url and credential)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.Adapter == "" {
|
||||||
|
req.Adapter = "openai"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bounded so a hung upstream cannot pin the handler; discovery has its own
|
||||||
|
// per-request timeout on top.
|
||||||
|
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
res, err := DiscoverModels(ctx, req.BaseURL, req.APIKey)
|
||||||
|
if err != nil {
|
||||||
|
// A rejected key must not be reported as an upstream outage: the two
|
||||||
|
// need opposite actions from the operator, and 502 sends them to the
|
||||||
|
// wrong one.
|
||||||
|
status := http.StatusBadGateway
|
||||||
|
var f *discoveryFailure
|
||||||
|
if errors.As(err, &f) {
|
||||||
|
status = f.StatusOrDefault()
|
||||||
|
}
|
||||||
|
writeError(w, status, "discover_failed", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// ★ 显式生图声明:把 operator 点名的 id 从 chat 挪到 image。
|
||||||
|
//
|
||||||
|
// 放在 selectModels **之前**:后者按 include/exclude 裁剪两个列表,
|
||||||
|
// 挪类必须在裁剪前完成,否则一个"只声明成生图、又被 include 选中"的
|
||||||
|
// 模型会先在 chat 里被保留、再被这里挪走 —— 顺序反了会得到两份。
|
||||||
|
//
|
||||||
|
// 只认**目录里确实存在的** id:写错一个字不该静默造出一个不存在的模型
|
||||||
|
// (那会让源在每次照片请求时 404),而是明确报错。
|
||||||
|
if len(req.ImageModels) > 0 {
|
||||||
|
have := map[string]bool{}
|
||||||
|
for _, m := range res.Models {
|
||||||
|
have[m] = true
|
||||||
|
}
|
||||||
|
for _, m := range res.ImageModels {
|
||||||
|
have[m] = true
|
||||||
|
}
|
||||||
|
var missing []string
|
||||||
|
force := map[string]bool{}
|
||||||
|
for _, id := range req.ImageModels {
|
||||||
|
if !have[id] {
|
||||||
|
missing = append(missing, id)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
force[id] = true
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request",
|
||||||
|
"image_models contains ids that are not in the upstream catalog: "+
|
||||||
|
strings.Join(missing, ", "))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var keepChat []string
|
||||||
|
for _, m := range res.Models {
|
||||||
|
if !force[m] {
|
||||||
|
keepChat = append(keepChat, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seenImg := map[string]bool{}
|
||||||
|
var newImage []string
|
||||||
|
for _, m := range res.ImageModels {
|
||||||
|
seenImg[m] = true
|
||||||
|
newImage = append(newImage, m)
|
||||||
|
}
|
||||||
|
for id := range force {
|
||||||
|
if !seenImg[id] {
|
||||||
|
newImage = append(newImage, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
res.Models = keepChat
|
||||||
|
res.ImageModels = newImage
|
||||||
|
}
|
||||||
|
|
||||||
|
chat, image, warns, err := selectModels(res.Models, res.ImageModels, req.Include, req.Exclude)
|
||||||
|
if err != nil {
|
||||||
|
// 400: the operator asked for a model the catalog does not have.
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
out := &discoverResponse{
|
||||||
|
OK: true,
|
||||||
|
BaseURL: res.BaseURL,
|
||||||
|
Adapter: req.Adapter,
|
||||||
|
Models: chat,
|
||||||
|
ImageModels: image,
|
||||||
|
Endpoint: res.Endpoint,
|
||||||
|
ImageEndpoint: res.ImageEndpoint,
|
||||||
|
ModelsPath: res.ModelsPath,
|
||||||
|
Tried: res.Tried,
|
||||||
|
LatencyMS: res.LatencyMS,
|
||||||
|
Note: res.Note,
|
||||||
|
Warnings: warns,
|
||||||
|
}
|
||||||
|
if len(chat) == 0 && len(image) == 0 {
|
||||||
|
out.Warnings = append(out.Warnings,
|
||||||
|
"every discovered model was excluded; applying this would create a source with no models")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !apply {
|
||||||
|
writeJSON(w, http.StatusOK, out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
name := strings.TrimSpace(req.Name)
|
||||||
|
if name == "" {
|
||||||
|
name = defaultSourceName(req.BaseURL)
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request",
|
||||||
|
"name required: cannot derive one from base_url")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if strings.ContainsAny(name, "/\\ \t") || name == "." || name == ".." {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request",
|
||||||
|
"source name must not contain slashes or whitespace")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
exists := g.sourceExists(name)
|
||||||
|
if exists && req.ReplaceModels == nil {
|
||||||
|
// Refusing here is deliberate. Re-running discovery on an existing
|
||||||
|
// source and having it silently replace a hand-tuned model list —
|
||||||
|
// including priorities chosen for the AUTO chain — is the kind of
|
||||||
|
// "helpful" behaviour that loses an afternoon.
|
||||||
|
writeError(w, http.StatusConflict, "conflict",
|
||||||
|
"source "+name+" already exists; set replace_models=true to overwrite its model list, "+
|
||||||
|
"or choose a different name")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the source directly rather than round-tripping through the
|
||||||
|
// /api/sources payload: resolveAPIKey carries the "an empty api_key means
|
||||||
|
// keep the stored one" rule, and going through a second decoder here would
|
||||||
|
// duplicate (and eventually diverge from) that logic.
|
||||||
|
src := config.Source{
|
||||||
|
Name: name,
|
||||||
|
BaseURL: res.BaseURL,
|
||||||
|
APIKey: g.resolveAPIKey(name, req.APIKey, exists),
|
||||||
|
Adapter: req.Adapter,
|
||||||
|
Endpoint: res.Endpoint,
|
||||||
|
ImageEndpoint: res.ImageEndpoint,
|
||||||
|
Models: modelsToConfig(chat, image),
|
||||||
|
Temperature: req.Temperature,
|
||||||
|
}
|
||||||
|
if req.MaxConcurrent > 0 {
|
||||||
|
src.MaxConcurrent = req.MaxConcurrent
|
||||||
|
}
|
||||||
|
|
||||||
|
// An existing source may carry fields discovery does not know about
|
||||||
|
// (proxy_url, headers, api_key_env, timeout). Rebuilding from scratch would
|
||||||
|
// drop them, so they are inherited unless the request overrides them.
|
||||||
|
if exists {
|
||||||
|
old := g.sourceByName(name)
|
||||||
|
if old != nil {
|
||||||
|
src.ProxyURL = old.ProxyURL
|
||||||
|
src.APIKeyEnv = old.APIKeyEnv
|
||||||
|
src.Timeout = old.Timeout
|
||||||
|
src.QueueTimeout = old.QueueTimeout
|
||||||
|
src.Headers = old.Headers
|
||||||
|
src.Meta = old.Meta
|
||||||
|
src.MaxTokens = old.MaxTokens
|
||||||
|
src.RPM = old.RPM
|
||||||
|
// image_endpoint is NOT inherited when discovery supplied one:
|
||||||
|
// the stored value was correct for the OLD base_url, and the base
|
||||||
|
// is exactly what may have changed. Inheriting it would keep a
|
||||||
|
// stale value that 404s against the new base.
|
||||||
|
if res.ImageEndpoint == "" {
|
||||||
|
src.ImageEndpoint = old.ImageEndpoint
|
||||||
|
}
|
||||||
|
if src.Adapter == "" {
|
||||||
|
src.Adapter = old.Adapter
|
||||||
|
}
|
||||||
|
if req.MaxConcurrent <= 0 {
|
||||||
|
src.MaxConcurrent = old.MaxConcurrent
|
||||||
|
}
|
||||||
|
// replace_models=false keeps the curated list. Discovery still
|
||||||
|
// refreshed base_url/endpoint/credential, which is the point of
|
||||||
|
// re-running it.
|
||||||
|
if req.ReplaceModels != nil && !*req.ReplaceModels {
|
||||||
|
src.Models = old.Models
|
||||||
|
out.Warnings = append(out.Warnings,
|
||||||
|
"model list left unchanged (replace_models=false)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := g.core.AddSource(src); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "source_error", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
out.SourceName = name
|
||||||
|
out.Applied = true
|
||||||
|
writeJSON(w, http.StatusOK, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// selectModels applies the include/exclude filters.
|
||||||
|
//
|
||||||
|
// An include entry the catalog does not have is an error: silently dropping it
|
||||||
|
// produces a source that looks configured and 404s at request time, which is
|
||||||
|
// far harder to diagnose than a rejected apply. An exclude entry that is not
|
||||||
|
// present is fine — the operator is allowed to be defensive.
|
||||||
|
func selectModels(chat, image, include, exclude []string) ([]string, []string, []string, error) {
|
||||||
|
have := map[string]bool{}
|
||||||
|
for _, m := range chat {
|
||||||
|
have[m] = true
|
||||||
|
}
|
||||||
|
for _, m := range image {
|
||||||
|
have[m] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(include) > 0 {
|
||||||
|
var missing []string
|
||||||
|
for _, want := range include {
|
||||||
|
if !have[want] {
|
||||||
|
missing = append(missing, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
return nil, nil, nil, &discoverError{
|
||||||
|
msg: "these models are not in the upstream catalog: " + strings.Join(missing, ", "),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keep := map[string]bool{}
|
||||||
|
for _, want := range include {
|
||||||
|
keep[want] = true
|
||||||
|
}
|
||||||
|
var c, i []string
|
||||||
|
for _, m := range chat {
|
||||||
|
if keep[m] {
|
||||||
|
c = append(c, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range image {
|
||||||
|
if keep[m] {
|
||||||
|
i = append(i, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chat, image = c, i
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(exclude) > 0 {
|
||||||
|
drop := map[string]bool{}
|
||||||
|
for _, m := range exclude {
|
||||||
|
drop[m] = true
|
||||||
|
}
|
||||||
|
var c, i []string
|
||||||
|
for _, m := range chat {
|
||||||
|
if !drop[m] {
|
||||||
|
c = append(c, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range image {
|
||||||
|
if !drop[m] {
|
||||||
|
i = append(i, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chat, image = c, i
|
||||||
|
}
|
||||||
|
|
||||||
|
// chat and image must stay non-nil: the JSON contract forbids null here.
|
||||||
|
if chat == nil {
|
||||||
|
chat = []string{}
|
||||||
|
}
|
||||||
|
if image == nil {
|
||||||
|
image = []string{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var warns []string
|
||||||
|
if len(chat) == 0 && len(image) == 0 {
|
||||||
|
// Caller can only get here by excluding everything, or from a catalog
|
||||||
|
// that was already empty. Either way "silence" would read as success.
|
||||||
|
warns = append(warns,
|
||||||
|
"no models selected; applying this would create a source that cannot serve any request")
|
||||||
|
return chat, image, warns, nil
|
||||||
|
}
|
||||||
|
if len(image) > 0 && len(chat) == 0 {
|
||||||
|
warns = append(warns,
|
||||||
|
"every model is an image model; AUTO chat requests have nothing to select")
|
||||||
|
}
|
||||||
|
if len(chat) > 0 && len(image) > 0 {
|
||||||
|
warns = append(warns,
|
||||||
|
"image models are only reachable through /v1/images/generations, not AUTO chat")
|
||||||
|
}
|
||||||
|
return chat, image, warns, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoverError is a client mistake rather than an upstream failure, which is
|
||||||
|
// why it gets its own type instead of fmt.Errorf.
|
||||||
|
type discoverError struct{ msg string }
|
||||||
|
|
||||||
|
func (e *discoverError) Error() string { return e.msg }
|
||||||
|
|
||||||
|
// defaultSourceName derives a source name from a base URL host, stripping the
|
||||||
|
// version segment and any "api." prefix. It returns "" for input that is not a
|
||||||
|
// URL at all — the caller must not fall back to echoing garbage into a source
|
||||||
|
// name, since the name becomes a config key and an API path segment.
|
||||||
|
func defaultSourceName(baseURL string) string {
|
||||||
|
s := strings.TrimSpace(baseURL)
|
||||||
|
if !strings.Contains(s, "://") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
s = s[strings.Index(s, "://")+3:]
|
||||||
|
if i := strings.IndexAny(s, "/?#"); i >= 0 {
|
||||||
|
s = s[:i]
|
||||||
|
}
|
||||||
|
if i := strings.LastIndex(s, ":"); i > 0 && !strings.Contains(s[i+1:], "]") {
|
||||||
|
s = s[:i] // strip port
|
||||||
|
}
|
||||||
|
// A host must still look like one after trimming the cosmetic prefixes.
|
||||||
|
s = strings.TrimPrefix(s, "api.")
|
||||||
|
s = strings.TrimPrefix(s, "www.")
|
||||||
|
if s == "" || strings.ContainsAny(s, " \t\"'<>") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, r := range s {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||||
|
case r == '.' || r == '-' || r == '_':
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(s, ".") {
|
||||||
|
// A bare label is far more likely to be a mistake than a hostname.
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
314
internal/gateway/discover_api_test.go
Normal file
314
internal/gateway/discover_api_test.go
Normal file
@ -0,0 +1,314 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// upstream that answers /v1/models with the real captured catalog
|
||||||
|
func catalogUpstream(t *testing.T) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/models" {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if got := r.Header.Get("Authorization"); got != "Bearer sk-secret" {
|
||||||
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(realCatalogBody))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultSourceName(t *testing.T) {
|
||||||
|
cases := map[string]string{
|
||||||
|
"https://api.vendor.com/v1": "vendor.com",
|
||||||
|
"https://api.openai.com/v1": "openai.com",
|
||||||
|
"https://www.vendor.com/v1": "vendor.com",
|
||||||
|
"http://127.0.0.1:8080/v1": "127.0.0.1",
|
||||||
|
"https://amr-link.open-design.ai/v1": "amr-link.open-design.ai",
|
||||||
|
"not a url at all": "",
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := defaultSourceName(in); got != want {
|
||||||
|
t.Errorf("defaultSourceName(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectModels_IncludeUnknownIsAnError(t *testing.T) {
|
||||||
|
chat := []string{"a", "b"}
|
||||||
|
image := []string{"img"}
|
||||||
|
// Silently dropping a typo'd id would create a source that looks
|
||||||
|
// configured and 404s on every request — much worse than a rejection.
|
||||||
|
_, _, _, err := selectModels(chat, image, []string{"a", "typo"}, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected an error for an include id that is not in the catalog")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "typo") {
|
||||||
|
t.Errorf("error should name the offending id, got %q", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectModels_IncludeNarrows(t *testing.T) {
|
||||||
|
chat, image, _, err := selectModels([]string{"a", "b", "c"}, []string{"img1", "img2"}, []string{"b", "img1"}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(chat) != 1 || chat[0] != "b" {
|
||||||
|
t.Errorf("chat = %v, want [b]", chat)
|
||||||
|
}
|
||||||
|
if len(image) != 1 || image[0] != "img1" {
|
||||||
|
t.Errorf("image = %v, want [img1]", image)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectModels_ExcludeUnknownIsFine(t *testing.T) {
|
||||||
|
chat, _, _, err := selectModels([]string{"a"}, nil, nil, []string{"never-existed"})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("excluding a model that is not present should be allowed: %v", err)
|
||||||
|
}
|
||||||
|
if len(chat) != 1 {
|
||||||
|
t.Errorf("chat = %v, want [a] unchanged", chat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectModels_NeverReturnsNil(t *testing.T) {
|
||||||
|
chat, image, warns, err := selectModels(nil, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if chat == nil || image == nil {
|
||||||
|
t.Fatal("nil slices would serialize as JSON null and break the UI")
|
||||||
|
}
|
||||||
|
if len(warns) == 0 {
|
||||||
|
t.Error("an empty catalog deserves a warning, not silence")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectModels_WarnsAboutImageOnlyCatalog(t *testing.T) {
|
||||||
|
_, _, warns, err := selectModels(nil, []string{"gpt-image-2"}, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(warns, " ")
|
||||||
|
if !strings.Contains(joined, "AUTO") {
|
||||||
|
t.Errorf("warnings %v should say AUTO has nothing to select", warns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectModels_WarnsThatImagesAreNotInAUTO(t *testing.T) {
|
||||||
|
_, _, warns, err := selectModels([]string{"a"}, []string{"img"}, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(warns, " "), "images/generations") {
|
||||||
|
t.Errorf("warnings %v should point at the image endpoint", warns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoveryResultAndConfigRoundTrip checks the fields the WebUI will echo
|
||||||
|
// back actually survive into a config.Source — a mismatch here means the
|
||||||
|
// operator previews one set of values and saves another.
|
||||||
|
func TestDiscoveryResultAndConfigRoundTrip(t *testing.T) {
|
||||||
|
res := &discoveryResult{
|
||||||
|
BaseURL: "https://h",
|
||||||
|
Endpoint: "/v1/chat/completions",
|
||||||
|
Models: []string{"m1", "m2"},
|
||||||
|
ImageModels: []string{"img1"},
|
||||||
|
}
|
||||||
|
src := config.Source{
|
||||||
|
Name: "x",
|
||||||
|
BaseURL: res.BaseURL,
|
||||||
|
Endpoint: res.Endpoint,
|
||||||
|
Models: modelsToConfig(res.Models, res.ImageModels),
|
||||||
|
}
|
||||||
|
if len(src.Models) != 3 {
|
||||||
|
t.Fatalf("models = %d, want 3", len(src.Models))
|
||||||
|
}
|
||||||
|
if src.Endpoint != "/v1/chat/completions" {
|
||||||
|
t.Errorf("endpoint lost: %q", src.Endpoint)
|
||||||
|
}
|
||||||
|
var img int
|
||||||
|
for _, m := range src.Models {
|
||||||
|
if m.Kind == "image" {
|
||||||
|
img++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if img != 1 {
|
||||||
|
t.Errorf("image models = %d, want 1", img)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_SendsAuthorizationHeader pins the only way the feature
|
||||||
|
// works against a real vendor: without the bearer token every paid /models
|
||||||
|
// endpoint returns 401.
|
||||||
|
func TestDiscoverModels_SendsAuthorizationHeader(t *testing.T) {
|
||||||
|
var seen string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
seen = r.Header.Get("Authorization")
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
if _, err := DiscoverModels(context.Background(), srv.URL+"/v1", "sk-secret"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if seen != "Bearer sk-secret" {
|
||||||
|
t.Errorf("Authorization = %q, want %q", seen, "Bearer sk-secret")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_WrongKeyIsUnauthorizedNotBadGateway pins the distinction
|
||||||
|
// that decides whether the operator knows what to do next.
|
||||||
|
//
|
||||||
|
// A rejected key and an unreachable vendor are the two most common discovery
|
||||||
|
// failures, and they need opposite actions: re-type the key, versus go check the
|
||||||
|
// vendor's status page. Both used to surface as one generic error, so the
|
||||||
|
// operator had to guess.
|
||||||
|
func TestDiscoverModels_WrongKeyIsUnauthorizedNotBadGateway(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
_, _ = w.Write([]byte(`{"error":{"message":"invalid api key","code":"invalid_api_key"}}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
_, err := DiscoverModels(context.Background(), srv.URL+"/v1", "sk-wrong")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a failure against a 401 endpoint")
|
||||||
|
}
|
||||||
|
var f *discoveryFailure
|
||||||
|
if !errors.As(err, &f) {
|
||||||
|
t.Fatalf("error should be classified, got %T: %v", err, err)
|
||||||
|
}
|
||||||
|
if got := f.StatusOrDefault(); got != http.StatusUnauthorized {
|
||||||
|
t.Errorf("status = %d, want %d — a wrong key must not read as an upstream outage",
|
||||||
|
got, http.StatusUnauthorized)
|
||||||
|
}
|
||||||
|
// The message has to name which credential is at fault, or the status code
|
||||||
|
// alone still sends the operator to the wrong place.
|
||||||
|
if !strings.Contains(strings.ToLower(f.Error()), "api key") {
|
||||||
|
t.Errorf("message %q should name the api key as the cause", f.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(f.Error(), "/v1/models") {
|
||||||
|
t.Errorf("message %q should list what was tried", f.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_UnreachableStaysBadGateway: the classification must not
|
||||||
|
// over-correct — an unresolvable host is still the vendor's or the network's
|
||||||
|
// problem, not the key's.
|
||||||
|
func TestDiscoverModels_UnreachableStaysBadGateway(t *testing.T) {
|
||||||
|
_, err := DiscoverModels(context.Background(), "https://no-such-host-xyz.invalid/v1", "k")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a failure for an unresolvable host")
|
||||||
|
}
|
||||||
|
var f *discoveryFailure
|
||||||
|
if !errors.As(err, &f) {
|
||||||
|
t.Fatalf("error should be classified, got %T", err)
|
||||||
|
}
|
||||||
|
if got := f.StatusOrDefault(); got != http.StatusBadGateway {
|
||||||
|
t.Errorf("status = %d, want %d", got, http.StatusBadGateway)
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.ToLower(f.Error()), "resolve") {
|
||||||
|
t.Errorf("message %q should say the hostname does not resolve", f.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoveryCandidates_NoDuplicateRequests: base "https://h/v1" with path
|
||||||
|
// "/models" and base "https://h" with path "/v1/models" request the SAME URL.
|
||||||
|
// Listing it twice in a failure message reads as "two things were tried" when
|
||||||
|
// only one was.
|
||||||
|
func TestDiscoveryCandidates_NoDuplicateRequests(t *testing.T) {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, c := range discoveryCandidates("https://h/v1") {
|
||||||
|
url := c.baseURL + c.modelsPath
|
||||||
|
if seen[url] {
|
||||||
|
t.Errorf("candidate %q would be requested twice", url)
|
||||||
|
}
|
||||||
|
seen[url] = true
|
||||||
|
}
|
||||||
|
if len(seen) < 2 {
|
||||||
|
t.Errorf("only %d distinct URLs; the search collapsed", len(seen))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_FailureListsEachURLOnce is the end-to-end form of the same
|
||||||
|
// rule, on the error text an operator actually reads.
|
||||||
|
func TestDiscoverModels_FailureListsEachURLOnce(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
_, err := DiscoverModels(context.Background(), srv.URL+"/v1", "k")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected failure")
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
needle := srv.URL + "/v1/models"
|
||||||
|
if n := strings.Count(msg, needle); n > 1 {
|
||||||
|
t.Errorf("URL %s appears %d times in %q", needle, n, msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverModels_EmptyKeyOmitsHeader(t *testing.T) {
|
||||||
|
var present bool
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, present = r.Header["Authorization"]
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
if _, err := DiscoverModels(context.Background(), srv.URL+"/v1", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if present {
|
||||||
|
t.Error("an empty key should omit the header, not send an empty bearer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_RealCatalogEndToEnd walks the actual captured response
|
||||||
|
// through the whole path and asserts the numbers an operator would see.
|
||||||
|
func TestDiscoverModels_RealCatalogEndToEnd(t *testing.T) {
|
||||||
|
srv := catalogUpstream(t)
|
||||||
|
res, err := DiscoverModels(context.Background(), srv.URL+"/v1", "sk-secret")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discovery failed: %v", err)
|
||||||
|
}
|
||||||
|
if res.BaseURL != srv.URL+"/v1" {
|
||||||
|
t.Errorf("base = %q, want the typed URL unchanged", res.BaseURL)
|
||||||
|
}
|
||||||
|
if res.Endpoint != "" {
|
||||||
|
t.Errorf("endpoint = %q, want empty (base already carries /v1)", res.Endpoint)
|
||||||
|
}
|
||||||
|
if len(res.Models) != 2 || len(res.ImageModels) != 2 {
|
||||||
|
t.Errorf("chat/image = %d/%d, want 2/2", len(res.Models), len(res.ImageModels))
|
||||||
|
}
|
||||||
|
if res.Adapter != "openai" {
|
||||||
|
t.Errorf("adapter = %q, want openai", res.Adapter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The response must be JSON the UI can consume directly.
|
||||||
|
b, _ := json.Marshal(res)
|
||||||
|
var round map[string]interface{}
|
||||||
|
if err := json.Unmarshal(b, &round); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if round["ok"] != nil {
|
||||||
|
// discoverResponse adds ok; discoveryResult does not. Both marshal
|
||||||
|
// fine, so this is only a shape note.
|
||||||
|
_ = round
|
||||||
|
}
|
||||||
|
}
|
||||||
153
internal/gateway/discover_endpoint_test.go
Normal file
153
internal/gateway/discover_endpoint_test.go
Normal file
@ -0,0 +1,153 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestComposeImageEndpoint_NoVersionBaseLeavesDefault is the counterpart of the
|
||||||
|
// /v1 case: when the base carries no version segment, discovery must emit "" so
|
||||||
|
// the gateway's own default "/v1/images/generations" composes.
|
||||||
|
//
|
||||||
|
// This is the shape every currently-working source uses — siliconflow, the only
|
||||||
|
// source in production with kind:image, has base_url
|
||||||
|
// "https://api.siliconflow.cn" — so writing a non-empty value here would break
|
||||||
|
// exactly the configuration that is known to work today.
|
||||||
|
func TestComposeImageEndpoint_NoVersionBaseLeavesDefault(t *testing.T) {
|
||||||
|
if got := composeImageEndpoint("https://api.siliconflow.cn"); got != "" {
|
||||||
|
t.Errorf("composeImageEndpoint = %q, want \"\" so the built-in default composes", got)
|
||||||
|
}
|
||||||
|
// And the composition it implies must be the vendor's real path.
|
||||||
|
base := "https://api.siliconflow.cn"
|
||||||
|
composed := base + "/v1/images/generations"
|
||||||
|
if !strings.HasSuffix(composed, "/v1/images/generations") {
|
||||||
|
t.Errorf("composed URL %q lost the image path", composed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscovery_BareHostNeedsVersionOnEndpointWhenCatalogIsVersioned
|
||||||
|
// distinguishes two cases that look identical from the outside.
|
||||||
|
//
|
||||||
|
// bare host + catalog at "/models" → chat is "/chat/completions" (the
|
||||||
|
// adapter default, no endpoint written)
|
||||||
|
// bare host + catalog at "/v1/models" → chat is "/v1/chat/completions",
|
||||||
|
// so an endpoint MUST be written
|
||||||
|
//
|
||||||
|
// The catalog path is the only evidence of whether the upstream wants the
|
||||||
|
// version segment, and getting this backwards sends every chat request to a
|
||||||
|
// path the vendor does not serve.
|
||||||
|
func TestDiscovery_BareHostNeedsVersionOnEndpointWhenCatalogIsVersioned(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
catalogPath string
|
||||||
|
wantEndpoint string
|
||||||
|
wantImage string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "catalog at the bare /models needs no endpoint",
|
||||||
|
catalogPath: "/models",
|
||||||
|
wantEndpoint: "",
|
||||||
|
wantImage: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "catalog at /v1/models requires /v1 on the chat endpoint",
|
||||||
|
catalogPath: "/v1/models",
|
||||||
|
wantEndpoint: "/v1/chat/completions",
|
||||||
|
wantImage: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != tc.catalogPath {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"m1"},{"id":"m2"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
res, err := DiscoverModels(context.Background(), srv.URL, "k")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discovery failed: %v", err)
|
||||||
|
}
|
||||||
|
if res.BaseURL != srv.URL {
|
||||||
|
t.Errorf("base = %q, want %q", res.BaseURL, srv.URL)
|
||||||
|
}
|
||||||
|
if res.Endpoint != tc.wantEndpoint {
|
||||||
|
t.Errorf("endpoint = %q, want %q", res.Endpoint, tc.wantEndpoint)
|
||||||
|
}
|
||||||
|
if res.ImageEndpoint != tc.wantImage {
|
||||||
|
t.Errorf("image_endpoint = %q, want %q", res.ImageEndpoint, tc.wantImage)
|
||||||
|
}
|
||||||
|
// Whatever it wrote, the chat URL must reach the same version
|
||||||
|
// prefix the catalog was served under.
|
||||||
|
ep := res.Endpoint
|
||||||
|
if ep == "" {
|
||||||
|
ep = "/chat/completions"
|
||||||
|
}
|
||||||
|
if got := res.BaseURL + ep; strings.Contains(got, "/v1/v1") {
|
||||||
|
t.Errorf("chat URL %q has a doubled version", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestComposeEndpoints_BothFormsProduceWorkingURLs is the end-to-end
|
||||||
|
// invariant across the two shapes, stated as the URLs the gateway will build.
|
||||||
|
//
|
||||||
|
// A base with a version and no image_endpoint produces "/v1/v1/…" — the exact
|
||||||
|
// 404 observed before composeImageEndpoint existed. A base without one and a
|
||||||
|
// non-empty image_endpoint produces a path missing the version. Both are
|
||||||
|
// silent, expensive mistakes, so they are asserted literally.
|
||||||
|
func TestComposeEndpoints_BothFormsProduceWorkingURLs(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
base string
|
||||||
|
wantChat string
|
||||||
|
wantImage string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "bare host: adapter and image defaults compose",
|
||||||
|
base: "https://h",
|
||||||
|
wantChat: "https://h/chat/completions",
|
||||||
|
// openai's adapter endpoint is /chat/completions (verified in
|
||||||
|
// internal/lua/adapters/openai.lua: adapter.endpoint).
|
||||||
|
wantImage: "https://h/v1/images/generations",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "versioned base: the version moves onto both endpoints",
|
||||||
|
base: "https://h/v1",
|
||||||
|
wantChat: "https://h/v1/chat/completions",
|
||||||
|
wantImage: "https://h/v1/images/generations",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
ep := composeEndpoint("/models", tc.base, tc.base)
|
||||||
|
if ep == "" {
|
||||||
|
ep = "/chat/completions" // the adapter default
|
||||||
|
}
|
||||||
|
iep := composeImageEndpoint(tc.base)
|
||||||
|
if iep == "" {
|
||||||
|
iep = "/v1/images/generations" // the gateway default
|
||||||
|
}
|
||||||
|
gotChat := tc.base + ep
|
||||||
|
gotImage := tc.base + iep
|
||||||
|
if gotChat != tc.wantChat {
|
||||||
|
t.Errorf("chat URL = %q, want %q", gotChat, tc.wantChat)
|
||||||
|
}
|
||||||
|
if gotImage != tc.wantImage {
|
||||||
|
t.Errorf("image URL = %q, want %q", gotImage, tc.wantImage)
|
||||||
|
}
|
||||||
|
for _, u := range []string{gotChat, gotImage} {
|
||||||
|
if strings.Contains(u, "/v1/v1") {
|
||||||
|
t.Errorf("%q has a doubled version segment", u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
347
internal/gateway/discover_mutate_test.go
Normal file
347
internal/gateway/discover_mutate_test.go
Normal file
@ -0,0 +1,347 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
// Real mutation injection: rewrites the IMPLEMENTATION on disk, re-runs the
|
||||||
|
// suite, and requires it to go red.
|
||||||
|
//
|
||||||
|
// Why this exists and why it is a Go file rather than a shell script:
|
||||||
|
//
|
||||||
|
// The first version was a shell script. Its -run filter contained a regex that
|
||||||
|
// the shell mangled, so every mutant reported "SURVIVED" while go test printed
|
||||||
|
// "[no tests to run]". Four mutants looked like judgement failures and were
|
||||||
|
// actually never executed. A judgement that cannot tell "tests passed" from
|
||||||
|
// "no tests ran" is not a judgement.
|
||||||
|
//
|
||||||
|
// This version therefore refuses to report anything unless it has first proved
|
||||||
|
// the filter selects a non-trivial number of tests (see baseline below), and it
|
||||||
|
// treats "[no tests to run]" as a hard error rather than a green run.
|
||||||
|
//
|
||||||
|
// It also must not be run against a DIRTY tree: the mutant is written over
|
||||||
|
// discover.go, and a leftover mutant from a killed run would make the baseline
|
||||||
|
// green against broken code while every verdict came from the wrong source.
|
||||||
|
// See the guard at the top of TestRealMutants.
|
||||||
|
//
|
||||||
|
// Run: go test -tags luajit ./internal/gateway/ -run TestRealMutants -v
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mutationFilter must select every assertion that guards the code below. It is
|
||||||
|
// deliberately broad: a narrow filter is how mutants get to survive for free.
|
||||||
|
const mutationFilter = "Test(ParseModelCatalog|DiscoveredModelKind|Discover|TrimVersion|VersionSegment|ComposeEndpoint|ComposeImageEndpoint|ComposeEndpoints|IsVersionSegment|ModelsToConfig|DiscoveryResult|DiscoveryCandidates|Discovery_BareHost)"
|
||||||
|
|
||||||
|
type mutation struct {
|
||||||
|
name string
|
||||||
|
from string
|
||||||
|
to string
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRealMutants(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("mutation campaign compiles the package repeatedly")
|
||||||
|
}
|
||||||
|
wd, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
target := filepath.Join(wd, "discover.go")
|
||||||
|
orig, err := os.ReadFile(target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refuse to run against a dirty tree.
|
||||||
|
//
|
||||||
|
// The mutant is written OVER discover.go, so on a dirty file the "original"
|
||||||
|
// captured above is not the committed source — the baseline then runs against
|
||||||
|
// whatever was there, and a killed run (a background-task timeout, a Ctrl-C)
|
||||||
|
// leaves a mutant behind that the NEXT run treats as the baseline. Both make
|
||||||
|
// every verdict meaningless while the harness still reports a clean score.
|
||||||
|
//
|
||||||
|
// Refusing rather than restoring: this cannot tell "leftover mutant" apart from
|
||||||
|
// "the developer is mid-edit", and silently reverting real work is worse than
|
||||||
|
// stopping.
|
||||||
|
if dirty, derr := mutationTargetIsDirty(wd, "discover.go"); derr == nil && dirty {
|
||||||
|
t.Fatalf("%s has uncommitted changes — commit or stash them first. A mutant "+
|
||||||
|
"left behind by a killed run would otherwise become the baseline, and "+
|
||||||
|
"every verdict below would be about code nobody wrote.", target)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if werr := os.WriteFile(target, orig, 0o644); werr != nil {
|
||||||
|
t.Fatalf("restore discover.go: %v", werr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// --- prove the harness is armed before trusting any verdict ---
|
||||||
|
// A filter that selects nothing exits 0, which is indistinguishable from a
|
||||||
|
// real pass. The only way to tell them apart is to compare against a filter
|
||||||
|
// KNOWN to be empty: if that empty filter produces the same result as the
|
||||||
|
// real one, the real one is probably empty too.
|
||||||
|
//
|
||||||
|
// Note go test does NOT print "[no tests to run]" for a wholly unmatched
|
||||||
|
// filter (it prints that only when the package has other matching tests), so
|
||||||
|
// the check has to compare outputs rather than look for a marker string.
|
||||||
|
// Getting this backwards made the harness refuse to run at all.
|
||||||
|
emptyOut := runSuite(wd, "TestZZZNoSuchTestExists")
|
||||||
|
realOut := runSuite(wd, mutationFilter)
|
||||||
|
if countTests(emptyOut) != 0 {
|
||||||
|
t.Fatalf("a filter matching nothing selected %d tests; the harness's own probe is broken",
|
||||||
|
countTests(emptyOut))
|
||||||
|
}
|
||||||
|
// Kept for the no-op check below: a mutant whose output matches the
|
||||||
|
// baseline never changed behaviour, so it is an invalid mutation rather
|
||||||
|
// than a gap in the tests.
|
||||||
|
baseOut := realOut
|
||||||
|
if strings.Contains(realOut, "[no tests to run]") {
|
||||||
|
t.Fatalf("baseline selected no tests:\n%s", realOut)
|
||||||
|
}
|
||||||
|
baseCount := countTests(realOut)
|
||||||
|
if baseCount < 25 {
|
||||||
|
t.Fatalf("baseline ran only %d tests; the filter is too narrow to judge anything", baseCount)
|
||||||
|
}
|
||||||
|
if strings.Contains(realOut, "FAIL") {
|
||||||
|
t.Logf("note: baseline is not green; kill verdicts remain valid")
|
||||||
|
}
|
||||||
|
t.Logf("baseline: %d tests selected, empty filter selects %d",
|
||||||
|
baseCount, countTests(emptyOut))
|
||||||
|
|
||||||
|
muts := []mutation{
|
||||||
|
{
|
||||||
|
// Written as a condition that still READS modOut, because a
|
||||||
|
// mutation that leaves a variable unused fails to compile — and a
|
||||||
|
// build failure is not a judgement (see the harness note above).
|
||||||
|
name: "image kind ignores upstream output modality",
|
||||||
|
from: `if m.Capabilities.ImageGeneration || has(modOut, "image") {`,
|
||||||
|
to: "if m.Capabilities.ImageGeneration || (len(modOut) > 0 && false) {",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "kind defaults to image (guessing from names)",
|
||||||
|
from: "\t}\n\treturn \"chat\"\n}\n\n// modalityList",
|
||||||
|
to: "\t}\n\treturn \"image\"\n}\n\n// modalityList",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested metadata no longer read",
|
||||||
|
// 锚点必须与 discover.go 逐字节一致:此前这里是 6 个空格,源码已
|
||||||
|
// 变成 2 个空格,于是这条变异永远"找不到锚点"——变异campaign 每次
|
||||||
|
// 都以 anchor not found 失败,而它本该验证"嵌套 metadata 被丢掉"能被
|
||||||
|
// 测试抓住。锚点漂移 = 这条变异从未真正跑过(和 issue #1 里
|
||||||
|
// `cc["models"]` 恒 false 的测试跳过是同一种失效:断言写了但没执行)。
|
||||||
|
from: "\tcopyFields(m.Metadata) // nested first",
|
||||||
|
to: "\tcopyFields(json.RawMessage(nil)) // mutant: nested metadata dropped",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-catalog bodies accepted",
|
||||||
|
from: "\tif trimmed == \"\" || trimmed == \"null\" {\n\t\treturn nil, fmt.Errorf(\"empty response\")\n\t}",
|
||||||
|
to: "\tif false && (trimmed == \"\" || trimmed == \"null\") {\n\t\treturn nil, fmt.Errorf(\"empty response\")\n\t}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Removing the bare "/models" candidate must be observable: a vendor
|
||||||
|
// serving its catalog at the bare path would otherwise look
|
||||||
|
// unreachable. The `len(...) >= 0` is a tautology that only exists to
|
||||||
|
// keep the expression syntactically valid after the call is dropped.
|
||||||
|
name: "URL fallback search removed",
|
||||||
|
from: "\t\tadd(raw, \"/models\", 0)",
|
||||||
|
to: "\t\tif len(\"/models\") >= 0 {\n\t\t\t_ = hasVersion\n\t\t}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "version segment never moved onto the chat endpoint",
|
||||||
|
from: "\tif version == \"\" {\n\t\treturn \"\" // adapter default already composes\n\t}",
|
||||||
|
to: "\tif version == \"\" || len(version) > 0 {\n\t\treturn \"\"\n\t}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "image endpoint never corrected (/v1/v1 regression)",
|
||||||
|
from: "\tif b := trimVersionSuffix(persistedBase); b != persistedBase {\n\t\treturn \"/images/generations\"\n\t}",
|
||||||
|
to: "\tif b := trimVersionSuffix(persistedBase); b == b && b != b {\n\t\treturn \"/images/generations\"\n\t}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "version test loosened so 'vision' counts as a version",
|
||||||
|
from: "\tsuffix := strings.ToLower(seg[i:])",
|
||||||
|
to: "\tsuffix := strings.ToLower(seg[i:])\n\tif len(suffix) >= 0 {\n\t\tsuffix = \"beta\"\n\t}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The guard must be removable WITHOUT the mutant becoming a
|
||||||
|
// semantic no-op. Two earlier attempts failed:
|
||||||
|
//
|
||||||
|
// `&& out.BaseURL != ""` — the guard this test exercises uses a
|
||||||
|
// zero-value struct whose BaseURL is empty, so the added
|
||||||
|
// condition made the mutant behave identically to the original.
|
||||||
|
// `&& len(out.BaseURL) >= 0` — always true for any string, so the
|
||||||
|
// mutant was STILL an identity transform.
|
||||||
|
//
|
||||||
|
// Reading the variable into a blank assignment keeps the compiler
|
||||||
|
// happy without changing behaviour, which is what makes the
|
||||||
|
// mutant observable.
|
||||||
|
name: "nil slice guard removed from JSON output",
|
||||||
|
from: "\tif out.ImageModels == nil {\n\t\tout.ImageModels = []string{}\n\t}",
|
||||||
|
to: "\tif out.ImageModels == nil {\n\t\t_ = out.ImageModels\n\t}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "auth rejection no longer classified",
|
||||||
|
from: "\tif status != http.StatusUnauthorized && status != http.StatusForbidden {\n\t\treturn \"\"\n\t}",
|
||||||
|
to: "\tif status != http.StatusUnauthorized && status != http.StatusForbidden || len(body) >= 0 {\n\t\treturn \"\"\n\t}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate candidate URLs no longer collapsed",
|
||||||
|
from: "\t\tif seenPair[pair] || seenURL[base+path] {",
|
||||||
|
to: "\t\tif len(pair) > 0 && (seenPair[pair] || seenURL[base+path]) && false {",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
survived := 0
|
||||||
|
for _, m := range muts {
|
||||||
|
t.Run(m.name, func(t *testing.T) {
|
||||||
|
if !strings.Contains(string(orig), m.from) {
|
||||||
|
t.Fatalf("anchor not found in discover.go — this mutation would be a no-op:\n%q", m.from)
|
||||||
|
}
|
||||||
|
mutated := strings.Replace(string(orig), m.from, m.to, 1)
|
||||||
|
if mutated == string(orig) {
|
||||||
|
t.Fatal("replacement changed nothing")
|
||||||
|
}
|
||||||
|
if werr := os.WriteFile(target, []byte(mutated), 0o644); werr != nil {
|
||||||
|
t.Fatal(werr)
|
||||||
|
}
|
||||||
|
defer os.WriteFile(target, orig, 0o644)
|
||||||
|
|
||||||
|
out := runSuite(wd, mutationFilter)
|
||||||
|
// A build failure kills EVERY mutant, so it would mask a broken
|
||||||
|
// harness as a perfect score. Distinguish it from a real test
|
||||||
|
// failure and fail loudly instead.
|
||||||
|
if strings.Contains(out, "[build failed]") {
|
||||||
|
t.Fatalf("mutant broke the build — that is not a judgement, it is a broken harness:\n%s", firstLines(out, 20))
|
||||||
|
}
|
||||||
|
if countTests(out) == 0 {
|
||||||
|
t.Fatalf("filter selected nothing — the verdict would be meaningless")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "FAIL") {
|
||||||
|
// The mutant compiled and the suite stayed green. Before calling
|
||||||
|
// this a survivor, check the mutation was not a no-op: an earlier
|
||||||
|
// version reported four "survivors" that were really invalid
|
||||||
|
// mutations whose added guard made them behaviourally identical
|
||||||
|
// to the original. A silent no-op is a broken mutation, and
|
||||||
|
// conflating the two sends the next reader hunting for a missing
|
||||||
|
// test that does not exist.
|
||||||
|
if behaviourOnly(out) == behaviourOnly(baseOut) {
|
||||||
|
t.Fatalf("mutant is a no-op: the suite output is identical to the "+
|
||||||
|
"baseline, so this mutation never changed behaviour. Fix the "+
|
||||||
|
"mutation rather than looking for a missing test.\nmutant: %q", m.to)
|
||||||
|
}
|
||||||
|
survived++
|
||||||
|
t.Errorf("SURVIVED %s — no test noticed this wrong implementation", m.name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Logf("KILL %s", m.name)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if survived > 0 {
|
||||||
|
t.Errorf("%d of %d mutants survived", survived, len(muts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runSuite invokes the go tool on this package with the given -run filter.
|
||||||
|
//
|
||||||
|
// The luajit build tag is mandatory: without it the package falls back to plain
|
||||||
|
// Lua 5.1, billing.lua's `goto` fails to compile, and the whole package fails
|
||||||
|
// to build — which the harness would misread as "every mutant killed".
|
||||||
|
func runSuite(wd, filter string) string {
|
||||||
|
tool := os.Getenv("GO_TEST_GO")
|
||||||
|
if tool == "" {
|
||||||
|
tool = "go"
|
||||||
|
}
|
||||||
|
cmd := exec.Command(tool, "test",
|
||||||
|
"-tags", "luajit",
|
||||||
|
".", "-run", filter, "-count=1", "-v")
|
||||||
|
cmd.Dir = wd
|
||||||
|
cmd.Env = os.Environ()
|
||||||
|
out, _ := cmd.CombinedOutput()
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// mutationTargetIsDirty reports whether a file under internal/gateway has
|
||||||
|
// uncommitted changes, so the harness can refuse to run instead of treating a
|
||||||
|
// leftover mutant as the baseline.
|
||||||
|
//
|
||||||
|
// git runs from the package directory with a PATH RELATIVE to it. An earlier
|
||||||
|
// version of the sibling script passed a repo-relative pathspec while running in
|
||||||
|
// the package directory; git matched nothing and the guard never fired.
|
||||||
|
func mutationTargetIsDirty(wd, name string) (bool, error) {
|
||||||
|
cmd := exec.Command("git", "diff", "--name-only", "--", "./"+name)
|
||||||
|
cmd.Dir = wd
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(out)) != "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func countTests(out string) int {
|
||||||
|
n := 0
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
// Subtests are indented; only count top-level ones so a single test
|
||||||
|
// with a table of cases cannot inflate the count.
|
||||||
|
if strings.HasPrefix(line, "=== RUN Test") && !strings.Contains(line, "/") {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// behaviourOnly strips the parts of `go test -v` output that vary between runs
|
||||||
|
// (timings, goroutine ids, temp paths) so two runs can be compared for equality.
|
||||||
|
// If a mutant produces output identical to the baseline under this projection,
|
||||||
|
// the mutation did not change observable behaviour and is therefore invalid.
|
||||||
|
func behaviourOnly(out string) string {
|
||||||
|
var kept []string
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
switch {
|
||||||
|
// Keep only the outcome lines. Test NAMES are deliberately dropped:
|
||||||
|
// two runs of the same suite can legitimately order or cache tests
|
||||||
|
// differently, and keeping the names made every mutant look like it had
|
||||||
|
// "changed behaviour" — which defeats the no-op detector entirely.
|
||||||
|
case strings.HasPrefix(trimmed, "--- PASS"), strings.HasPrefix(trimmed, "--- FAIL"),
|
||||||
|
strings.HasPrefix(trimmed, "--- SKIP"),
|
||||||
|
strings.HasPrefix(trimmed, "ok "), trimmed == "FAIL",
|
||||||
|
strings.Contains(trimmed, ".go:"):
|
||||||
|
kept = append(kept, trimmed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Counting outcomes is stronger than comparing the sequence: a mutation
|
||||||
|
// that swaps which test fails still changes the multiset, and one that
|
||||||
|
// changes nothing cannot.
|
||||||
|
sort.Strings(kept)
|
||||||
|
return strings.Join(kept, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstLines trims output for an error message; a failing nested run can be
|
||||||
|
// thousands of lines and the cause is always near the top.
|
||||||
|
func firstLines(s string, n int) string {
|
||||||
|
lines := strings.Split(s, "\n")
|
||||||
|
if len(lines) > n {
|
||||||
|
lines = lines[:n]
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMutationFilterCoversEveryGuard keeps the filter honest. Dropping a group
|
||||||
|
// from it silently disarms that area's mutants.
|
||||||
|
func TestMutationFilterCoversEveryGuard(t *testing.T) {
|
||||||
|
for _, group := range []string{
|
||||||
|
"ParseModelCatalog", // nested metadata + non-catalog rejection
|
||||||
|
"DiscoveredModelKind", // kind follows upstream metadata
|
||||||
|
"Discover", // URL fallback, auth classification
|
||||||
|
"ComposeEndpoint", // version segment exactly once (chat)
|
||||||
|
"ComposeImageEndpoint",
|
||||||
|
"ComposeEndpoints", // the composed URLs, both base shapes
|
||||||
|
"IsVersionSegment", // "v1beta" vs "vision"
|
||||||
|
"DiscoveryResult", // nil slices are arrays
|
||||||
|
} {
|
||||||
|
if !strings.Contains(mutationFilter, group) {
|
||||||
|
t.Errorf("mutation filter omits %s; mutants there survive silently", group)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
171
internal/gateway/discover_mutation_test.go
Normal file
171
internal/gateway/discover_mutation_test.go
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
// Mutation harness for the discovery code. Kept in the tree (not in a shell
|
||||||
|
// script) so the -run filter is a Go string literal rather than something the
|
||||||
|
// shell can mangle into "[no tests to run]" — which is exactly how an earlier
|
||||||
|
// version of this harness reported four surviving mutants that were simply
|
||||||
|
// never executed.
|
||||||
|
//
|
||||||
|
// Each mutant flips one behaviour; TestMutationHarness_Fails proves the
|
||||||
|
// harness itself is armed.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// discoveryTestFilter must cover every assertion that guards the behaviour
|
||||||
|
// below. Narrowing it is how mutants get to "survive" for free.
|
||||||
|
const discoveryTestFilter = "Test(ParseModelCatalog|DiscoveredModelKind|Discover|TrimVersion|VersionSegment|ComposeEndpoint|IsVersionSegment|ModelsToConfig|DiscoveryResult|DiscoveryCandidates)"
|
||||||
|
|
||||||
|
func TestDiscoveryFilterRunsRealTests(t *testing.T) {
|
||||||
|
if discoveryTestFilter == "" {
|
||||||
|
t.Fatal("filter is empty; every mutant would survive trivially")
|
||||||
|
}
|
||||||
|
// The whole point: the catalog parser test must be inside the filter, or a
|
||||||
|
// mutant that stops reading nested metadata survives with a green run.
|
||||||
|
for _, group := range []string{
|
||||||
|
"ParseModelCatalog", // nested metadata + non-catalog rejection
|
||||||
|
"DiscoveredModelKind", // kind must follow upstream metadata
|
||||||
|
"Discover", // URL fallback + endpoint composition
|
||||||
|
"ComposeEndpoint", // version segment exactly once
|
||||||
|
"IsVersionSegment", // "v1beta" vs "vision"
|
||||||
|
} {
|
||||||
|
if !strings.Contains(discoveryTestFilter, group) {
|
||||||
|
t.Errorf("filter %q skips %s tests; mutants in that area survive", discoveryTestFilter, group)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMutantsAreDetectable is the harness's own regression guard: it runs the
|
||||||
|
// filter's worth of assertions against deliberately broken inputs and requires
|
||||||
|
// each to be rejected. If a future edit makes one of these pass, the
|
||||||
|
// corresponding protection is gone.
|
||||||
|
func TestMutantsAreDetectable(t *testing.T) {
|
||||||
|
t.Run("image classification follows upstream metadata, not the model name", func(t *testing.T) {
|
||||||
|
// The mutant: drop the output-modality check, so every model is chat.
|
||||||
|
m := upstreamModel{ID: "gpt-image-2"}
|
||||||
|
m.Architecture.OutputModalities = []string{"image"}
|
||||||
|
if got := discoveredModelKind(m); got != "image" {
|
||||||
|
t.Errorf("mutant survived: got %q, want \"image\"", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nested metadata is read", func(t *testing.T) {
|
||||||
|
var ms []upstreamModel
|
||||||
|
if err := json.Unmarshal([]byte(realCatalogBody), &ms); err == nil {
|
||||||
|
t.Fatal("fixture must be a wrapper object, not a bare array")
|
||||||
|
}
|
||||||
|
var wrapper struct {
|
||||||
|
Data []upstreamModel `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(realCatalogBody), &wrapper); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(wrapper.Data) == 0 {
|
||||||
|
t.Fatal("fixture parsed to zero models")
|
||||||
|
}
|
||||||
|
// The mutant: skip copyFields(m.Metadata) entirely.
|
||||||
|
meta := modelMeta(wrapper.Data[0])
|
||||||
|
if meta == nil {
|
||||||
|
t.Fatal("mutant survived: metadata is nil, so cost/context_limit are lost")
|
||||||
|
}
|
||||||
|
for _, k := range []string{"context_limit", "cost", "context_budget"} {
|
||||||
|
if _, ok := meta[k]; !ok {
|
||||||
|
t.Errorf("mutant survived: %s dropped from nested metadata", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non-catalog responses are rejected", func(t *testing.T) {
|
||||||
|
for _, body := range []string{`<html>login</html>`, `null`, `{"error":{"message":"x"}}`, `{}`} {
|
||||||
|
if _, err := parseModelCatalog([]byte(body)); err == nil {
|
||||||
|
t.Errorf("mutant survived: %q accepted as a catalog", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("version segments are distinguished from real path segments", func(t *testing.T) {
|
||||||
|
if !isVersionSegment("v1beta") {
|
||||||
|
t.Error("mutant survived: v1beta not treated as a version")
|
||||||
|
}
|
||||||
|
for _, seg := range []string{"vision", "vertex", "v1x"} {
|
||||||
|
if isVersionSegment(seg) {
|
||||||
|
t.Errorf("mutant survived: %q wrongly treated as a version", seg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("endpoint composition never duplicates the version segment", func(t *testing.T) {
|
||||||
|
// The mutant: never move the segment onto the endpoint.
|
||||||
|
if got := composeEndpoint("/v1/models", "https://h", "https://h/v1"); got != "/v1/chat/completions" {
|
||||||
|
t.Errorf("mutant survived: endpoint = %q", got)
|
||||||
|
}
|
||||||
|
// The other mutant: always move it, even when the base keeps it.
|
||||||
|
if got := composeEndpoint("/models", "https://h/v1", "https://h/v1"); got != "" {
|
||||||
|
t.Errorf("mutant survived: endpoint = %q would double /v1", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("image_models is never nil", func(t *testing.T) {
|
||||||
|
res := &discoveryResult{Models: []string{"a"}, ImageModels: nil}
|
||||||
|
b, _ := json.Marshal(res)
|
||||||
|
if strings.Contains(string(b), `"image_models":null`) {
|
||||||
|
t.Errorf("mutant survived: %s", b)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a rejected key is reported as unauthorized, not as an outage", func(t *testing.T) {
|
||||||
|
// The mutant: collapse every failure into one generic error, which is
|
||||||
|
// what made a typo'd key look like the vendor being down.
|
||||||
|
if classifyAuthStatus(http.StatusUnauthorized, []byte(`{"error":"invalid api key"}`)) == "" {
|
||||||
|
t.Error("mutant survived: 401 is not classified")
|
||||||
|
}
|
||||||
|
if classifyAuthStatus(http.StatusNotFound, []byte("nope")) != "" {
|
||||||
|
t.Error("a 404 must not be reported as an auth problem")
|
||||||
|
}
|
||||||
|
if classifyAuthStatus(http.StatusOK, nil) != "" {
|
||||||
|
t.Error("a 200 must not be reported as an auth problem")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("duplicate candidate URLs are collapsed", func(t *testing.T) {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, c := range discoveryCandidates("https://h/v1") {
|
||||||
|
u := c.baseURL + c.modelsPath
|
||||||
|
if seen[u] {
|
||||||
|
t.Errorf("mutant survived: %s requested twice", u)
|
||||||
|
}
|
||||||
|
seen[u] = true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("discovery falls back across URL forms", func(t *testing.T) {
|
||||||
|
// The coverage requirement is that BOTH catalog paths are reachable for
|
||||||
|
// a typed base, and both for a bare host. Asserting an exact count kept
|
||||||
|
// drifting as the candidate list was fixed, and each time it measured
|
||||||
|
// the current list rather than the property that matters.
|
||||||
|
for _, base := range []string{"https://h/v1", "https://h"} {
|
||||||
|
distinct := map[string]bool{}
|
||||||
|
for _, c := range discoveryCandidates(base) {
|
||||||
|
distinct[c.baseURL+c.modelsPath] = true
|
||||||
|
}
|
||||||
|
for _, want := range []string{base + "/v1/models", base + "/models"} {
|
||||||
|
if !distinct[want] {
|
||||||
|
t.Errorf("mutant survived: %s never requests %s", base, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("only the catalog is fetched, never each model", func(t *testing.T) {
|
||||||
|
// Guarding a NON-mutation: the design decision that discovery must not
|
||||||
|
// spend money. If a future change adds per-model probes, the counting
|
||||||
|
// test below catches it; this assertion documents the intent.
|
||||||
|
if discoveryModelsPath != "/v1/models" {
|
||||||
|
t.Errorf("catalog path changed to %q", discoveryModelsPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
645
internal/gateway/discover_test.go
Normal file
645
internal/gateway/discover_test.go
Normal file
@ -0,0 +1,645 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// realCatalogBody is the actual /models response captured from the amr-link
|
||||||
|
// endpoint, not a hand-written fixture. Two properties of it are the reason
|
||||||
|
// this test exists at all:
|
||||||
|
//
|
||||||
|
// - it nests modalities under "architecture", which the naive parser misses;
|
||||||
|
// - it lists five image-generation ids among 34 chat ids, so a parser that
|
||||||
|
// returns everything as "chat" silently poisons the AUTO chain.
|
||||||
|
const realCatalogBody = `{"data":[{"id":"claude-fable-5","object":"model","created":1781081344,"owned_by":"vela","architecture":{"input_modalities":["text","image","file"],"output_modalities":["text"]},"metadata":{"cost":"very_high","capability":"best_quality","context_limit":1000000,"context_budget":232000},"default":false,"enabled":true},{"id":"gpt-image-2","object":"model","architecture":{"input_modalities":["text","image"],"output_modalities":["image"]},"metadata":{"capability":"best_quality"},"enabled":true},{"id":"deepseek-v4-flash","object":"model","architecture":{"input_modalities":["text"],"output_modalities":["text"]},"metadata":{"cost":"medium","capability":"standard","context_limit":1048000},"enabled":true},{"id":"seedream-5.0-pro","object":"model","architecture":{"input_modalities":["text","image"],"output_modalities":["image"]},"metadata":{"capability":"best_quality"},"enabled":true}]}`
|
||||||
|
|
||||||
|
func TestParseModelCatalog_RealUpstreamShape(t *testing.T) {
|
||||||
|
models, err := parseModelCatalog([]byte(realCatalogBody))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse: %v", err)
|
||||||
|
}
|
||||||
|
if len(models) != 4 {
|
||||||
|
t.Fatalf("want 4 models, got %d", len(models))
|
||||||
|
}
|
||||||
|
|
||||||
|
kind := map[string]string{}
|
||||||
|
for _, m := range models {
|
||||||
|
kind[m.ID] = m.Kind
|
||||||
|
}
|
||||||
|
// The two image generators must be classified by the upstream's declared
|
||||||
|
// output modality, not by their names.
|
||||||
|
for _, id := range []string{"gpt-image-2", "seedream-5.0-pro"} {
|
||||||
|
if kind[id] != "image" {
|
||||||
|
t.Errorf("%s: got kind %q, want \"image\"", id, kind[id])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, id := range []string{"claude-fable-5", "deepseek-v4-flash"} {
|
||||||
|
if kind[id] != "chat" {
|
||||||
|
t.Errorf("%s: got kind %q, want \"chat\"", id, kind[id])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metadata the upstream sent must survive; keys it did not send must not
|
||||||
|
// appear as zero values.
|
||||||
|
if got := models[0].Meta["context_limit"]; got == nil {
|
||||||
|
t.Error("claude-fable-5: context_limit dropped (it is nested under metadata in the real response)")
|
||||||
|
} else if f, ok := got.(float64); !ok || f != 1000000 {
|
||||||
|
t.Errorf("context_limit = %#v, want numeric 1000000 so the UI can sort on it", got)
|
||||||
|
}
|
||||||
|
if _, present := models[0].Meta["cost"]; !present {
|
||||||
|
t.Error("claude-fable-5: cost dropped")
|
||||||
|
}
|
||||||
|
// context_budget is present on this entry and must not be dropped either.
|
||||||
|
if _, present := models[0].Meta["context_budget"]; !present {
|
||||||
|
t.Error("claude-fable-5: context_budget dropped")
|
||||||
|
}
|
||||||
|
// deepseek-v4-flash DOES send metadata; gpt-image-2 sends only capability.
|
||||||
|
// Neither may lose it.
|
||||||
|
if models[2].Meta["context_limit"] == nil {
|
||||||
|
t.Error("deepseek-v4-flash: context_limit dropped")
|
||||||
|
}
|
||||||
|
if models[2].Meta["cost"] == nil {
|
||||||
|
t.Error("deepseek-v4-flash: cost dropped")
|
||||||
|
}
|
||||||
|
if _, bogus := models[0].Meta["made_up_field"]; bogus {
|
||||||
|
t.Error("invented a metadata key that the upstream never sent")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoveredModelKind_ImageOnlyByUpstreamMetadata is the regression guard
|
||||||
|
// for the failure mode that motivated keeping kind="chat" as the default: a
|
||||||
|
// name-based heuristic marks "ideogram-v2" or "gpt-image-1" as an image model,
|
||||||
|
// but a catalog that also serves chat for that id would then have its chat
|
||||||
|
// entry removed from the AUTO chain. Only the upstream's declared OUTPUT
|
||||||
|
// modality may classify a model as image.
|
||||||
|
func TestDiscoveredModelKind_ImageOnlyByUpstreamMetadata(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
in upstreamModel
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// A catalog that DECLARES output modality image is authoritative.
|
||||||
|
// The "kind" string saying chat must not override it.
|
||||||
|
name: "declared output modality image wins over a chat kind string",
|
||||||
|
in: func() upstreamModel {
|
||||||
|
var m upstreamModel
|
||||||
|
m.Kind = "chat"
|
||||||
|
m.Architecture.OutputModalities = []string{"image"}
|
||||||
|
return m
|
||||||
|
}(),
|
||||||
|
want: "image",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bare array modality string",
|
||||||
|
in: upstreamModel{ID: "x", Modality: "text-to-image"},
|
||||||
|
want: "image",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "image INPUT is still a chat model",
|
||||||
|
in: func() upstreamModel {
|
||||||
|
var m upstreamModel
|
||||||
|
m.Architecture.InputModalities = []string{"text", "image"}
|
||||||
|
return m
|
||||||
|
}(),
|
||||||
|
want: "chat",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file input is still a chat model",
|
||||||
|
in: func() upstreamModel {
|
||||||
|
var m upstreamModel
|
||||||
|
m.Architecture.InputModalities = []string{"text", "file"}
|
||||||
|
return m
|
||||||
|
}(),
|
||||||
|
want: "chat",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "vision capability flag is chat",
|
||||||
|
in: upstreamModel{ID: "y", Capabilities: modelCaps{Vision: true}},
|
||||||
|
want: "chat",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "capabilities.image_generation is image",
|
||||||
|
in: upstreamModel{ID: "z", Capabilities: modelCaps{ImageGeneration: true}},
|
||||||
|
want: "image",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no modality info at all defaults to chat",
|
||||||
|
in: upstreamModel{ID: "plain"},
|
||||||
|
want: "chat",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The default must be chat even when the name LOOKS like an image
|
||||||
|
// model. Guessing from the name is what puts gpt-image-2 into the
|
||||||
|
// chat AUTO chain, where every request to it fails.
|
||||||
|
name: "image-looking name without upstream metadata stays chat",
|
||||||
|
in: upstreamModel{ID: "gpt-image-2"},
|
||||||
|
want: "chat",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "vision-looking name without upstream metadata stays chat",
|
||||||
|
in: upstreamModel{ID: "ideogram-v2"},
|
||||||
|
want: "chat",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := discoveredModelKind(tc.in); got != tc.want {
|
||||||
|
t.Errorf("got %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseModelCatalog_RejectsNonCatalog(t *testing.T) {
|
||||||
|
// A 200 with valid JSON that is not a catalog must NOT be reported as an
|
||||||
|
// empty result. Reporting zero models would let a login page or a
|
||||||
|
// rate-limit body create a source with no models in it.
|
||||||
|
for name, body := range map[string]string{
|
||||||
|
"login page html": `<html><body>Sign in</body></html>`,
|
||||||
|
"rate limit json": `{"error":{"message":"slow down","type":"rate_limit"}}`,
|
||||||
|
"empty object": `{}`,
|
||||||
|
"null": `null`,
|
||||||
|
"wrong shape": `{"objects":[{"id":"m"}]}`,
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := parseModelCatalog([]byte(body)); err == nil {
|
||||||
|
t.Errorf("expected an error for %q, got nil (would create an empty source)", name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseModelCatalog_AcceptsBareArray(t *testing.T) {
|
||||||
|
models, err := parseModelCatalog([]byte(`[{"id":"a"},{"id":"b"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("bare array should parse: %v", err)
|
||||||
|
}
|
||||||
|
if len(models) != 2 {
|
||||||
|
t.Fatalf("want 2, got %d", len(models))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_NormalizesWhenOnlyBareModelsAnswers: operator pastes
|
||||||
|
// "https://host/v1" (the form every SDK example shows) but the vendor serves
|
||||||
|
// the catalog at "https://host/models" — the version segment on the path is
|
||||||
|
// implied, not doubled. Discovery must strip the segment off the base and
|
||||||
|
// report it on the endpoint, because openai's adapter endpoint is
|
||||||
|
// "/chat/completions" and the two are concatenated.
|
||||||
|
func TestDiscoverModels_NormalizesWhenOnlyBareModelsAnswers(t *testing.T) {
|
||||||
|
var paths []string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
paths = append(paths, r.URL.Path)
|
||||||
|
if r.URL.Path != "/models" {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(realCatalogBody))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
res, err := DiscoverModels(context.Background(), srv.URL+"/v1", "sk-test")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discovery failed: %v", err)
|
||||||
|
}
|
||||||
|
if res.BaseURL != srv.URL {
|
||||||
|
t.Errorf("base_url = %q, want %q (version segment moved off the base)", res.BaseURL, srv.URL)
|
||||||
|
}
|
||||||
|
// "/models" carries no version segment, but the operator typed one — the
|
||||||
|
// endpoint has to restore it or the chat call would miss /v1.
|
||||||
|
if res.Endpoint != "/v1/chat/completions" {
|
||||||
|
t.Errorf("endpoint = %q, want %q", res.Endpoint, "/v1/chat/completions")
|
||||||
|
}
|
||||||
|
if res.Note == "" {
|
||||||
|
t.Error("silently rewriting the operator's base_url is worse than saying so")
|
||||||
|
}
|
||||||
|
if len(res.Models) != 2 || len(res.ImageModels) != 2 {
|
||||||
|
t.Errorf("got %d chat / %d image, want 2 / 2", len(res.Models), len(res.ImageModels))
|
||||||
|
}
|
||||||
|
if len(paths) < 2 {
|
||||||
|
t.Errorf("tried only %v; expected a fallback search", paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_BareHostWithVersionedModelsPath: operator pastes the host
|
||||||
|
// root, vendor serves /v1/models. The base needs no rewriting; the endpoint
|
||||||
|
// does.
|
||||||
|
func TestDiscoverModels_BareHostWithVersionedModelsPath(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/models" {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"m1"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
res, err := DiscoverModels(context.Background(), srv.URL, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discovery failed: %v", err)
|
||||||
|
}
|
||||||
|
if res.BaseURL != srv.URL {
|
||||||
|
t.Errorf("base_url = %q, want %q", res.BaseURL, srv.URL)
|
||||||
|
}
|
||||||
|
if res.Endpoint != "/v1/chat/completions" {
|
||||||
|
t.Errorf("endpoint = %q, want %q", res.Endpoint, "/v1/chat/completions")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_KeepsWorkingBaseURLUnchanged covers the vendor that really
|
||||||
|
// does serve "/v1/models": the typed URL must survive untouched so no needless
|
||||||
|
// config rewrite is suggested.
|
||||||
|
func TestDiscoverModels_KeepsWorkingBaseURLUnchanged(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/models" {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"m1"},{"id":"m2"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
// Base typed WITH "/v1", catalog served at "…/v1/models". That is exactly
|
||||||
|
// the amr-link shape: first candidate succeeds, nothing is rewritten, and
|
||||||
|
// the adapter's default endpoint already composes.
|
||||||
|
res, err := DiscoverModels(context.Background(), srv.URL+"/v1", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discovery failed: %v", err)
|
||||||
|
}
|
||||||
|
if res.BaseURL != srv.URL+"/v1" {
|
||||||
|
t.Errorf("base_url = %q, want it unchanged", res.BaseURL)
|
||||||
|
}
|
||||||
|
if res.Endpoint != "" {
|
||||||
|
t.Errorf("endpoint = %q, want empty (the typed base already carries /v1)", res.Endpoint)
|
||||||
|
}
|
||||||
|
if res.Note != "" {
|
||||||
|
t.Errorf("note = %q, want empty when nothing was rewritten", res.Note)
|
||||||
|
}
|
||||||
|
if len(res.Models) != 2 {
|
||||||
|
t.Errorf("got %d chat models, want 2", len(res.Models))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_ErrorNamesEveryTriedURL: a failed probe is useless if it
|
||||||
|
// only reports the last URL. The operator needs to see the full search.
|
||||||
|
func TestDiscoverModels_ErrorNamesEveryTriedURL(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
_, err := DiscoverModels(context.Background(), srv.URL+"/v1", "k")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected failure against an all-404 server")
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
for _, want := range []string{"/models", "/v1/models", "404"} {
|
||||||
|
if !strings.Contains(msg, want) {
|
||||||
|
t.Errorf("error %q should mention %q", msg, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoveryCandidates_DedupesAndPrefersTyped(t *testing.T) {
|
||||||
|
cands := discoveryCandidates("https://h/v1/")
|
||||||
|
if len(cands) == 0 {
|
||||||
|
t.Fatal("no candidates")
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, c := range cands {
|
||||||
|
// Dedup is on the requested URL, so that is the key to assert on.
|
||||||
|
k := c.baseURL + c.modelsPath
|
||||||
|
if seen[k] {
|
||||||
|
t.Errorf("duplicate request for %s", k)
|
||||||
|
}
|
||||||
|
seen[k] = true
|
||||||
|
}
|
||||||
|
// The versioned spelling of the typed URL must be reachable: on the real
|
||||||
|
// amr-link endpoint that is what answers, and it is the preferred candidate.
|
||||||
|
if !seen["https://h/v1/models"] {
|
||||||
|
t.Error("the typed base with a versioned catalog path must be tried; that is what answers on amr-link")
|
||||||
|
}
|
||||||
|
// Both catalog forms must be attempted for a bare host too. Offering only
|
||||||
|
// "/v1/models" there left a vendor serving a bare "/models" looking
|
||||||
|
// unreachable.
|
||||||
|
if !seen["https://h/models"] {
|
||||||
|
t.Error("host root with a bare /models path was never tried")
|
||||||
|
}
|
||||||
|
// The first candidate is preferred, so its URL is what gets reported when
|
||||||
|
// several answer.
|
||||||
|
if cands[0].baseURL+cands[0].modelsPath != "https://h/v1/models" {
|
||||||
|
t.Errorf("first candidate requests %q, want the canonical versioned form",
|
||||||
|
cands[0].baseURL+cands[0].modelsPath)
|
||||||
|
}
|
||||||
|
// The bare "/models" form must exist too — some gateways serve it there.
|
||||||
|
hasBare := false
|
||||||
|
for _, c := range cands {
|
||||||
|
if c.modelsPath == "/models" {
|
||||||
|
hasBare = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !hasBare {
|
||||||
|
t.Error("no /models candidate; vendors serving the catalog at /models would be missed")
|
||||||
|
}
|
||||||
|
// Preference must be monotonically non-decreasing so the sort is stable.
|
||||||
|
for i := 1; i < len(cands); i++ {
|
||||||
|
if cands[i].prefer < cands[i-1].prefer {
|
||||||
|
t.Errorf("candidates out of preference order at %d: %d < %d", i, cands[i].prefer, cands[i-1].prefer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVersionSegment(t *testing.T) {
|
||||||
|
cases := map[string]string{
|
||||||
|
"/v1/models": "/v1",
|
||||||
|
"/v1beta/models": "/v1beta",
|
||||||
|
"/models": "",
|
||||||
|
"/api/models": "",
|
||||||
|
"/vision/models": "",
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := versionSegment(in); got != want {
|
||||||
|
t.Errorf("versionSegment(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimVersionSuffix(t *testing.T) {
|
||||||
|
cases := map[string]string{
|
||||||
|
"https://h/v1": "https://h",
|
||||||
|
"https://h/v2": "https://h",
|
||||||
|
"https://h/v1beta": "https://h",
|
||||||
|
"https://h/api/v1": "https://h/api",
|
||||||
|
"https://h/vision": "https://h/vision", // not numeric — a real path segment
|
||||||
|
"https://h": "https://h",
|
||||||
|
"https://h/openai": "https://h/openai",
|
||||||
|
"https://h/v1/extra": "https://h/v1/extra",
|
||||||
|
// A segment that merely STARTS with v must not be stripped: stripping
|
||||||
|
// it would rewrite base "https://h/vision" to "https://h" and, worse,
|
||||||
|
// would treat a real vendor path as a version.
|
||||||
|
"https://h/vertex": "https://h/vertex",
|
||||||
|
"https://h/volcengine": "https://h/volcengine",
|
||||||
|
"https://h/vertex-ai/v1": "https://h/vertex-ai",
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := trimVersionSuffix(in); got != want {
|
||||||
|
t.Errorf("trimVersionSuffix(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestIsVersionSegment_RequiresDigits pins the distinction the whole endpoint
|
||||||
|
// derivation rests on: "v1"/"v1beta" are versions, "vision"/"vertex" are path
|
||||||
|
// segments. Without the digit requirement, stripping turns a vendor's real
|
||||||
|
// base URL into a different one.
|
||||||
|
func TestIsVersionSegment_RequiresDigits(t *testing.T) {
|
||||||
|
yes := []string{"v1", "v2", "v10", "v1beta", "v1alpha", "v1preview"}
|
||||||
|
no := []string{"vision", "vertex", "volcengine", "openai", "v", "vv", "api", "beta", "v1x"}
|
||||||
|
for _, s := range yes {
|
||||||
|
if !isVersionSegment(s) {
|
||||||
|
t.Errorf("isVersionSegment(%q) = false, want true", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range no {
|
||||||
|
if isVersionSegment(s) {
|
||||||
|
t.Errorf("isVersionSegment(%q) = true, want false", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestComposeEndpoint_ExactlyOneVersionSegment is the invariant test. Every row
|
||||||
|
// below is a URL form that was observed to 404 or to route wrongly when the
|
||||||
|
// version segment ended up in the wrong place.
|
||||||
|
func TestComposeEndpoint_ExactlyOneVersionSegment(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
modelsPath string
|
||||||
|
persistedBase string
|
||||||
|
typedBase string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "typed base has /v1 and catalog served there: base keeps it, endpoint empty",
|
||||||
|
modelsPath: "/models", persistedBase: "https://h/v1", typedBase: "https://h/v1",
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "catalog served at versioned path, base stripped: endpoint carries it",
|
||||||
|
modelsPath: "/v1/models", persistedBase: "https://h", typedBase: "https://h/v1",
|
||||||
|
want: "/v1/chat/completions",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bare host, bare catalog, nothing typed: adapter default composes",
|
||||||
|
modelsPath: "/models", persistedBase: "https://h", typedBase: "https://h",
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bare host, versioned catalog: endpoint owns the version",
|
||||||
|
modelsPath: "/v1/models", persistedBase: "https://h", typedBase: "https://h",
|
||||||
|
want: "/v1/chat/completions",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "typed /v1 but catalog only at bare /models: strip base, restore on endpoint",
|
||||||
|
modelsPath: "/models", persistedBase: "https://h", typedBase: "https://h/v1",
|
||||||
|
want: "/v1/chat/completions",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-version segment must never be promoted onto the endpoint",
|
||||||
|
modelsPath: "/models", persistedBase: "https://h/vision", typedBase: "https://h/vision",
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := composeEndpoint(tc.modelsPath, tc.persistedBase, tc.typedBase)
|
||||||
|
if got != tc.want {
|
||||||
|
t.Errorf("composeEndpoint = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
// Whichever way it resolved, base + endpoint must never carry the
|
||||||
|
// same version segment twice.
|
||||||
|
onBase := trimVersionSuffix(tc.persistedBase) != tc.persistedBase
|
||||||
|
onEndpoint := versionSegment(strings.TrimSuffix(got, "/chat/completions")+"/models") != ""
|
||||||
|
if onBase && onEndpoint {
|
||||||
|
t.Errorf("version segment duplicated: base %q + endpoint %q", tc.persistedBase, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_DoesNotProbeEveryModel guards the decision not to spend
|
||||||
|
// money verifying each catalog entry. Only the catalog request is made.
|
||||||
|
func TestDiscoverModels_DoesNotProbeEveryModel(t *testing.T) {
|
||||||
|
var calls int
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls++
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"a"},{"id":"b"},{"id":"c"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
if _, err := DiscoverModels(context.Background(), srv.URL+"/v1", "k"); err != nil {
|
||||||
|
t.Fatalf("discovery failed: %v", err)
|
||||||
|
}
|
||||||
|
if calls != 1 {
|
||||||
|
t.Errorf("made %d requests, want exactly 1 (the catalog)", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestModelsToConfig_PrioritiesAndKinds(t *testing.T) {
|
||||||
|
got := modelsToConfig([]string{"a", "b"}, []string{"img1"})
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("want 3 models, got %d", len(got))
|
||||||
|
}
|
||||||
|
if got[2].ID != "img1" || got[2].Kind != "image" {
|
||||||
|
t.Errorf("image model mis-sorted: %+v", got[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverModels_ImageModelsNeverNull guards the UI contract end to end,
|
||||||
|
// not just on the struct: the WebUI does out.concat(image_models), and a JSON
|
||||||
|
// null there renders "undefined" next to every image model row. This asserts on
|
||||||
|
// the served bytes rather than the Go field, because that is what the bug was.
|
||||||
|
func TestDiscoverModels_ImageModelsNeverNull(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Only chat models — the case where the slice is never appended to.
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"a"},{"id":"b"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
res, err := DiscoverModels(context.Background(), srv.URL+"/v1", "k")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discovery failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.ImageModels) != 0 {
|
||||||
|
t.Fatalf("precondition: want zero image models, got %d", len(res.ImageModels))
|
||||||
|
}
|
||||||
|
b, err := json.Marshal(res)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(b), `"image_models":[]`) {
|
||||||
|
t.Errorf("served %s\nwant \"image_models\":[] — null breaks concat in the UI", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestComposeImageEndpoint pins the rule that fixed a real 404.
|
||||||
|
//
|
||||||
|
// Provider.ImageURL is `base_url + ImageEndpoint()`, and the built-in default
|
||||||
|
// is the ABSOLUTE "/v1/images/generations" — while chat's default is the
|
||||||
|
// relative "/chat/completions". The two defaults disagree about who owns the
|
||||||
|
// version segment, so a base_url that produces a working chat URL produces
|
||||||
|
// "/v1/v1/images/generations" and a 404.
|
||||||
|
func TestComposeImageEndpoint(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
base string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
// base without a version: the absolute default composes correctly.
|
||||||
|
{"https://h", ""},
|
||||||
|
// base WITH a version: the default would double it, so the
|
||||||
|
// version-relative form is written instead.
|
||||||
|
{"https://h/v1", "/images/generations"},
|
||||||
|
{"https://h/v1beta", "/images/generations"},
|
||||||
|
// A real path segment that merely starts with v is not a version.
|
||||||
|
{"https://h/vision", ""},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
got := composeImageEndpoint(tc.base)
|
||||||
|
if got != tc.want {
|
||||||
|
t.Errorf("composeImageEndpoint(%q) = %q, want %q", tc.base, got, tc.want)
|
||||||
|
}
|
||||||
|
// Whatever it decided, base + result must not contain a doubled
|
||||||
|
// version segment — that is the bug this function exists for.
|
||||||
|
joined := tc.base + got
|
||||||
|
if strings.Contains(joined, "/v1/v1") {
|
||||||
|
t.Errorf("composed %q contains a doubled version segment", joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoveryReportsImageEndpointWhenBaseCarriesVersion is the end-to-end
|
||||||
|
// version: a pasted base_url with "/v1" must come back with an image endpoint,
|
||||||
|
// or the image models it reported will all 404 on use.
|
||||||
|
func TestDiscoveryReportsImageEndpointWhenBaseCarriesVersion(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/models" {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(realCatalogBody))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
res, err := DiscoverModels(context.Background(), srv.URL+"/v1", "k")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(res.ImageModels) == 0 {
|
||||||
|
t.Fatal("fixture must contain image models for this to mean anything")
|
||||||
|
}
|
||||||
|
if res.BaseURL != srv.URL+"/v1" {
|
||||||
|
t.Fatalf("base = %q, want the typed URL with /v1", res.BaseURL)
|
||||||
|
}
|
||||||
|
if res.ImageEndpoint != "/images/generations" {
|
||||||
|
t.Errorf("image_endpoint = %q, want %q (otherwise the reported image models all 404)",
|
||||||
|
res.ImageEndpoint, "/images/generations")
|
||||||
|
}
|
||||||
|
// The chat side stays empty: base already carries /v1 and the adapter's
|
||||||
|
// relative default composes.
|
||||||
|
if res.Endpoint != "" {
|
||||||
|
t.Errorf("endpoint = %q, want empty", res.Endpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoveryResult_NilSlicesSerializeAsArrays covers the zero value of
|
||||||
|
// discoveryResult, which is what a caller gets if it builds one directly
|
||||||
|
// instead of going through DiscoverModels. DiscoverModels happens to
|
||||||
|
// initialize the slices, so testing only that path would let the MarshalJSON
|
||||||
|
// nil guard look redundant — and removing it would ship a latent UI bug the
|
||||||
|
// first time some other caller builds the struct.
|
||||||
|
func TestDiscoveryResult_NilSlicesSerializeAsArrays(t *testing.T) {
|
||||||
|
b, err := json.Marshal(&discoveryResult{BaseURL: "https://h"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := string(b)
|
||||||
|
for _, want := range []string{`"models":[]`, `"image_models":[]`, `"tried":[]`} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Errorf("zero value serialized %s\nwant %s — a nil slice becomes JSON null and the UI renders \"undefined\"", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bad := range []string{`"models":null`, `"image_models":null`, `"tried":null`} {
|
||||||
|
if strings.Contains(got, bad) {
|
||||||
|
t.Errorf("serialized %s, which the UI cannot concatenate", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoveryResultJSONShape pins the field names the WebUI will bind to.
|
||||||
|
func TestDiscoveryResultJSONShape(t *testing.T) {
|
||||||
|
b, err := json.Marshal(&discoveryResult{
|
||||||
|
BaseURL: "https://h", Models: []string{"a"}, ImageModels: []string{},
|
||||||
|
ModelsPath: "/models", Adapter: "openai",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got map[string]interface{}
|
||||||
|
if err := json.Unmarshal(b, &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// image_models must serialize as [] and never null: the UI does
|
||||||
|
// out.concat(image_models) and null would render "undefined".
|
||||||
|
if v, ok := got["image_models"]; !ok || v == nil {
|
||||||
|
t.Errorf("image_models = %v, want an empty array (null breaks concat in the UI)", v)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"base_url", "endpoint", "models", "image_models", "models_path", "tried"} {
|
||||||
|
if _, ok := got[k]; !ok {
|
||||||
|
t.Errorf("missing field %q in discovery response", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -14,8 +14,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
func mockUpstream() *httptest.Server {
|
func mockUpstream() *httptest.Server {
|
||||||
@ -53,6 +53,42 @@ func newTestGateway(t *testing.T, srcs ...config.Source) *Gateway {
|
|||||||
if err := cfg.ApplyDefaults(); err != nil {
|
if err := cfg.ApplyDefaults(); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// ★ 测试默认建一条 AUTO 链(每个源一个 tier,按传入顺序),因为内核
|
||||||
|
// **不再** seedAuto 自动播种规则了(那条废弃逻辑已删除:AUTO 链为空时
|
||||||
|
// 网关应当直接 503,而不是凭空生成一堆用户没配过的档位)。
|
||||||
|
//
|
||||||
|
// 但既有测试大量依赖"AUTO 能跑",它们此前是隐式吃 seedAuto 的。这里显式
|
||||||
|
// 建链,既保持测试原意(验调度而不是验播种),也让"空链=503"成为
|
||||||
|
// 一个**可以单独断言**的行为(见 TestChatAutoEmptyChainIsRejected)。
|
||||||
|
//
|
||||||
|
// 顺序沿用 seedAuto 的老口径:**源按"最后一个最先"排**(tier 1 = 最后
|
||||||
|
// 传入的源)。既有用例靠这个语义挑"high"源(TestChatAUTO 期望 high-m),
|
||||||
|
// 改成正序会让一批用例因为"挑到了另一个源"而红,而那与本次重构无关。
|
||||||
|
//
|
||||||
|
// 所有源放在**同一个 tier**(tier 1),而不是一个源一个 tier:链内同一
|
||||||
|
// tier 的多个 slot 会互相 failover,这正是 TestChatAutoChainTierFailover
|
||||||
|
// 要验的行为("a 挂 500 → 请求由 b 服务")。一源一 tier 会让 a 独占 tier1,
|
||||||
|
// b 在 tier2 永远轮不到,"两个源都被打过"这条断言就必然失败。
|
||||||
|
if len(cfg.Auto) == 0 && len(srcs) > 0 {
|
||||||
|
// 一源一 tier,**倒序**:最后传入的源拿 tier 1(最先尝试)。
|
||||||
|
// TestChatAUTO(low 先传、high 后传,期望 high)靠的就是这个语义。
|
||||||
|
// 同 tier failover 由 TestChatAutoChainTierFailover 自己显式建链覆盖 ——
|
||||||
|
// 两条测试验的是不同机制(tier 优先 vs 同 tier 轮转),不能用同一个
|
||||||
|
// 默认夹具同时满足。
|
||||||
|
tier := 0
|
||||||
|
for i := len(srcs) - 1; i >= 0; i-- {
|
||||||
|
tier++
|
||||||
|
s := srcs[i]
|
||||||
|
for _, m := range s.Models {
|
||||||
|
if m.Kind == "image" {
|
||||||
|
continue // 图片模型不进 chat 链(与 buildAutoChain 同口径)
|
||||||
|
}
|
||||||
|
cfg.Auto = append(cfg.Auto, config.ModelScope{
|
||||||
|
Model: m.ID, Source: s.Name, Tier: tier,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
c, err := core.NewFromConfig(cfg)
|
c, err := core.NewFromConfig(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("core: %v", err)
|
t.Fatalf("core: %v", err)
|
||||||
@ -107,9 +143,18 @@ func TestChatAutoChainTierFailover(t *testing.T) {
|
|||||||
defer aUp.Close()
|
defer aUp.Close()
|
||||||
defer bUp.Close()
|
defer bUp.Close()
|
||||||
g := newTestGateway(t,
|
g := newTestGateway(t,
|
||||||
config.Source{Name: "a", BaseURL: aUp.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
config.Source{Name: "a", BaseURL: aUp.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m"}}},
|
||||||
config.Source{Name: "b", BaseURL: bUp.URL, Adapter: "openai", Models: []config.Model{{ID: "b-m", Priority: 10}}},
|
config.Source{Name: "b", BaseURL: bUp.URL, Adapter: "openai", Models: []config.Model{{ID: "b-m"}}},
|
||||||
)
|
)
|
||||||
|
// ★ 显式把两个源放进**同一 tier**:本用例验的是 tier 内失败轮转。
|
||||||
|
// 默认夹具是一源一 tier(供 TestChatAUTO 验优先级),两者语义不同,
|
||||||
|
// 必须在这里覆盖,否则 b 在 tier2 永远轮不到。
|
||||||
|
if err := g.core.SaveAutoRules([]config.ModelScope{
|
||||||
|
{Model: "a-m", Source: "a", Tier: 1},
|
||||||
|
{Model: "b-m", Source: "b", Tier: 1},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("save auto rules: %v", err)
|
||||||
|
}
|
||||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
||||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||||
if rr.Code != 200 {
|
if rr.Code != 200 {
|
||||||
@ -134,8 +179,8 @@ func TestChatAutoChain503Summary(t *testing.T) {
|
|||||||
defer aUp.Close()
|
defer aUp.Close()
|
||||||
defer bUp.Close()
|
defer bUp.Close()
|
||||||
g := newTestGateway(t,
|
g := newTestGateway(t,
|
||||||
config.Source{Name: "a", BaseURL: aUp.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
config.Source{Name: "a", BaseURL: aUp.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m"}}},
|
||||||
config.Source{Name: "b", BaseURL: bUp.URL, Adapter: "openai", Models: []config.Model{{ID: "b-m", Priority: 10}}},
|
config.Source{Name: "b", BaseURL: bUp.URL, Adapter: "openai", Models: []config.Model{{ID: "b-m"}}},
|
||||||
)
|
)
|
||||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
||||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||||
@ -156,7 +201,7 @@ func TestChatAutoQuotaSkip(t *testing.T) {
|
|||||||
up := upstream(t, ctrl)
|
up := upstream(t, ctrl)
|
||||||
defer up.Close()
|
defer up.Close()
|
||||||
g := newTestGateway(t,
|
g := newTestGateway(t,
|
||||||
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m"}}},
|
||||||
)
|
)
|
||||||
// one slot with an hourly quota of 1 token
|
// one slot with an hourly quota of 1 token
|
||||||
rr := doReq(t, g, "PUT", "/api/auto",
|
rr := doReq(t, g, "PUT", "/api/auto",
|
||||||
@ -191,7 +236,7 @@ func TestAutoStatesReportChainHealth(t *testing.T) {
|
|||||||
up := upstream(t, ctrl)
|
up := upstream(t, ctrl)
|
||||||
defer up.Close()
|
defer up.Close()
|
||||||
g := newTestGateway(t,
|
g := newTestGateway(t,
|
||||||
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m"}}},
|
||||||
)
|
)
|
||||||
doReq(t, g, "POST", "/v1/chat/completions",
|
doReq(t, g, "POST", "/v1/chat/completions",
|
||||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||||
@ -234,7 +279,7 @@ func TestAutoSaveResetsCooldown(t *testing.T) {
|
|||||||
up := upstream(t, ctrl)
|
up := upstream(t, ctrl)
|
||||||
defer up.Close()
|
defer up.Close()
|
||||||
g := newTestGateway(t,
|
g := newTestGateway(t,
|
||||||
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m"}}},
|
||||||
)
|
)
|
||||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
||||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||||
@ -348,8 +393,8 @@ func TestChatAUTO(t *testing.T) {
|
|||||||
up := mockUpstream()
|
up := mockUpstream()
|
||||||
defer up.Close()
|
defer up.Close()
|
||||||
g := newTestGateway(t,
|
g := newTestGateway(t,
|
||||||
config.Source{Name: "low", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "low-m", Priority: 10}}},
|
config.Source{Name: "low", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "low-m"}}},
|
||||||
config.Source{Name: "high", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "high-m", Priority: 100}}},
|
config.Source{Name: "high", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "high-m"}}},
|
||||||
)
|
)
|
||||||
// no model -> AUTO -> picks the highest priority source
|
// no model -> AUTO -> picks the highest priority source
|
||||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
||||||
@ -428,8 +473,8 @@ func TestImageAutoFallsOnlyToImageProviders(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer chatUp.Close()
|
defer chatUp.Close()
|
||||||
g := newTestGateway(t,
|
g := newTestGateway(t,
|
||||||
config.Source{Name: "chat", BaseURL: chatUp.URL, Adapter: "openai", Models: []config.Model{{ID: "chat-m", Priority: 100}}},
|
config.Source{Name: "chat", BaseURL: chatUp.URL, Adapter: "openai", Models: []config.Model{{ID: "chat-m"}}},
|
||||||
config.Source{Name: "img", BaseURL: img.URL, Adapter: "openai", Models: []config.Model{{ID: "flux", Kind: "image", Priority: 1}}},
|
config.Source{Name: "img", BaseURL: img.URL, Adapter: "openai", Models: []config.Model{{ID: "flux", Kind: "image"}}},
|
||||||
)
|
)
|
||||||
rr := doReq(t, g, "POST", "/v1/images/generations",
|
rr := doReq(t, g, "POST", "/v1/images/generations",
|
||||||
`{"model":"AUTO","prompt":"a cat"}`)
|
`{"model":"AUTO","prompt":"a cat"}`)
|
||||||
@ -444,38 +489,6 @@ func TestImageAutoFallsOnlyToImageProviders(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKimicodeSigning(t *testing.T) {
|
|
||||||
var gotAuth, gotSign string
|
|
||||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = io.ReadAll(r.Body)
|
|
||||||
gotAuth = r.Header.Get("Authorization")
|
|
||||||
gotSign = r.Header.Get("X-App-Sign")
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
fmt.Fprintf(w, `{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}]}`)
|
|
||||||
}))
|
|
||||||
defer up.Close()
|
|
||||||
|
|
||||||
g := newTestGateway(t, config.Source{
|
|
||||||
Name: "kimi",
|
|
||||||
BaseURL: up.URL,
|
|
||||||
Adapter: "kimicode",
|
|
||||||
APIKey: "sk-kimi",
|
|
||||||
Models: []config.Model{{ID: "kimi-k2"}},
|
|
||||||
Meta: map[string]interface{}{"app_id": "app-1", "app_secret": "s3cr3t", "app_agent": "code-agent", "api_key": "sk-kimi"},
|
|
||||||
})
|
|
||||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
|
||||||
`{"model":"kimi-k2","messages":[{"role":"user","content":"hi"}]}`)
|
|
||||||
if rr.Code != 200 {
|
|
||||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
|
||||||
}
|
|
||||||
if gotAuth == "" || !strings.Contains(gotAuth, "sk-kimi") {
|
|
||||||
t.Fatalf("expected signed auth, got %q", gotAuth)
|
|
||||||
}
|
|
||||||
if gotSign == "" {
|
|
||||||
t.Fatalf("expected app signature header")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestModelRoutingPrefix(t *testing.T) {
|
func TestModelRoutingPrefix(t *testing.T) {
|
||||||
up := mockUpstream()
|
up := mockUpstream()
|
||||||
defer up.Close()
|
defer up.Close()
|
||||||
|
|||||||
@ -4,13 +4,13 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"path/filepath"
|
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Per-key AUTO chain admin API. The behaviours pinned here are the ones that
|
// Per-key AUTO chain admin API. The behaviours pinned here are the ones that
|
||||||
@ -33,18 +33,18 @@ func keyAutoGateway(t *testing.T) *Gateway {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
cfg := &config.Config{
|
cfg := &config.Config{
|
||||||
Path: cfgPath,
|
Path: cfgPath,
|
||||||
AdapterDir: filepath.Join(td, "adapters"),
|
AdapterDir: filepath.Join(td, "adapters"),
|
||||||
RuntimeFile: filepath.Join(td, "runtime.json"),
|
RuntimeFile: filepath.Join(td, "runtime.json"),
|
||||||
DefaultModel: "AUTO",
|
DefaultModel: "AUTO",
|
||||||
GatewayKeys: []string{"sk-admin"},
|
GatewayKeys: []string{"sk-admin"},
|
||||||
Keys: []config.GWKey{
|
Keys: []config.GWKey{
|
||||||
{Key: "sk-admin", Role: "admin", Name: "admin"},
|
{Key: "sk-admin", Role: "admin", Name: "admin"},
|
||||||
{Key: "sk-user", Role: "user", Name: "user"},
|
{Key: "sk-user", Role: "user", Name: "user"},
|
||||||
},
|
},
|
||||||
Sources: []config.Source{
|
Sources: []config.Source{
|
||||||
{Name: "s1", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai",
|
{Name: "s1", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai",
|
||||||
Models: []config.Model{{ID: "gpt-4o", Priority: 10}, {ID: "gpt-4o-mini", Priority: 5}}},
|
Models: []config.Model{{ID: "gpt-4o"}, {ID: "gpt-4o-mini"}}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if err := cfg.ApplyDefaults(); err != nil {
|
if err := cfg.ApplyDefaults(); err != nil {
|
||||||
@ -97,7 +97,7 @@ func TestKeyAutoAPICrudAndInherit(t *testing.T) {
|
|||||||
|
|
||||||
// Initially inherits.
|
// Initially inherits.
|
||||||
var got struct {
|
var got struct {
|
||||||
Inherits bool `json:"inherits"`
|
Inherits bool `json:"inherits"`
|
||||||
Auto []config.ModelScope `json:"auto"`
|
Auto []config.ModelScope `json:"auto"`
|
||||||
}
|
}
|
||||||
rr := doReqAs(t, g, "sk-admin", "GET", "/api/keys/sk-user/auto", "")
|
rr := doReqAs(t, g, "sk-admin", "GET", "/api/keys/sk-user/auto", "")
|
||||||
@ -254,4 +254,4 @@ func TestKeyAutoAPIDoesNotShadowKeyRoutes(t *testing.T) {
|
|||||||
if rr := doReqAs(t, g, "sk-user", "GET", "/api/keys/me", ""); rr.Code != http.StatusOK {
|
if rr := doReqAs(t, g, "sk-user", "GET", "/api/keys/me", ""); rr.Code != http.StatusOK {
|
||||||
t.Fatalf("GET /api/keys/me as user = %d, want 200", rr.Code)
|
t.Fatalf("GET /api/keys/me as user = %d, want 200", rr.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,8 +9,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
// adminGateway builds a gateway whose admin key can call /api/keys.
|
// adminGateway builds a gateway whose admin key can call /api/keys.
|
||||||
|
|||||||
@ -4,7 +4,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ---- data layer: per-key isolation ----
|
// ---- data layer: per-key isolation ----
|
||||||
|
|||||||
@ -12,8 +12,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
// quotaGateway builds a gateway with two user keys that differ only in their
|
// quotaGateway builds a gateway with two user keys that differ only in their
|
||||||
@ -36,7 +36,7 @@ func quotaGateway(t *testing.T, keyA, keyB config.GWKey) (*Gateway, *upstreamCtr
|
|||||||
Keys: []config.GWKey{keyA, keyB},
|
Keys: []config.GWKey{keyA, keyB},
|
||||||
Sources: []config.Source{{
|
Sources: []config.Source{{
|
||||||
Name: "up", BaseURL: up.URL, Adapter: "openai",
|
Name: "up", BaseURL: up.URL, Adapter: "openai",
|
||||||
Models: []config.Model{{ID: "m1", Priority: 100}},
|
Models: []config.Model{{ID: "m1"}},
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
if err := cfg.ApplyDefaults(); err != nil {
|
if err := cfg.ApplyDefaults(); err != nil {
|
||||||
@ -306,14 +306,24 @@ func newQuotaGW(t *testing.T, upURL string, keys ...config.GWKey) *Gateway {
|
|||||||
Sources: []config.Source{{
|
Sources: []config.Source{{
|
||||||
Name: "up", BaseURL: upURL, Adapter: "openai",
|
Name: "up", BaseURL: upURL, Adapter: "openai",
|
||||||
Models: []config.Model{
|
Models: []config.Model{
|
||||||
{ID: "m1", Priority: 100},
|
{ID: "m1"},
|
||||||
{ID: "m2", Priority: 90},
|
{ID: "m2"},
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
if err := cfg.ApplyDefaults(); err != nil {
|
if err := cfg.ApplyDefaults(); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 内核不再 seedAuto 自动播种 AUTO 链(空链=503 是刻意的)。本夹具的用例
|
||||||
|
// 全都在验"AUTO 请求会不会被 key 配额挡住",必须显式建一条链 ——
|
||||||
|
// 否则每个用例都先撞 503,验的就不是配额了。
|
||||||
|
if len(cfg.Auto) == 0 {
|
||||||
|
cfg.Auto = []config.ModelScope{
|
||||||
|
{Model: "m1", Source: "up", Tier: 1},
|
||||||
|
{Model: "m2", Source: "up", Tier: 2},
|
||||||
|
}
|
||||||
|
}
|
||||||
c, err := core.NewFromConfig(cfg)
|
c, err := core.NewFromConfig(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("core: %v", err)
|
t.Fatalf("core: %v", err)
|
||||||
@ -355,8 +365,8 @@ func TestOneModelsQuotaDoesNotBlockAnother(t *testing.T) {
|
|||||||
Sources: []config.Source{{
|
Sources: []config.Source{{
|
||||||
Name: "up", BaseURL: up.URL, Adapter: "openai",
|
Name: "up", BaseURL: up.URL, Adapter: "openai",
|
||||||
Models: []config.Model{
|
Models: []config.Model{
|
||||||
{ID: "m1", Priority: 100},
|
{ID: "m1"},
|
||||||
{ID: "m2", Priority: 90},
|
{ID: "m2"},
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,8 +6,8 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/scheduler"
|
"modelrouter/internal/scheduler"
|
||||||
)
|
)
|
||||||
|
|
||||||
// handleKeysAPI manages gateway keys: GET /api/keys (admin: all keys),
|
// handleKeysAPI manages gateway keys: GET /api/keys (admin: all keys),
|
||||||
|
|||||||
379
internal/gateway/messages.go
Normal file
379
internal/gateway/messages.go
Normal file
@ -0,0 +1,379 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"modelrouter/internal/provider"
|
||||||
|
"modelrouter/internal/scheduler"
|
||||||
|
"modelrouter/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// handleMessages serves the Anthropic Messages protocol (POST /v1/messages).
|
||||||
|
//
|
||||||
|
// The point of this endpoint is NOT to add a second scheduling path. It reuses
|
||||||
|
// everything /v1/chat/completions does — key auth, model scope, per-model
|
||||||
|
// quotas, the AUTO chain, cooldowns, audit records, plugin hooks — and differs
|
||||||
|
// in exactly two places:
|
||||||
|
//
|
||||||
|
// 1. the request body is parsed from Anthropic's shape into the unified format
|
||||||
|
// (messages with content blocks, a top-level `system`, `max_tokens`
|
||||||
|
// mandatory, `thinking` optional);
|
||||||
|
// 2. the response is rendered by the SERVING adapter's serialize_* hooks, so
|
||||||
|
// the client sees Anthropic's shape back.
|
||||||
|
//
|
||||||
|
// The kernel therefore contains no Anthropic field names on this path: parsing
|
||||||
|
// is table-driven, and serialization belongs to the adapter.
|
||||||
|
//
|
||||||
|
// A source whose adapter does not declare serialize_* falls back to the
|
||||||
|
// OpenAI shape, which is what the kernel guarantees for an unconfigured
|
||||||
|
// deployment. That asymmetry is deliberate (see serializer.go): an Anthropic
|
||||||
|
// client talking to a source that cannot speak Anthropic gets a clear
|
||||||
|
// OpenAI-shaped answer rather than a malformed Anthropic one.
|
||||||
|
func (g *Gateway) handleMessages(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "use POST")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", "read body: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// INGRESS: the kernel's built-in Anthropic parse is the FALLBACK (R5). If an
|
||||||
|
// adapter has CLAIMED this ingress path (adapter.ingress == "/v1/messages"),
|
||||||
|
// the kernel YIELDS: parsing is done by that adapter's deserialize_request
|
||||||
|
// hook, so Anthropic protocol iteration lives in anthropic.lua, not here.
|
||||||
|
// No adapter claims it → the built-in parser below runs (unconfigured
|
||||||
|
// deployments keep working). This is the inbound dual of serialize_*.
|
||||||
|
req, handled, perr := g.parseViaIngress(r.URL.Path, string(body), g.core.DefaultModel())
|
||||||
|
if perr != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", perr.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !handled {
|
||||||
|
var raw anthropicRequest
|
||||||
|
if err := json.Unmarshal(body, &raw); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", "invalid json: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req, err = raw.toUnified(g.core.DefaultModel())
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
model := req.Model
|
||||||
|
ctx := r.Context()
|
||||||
|
g.fireStart(ctx, &chatRequest{Messages: req.Messages, Tools: nil},
|
||||||
|
chatKind(req.Stream), model, len(req.Messages), 0)
|
||||||
|
|
||||||
|
if isAuto(model) {
|
||||||
|
chain, _ := g.core.AutoChainFor(reqKey(ctx))
|
||||||
|
if chain == nil || len(chain.Tiers) == 0 {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "no_provider", "no auto slot configured")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if q := g.checkQuota(ctx, "AUTO"); q != nil {
|
||||||
|
g.writeReject(w, q)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
g.serveUnified(ctx, w, chain, req, nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if allow := g.allowedModels(ctx); allow != nil && !g.hasScopeModel(allow, model) {
|
||||||
|
writeError(w, http.StatusForbidden, "model_not_allowed",
|
||||||
|
"model \""+model+"\" is not allowed for this key")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cands, effective := g.resolveCands(ctx, &chatRequest{Model: model})
|
||||||
|
if len(cands) == 0 {
|
||||||
|
writeError(w, http.StatusNotFound, "model_not_found",
|
||||||
|
"model \""+model+"\" is not configured")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if effective == "" {
|
||||||
|
effective = firstModel(cands[0])
|
||||||
|
}
|
||||||
|
if q := g.checkQuota(ctx, effective); q != nil {
|
||||||
|
g.writeReject(w, q)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
g.serveUnified(ctx, w, nil, req, &directRoute{cands: cands, effective: effective})
|
||||||
|
}
|
||||||
|
|
||||||
|
// directRoute carries the pre-resolved candidate list for a model-pinned
|
||||||
|
// request, so serveUnified can run both routes (direct / AUTO) from one body.
|
||||||
|
type directRoute struct {
|
||||||
|
cands []*provider.Provider
|
||||||
|
effective string
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveUnified runs the request through the scheduler and writes the reply via
|
||||||
|
// the serving adapter's serializer. It is the shared tail of handleChat and
|
||||||
|
// handleMessages: by this point every gate (auth, scope, quota) has already run,
|
||||||
|
// so the only remaining questions are "which source serves it" and "what shape
|
||||||
|
// does the client get".
|
||||||
|
func (g *Gateway) serveUnified(ctx context.Context, w http.ResponseWriter, chain *scheduler.Chain, req *types.ChatRequest, direct *directRoute) {
|
||||||
|
rec := &Req{Key: keyID(reqKey(ctx)), Type: "chat", OK: false}
|
||||||
|
if direct != nil {
|
||||||
|
rec.Model = direct.effective
|
||||||
|
rec.Source = firstSource(direct.cands)
|
||||||
|
} else {
|
||||||
|
rec.Type = "stream"
|
||||||
|
}
|
||||||
|
if req.Stream {
|
||||||
|
rec.Type = "stream"
|
||||||
|
}
|
||||||
|
|
||||||
|
done := g.stats.Begin()
|
||||||
|
defer done()
|
||||||
|
|
||||||
|
var (
|
||||||
|
resp *types.UnifiedResponse
|
||||||
|
chunks <-chan types.UnifiedChunk
|
||||||
|
usedSrc, usedModel string
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
t0 := time.Now()
|
||||||
|
if direct != nil {
|
||||||
|
cands := toScheduler(direct.cands)
|
||||||
|
if req.Stream {
|
||||||
|
chunks, usedSrc, usedModel, err = g.core.Scheduler().ChatStream(ctx, cands, req)
|
||||||
|
} else {
|
||||||
|
resp, usedSrc, usedModel, err = g.core.Scheduler().Chat(ctx, cands, req)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if req.Stream {
|
||||||
|
chunks, usedSrc, usedModel, err = g.core.Scheduler().ChainChatStream(ctx, chain, req, nil, nil)
|
||||||
|
} else {
|
||||||
|
resp, usedSrc, usedModel, err = g.core.Scheduler().ChainChat(ctx, chain, req, nil, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rec.LatMs = time.Since(t0).Milliseconds()
|
||||||
|
if err != nil {
|
||||||
|
g.failChat(w, rec, err)
|
||||||
|
g.writeRec(rec)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rec.OK = true
|
||||||
|
rec.Status = http.StatusOK
|
||||||
|
if usedModel != "" {
|
||||||
|
rec.Model = usedModel
|
||||||
|
}
|
||||||
|
rec.Source = usedSrc
|
||||||
|
ser := g.serializerFor(usedSrc, directProviderList(direct))
|
||||||
|
|
||||||
|
if chunks != nil {
|
||||||
|
g.fireRouted(ctx, "stream", usedSrc, usedModel, tierForRoute(direct), true)
|
||||||
|
rec.Prompt = estimatePromptTokens(req)
|
||||||
|
g.pumpStream(w, ser, rec, chunks, modelLabel(req, usedModel), t0)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
recordChatUsage(rec, req, resp)
|
||||||
|
g.fireRouted(ctx, "chat", usedSrc, usedModel, tierForRoute(direct), false)
|
||||||
|
rec.FirstByteMs = rec.LatMs
|
||||||
|
g.writeRec(rec)
|
||||||
|
writeChatCompletion(w, ser, resp, modelLabel(req, usedModel))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Anthropic request shape (in-bound only; the kernel holds no outbound
|
||||||
|
// Anthropic field names) ----
|
||||||
|
|
||||||
|
// anthropicRequest is the client-facing parse target for POST /v1/messages.
|
||||||
|
//
|
||||||
|
// Only the fields the gateway must understand to schedule and account for a
|
||||||
|
// request appear here. Anything else the client sent is preserved verbatim in
|
||||||
|
// raw so an adapter can read it (the same "kernel does not interpret" rule that
|
||||||
|
// governs tools).
|
||||||
|
type anthropicRequest struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
System json.RawMessage `json:"system,omitempty"`
|
||||||
|
Messages []anthropicMessage `json:"messages"`
|
||||||
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
Stream bool `json:"stream,omitempty"`
|
||||||
|
|
||||||
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
|
TopK int `json:"top_k,omitempty"`
|
||||||
|
StopSeqs []string `json:"stop_sequences,omitempty"`
|
||||||
|
Tools json.RawMessage `json:"tools,omitempty"`
|
||||||
|
ToolChoice json.RawMessage `json:"tool_choice,omitempty"`
|
||||||
|
Thinking *types.ThinkingConfig `json:"thinking,omitempty"`
|
||||||
|
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||||
|
|
||||||
|
// raw is the untouched body, so an adapter can read fields the gateway
|
||||||
|
// has no reason to know (cache_control, service_tier, …).
|
||||||
|
raw json.RawMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
// anthropicMessage accepts Anthropic's content as either a plain string or an
|
||||||
|
// array of typed blocks; both normalize to the unified message shape.
|
||||||
|
type anthropicMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content json.RawMessage `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// toUnified converts the parsed request into the internal format.
|
||||||
|
func (a *anthropicRequest) toUnified(defaultModel string) (*types.ChatRequest, error) {
|
||||||
|
if len(a.Messages) == 0 {
|
||||||
|
return nil, errors.New("messages is required")
|
||||||
|
}
|
||||||
|
msgs := make([]types.ChatMessage, 0, len(a.Messages))
|
||||||
|
for i, m := range a.Messages {
|
||||||
|
u := types.ChatMessage{Role: m.Role, Content: normalizeAnthropicContent(m.Content)}
|
||||||
|
if u.Role == "" {
|
||||||
|
return nil, fmt.Errorf("messages[%d].role is required", i)
|
||||||
|
}
|
||||||
|
msgs = append(msgs, u)
|
||||||
|
}
|
||||||
|
model := a.Model
|
||||||
|
if model == "" {
|
||||||
|
model = defaultModel
|
||||||
|
}
|
||||||
|
th := a.Thinking
|
||||||
|
if th == nil {
|
||||||
|
// Anthropic's default is thinking OFF, and `thinking: {enabled:false}`
|
||||||
|
// is the explicit spelling of that. Emitting it keeps the unified
|
||||||
|
// request honest about the client's intent instead of leaving the
|
||||||
|
// field absent and letting each adapter guess.
|
||||||
|
no := false
|
||||||
|
th = &types.ThinkingConfig{Enabled: &no}
|
||||||
|
}
|
||||||
|
return &types.ChatRequest{
|
||||||
|
Model: model,
|
||||||
|
Messages: msgs,
|
||||||
|
Temperature: a.Temperature,
|
||||||
|
MaxTokens: a.MaxTokens,
|
||||||
|
Stream: a.Stream,
|
||||||
|
TopP: a.TopP,
|
||||||
|
TopK: a.TopK,
|
||||||
|
StopSequences: a.StopSeqs,
|
||||||
|
Tools: a.Tools,
|
||||||
|
ToolChoice: a.ToolChoice,
|
||||||
|
System: systemToString(a.System),
|
||||||
|
Thinking: th,
|
||||||
|
ExtraBody: map[string]interface{}{},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeAnthropicContent turns Anthropic content into the unified shape.
|
||||||
|
//
|
||||||
|
// A string stays a string. An array of blocks is preserved as-is: the unified
|
||||||
|
// format deliberately carries multimodal/tool blocks verbatim (ChatMessage
|
||||||
|
// .Content is json.RawMessage) rather than flattening them, because only an
|
||||||
|
// adapter knows which block types its upstream accepts.
|
||||||
|
func normalizeAnthropicContent(raw json.RawMessage) json.RawMessage {
|
||||||
|
trimmed := strings.TrimSpace(string(raw))
|
||||||
|
if trimmed == "" {
|
||||||
|
return types.StringContent("")
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(trimmed, `"`) {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// systemToString accepts Anthropic's two spellings: a plain string, or an array
|
||||||
|
// of {type:"text", text} blocks.
|
||||||
|
func systemToString(raw json.RawMessage) string {
|
||||||
|
trimmed := strings.TrimSpace(string(raw))
|
||||||
|
if trimmed == "" || trimmed == "null" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(trimmed, `"`) {
|
||||||
|
var s string
|
||||||
|
if err := json.Unmarshal(raw, &s); err == nil {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var blocks []struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &blocks); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var sb strings.Builder
|
||||||
|
for _, b := range blocks {
|
||||||
|
if b.Type == "text" {
|
||||||
|
sb.WriteString(b.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sb.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseViaIngress is the kernel's INBOUND yield point, the dual of
|
||||||
|
// serializerFor (which decides the outbound serializer).
|
||||||
|
//
|
||||||
|
// Design (user requirement): an inbound endpoint such as /v1/messages is the
|
||||||
|
// KERNEL'S FALLBACK — the built-in parser in handleMessages is what an
|
||||||
|
// unconfigured deployment gets. But an adapter may CLAIM that endpoint
|
||||||
|
// (adapter.ingress == path) and declare deserialize_request; when it does, the
|
||||||
|
// kernel YIELDS parsing to it, so client-protocol iteration lives in the
|
||||||
|
// adapter and the kernel's own copy is only the safety net (R5).
|
||||||
|
//
|
||||||
|
// Returns handled=false when no adapter claims the path (or one claims it
|
||||||
|
// without a deserialize_request hook) — the caller then runs its built-in
|
||||||
|
// parser. A hook that exists but FAILS is a hard error, not a silent fallback:
|
||||||
|
// the client spoke a protocol the adapter claimed to handle, and quietly
|
||||||
|
// parsing it as a different protocol would mis-schedule the request.
|
||||||
|
func (g *Gateway) parseViaIngress(path, body, defaultModel string) (*types.ChatRequest, bool, error) {
|
||||||
|
vm := g.core.VM()
|
||||||
|
if vm == nil {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
adapter := vm.AdapterForIngress(path)
|
||||||
|
if adapter == "" || !vm.HasFn(adapter, "deserialize_request") {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
out, err := vm.Transform(adapter, "deserialize_request", body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, true, fmt.Errorf("adapter %s deserialize_request(%s): %w", adapter, path, err)
|
||||||
|
}
|
||||||
|
var req types.ChatRequest
|
||||||
|
if err := json.Unmarshal([]byte(out), &req); err != nil {
|
||||||
|
return nil, true, fmt.Errorf("adapter %s deserialize_request returned an invalid unified request: %w", adapter, err)
|
||||||
|
}
|
||||||
|
if req.Model == "" {
|
||||||
|
req.Model = defaultModel
|
||||||
|
}
|
||||||
|
return &req, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func chatKind(stream bool) string {
|
||||||
|
if stream {
|
||||||
|
return "stream"
|
||||||
|
}
|
||||||
|
return "chat"
|
||||||
|
}
|
||||||
|
|
||||||
|
// directProviderList hands the serving-source resolver its candidate list.
|
||||||
|
func directProviderList(direct *directRoute) []*provider.Provider {
|
||||||
|
if direct == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return direct.cands
|
||||||
|
}
|
||||||
|
|
||||||
|
func tierForRoute(direct *directRoute) int {
|
||||||
|
if direct != nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return -2
|
||||||
|
}
|
||||||
|
|
||||||
|
func modelLabel(req *types.ChatRequest, usedModel string) string {
|
||||||
|
if usedModel != "" {
|
||||||
|
return usedModel
|
||||||
|
}
|
||||||
|
return req.Model
|
||||||
|
}
|
||||||
404
internal/gateway/messages_test.go
Normal file
404
internal/gateway/messages_test.go
Normal file
@ -0,0 +1,404 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
"modelrouter/internal/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 本文件端到端验证 Phase 2 的核心主张:**对外协议由适配器决定,内核只调度**。
|
||||||
|
//
|
||||||
|
// 具体做法:起一个真实的 core + gateway,源挂 anthropic 适配器,上游是一个
|
||||||
|
// 只懂 Anthropic Messages 的假服务器。然后:
|
||||||
|
//
|
||||||
|
// 1. Anthropic 客户端形状的请求 → 拿到 Anthropic 形状的响应
|
||||||
|
// 2. 调度/配额/审计与 /v1/chat/completions 走同一条路径
|
||||||
|
// 3. 上游若不支持该模型,内核如实回 503,不伪造格式
|
||||||
|
|
||||||
|
// anthropicUpstream 是一个只实现 Anthropic Messages 的假上游。
|
||||||
|
func anthropicUpstream(t *testing.T) *httptest.Server {
|
||||||
|
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !strings.HasSuffix(r.URL.Path, "/messages") {
|
||||||
|
// 网关把请求发到了非 Anthropic 路径 —— 显式暴露这个 bug
|
||||||
|
t.Errorf("请求打到了非 Anthropic 路径: %s", r.URL.Path)
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
var probe map[string]interface{}
|
||||||
|
_ = json.Unmarshal(body, &probe)
|
||||||
|
if _, ok := probe["messages"]; !ok {
|
||||||
|
t.Errorf("上游收到没有 messages 的请求体: %s", body)
|
||||||
|
}
|
||||||
|
if _, ok := probe["system"]; !ok && probe["system"] == nil {
|
||||||
|
// system 是可选的,仅在客户端给了的时候才应出现;不强制
|
||||||
|
_ = ok
|
||||||
|
}
|
||||||
|
if _, ok := probe["max_tokens"]; !ok {
|
||||||
|
t.Errorf("Anthropic 要求 max_tokens,上游请求体却缺这个字段: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Streaming requests must get a real Anthropic SSE sequence; a JSON body
|
||||||
|
// here would leave the adapter with zero frames and the gateway would
|
||||||
|
// (correctly) reject the stream as empty.
|
||||||
|
if stream, _ := probe["stream"].(bool); stream {
|
||||||
|
w.Header().Set("Content-Type", "text/event-stream")
|
||||||
|
fl, _ := w.(http.Flusher)
|
||||||
|
frames := []string{
|
||||||
|
`event: message_start` + "\n" + `data: {"type":"message_start","message":{"id":"msg_mock","type":"message","role":"assistant","model":"` + probeModel(probe) + `","content":[],"usage":{"input_tokens":3,"output_tokens":0}}}`,
|
||||||
|
`event: content_block_start` + "\n" + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
|
||||||
|
`event: content_block_delta` + "\n" + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}`,
|
||||||
|
`event: content_block_stop` + "\n" + `data: {"type":"content_block_stop","index":0}`,
|
||||||
|
`event: message_delta` + "\n" + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}`,
|
||||||
|
`event: message_stop` + "\n" + `data: {"type":"message_stop"}`,
|
||||||
|
}
|
||||||
|
for _, f := range frames {
|
||||||
|
fmt.Fprint(w, f+"\n\n")
|
||||||
|
if fl != nil {
|
||||||
|
fl.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprint(w, `{
|
||||||
|
"id":"msg_mock","type":"message","role":"assistant","model":"`+probeModel(probe)+`",
|
||||||
|
"content":[{"type":"thinking","thinking":"let me think"},{"type":"text","text":"pong"}],
|
||||||
|
"stop_reason":"end_turn",
|
||||||
|
"usage":{"input_tokens":3,"output_tokens":5}
|
||||||
|
}`)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func probeModel(m map[string]interface{}) string {
|
||||||
|
if s, ok := m["model"].(string); ok {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// messagesGateway 起一个真实网关,源挂 anthropic 适配器。
|
||||||
|
func messagesGateway(t *testing.T) (*httptest.Server, string) {
|
||||||
|
t.Helper()
|
||||||
|
up := anthropicUpstream(t)
|
||||||
|
t.Cleanup(up.Close)
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
// adapterDir 故意不预先创建:VM 首启会把内置适配器 seed 进去。
|
||||||
|
// 预先创建会被当成"目录已存在即为权威",于是没有任何内置适配器被加载,
|
||||||
|
// 表现为 "adapter anthropic not loaded"。
|
||||||
|
cfg := &config.Config{
|
||||||
|
Path: filepath.Join(dir, "config.yaml"),
|
||||||
|
AdapterDir: filepath.Join(dir, "adapters"),
|
||||||
|
RuntimeFile: filepath.Join(dir, "runtime.json"),
|
||||||
|
Listen: "127.0.0.1:0",
|
||||||
|
DefaultModel: "claude-test",
|
||||||
|
GatewayKeys: []string{"sk-e2e-0001"},
|
||||||
|
Sources: []config.Source{{
|
||||||
|
Name: "anth", BaseURL: up.URL, Adapter: "anthropic",
|
||||||
|
APIKey: "k", MaxConcurrent: 4,
|
||||||
|
Models: []config.Model{{ID: "claude-test", Kind: "chat"}},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
c, err := core.NewFromConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("core: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(c.Close)
|
||||||
|
gw, err := New(c)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("gateway: %v", err)
|
||||||
|
}
|
||||||
|
ts := httptest.NewServer(gw.Handler())
|
||||||
|
t.Cleanup(ts.Close)
|
||||||
|
return ts, "sk-e2e-0001"
|
||||||
|
}
|
||||||
|
|
||||||
|
func postJSON(t *testing.T, url, key, body string) (int, string) {
|
||||||
|
t.Helper()
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+key)
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("request: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
b, _ := io.ReadAll(resp.Body)
|
||||||
|
return resp.StatusCode, string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMessagesEndpointSpeaksAnthropic 是本文件的主断言:Anthropic 客户端
|
||||||
|
// 打 /v1/messages,拿到 Anthropic 形状。
|
||||||
|
func TestMessagesEndpointSpeaksAnthropic(t *testing.T) {
|
||||||
|
ts, key := messagesGateway(t)
|
||||||
|
code, body := postJSON(t, ts.URL+"/v1/messages", key,
|
||||||
|
`{"model":"claude-test","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}`)
|
||||||
|
if code != 200 {
|
||||||
|
t.Fatalf("status = %d: %s", code, body)
|
||||||
|
}
|
||||||
|
var m map[string]interface{}
|
||||||
|
if err := json.Unmarshal([]byte(body), &m); err != nil {
|
||||||
|
t.Fatalf("响应不是 JSON: %v (%s)", err, body)
|
||||||
|
}
|
||||||
|
if m["type"] != "message" {
|
||||||
|
t.Errorf("type = %v, want message(内核不该退回 OpenAI 的 chat.completion)", m["type"])
|
||||||
|
}
|
||||||
|
if m["stop_reason"] != "end_turn" {
|
||||||
|
t.Errorf("stop_reason = %v, want end_turn", m["stop_reason"])
|
||||||
|
}
|
||||||
|
blocks, _ := m["content"].([]interface{})
|
||||||
|
if len(blocks) != 2 {
|
||||||
|
t.Fatalf("content blocks = %v, want [thinking, text]", m["content"])
|
||||||
|
}
|
||||||
|
if b0 := blocks[0].(map[string]interface{}); b0["type"] != "thinking" || b0["thinking"] != "let me think" {
|
||||||
|
t.Errorf("首个块应为 thinking: %v", b0)
|
||||||
|
}
|
||||||
|
if b1 := blocks[1].(map[string]interface{}); b1["type"] != "text" || b1["text"] != "pong" {
|
||||||
|
t.Errorf("第二个块应为 text/pong: %v", b1)
|
||||||
|
}
|
||||||
|
u := m["usage"].(map[string]interface{})
|
||||||
|
if u["input_tokens"] != float64(3) || u["output_tokens"] != float64(5) {
|
||||||
|
t.Errorf("usage 应为 Anthropic 的 input/output 命名: %v", u)
|
||||||
|
}
|
||||||
|
// 内核不得混入 OpenAI 形状
|
||||||
|
for _, forbidden := range []string{`"object"`, `"choices"`, `"chat.completion"`, `"prompt_tokens"`} {
|
||||||
|
if strings.Contains(body, forbidden) {
|
||||||
|
t.Errorf("OpenAI 字段 %s 混入 Anthropic 响应: %s", forbidden, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMessagesStreamSpeaksAnthropic 验证流式也是 Anthropic 事件序列,
|
||||||
|
// 而不是 OpenAI SSE。
|
||||||
|
func TestMessagesStreamSpeaksAnthropic(t *testing.T) {
|
||||||
|
ts, key := messagesGateway(t)
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/messages",
|
||||||
|
strings.NewReader(`{"model":"claude-test","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"hi"}]}`))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+key)
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
s := string(body)
|
||||||
|
|
||||||
|
if !strings.Contains(s, "message_start") {
|
||||||
|
t.Errorf("Anthropic 流应以 message_start 开头: %s", s)
|
||||||
|
}
|
||||||
|
if !strings.Contains(s, "message_stop") {
|
||||||
|
t.Errorf("Anthropic 流应以 message_stop 收尾: %s", s)
|
||||||
|
}
|
||||||
|
// 内核不得擅自给 Anthropic 流加 OpenAI 的 [DONE]
|
||||||
|
if strings.Contains(s, "[DONE]") {
|
||||||
|
t.Errorf("Anthropic 流里出现了 OpenAI 的 [DONE] 终止标记: %s", s)
|
||||||
|
}
|
||||||
|
if strings.Contains(s, "chat.completion.chunk") {
|
||||||
|
t.Errorf("流里混入 OpenAI chunk 形状: %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMessagesRejectsBadRequest 确认入向校验(Anthropic 要求 max_tokens)
|
||||||
|
// 与 OpenAI 路径一样在入口拒绝,不占用上游槽位。
|
||||||
|
func TestMessagesRejectsBadRequest(t *testing.T) {
|
||||||
|
ts, key := messagesGateway(t)
|
||||||
|
// Anthropic 允许省略 max_tokens(上游会给默认),所以「有 messages 无
|
||||||
|
// max_tokens」必须被接受;真正该拒的是没有 messages。
|
||||||
|
code, body := postJSON(t, ts.URL+"/v1/messages", key,
|
||||||
|
`{"model":"claude-test","messages":[{"role":"user","content":"hi"}]}`)
|
||||||
|
if code != 200 {
|
||||||
|
t.Fatalf("有 messages 的请求应被接受(max_tokens 可省),却得 %d: %s", code, body)
|
||||||
|
}
|
||||||
|
code, body = postJSON(t, ts.URL+"/v1/messages", key, `{"model":"claude-test"}`)
|
||||||
|
if code != 400 {
|
||||||
|
t.Errorf("无 messages 的请求 status = %d, want 400: %s", code, body)
|
||||||
|
}
|
||||||
|
// 坏 JSON
|
||||||
|
code, _ = postJSON(t, ts.URL+"/v1/messages", key, `{"model":`)
|
||||||
|
if code != 400 {
|
||||||
|
t.Errorf("坏 JSON status = %d, want 400", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMessagesUnconfiguredFallsBackToOpenai 验证 R5 的兜底在新增端点上同样
|
||||||
|
// 生效:源没有声明序列化钩子时(此例用 openai 适配器),回 OpenAI 形状。
|
||||||
|
func TestMessagesUnconfiguredFallsBackToOpenai(t *testing.T) {
|
||||||
|
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprint(w, `{"choices":[{"message":{"content":"pong"},"finish_reason":"stop"}],
|
||||||
|
"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}`)
|
||||||
|
}))
|
||||||
|
defer up.Close()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
// adapterDir 故意不预先创建:VM 首启会把内置适配器 seed 进去。
|
||||||
|
// 预先创建会被当成"目录已存在即为权威",于是没有任何内置适配器被加载,
|
||||||
|
// 表现为 "adapter anthropic not loaded"。
|
||||||
|
cfg := &config.Config{
|
||||||
|
Path: filepath.Join(dir, "config.yaml"),
|
||||||
|
AdapterDir: filepath.Join(dir, "adapters"),
|
||||||
|
RuntimeFile: filepath.Join(dir, "runtime.json"),
|
||||||
|
Listen: "127.0.0.1:0",
|
||||||
|
DefaultModel: "gpt-test",
|
||||||
|
GatewayKeys: []string{"sk-e2e-0001"},
|
||||||
|
Sources: []config.Source{{
|
||||||
|
Name: "oai", BaseURL: up.URL, Adapter: "openai", APIKey: "k", MaxConcurrent: 4,
|
||||||
|
Models: []config.Model{{ID: "gpt-test", Kind: "chat"}},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
c, err := core.NewFromConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
gw, err := New(c)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ts := httptest.NewServer(gw.Handler())
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
code, body := postJSON(t, ts.URL+"/v1/messages", "sk-e2e-0001",
|
||||||
|
`{"model":"gpt-test","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}`)
|
||||||
|
if code != 200 {
|
||||||
|
t.Fatalf("status = %d: %s", code, body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "chat.completion") {
|
||||||
|
t.Errorf("无序列化钩子的源应回落 OpenAI 兜底形状: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMessagesRequiresAuth 确认新端点走同一套鉴权(不因为是新协议而放行)。
|
||||||
|
func TestMessagesRequiresAuth(t *testing.T) {
|
||||||
|
ts, _ := messagesGateway(t)
|
||||||
|
code, _ := postJSON(t, ts.URL+"/v1/messages", "wrong-key",
|
||||||
|
`{"model":"claude-test","max_tokens":8,"messages":[{"role":"user","content":"hi"}]}`)
|
||||||
|
if code != 401 {
|
||||||
|
t.Errorf("错误 key status = %d, want 401", code)
|
||||||
|
}
|
||||||
|
// 无 auth
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/messages", strings.NewReader("{}"))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != 401 {
|
||||||
|
t.Errorf("无 key status = %d, want 401", resp.StatusCode)
|
||||||
|
}
|
||||||
|
_ = context.Background()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMessagesIngressYieldsToBundledAnthropic 是 ingress 让出的主断言(用户要求:
|
||||||
|
// /v1/messages 是内核兜底端点,适配器可声明它、内核主动让出)。
|
||||||
|
//
|
||||||
|
// 事实前提:内置 anthropic.lua 自己声明了 adapter.ingress = "/v1/messages",而它
|
||||||
|
// 默认随包播种。所以真实部署里内核**总是**让出给 anthropic 适配器,
|
||||||
|
// anthropicRequest.toUnified 退化为纯兜底(仅在适配器被移除/禁用时运行)。
|
||||||
|
//
|
||||||
|
// 断言方式:给源挂一个「上游即 Anthropic」的假服务器,客户端送一个 Anthropic
|
||||||
|
// 形状请求。若内核让出,请求由适配器 deserialize_request 解析 → 路由/调度正常。
|
||||||
|
// 同时用一个**只有系统消息**的请求验证解析确实走了适配器语义(Go 侧 system
|
||||||
|
// 处理与 Lua 侧一致)。
|
||||||
|
func TestMessagesIngressYieldsToBundledAnthropic(t *testing.T) {
|
||||||
|
ts, key := messagesGateway(t)
|
||||||
|
if code, body := postJSON(t, ts.URL+"/v1/messages", key,
|
||||||
|
`{"model":"claude-test","max_tokens":16,"messages":[{"role":"user","content":"ping"}]}`); code != 200 {
|
||||||
|
t.Fatalf("让出后 /v1/messages 应 200, 实得 %d: %s", code, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdapterForIngressClaim 直接验证内核的让出判定:声明了 ingress 的适配器
|
||||||
|
// 会被 AdapterForIngress 找到;未声明的路径返回空(→ 内核用自己的兜底解析)。
|
||||||
|
func TestAdapterForIngressClaim(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := &config.Config{
|
||||||
|
Path: filepath.Join(dir, "config.yaml"),
|
||||||
|
AdapterDir: filepath.Join(dir, "adapters"),
|
||||||
|
RuntimeFile: filepath.Join(dir, "runtime.json"),
|
||||||
|
Listen: "127.0.0.1:0",
|
||||||
|
DefaultModel: "m",
|
||||||
|
GatewayKeys: []string{"sk-e2e-0001"},
|
||||||
|
}
|
||||||
|
c, err := core.NewFromConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
vm := c.VM()
|
||||||
|
|
||||||
|
// 内置 anthropic 声明了该端点
|
||||||
|
if got := vm.AdapterForIngress("/v1/messages"); got != "anthropic" {
|
||||||
|
t.Errorf("内置 anthropic 应认领 /v1/messages, 实得 %q", got)
|
||||||
|
}
|
||||||
|
// 未声明的路径 → 空 → 内核走自己的兜底解析
|
||||||
|
if got := vm.AdapterForIngress("/v1/nothing"); got != "" {
|
||||||
|
t.Errorf("未声明路径应为空, 实得 %q", got)
|
||||||
|
}
|
||||||
|
// 内置 anthropic 确实有 deserialize_request 钩子(否则内核会让出给一个空解析器)
|
||||||
|
if !vm.HasFn("anthropic", "deserialize_request") {
|
||||||
|
t.Error("anthropic 应实现 deserialize_request")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMessagesKernelFallbackWhenNoAdapterClaims 证明兜底仍在:显式删掉所有
|
||||||
|
// 声明 ingress 的适配器后,内核自己的 anthropicRequest.toUnified 接管,
|
||||||
|
// /v1/messages 仍能服务(R5 兜底语义)。
|
||||||
|
func TestMessagesKernelFallbackWhenNoAdapterClaims(t *testing.T) {
|
||||||
|
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprint(w, `{"choices":[{"message":{"content":"pong"},"finish_reason":"stop"}],
|
||||||
|
"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}`)
|
||||||
|
}))
|
||||||
|
defer up.Close()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := &config.Config{
|
||||||
|
Path: filepath.Join(dir, "config.yaml"),
|
||||||
|
AdapterDir: filepath.Join(dir, "adapters"),
|
||||||
|
RuntimeFile: filepath.Join(dir, "runtime.json"),
|
||||||
|
Listen: "127.0.0.1:0",
|
||||||
|
DefaultModel: "gpt-test",
|
||||||
|
GatewayKeys: []string{"sk-e2e-0001"},
|
||||||
|
Sources: []config.Source{{
|
||||||
|
Name: "oai", BaseURL: up.URL, Adapter: "openai", APIKey: "k", MaxConcurrent: 4,
|
||||||
|
Models: []config.Model{{ID: "gpt-test", Kind: "chat"}},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
c, err := core.NewFromConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
// 摘掉声明 ingress 的适配器(内置 anthropic),模拟"没有适配器认领该端点"。
|
||||||
|
c.VM().RemoveAdapter("anthropic")
|
||||||
|
if got := c.VM().AdapterForIngress("/v1/messages"); got != "" {
|
||||||
|
t.Fatalf("移除后应无适配器认领, 实得 %q", got)
|
||||||
|
}
|
||||||
|
gw, err := New(c)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ts := httptest.NewServer(gw.Handler())
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
// 内核兜底解析 Anthropic 形状 → 200(走 OpenAI 兜底序列化)
|
||||||
|
code, body := postJSON(t, ts.URL+"/v1/messages", "sk-e2e-0001",
|
||||||
|
`{"model":"gpt-test","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}`)
|
||||||
|
if code != 200 {
|
||||||
|
t.Fatalf("无适配器认领时内核兜底应仍 200, 实得 %d: %s", code, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
231
internal/gateway/mount.go
Normal file
231
internal/gateway/mount.go
Normal file
@ -0,0 +1,231 @@
|
|||||||
|
// Package gateway — 对外挂载点(R3 / Phase 6)
|
||||||
|
//
|
||||||
|
// 用户需求:插件与适配器可以注册**对外服务的端口**。
|
||||||
|
//
|
||||||
|
// 现状:插件已经能注册 HTTP 端点,但被固定在 /api/plugins/http/ 前缀下。
|
||||||
|
// 那条前缀限制是有理由的(可 grep、可整体撤销、插件无法影子内核路由),
|
||||||
|
// 但它也意味着插件无法提供"看起来像内核一部分"的接口——而对外服务恰恰
|
||||||
|
// 需要这个(例如给上游回调用的固定路径 /v1/plugins/<name>/callback)。
|
||||||
|
//
|
||||||
|
// 本文件放开这一点,同时把当初那条限制的**三个理由**逐条补回来:
|
||||||
|
//
|
||||||
|
// 1. 可 grep / 可整体撤销 → 每条自定义挂载都必须落在 /plugins/ 或
|
||||||
|
// /v1/plugins/ 命名空间内(不接受任意前缀)
|
||||||
|
// 2. 不能影子内核路由 → 与内核保留路由表做前缀冲突检测,撞了直接拒绝
|
||||||
|
// 3. 鉴权默认关闭 → 沿用 HTTPHandler 的 roles/keys/public 语义
|
||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"modelrouter/internal/lua"
|
||||||
|
)
|
||||||
|
|
||||||
|
// allowedMountPrefixes are the only namespaces a plugin/adapters may claim.
|
||||||
|
//
|
||||||
|
// A free-form prefix would let a plugin register "/v1/chat/completions" and
|
||||||
|
// silently intercept every inference request on the gateway — the exact
|
||||||
|
// shadowing the narrow default prefix was protecting against. Constraining the
|
||||||
|
// namespace keeps the security property while still allowing a stable public
|
||||||
|
// path.
|
||||||
|
var allowedMountPrefixes = []string{"/plugins/", "/v1/plugins/"}
|
||||||
|
|
||||||
|
// kernelReservedPrefixes are prefixes the kernel owns. A custom mount that
|
||||||
|
// overlaps any of them is refused, so an operator's route table can never
|
||||||
|
// change meaning because a plugin was installed.
|
||||||
|
//
|
||||||
|
// This list is deliberately explicit rather than derived from the router's
|
||||||
|
// switch: the switch is code, and a future branch added there would silently
|
||||||
|
// become shadowable if this were inferred. The test
|
||||||
|
// TestKernelRoutesAreAllCoveredByReservedPrefixes keeps the two in sync.
|
||||||
|
var kernelReservedPrefixes = []string{
|
||||||
|
"/v1/chat/completions",
|
||||||
|
"/v1/messages",
|
||||||
|
"/v1/images/generations",
|
||||||
|
"/v1/models",
|
||||||
|
"/api/",
|
||||||
|
"/login",
|
||||||
|
"/ui",
|
||||||
|
"/",
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateCustomMount checks that a plugin-declared mount point is allowed.
|
||||||
|
//
|
||||||
|
// Returns the normalized path (leading slash, no trailing slash) or an error
|
||||||
|
// naming the conflict. The error text is what an operator sees in the plugin's
|
||||||
|
// load-error field, so it says WHICH rule was hit.
|
||||||
|
func ValidateCustomMount(p string) (string, error) {
|
||||||
|
p = strings.TrimSpace(p)
|
||||||
|
if p == "" {
|
||||||
|
return "", fmt.Errorf("empty mount path")
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(p, "/") {
|
||||||
|
return "", fmt.Errorf("mount path %q must start with /", p)
|
||||||
|
}
|
||||||
|
if strings.Contains(p, "..") || strings.Contains(p, "//") {
|
||||||
|
return "", fmt.Errorf("mount path %q contains an empty or escaping segment", p)
|
||||||
|
}
|
||||||
|
norm := strings.TrimRight(p, "/")
|
||||||
|
if norm == "" {
|
||||||
|
norm = "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rule 1: must live in one of the allowed namespaces.
|
||||||
|
ok := false
|
||||||
|
for _, pre := range allowedMountPrefixes {
|
||||||
|
if strings.HasPrefix(norm+"/", pre) {
|
||||||
|
ok = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("mount path %q must be under %s",
|
||||||
|
norm, strings.Join(allowedMountPrefixes, " or "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rule 2: must not overlap a kernel route.
|
||||||
|
for _, res := range kernelReservedPrefixes {
|
||||||
|
if res == "/" {
|
||||||
|
continue // the catch-all is handled by the namespace rule above
|
||||||
|
}
|
||||||
|
if norm == strings.TrimRight(res, "/") || strings.HasPrefix(norm, res) {
|
||||||
|
return "", fmt.Errorf("mount path %q collides with the kernel route %q", norm, res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return norm, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RouteOwner describes one occupied route, for conflict reporting.
|
||||||
|
type RouteOwner struct {
|
||||||
|
Path string
|
||||||
|
Owner string // "kernel", "plugin:name", "adapter:name"
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildRouteTable assembles the set of paths a plugin/adapters must not take.
|
||||||
|
// It is used both to validate new mounts and to explain a conflict to an
|
||||||
|
// operator (a bare "conflict" is not actionable; "already served by plugin X" is).
|
||||||
|
func BuildRouteTable(pluginMounts, adapterMounts map[string]string) []RouteOwner {
|
||||||
|
var out []RouteOwner
|
||||||
|
for _, p := range kernelReservedPrefixes {
|
||||||
|
if p == "/" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, RouteOwner{Path: strings.TrimRight(p, "/"), Owner: "kernel"})
|
||||||
|
}
|
||||||
|
for path, owner := range pluginMounts {
|
||||||
|
out = append(out, RouteOwner{Path: path, Owner: "plugin:" + owner})
|
||||||
|
}
|
||||||
|
for path, owner := range adapterMounts {
|
||||||
|
out = append(out, RouteOwner{Path: path, Owner: "adapter:" + owner})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindMountConflict reports whether candidate overlaps an occupied route and, if
|
||||||
|
// so, who holds it.
|
||||||
|
//
|
||||||
|
// Overlap is checked in BOTH directions: a new mount that CONTAINS an existing
|
||||||
|
// one shadows it just as effectively as one nested inside it, and only checking
|
||||||
|
// the prefix direction would let "/plugins/x/sub" be registered before
|
||||||
|
// "/plugins/x" and then silently swallow it.
|
||||||
|
func FindMountConflict(candidate string, table []RouteOwner) (RouteOwner, bool) {
|
||||||
|
c := strings.TrimRight(candidate, "/")
|
||||||
|
for _, r := range table {
|
||||||
|
rp := strings.TrimRight(r.Path, "/")
|
||||||
|
if rp == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == rp || strings.HasPrefix(c, rp+"/") || strings.HasPrefix(rp, c+"/") {
|
||||||
|
return r, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return RouteOwner{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleCustomMount dispatches a request to a plugin that claimed a custom mount
|
||||||
|
// path (/plugins/<name>/… or /v1/plugins/<name>/…).
|
||||||
|
//
|
||||||
|
// It deliberately REUSES handlePluginHTTP rather than reimplementing dispatch:
|
||||||
|
// that function already does longest-match resolution, per-endpoint
|
||||||
|
// roles/keys/public authorization and the Lua call protocol. Duplicating it here
|
||||||
|
// would create two dispatch paths that must be kept in agreement — and a
|
||||||
|
// security rule that is enforced on only one of them is worse than none.
|
||||||
|
//
|
||||||
|
// The rewrite below maps a claimed mount onto the handler table's canonical
|
||||||
|
// form: handler paths are declared as "<name>/<rest>", so the request path is
|
||||||
|
// reduced to exactly that. Anything that does not reduce cleanly (a missing
|
||||||
|
// name, a name with no registered handler) is a 404, not a fallthrough — a
|
||||||
|
// custom namespace that silently answered from the default prefix would make
|
||||||
|
// the mount validation meaningless.
|
||||||
|
func (g *Gateway) handleCustomMount(w http.ResponseWriter, r *http.Request) {
|
||||||
|
owner, rest, ok := parseMountRef(r.URL.Path)
|
||||||
|
if !ok || owner == "" {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "malformed plugin mount path")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ps := g.core.Plugins()
|
||||||
|
if ps == nil {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "no plugins loaded")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Resolve which handler owns this mount, so an unclaimed namespace is a 404.
|
||||||
|
handlers := ps.HTTPHandlers()
|
||||||
|
candidate := owner
|
||||||
|
if rest != "" {
|
||||||
|
candidate = owner + "/" + rest
|
||||||
|
}
|
||||||
|
if !mountClaimed(handlers, candidate) {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found",
|
||||||
|
"no plugin serves "+r.URL.Path)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Reuse the default dispatcher by presenting the path in its own shape.
|
||||||
|
r2 := r.Clone(r.Context())
|
||||||
|
r2.URL.Path = pluginHTTPPrefix + candidate
|
||||||
|
g.handlePluginHTTP(w, r2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// mountClaimed reports whether any registered handler's path is a prefix of the
|
||||||
|
// candidate (longest-match semantics: /plugins/auth/token is served by a
|
||||||
|
// handler declared as "auth").
|
||||||
|
func mountClaimed(handlers []lua.HTTPHandler, candidate string) bool {
|
||||||
|
for i := range handlers {
|
||||||
|
h := strings.Trim(handlers[i].Path, "/")
|
||||||
|
if h == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if candidate == h || strings.HasPrefix(candidate, h+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseMountRef splits "/plugins/<name>/<rest>" or "/v1/plugins/<name>/<rest>"
|
||||||
|
// into its owner name and remaining path.
|
||||||
|
//
|
||||||
|
// The "/v1/plugins/" form is checked first because it has the longer prefix:
|
||||||
|
// testing "/plugins/" first would match nothing for a /v1/ path, but keeping the
|
||||||
|
// ordering explicit avoids a future prefix being added that nests inside
|
||||||
|
// another.
|
||||||
|
func parseMountRef(p string) (owner, rest string, ok bool) {
|
||||||
|
prefixes := []string{"/v1/plugins/", "/plugins/"}
|
||||||
|
for _, prefix := range prefixes {
|
||||||
|
if !strings.HasPrefix(p, prefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
trimmed := strings.TrimPrefix(p, prefix)
|
||||||
|
if trimmed == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
if i := strings.Index(trimmed, "/"); i >= 0 {
|
||||||
|
return trimmed[:i], trimmed[i+1:], true
|
||||||
|
}
|
||||||
|
return trimmed, "", true
|
||||||
|
}
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
324
internal/gateway/mount_test.go
Normal file
324
internal/gateway/mount_test.go
Normal file
@ -0,0 +1,324 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"modelrouter/internal/config"
|
||||||
|
"modelrouter/internal/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 本文件锁定 R3 的两条不变量:
|
||||||
|
// 1. 自定义挂载只能落在允许的命名空间内
|
||||||
|
// 2. 任何内核路由都不能被插件声明覆盖 —— 且这条在新增内核路由时**不会失效**
|
||||||
|
|
||||||
|
// TestValidateCustomMountNamespace 只有 /plugins/ 与 /v1/plugins/ 可被占用。
|
||||||
|
func TestValidateCustomMountNamespace(t *testing.T) {
|
||||||
|
ok := []string{
|
||||||
|
"/plugins/auth/token",
|
||||||
|
"/plugins/auth",
|
||||||
|
"/v1/plugins/callback",
|
||||||
|
"/v1/plugins/callback/recv",
|
||||||
|
"/plugins/nested/deep/path",
|
||||||
|
}
|
||||||
|
for _, p := range ok {
|
||||||
|
if _, err := ValidateCustomMount(p); err != nil {
|
||||||
|
t.Errorf("合法挂载 %q 被拒: %v", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bad := []string{
|
||||||
|
"",
|
||||||
|
"plugins/x", // 没有前导斜杠
|
||||||
|
"/v1/chat/completions", // 影子内核路由(且不在允许命名空间)
|
||||||
|
"/api/anything", // 内核命名空间
|
||||||
|
"/callback", // 任意前缀
|
||||||
|
"/plugins/../etc", // 逃逸
|
||||||
|
"/plugins//double", // 空段
|
||||||
|
"/v1/plugins/../../x",
|
||||||
|
}
|
||||||
|
for _, p := range bad {
|
||||||
|
if _, err := ValidateCustomMount(p); err == nil {
|
||||||
|
t.Errorf("非法挂载 %q 被接受", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCustomMountCannotShadowKernelRoute 逐条核对内核保留前缀。
|
||||||
|
func TestCustomMountCannotShadowKernelRoute(t *testing.T) {
|
||||||
|
// 这些是要保护的公开接口;任何能覆盖它们的挂载都必须被拒。
|
||||||
|
for _, kernel := range []string{"/v1/chat/completions", "/v1/messages", "/v1/models"} {
|
||||||
|
// 尝试用允许命名空间里的路径去"贴近"内核路由 —— 例如
|
||||||
|
// /v1/plugins/../../v1/models(逃逸),必须被拒。
|
||||||
|
cand := "/v1/plugins/../../" + strings.TrimPrefix(kernel, "/")
|
||||||
|
if _, err := ValidateCustomMount(cand); err == nil {
|
||||||
|
t.Errorf("逃逸路径 %q 应被拒", cand)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKernelRoutesAreAllCoveredByReservedPrefixes 是这道防线**不腐坏**的保证。
|
||||||
|
//
|
||||||
|
// 它从 server.go 里把路由 switch 的分支字符串刮出来,逐个检查是否被
|
||||||
|
// kernelReservedPrefixes 覆盖。将来有人新增一条内核路由却忘了更新保留表时,
|
||||||
|
// 这条测试会失败 —— 否则新路由会静默变成可被插件覆盖的。
|
||||||
|
func TestKernelRoutesAreAllCoveredByReservedPrefixes(t *testing.T) {
|
||||||
|
src, err := os.ReadFile("server.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 抓 switch 里的字面量路径:case r.URL.Path == "/x" / HasPrefix(..., "/x")
|
||||||
|
re := regexp.MustCompile(`r\.URL\.Path\s*==\s*"(/[^"]*)"`)
|
||||||
|
matches := re.FindAllStringSubmatch(string(src), -1)
|
||||||
|
if len(matches) == 0 {
|
||||||
|
t.Fatal("没有从 server.go 抓到任何路由字面量 —— 测试自身失效")
|
||||||
|
}
|
||||||
|
covered := func(p string) bool {
|
||||||
|
// "/" is the UI catch-all (serveUI). It is the default branch, not a
|
||||||
|
// path a plugin could shadow, and it is covered by the namespace rule:
|
||||||
|
// a custom mount must start with /plugins/ or /v1/plugins/, neither of
|
||||||
|
// which is "/".
|
||||||
|
if p == "/" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, res := range kernelReservedPrefixes {
|
||||||
|
if res == "/" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rp := strings.TrimRight(res, "/")
|
||||||
|
if p == rp || strings.HasPrefix(p, rp+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// /api/plugins/... 由 pluginHTTPPrefix 与 /api/ 覆盖
|
||||||
|
if strings.HasPrefix(p, "/api/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, m := range matches {
|
||||||
|
p := m[1]
|
||||||
|
if !covered(p) {
|
||||||
|
t.Errorf("内核路由 %q 未被 kernelReservedPrefixes 覆盖 —— "+
|
||||||
|
"新增路由时必须同步更新保留表,否则插件可影子它", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMountConflictDetectionBothDirections 冲突检测必须双向。
|
||||||
|
//
|
||||||
|
// 只查一个方向会让「先注册 /plugins/x,后注册 /plugins/x/sub」或反过来时的
|
||||||
|
// 一个被另一个静默吞掉。
|
||||||
|
func TestMountConflictDetectionBothDirections(t *testing.T) {
|
||||||
|
table := []RouteOwner{{Path: "/plugins/x", Owner: "plugin:a"}}
|
||||||
|
// 被包含
|
||||||
|
if _, conflict := FindMountConflict("/plugins/x/sub", table); !conflict {
|
||||||
|
t.Error("嵌套在已有挂载下的路径应判冲突")
|
||||||
|
}
|
||||||
|
// 包含
|
||||||
|
if _, conflict := FindMountConflict("/plugins", table); !conflict {
|
||||||
|
t.Error("包含已有挂载的路径应判冲突")
|
||||||
|
}
|
||||||
|
// 相同
|
||||||
|
if _, conflict := FindMountConflict("/plugins/x", table); !conflict {
|
||||||
|
t.Error("完全相同的路径应判冲突")
|
||||||
|
}
|
||||||
|
// 无关(前缀相似但不是路径段)
|
||||||
|
if _, conflict := FindMountConflict("/plugins/xy", table); conflict {
|
||||||
|
t.Error("/plugins/xy 与 /plugins/x 不是同一个路径段,不该判冲突")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMountConflictNamesTheOwner 冲突信息必须点名持有者(否则不可行动)。
|
||||||
|
func TestMountConflictNamesTheOwner(t *testing.T) {
|
||||||
|
table := []RouteOwner{{Path: "/plugins/taken", Owner: "plugin:other"}}
|
||||||
|
owner, conflict := FindMountConflict("/plugins/taken/deep", table)
|
||||||
|
if !conflict {
|
||||||
|
t.Fatal("应判冲突")
|
||||||
|
}
|
||||||
|
if owner.Owner != "plugin:other" {
|
||||||
|
t.Errorf("应点名持有者 plugin:other, 实得 %q", owner.Owner)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseMountRef 解析两种命名空间。
|
||||||
|
func TestParseMountRef(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
in, owner, rest string
|
||||||
|
ok bool
|
||||||
|
}{
|
||||||
|
{"/plugins/auth/token", "auth", "token", true},
|
||||||
|
{"/plugins/auth", "auth", "", true},
|
||||||
|
{"/v1/plugins/cb/recv", "cb", "recv", true},
|
||||||
|
{"/v1/plugins/cb", "cb", "", true},
|
||||||
|
{"/plugins/", "", "", false},
|
||||||
|
{"/plugins", "", "", false},
|
||||||
|
{"/other/x", "", "", false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
owner, rest, ok := parseMountRef(c.in)
|
||||||
|
if ok != c.ok || owner != c.owner || rest != c.rest {
|
||||||
|
t.Errorf("parseMountRef(%q) = (%q,%q,%v), want (%q,%q,%v)",
|
||||||
|
c.in, owner, rest, ok, c.owner, c.rest, c.ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBuildRouteTableIncludesKernel 路由表含内核保留项(供冲突提示用)。
|
||||||
|
func TestBuildRouteTableIncludesKernel(t *testing.T) {
|
||||||
|
table := BuildRouteTable(
|
||||||
|
map[string]string{"/plugins/a": "alpha"},
|
||||||
|
map[string]string{"/v1/plugins/b": "beta"},
|
||||||
|
)
|
||||||
|
var hasKernel, hasPlugin, hasAdapter bool
|
||||||
|
for _, r := range table {
|
||||||
|
if r.Owner == "kernel" {
|
||||||
|
hasKernel = true
|
||||||
|
}
|
||||||
|
if r.Owner == "plugin:alpha" {
|
||||||
|
hasPlugin = true
|
||||||
|
}
|
||||||
|
if r.Owner == "adapter:beta" {
|
||||||
|
hasAdapter = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !hasKernel || !hasPlugin || !hasAdapter {
|
||||||
|
t.Errorf("路由表应含 kernel/plugin/adapter 三类: %+v", table)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPluginServesCustomMountEndToEnd 是 R3 的端到端验收:
|
||||||
|
// 插件声明一个自定义对外挂载点,客户端按该路径访问,得到插件的应答。
|
||||||
|
func TestPluginServesCustomMountEndToEnd(t *testing.T) {
|
||||||
|
ts, key := messagesGateway(t)
|
||||||
|
// messagesGateway 用的 core 没有插件目录配置,这里直接建一个带插件的。
|
||||||
|
_ = ts
|
||||||
|
_ = key
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := &config.Config{
|
||||||
|
Path: filepath.Join(dir, "config.yaml"),
|
||||||
|
AdapterDir: filepath.Join(dir, "adapters"),
|
||||||
|
PluginDir: filepath.Join(dir, "plugins"),
|
||||||
|
RuntimeFile: filepath.Join(dir, "runtime.json"),
|
||||||
|
Listen: "127.0.0.1:0",
|
||||||
|
DefaultModel: "AUTO",
|
||||||
|
GatewayKeys: []string{"sk-mount-0001"},
|
||||||
|
Sources: []config.Source{{
|
||||||
|
Name: "s", BaseURL: "http://127.0.0.1:1/v1", Adapter: "openai",
|
||||||
|
APIKey: "k", MaxConcurrent: 1,
|
||||||
|
Models: []config.Model{{ID: "m", Kind: "chat"}},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
c, err := core.NewFromConfig(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
gw, err := New(c)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ts2 := httptest.NewServer(gw.Handler())
|
||||||
|
defer ts2.Close()
|
||||||
|
|
||||||
|
// 装一个声明自定义挂载的插件
|
||||||
|
code := `
|
||||||
|
local plugin = { name = "publicapi", version = "1.0" }
|
||||||
|
plugin.http = {
|
||||||
|
{ path = "publicapi/token", method = "GET", handler = "token",
|
||||||
|
roles = { "admin", "user" }, description = "对外 token 端点" },
|
||||||
|
}
|
||||||
|
function plugin.token(req)
|
||||||
|
return { status = 200, body = '{"token":"abc"}' }
|
||||||
|
end
|
||||||
|
return plugin
|
||||||
|
`
|
||||||
|
if err := c.Plugins().LoadSource("publicapi", code); err != nil {
|
||||||
|
t.Fatalf("load plugin: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 按自定义挂载访问
|
||||||
|
req, _ := http.NewRequest("GET", ts2.URL+"/plugins/publicapi/token", nil)
|
||||||
|
req.Header.Set("Authorization", "Bearer "+cfg.GatewayKeys[0])
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
t.Fatalf("自定义挂载 status = %d: %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(body), "abc") {
|
||||||
|
t.Errorf("自定义挂载未返回插件应答: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// /v1/plugins/ 形式同样可用
|
||||||
|
req2, _ := http.NewRequest("GET", ts2.URL+"/v1/plugins/publicapi/token", nil)
|
||||||
|
req2.Header.Set("Authorization", "Bearer "+cfg.GatewayKeys[0])
|
||||||
|
resp2, err := http.DefaultClient.Do(req2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp2.Body.Close()
|
||||||
|
if resp2.StatusCode != 200 {
|
||||||
|
t.Errorf("/v1/plugins/ 形式 status = %d", resp2.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 未声明的命名空间 404(不能因为落在允许命名空间就放行)
|
||||||
|
req3, _ := http.NewRequest("GET", ts2.URL+"/plugins/nobody/here", nil)
|
||||||
|
req3.Header.Set("Authorization", "Bearer "+cfg.GatewayKeys[0])
|
||||||
|
resp3, err := http.DefaultClient.Do(req3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp3.Body.Close()
|
||||||
|
if resp3.StatusCode != 404 {
|
||||||
|
t.Errorf("未注册的挂载应 404, 实得 %d", resp3.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 鉴权仍生效:无 key 时,
|
||||||
|
// - API 路径(/v1/plugins/…)应 401(isAPIPath 判定)
|
||||||
|
// - 浏览器路径(/plugins/…)应 302 跳登录页
|
||||||
|
req4, _ := http.NewRequest("GET", ts2.URL+"/v1/plugins/publicapi/token", nil)
|
||||||
|
noRedirect := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
|
return http.ErrUseLastResponse
|
||||||
|
}}
|
||||||
|
resp4, err := noRedirect.Do(req4)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp4.Body.Close()
|
||||||
|
if resp4.StatusCode != 401 {
|
||||||
|
t.Errorf("无 key 访问 /v1/plugins/ 应 401, 实得 %d", resp4.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
req5, _ := http.NewRequest("GET", ts2.URL+"/plugins/publicapi/token", nil)
|
||||||
|
resp5, err := noRedirect.Do(req5)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp5.Body.Close()
|
||||||
|
if resp5.StatusCode != 302 {
|
||||||
|
t.Errorf("无 key 访问 /plugins/ 应 302 跳登录, 实得 %d", resp5.StatusCode)
|
||||||
|
}
|
||||||
|
if loc := resp5.Header.Get("Location"); !strings.Contains(loc, "/login") {
|
||||||
|
t.Errorf("应跳转到 /login, 实得 %q", loc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCustomMountCannotInterceptKernelRoute 端到端确认内核路由不受影响:
|
||||||
|
// 即使有插件在用 /plugins/,/v1/chat/completions 与 /v1/models 仍归内核。
|
||||||
|
func TestCustomMountCannotInterceptKernelRoute(t *testing.T) {
|
||||||
|
for _, p := range []string{"/v1/chat/completions", "/v1/models", "/v1/messages", "/api/stats"} {
|
||||||
|
if _, err := ValidateCustomMount(p); err == nil {
|
||||||
|
t.Errorf("内核路由 %q 必须不能被声明为自定义挂载", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
71
internal/gateway/mutation_guard_test.go
Normal file
71
internal/gateway/mutation_guard_test.go
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 变异测试会**改写生产源码** `discover.go`(discover_mutate_test.go),
|
||||||
|
// 靠 defer 把原文件写回。这在进程被强杀时不成立:
|
||||||
|
//
|
||||||
|
// - `go test -timeout` 超时 → 测试进程被杀
|
||||||
|
// - Ctrl-C / 后台任务被 kill
|
||||||
|
// - 跑测试的 shell 被误杀(本仓真实踩过:`pkill -f` 匹配到执行它的那条命令行)
|
||||||
|
//
|
||||||
|
// 后果不是"这次测试没跑完",而是**变异体被留在工作区里**:下一次运行会把
|
||||||
|
// 它当成"原始源码"捕获,于是基线本身就是变异体,所有裁决静默失效;更糟的是
|
||||||
|
// 开发者可能把它当成自己的改动提交上去(真实踩过两次:`b == b && b != b`
|
||||||
|
// 这种恒假的变异体出现在 `git status` 里)。
|
||||||
|
//
|
||||||
|
// 兜底:每次进入本包的测试都先把已知变异体的痕迹清掉(回滚到提交版本),
|
||||||
|
// 这样即使上一次被强杀,本次运行也不会建立在污染之上。TestRealMutants 自己
|
||||||
|
// 也会在脏树上拒绝运行,但那是"拒绝",这里是"自愈"。
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
restoreLeakedMutants()
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
|
||||||
|
// leakedMutants 列出「变异体特征 → 它对应的正确原文」。
|
||||||
|
// 只处理能**确定判定**的形态:出现特征即说明上一次没回滚干净。
|
||||||
|
var leakedMutants = []struct {
|
||||||
|
file string
|
||||||
|
bad string
|
||||||
|
good string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
file: "discover.go",
|
||||||
|
bad: "\tif b := trimVersionSuffix(persistedBase); b == b && b != b {",
|
||||||
|
good: "\tif b := trimVersionSuffix(persistedBase); b != persistedBase {",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: "discover.go",
|
||||||
|
bad: "\t\tif len(pair) > 0 && (seenPair[pair] || seenURL[base+path]) && false {",
|
||||||
|
good: "\t\tif seenPair[pair] || seenURL[base+path] {",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreLeakedMutants() {
|
||||||
|
wd, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, m := range leakedMutants {
|
||||||
|
p := filepath.Join(wd, m.file)
|
||||||
|
b, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s := string(b)
|
||||||
|
if !strings.Contains(s, m.bad) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// 只替换回原文,不猜别的差异 —— 保守,避免把开发中的改动一起冲掉。
|
||||||
|
fixed := strings.Replace(s, m.bad, m.good, 1)
|
||||||
|
if fixed == s {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(p, []byte(fixed), 0o644)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -6,8 +6,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/scheduler"
|
"modelrouter/internal/scheduler"
|
||||||
"llmsproxy/internal/types"
|
"modelrouter/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// piOverflowRe 是 @earendil-works/pi-ai 的 OVERFLOW_PATTERNS 中对应的一条。
|
// piOverflowRe 是 @earendil-works/pi-ai 的 OVERFLOW_PATTERNS 中对应的一条。
|
||||||
|
|||||||
@ -17,8 +17,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
// callPlugin exposes exactly one export plus a function that is NOT exported.
|
// callPlugin exposes exactly one export plus a function that is NOT exported.
|
||||||
@ -147,4 +147,4 @@ func TestPluginCallIsAdminOnly(t *testing.T) {
|
|||||||
t.Fatalf("a non-admin key reached the call endpoint: HTTP %d %s",
|
t.Fatalf("a non-admin key reached the call endpoint: HTTP %d %s",
|
||||||
rr.Code, rr.Body.String())
|
rr.Code, rr.Body.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -17,10 +17,11 @@ package gateway
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"llmsproxy/internal/lua"
|
"modelrouter/internal/lua"
|
||||||
)
|
)
|
||||||
|
|
||||||
// pluginHTTPPrefix is the mount point. It is deliberately narrow: a plugin's
|
// pluginHTTPPrefix is the mount point. It is deliberately narrow: a plugin's
|
||||||
@ -45,19 +46,48 @@ func (g *Gateway) handlePluginHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
matches := ps.HTTPHandlers()
|
matches := ps.HTTPHandlers()
|
||||||
var chosen *lua.HTTPHandler
|
var chosen *lua.HTTPHandler
|
||||||
var params map[string]string
|
var params map[string]string
|
||||||
|
// contenders holds the handlers that matched at the current best depth with
|
||||||
|
// the SAME literal route pattern — i.e. two plugins that both declared this
|
||||||
|
// exact path. That is the contested case the operator must arbitrate (user
|
||||||
|
// requirement A): rather than let load order pick a silent winner, a
|
||||||
|
// contested path with no recorded decision returns 409 naming the candidates.
|
||||||
|
var contenders []lua.HTTPHandler
|
||||||
for i := range matches {
|
for i := range matches {
|
||||||
h := matches[i]
|
h := matches[i]
|
||||||
if p, ok := matchHandlerPath(h.Path, rest); ok {
|
p, ok := matchHandlerPath(h.Path, rest)
|
||||||
if chosen == nil || segmentCount(h.Path) > segmentCount(chosen.Path) {
|
if h.Prefix {
|
||||||
chosen, params = &h, p
|
pattern := strings.Trim(h.Path, "/")
|
||||||
|
parts := strings.Split(strings.Trim(rest, "/"), "/")
|
||||||
|
n := len(strings.Split(pattern, "/"))
|
||||||
|
ok = len(parts) >= n
|
||||||
|
if ok {
|
||||||
|
p, ok = matchHandlerPath(pattern, strings.Join(parts[:n], "/"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case chosen == nil || segmentCount(h.Path) > segmentCount(chosen.Path):
|
||||||
|
chosen, params = &matches[i], p
|
||||||
|
contenders = []lua.HTTPHandler{matches[i]}
|
||||||
|
case segmentCount(h.Path) == segmentCount(chosen.Path) && h.Path == chosen.Path:
|
||||||
|
// Same route pattern, a different plugin. The winner is arbitrated below.
|
||||||
|
contenders = append(contenders, matches[i])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if chosen == nil {
|
if chosen == nil {
|
||||||
writeError(w, http.StatusNotFound, "not_found",
|
writeError(w, http.StatusNotFound, "not_found",
|
||||||
"no plugin serves this path")
|
"no plugin serves this path")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if len(contenders) > 1 {
|
||||||
|
resolved := g.resolveContestedRoute(w, ps, chosen.Path, contenders)
|
||||||
|
if resolved == nil {
|
||||||
|
return // 409 already written, naming the candidates
|
||||||
|
}
|
||||||
|
chosen = resolved
|
||||||
|
}
|
||||||
|
|
||||||
if !strings.EqualFold(r.Method, chosen.Method) {
|
if !strings.EqualFold(r.Method, chosen.Method) {
|
||||||
w.Header().Set("Allow", chosen.Method)
|
w.Header().Set("Allow", chosen.Method)
|
||||||
@ -95,9 +125,57 @@ func (g *Gateway) handlePluginHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeError(w, http.StatusBadGateway, "plugin_error", err.Error())
|
writeError(w, http.StatusBadGateway, "plugin_error", err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if resp.Proxy != "" {
|
||||||
|
if !chosen.Prefix {
|
||||||
|
writeError(w, http.StatusBadGateway, "plugin_error", "proxy response requires prefix=true")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
parts := strings.Split(strings.Trim(rest, "/"), "/")
|
||||||
|
n := len(strings.Split(strings.Trim(chosen.Path, "/"), "/"))
|
||||||
|
g.servePluginProxy(w, r, resp.Proxy, pluginHTTPPrefix+strings.Join(parts[:n], "/"))
|
||||||
|
return
|
||||||
|
}
|
||||||
lua.WriteHandlerResponse(w, resp)
|
lua.WriteHandlerResponse(w, resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveContestedRoute arbitrates a service path that more than one plugin
|
||||||
|
// declared (user requirement A). It returns the winning handler, or nil after
|
||||||
|
// writing a 409 when the operator has not yet chosen.
|
||||||
|
//
|
||||||
|
// This is the crux of the "report, don't crash, don't silently pick" rule: with
|
||||||
|
// two plugins both serving e.g. "stats", load order used to decide the winner.
|
||||||
|
// Now an unresolved contest is a 409 that NAMES the candidates, and the operator
|
||||||
|
// picks one in the WebUI (POST /api/plugins/conflicts). A recorded decision is
|
||||||
|
// honoured here on every request thereafter.
|
||||||
|
func (g *Gateway) resolveContestedRoute(w http.ResponseWriter, ps *lua.Plugins, path string, contenders []lua.HTTPHandler) *lua.HTTPHandler {
|
||||||
|
winner := ps.RouteWinner(path)
|
||||||
|
if winner != "" {
|
||||||
|
for i := range contenders {
|
||||||
|
if contenders[i].Plugin == winner {
|
||||||
|
return &contenders[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A decision exists but points at a plugin that no longer serves this
|
||||||
|
// path (it was removed or edited). Fall through to 409 rather than honour
|
||||||
|
// a stale winner that cannot answer.
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(contenders))
|
||||||
|
for i := range contenders {
|
||||||
|
names = append(names, contenders[i].Plugin)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||||
|
writeJSON(w, http.StatusConflict, map[string]interface{}{
|
||||||
|
"error": map[string]interface{}{
|
||||||
|
"type": "route_conflict",
|
||||||
|
"message": "service path \"" + path + "\" is claimed by " + strconv.Itoa(len(names)) + " plugins (" + strings.Join(names, ", ") + "); an operator must choose one in the admin UI (Conflicts) or POST /api/plugins/conflicts",
|
||||||
|
"path": path,
|
||||||
|
"candidates": names,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// handlerAccessHint explains, in the refusal, exactly which declaration would
|
// handlerAccessHint explains, in the refusal, exactly which declaration would
|
||||||
// open this endpoint. A 403 with no reason is the kind of thing an operator
|
// open this endpoint. A 403 with no reason is the kind of thing an operator
|
||||||
// opens the source to work out, and then the fix is a guess.
|
// opens the source to work out, and then the fix is a guess.
|
||||||
@ -204,4 +282,4 @@ type bodyTooLarge struct{}
|
|||||||
func (bodyTooLarge) Error() string { return "request body exceeds 4 MiB" }
|
func (bodyTooLarge) Error() string { return "request body exceeds 4 MiB" }
|
||||||
|
|
||||||
// limitBody is kept for the numeric form some callers want in a log line.
|
// limitBody is kept for the numeric form some callers want in a log line.
|
||||||
var _ = strconv.Itoa
|
var _ = strconv.Itoa
|
||||||
|
|||||||
54
internal/gateway/plugin_proxy.go
Normal file
54
internal/gateway/plugin_proxy.go
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httputil"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// servePluginProxy forwards a plugin-owned prefix to an explicit loopback
|
||||||
|
// origin. No destination or path may be supplied by the client. Authentication
|
||||||
|
// and endpoint authorization have already completed before this is called.
|
||||||
|
func (g *Gateway) servePluginProxy(w http.ResponseWriter, r *http.Request, target, prefix string) {
|
||||||
|
u, err := url.Parse(target)
|
||||||
|
if err != nil || u.Scheme != "http" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") {
|
||||||
|
writeError(w, http.StatusBadGateway, "plugin_proxy", "proxy must name a loopback HTTP origin")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ip := net.ParseIP(u.Hostname())
|
||||||
|
if ip == nil || !ip.IsLoopback() || u.Port() == "" {
|
||||||
|
writeError(w, http.StatusBadGateway, "plugin_proxy", "proxy requires a literal loopback address and port")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// WebSocket requests are browser-initiated and carry ambient cookies. Do not
|
||||||
|
// allow a foreign website to use an administrator's session to open a tunnel.
|
||||||
|
if origin := r.Header.Get("Origin"); origin != "" {
|
||||||
|
o, e := url.Parse(origin)
|
||||||
|
if e != nil || !strings.EqualFold(o.Host, r.Host) {
|
||||||
|
writeError(w, http.StatusForbidden, "forbidden", "cross-origin proxy request refused")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
proxy := httputil.NewSingleHostReverseProxy(u)
|
||||||
|
director := proxy.Director
|
||||||
|
proxy.Director = func(out *http.Request) {
|
||||||
|
director(out)
|
||||||
|
out.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
|
||||||
|
if out.URL.Path == "" {
|
||||||
|
out.URL.Path = "/"
|
||||||
|
}
|
||||||
|
out.URL.RawPath = ""
|
||||||
|
out.Host = u.Host
|
||||||
|
out.Header.Del("Authorization")
|
||||||
|
out.Header.Del("Cookie")
|
||||||
|
q := out.URL.Query()
|
||||||
|
q.Del("api_key")
|
||||||
|
out.URL.RawQuery = q.Encode()
|
||||||
|
}
|
||||||
|
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
|
||||||
|
writeError(w, http.StatusBadGateway, "plugin_proxy", "plugin service unavailable")
|
||||||
|
}
|
||||||
|
proxy.ServeHTTP(w, r)
|
||||||
|
}
|
||||||
66
internal/gateway/plugin_proxy_security_test.go
Normal file
66
internal/gateway/plugin_proxy_security_test.go
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPluginProxyRejectsUnsafeTargetsAndOrigin(t *testing.T) {
|
||||||
|
g, _ := gatewayWithSpy(t)
|
||||||
|
for _, target := range []string{"http://example.com:80", "http://localhost:80", "https://127.0.0.1:80", "http://127.0.0.1", "http://u:p@127.0.0.1:80", "http://127.0.0.1:80/path", "http://127.0.0.1:80/?x=1"} {
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
g.servePluginProxy(rr, httptest.NewRequest("GET", "http://gateway/api/plugins/http/relay/s/live.js", nil), target, "/api/plugins/http/relay/s")
|
||||||
|
if rr.Code != 502 {
|
||||||
|
t.Errorf("unsafe target %q accepted: %d", target, rr.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest("GET", "http://gateway/api/plugins/http/relay/s/websockify", nil)
|
||||||
|
req.Header.Set("Origin", "https://evil.example")
|
||||||
|
g.servePluginProxy(rr, req, "http://127.0.0.1:12345", "/api/plugins/http/relay/s")
|
||||||
|
if rr.Code != 403 {
|
||||||
|
t.Fatalf("foreign origin accepted: %d", rr.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPluginPrefixProxyAuthorizationAndRevocation(t *testing.T) {
|
||||||
|
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Error("denied request reached backend") }))
|
||||||
|
defer backend.Close()
|
||||||
|
g, _ := gatewayWithSpy(t)
|
||||||
|
src := `local p={name="relaydeny",http={{path="relaydeny/:session",prefix=true,roles={"admin"},handler="proxy"}}}
|
||||||
|
function p.proxy(req) return {status=404,body="expired session"} end
|
||||||
|
return p`
|
||||||
|
if err := g.core.Plugins().LoadSource("relaydeny", src); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
server := httptest.NewServer(g.Handler())
|
||||||
|
defer server.Close()
|
||||||
|
for _, tc := range []struct {
|
||||||
|
path, key string
|
||||||
|
status int
|
||||||
|
}{
|
||||||
|
{"/api/plugins/http/relaydeny/expired/vnc.html", "", 401},
|
||||||
|
{"/api/plugins/http/relaydeny/expired/vnc.html", "sk-test", 404},
|
||||||
|
{"/api/plugins/http/relaydeny-not-a-child/expired/vnc.html", "sk-test", 404},
|
||||||
|
} {
|
||||||
|
req, _ := http.NewRequest("GET", server.URL+tc.path, nil)
|
||||||
|
if tc.key != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+tc.key)
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, _ := io.ReadAll(resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != tc.status {
|
||||||
|
t.Fatalf("%s got %d %s", tc.path, resp.StatusCode, b)
|
||||||
|
}
|
||||||
|
if tc.key != "" && strings.Contains(tc.path, "/expired/") && strings.Contains(tc.path, "/relaydeny/") && string(b) != "expired session" {
|
||||||
|
t.Fatalf("revocation response=%s", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
82
internal/gateway/plugin_proxy_test.go
Normal file
82
internal/gateway/plugin_proxy_test.go
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Drive the real authenticated plugin route, including the access-log wrapper.
|
||||||
|
// A successful static request alone does not prove WebSocket can upgrade.
|
||||||
|
func TestPluginPrefixProxyHTTPAndUpgrade(t *testing.T) {
|
||||||
|
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" || r.URL.Query().Get("api_key") != "" {
|
||||||
|
t.Error("gateway credentials leaked to plugin backend")
|
||||||
|
}
|
||||||
|
if r.URL.Path == "/websockify" {
|
||||||
|
c, b, err := w.(http.Hijacker).Hijack()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
fmt.Fprint(b, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n")
|
||||||
|
b.Flush()
|
||||||
|
io.Copy(c, c)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprint(w, r.URL.Path+"?"+r.URL.RawQuery)
|
||||||
|
}))
|
||||||
|
defer backend.Close()
|
||||||
|
g, _ := gatewayWithSpy(t)
|
||||||
|
src := fmt.Sprintf(`local p={name="relay",http={{path="relay/:session",prefix=true,method="GET",roles={"admin"},handler="proxy"}}}
|
||||||
|
function p.proxy(req)
|
||||||
|
if req.params.session ~= "live" then return {status=404,body="expired"} end
|
||||||
|
return {proxy=%q}
|
||||||
|
end
|
||||||
|
return p`, backend.URL)
|
||||||
|
if err := g.core.Plugins().LoadSource("relay", src); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
server := httptest.NewServer(g.Handler())
|
||||||
|
defer server.Close()
|
||||||
|
req, _ := http.NewRequest("GET", server.URL+"/api/plugins/http/relay/live/app/main.js?x=1&api_key=sk-test", nil)
|
||||||
|
req.Header.Set("Authorization", "Bearer sk-test")
|
||||||
|
req.AddCookie(&http.Cookie{Name: "gw_key", Value: "sk-test"})
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != 200 || string(body) != "/app/main.js?x=1" {
|
||||||
|
t.Fatalf("HTTP proxy: %d %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
c, err := net.Dial("tcp", strings.TrimPrefix(server.URL, "http://"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
fmt.Fprintf(c, "GET /api/plugins/http/relay/live/websockify HTTP/1.1\r\nHost: %s\r\nAuthorization: Bearer sk-test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", strings.TrimPrefix(server.URL, "http://"))
|
||||||
|
b := bufio.NewReader(c)
|
||||||
|
reply, err := http.ReadResponse(b, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if reply.StatusCode != 101 {
|
||||||
|
t.Fatalf("upgrade failed: %s", reply.Status)
|
||||||
|
}
|
||||||
|
c.Write([]byte("ping"))
|
||||||
|
got := make([]byte, 4)
|
||||||
|
if _, err = io.ReadFull(b, got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(got) != "ping" {
|
||||||
|
t.Fatalf("tunnel payload=%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,8 +9,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
// newFailingServer serves handler and returns its test server.
|
// newFailingServer serves handler and returns its test server.
|
||||||
|
|||||||
@ -9,7 +9,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"llmsproxy/internal/lua"
|
"modelrouter/internal/lua"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -82,6 +82,57 @@ func (g *Gateway) handlePluginsAPI(w http.ResponseWriter, r *http.Request) {
|
|||||||
path := strings.TrimPrefix(r.URL.Path, "/api/plugins")
|
path := strings.TrimPrefix(r.URL.Path, "/api/plugins")
|
||||||
path = strings.Trim(path, "/")
|
path = strings.Trim(path, "/")
|
||||||
|
|
||||||
|
// /api/plugins/conflicts — the arbitration surface (user requirement).
|
||||||
|
// GET lists every contested route (A) and every suspended adapter-name
|
||||||
|
// install (B), each with its candidates/incumbent and any recorded
|
||||||
|
// decision.
|
||||||
|
// POST {kind,path|name,winner} records the operator's choice and persists
|
||||||
|
// it. Must be routed BEFORE the single-segment {name} handler below,
|
||||||
|
// which would otherwise read "conflicts" as a plugin name.
|
||||||
|
if path == "conflicts" {
|
||||||
|
if reqRole(r.Context()) != "admin" {
|
||||||
|
writeError(w, http.StatusForbidden, "forbidden", "admin role required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodGet:
|
||||||
|
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||||
|
"routes": ps.RouteConflicts(),
|
||||||
|
"adapters": ps.AdapterConflicts(),
|
||||||
|
})
|
||||||
|
case http.MethodPost:
|
||||||
|
var body struct {
|
||||||
|
Kind string `json:"kind"` // "route" or "adapter"
|
||||||
|
Path string `json:"path"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Winner string `json:"winner"` // route: plugin name; adapter: "incumbent"|"candidate"
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", "invalid json: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch body.Kind {
|
||||||
|
case "route":
|
||||||
|
if err := ps.ResolveRouteConflict(body.Path, body.Winner); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "conflict_error", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]interface{}{"ok": true, "path": body.Path, "winner": body.Winner})
|
||||||
|
case "adapter":
|
||||||
|
if err := ps.ResolveAdapterConflict(body.Name, body.Winner); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "conflict_error", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]interface{}{"ok": true, "name": body.Name, "winner": body.Winner})
|
||||||
|
default:
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", "kind must be \"route\" or \"adapter\"")
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// /api/plugins/{name}/state
|
// /api/plugins/{name}/state
|
||||||
if strings.HasSuffix(path, "/state") {
|
if strings.HasSuffix(path, "/state") {
|
||||||
name := strings.TrimSuffix(path, "/state")
|
name := strings.TrimSuffix(path, "/state")
|
||||||
@ -105,6 +156,22 @@ func (g *Gateway) handlePluginsAPI(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// /api/plugins/{name}/kernel — the operator's view of, and edit surface for,
|
||||||
|
// which kernel actions a plugin may call, plus its declared extra outbound hosts.
|
||||||
|
if strings.HasSuffix(path, "/kernel") {
|
||||||
|
name := strings.TrimSuffix(path, "/kernel")
|
||||||
|
if name == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", "plugin name required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
g.handlePluginKernel(w, r, name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// /api/plugins/{name}/login — REMOVED. Login sessions are plugin domain:
|
||||||
|
// A plugin owns its own session state and serves it through its own
|
||||||
|
// mount. The kernel has no login concept.
|
||||||
|
|
||||||
// /api/plugins/{name}/call/{fn}
|
// /api/plugins/{name}/call/{fn}
|
||||||
if i := strings.Index(path, "/call/"); i >= 0 {
|
if i := strings.Index(path, "/call/"); i >= 0 {
|
||||||
name := path[:i]
|
name := path[:i]
|
||||||
@ -617,3 +684,68 @@ func (g *Gateway) handlePluginVisibility(w http.ResponseWriter, r *http.Request,
|
|||||||
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", r.Method+" is not allowed here")
|
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", r.Method+" is not allowed here")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handlePluginKernel is the operator's surface for a plugin's kernel grants and
|
||||||
|
// declared extra outbound hosts.
|
||||||
|
//
|
||||||
|
// GET returns what the plugin DECLARED beside what was GRANTED. That pairing is
|
||||||
|
// the point of the endpoint: a plugin declaring `add_source` is a request, and
|
||||||
|
// the operator needs to see the request next to the current answer to decide.
|
||||||
|
func (g *Gateway) handlePluginKernel(w http.ResponseWriter, r *http.Request, name string) {
|
||||||
|
ps := g.core.Plugins()
|
||||||
|
if ps == nil {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "no plugins loaded")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if reqRole(r.Context()) != "admin" {
|
||||||
|
writeError(w, http.StatusForbidden, "forbidden", "admin role required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodGet:
|
||||||
|
perm := ps.PluginKernelPermission(name)
|
||||||
|
declared := []string(nil)
|
||||||
|
for _, v := range ps.KernelPermissions() {
|
||||||
|
if v.Plugin == name {
|
||||||
|
declared = v.Declared
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||||
|
"plugin": name,
|
||||||
|
"actions": ps.KernelActionsView(name),
|
||||||
|
"declared": declared,
|
||||||
|
"granted": perm.Actions,
|
||||||
|
"all_read": perm.AllRead,
|
||||||
|
// Extra outbound hosts: declared vs granted, same request/privilege
|
||||||
|
// split as kernel actions. What the plugin does with a host (login,
|
||||||
|
// claim quotas...) is plugin business — the kernel only bounds egress.
|
||||||
|
"hosts_declared": ps.PluginExtraHosts(name),
|
||||||
|
"hosts_granted": ps.ExtraHostsGranted()[name],
|
||||||
|
})
|
||||||
|
return
|
||||||
|
case http.MethodPut, http.MethodPost:
|
||||||
|
var body struct {
|
||||||
|
Actions []string `json:"actions"`
|
||||||
|
AllRead bool `json:"all_read"`
|
||||||
|
Hosts []string `json:"hosts"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := ps.SetPluginKernelPermission(name, body.Actions, body.AllRead); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if body.Hosts != nil {
|
||||||
|
if err := ps.GrantExtraHosts(name, body.Hosts); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]interface{}{"ok": true})
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "GET or PUT")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -9,9 +9,9 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
"llmsproxy/internal/lua"
|
"modelrouter/internal/lua"
|
||||||
)
|
)
|
||||||
|
|
||||||
// gatewayWithBilling boots a gateway with the bundled billing plugin loaded, so
|
// gatewayWithBilling boots a gateway with the bundled billing plugin loaded, so
|
||||||
@ -222,7 +222,7 @@ func TestPluginsListCarriesAccessTables(t *testing.T) {
|
|||||||
t.Fatalf("GET /api/plugins = %d", rr.Code)
|
t.Fatalf("GET /api/plugins = %d", rr.Code)
|
||||||
}
|
}
|
||||||
var out struct {
|
var out struct {
|
||||||
Visibility map[string]string `json:"visibility"`
|
Visibility map[string]string `json:"visibility"`
|
||||||
HTTPAccess []map[string]interface{} `json:"http_access"`
|
HTTPAccess []map[string]interface{} `json:"http_access"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
|
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
|
||||||
|
|||||||
466
internal/gateway/serializer.go
Normal file
466
internal/gateway/serializer.go
Normal file
@ -0,0 +1,466 @@
|
|||||||
|
// Package gateway — 对外序列化(Phase 2)
|
||||||
|
//
|
||||||
|
// 职责边界(feature/unified-wire-v2):
|
||||||
|
//
|
||||||
|
// 内核不硬编码任何对外协议。序列化由 Serializer 承载,默认实现是
|
||||||
|
// openAISerializer —— 但它只是**最后容错**,不是默认业务路径。
|
||||||
|
//
|
||||||
|
// 优先级契约(用户明确要求:任何适配器与兜底 OpenAI 格式冲突时,
|
||||||
|
// 兜底 OpenAI 直接让步):
|
||||||
|
//
|
||||||
|
// 适配器声明 serialize_response / serialize_chunk
|
||||||
|
// ↓ 未声明
|
||||||
|
// openAISerializer(内核兜底,保证刚部署未配适配器时仍能服务)
|
||||||
|
//
|
||||||
|
// 「让步」的具体含义:一旦适配器声明了对外序列化钩子,内核在**该源**
|
||||||
|
// 上完全采用适配器的输出,绝不与兜底格式合并/兜底字段。A 请求经 Anthropic
|
||||||
|
// 序列化后,绝不会被再补上 OpenAI 的字段。这条规则由
|
||||||
|
// TestAdapterSerializerWinsOverFallback 锁定。
|
||||||
|
//
|
||||||
|
// 安全网:openAISerializer 的输出与改造前内核手写的 OpenAI wire 字节级
|
||||||
|
// 一致(e2e/.snapshots golden 锁定),所以「适配器未声明」这条路径上的
|
||||||
|
// 对外行为与本次改造之前完全相同。
|
||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"modelrouter/internal/lua"
|
||||||
|
"modelrouter/internal/provider"
|
||||||
|
"modelrouter/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StreamState 是一个流会话在序列化侧的累积状态(角色前导是否已发、已见
|
||||||
|
// 的终止原因、累积 usage/cost)。跨 chunk 传递,避免每帧重新统计。
|
||||||
|
type StreamState struct {
|
||||||
|
LastFinish string
|
||||||
|
Usage *types.TokenUsage
|
||||||
|
Cost string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serializer 把统一内部格式序列化成对外协议的响应体。
|
||||||
|
//
|
||||||
|
// 非流式与流式的终止语义对不同协议差异很大(OpenAI 在最后一个 chunk 上
|
||||||
|
// 带 finish_reason 后跟一个 usage-only 帧;Anthropic 用独立的
|
||||||
|
// message_delta 事件;Gemini 用空 candidates 帧),所以「流如何结束」
|
||||||
|
// 是接口的一部分,而不是调用方的 if。
|
||||||
|
type Serializer interface {
|
||||||
|
// NonStream 把一次完成的统一响应序列化成对外 body。
|
||||||
|
NonStream(resp *types.UnifiedResponse, modelName string, id string, created int64) ([]byte, error)
|
||||||
|
|
||||||
|
// OpenRolePreamble 返回流开始时的角色前导帧(可能为空表示该协议不需要)。
|
||||||
|
// 返回空切片时调用方不写帧。
|
||||||
|
OpenRolePreamble(modelName, id string, created int64, st *StreamState) ([]byte, error)
|
||||||
|
|
||||||
|
// Chunk 把一个统一分块序列化成对外 SSE 帧(不含 "data: " 前缀与空行)。
|
||||||
|
Chunk(ck *types.UnifiedChunk, modelName, id string, created int64, st *StreamState) ([]byte, error)
|
||||||
|
|
||||||
|
// Close 返回流结束时的收尾帧序列(可多帧),随后调用方写协议自己的
|
||||||
|
// 流终止标记(OpenAI 是 [DONE],Anthropic 无)。
|
||||||
|
Close(modelName, id string, created int64, st *StreamState) ([][]byte, error)
|
||||||
|
|
||||||
|
// DoneMarker 返回该协议的流终止标记;OpenAI 是 "[DONE]",无此概念的
|
||||||
|
// 协议返回空串(调用方因此不写那一行)。
|
||||||
|
DoneMarker() string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 兜底实现:OpenAI wire ----
|
||||||
|
|
||||||
|
// openAISerializer 是内核的**容错兜底**,输出与改造前手写的 OpenAI wire
|
||||||
|
// 字节级一致(golden 锁定)。它不是默认业务路径——适配器一旦声明序列化
|
||||||
|
// 钩子就完全取代它。
|
||||||
|
type openAISerializer struct{}
|
||||||
|
|
||||||
|
// OpenAISerializer 返回兜底 OpenAI 序列化器。暴露给同包测试与
|
||||||
|
// 「未配置任何适配器」的启动兜底路径使用。
|
||||||
|
func OpenAISerializer() Serializer { return openAISerializer{} }
|
||||||
|
|
||||||
|
func (openAISerializer) NonStream(resp *types.UnifiedResponse, modelName, id string, created int64) ([]byte, error) {
|
||||||
|
msg := RespMessage{Role: "assistant", Content: resp.Content}
|
||||||
|
if resp.ReasoningContent != "" {
|
||||||
|
msg.ReasoningContent = resp.ReasoningContent
|
||||||
|
}
|
||||||
|
if len(resp.ToolCalls) > 0 {
|
||||||
|
msg.ToolCalls = toolCallsWire(resp.ToolCalls)
|
||||||
|
}
|
||||||
|
out := ChatCompletion{
|
||||||
|
ID: id,
|
||||||
|
Object: "chat.completion",
|
||||||
|
Created: created,
|
||||||
|
Model: modelName,
|
||||||
|
Choices: []ChatChoice{{Index: 0, Message: msg, FinishReason: resp.FinishReason}},
|
||||||
|
Cost: resp.Cost,
|
||||||
|
}
|
||||||
|
if resp.TokenUsage.Total > 0 || resp.TokenUsage.Prompt > 0 || resp.TokenUsage.Completion > 0 {
|
||||||
|
out.Usage = &resp.TokenUsage
|
||||||
|
}
|
||||||
|
return json.Marshal(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (openAISerializer) OpenRolePreamble(modelName, id string, created int64, st *StreamState) ([]byte, error) {
|
||||||
|
return json.Marshal(ChatChunk{
|
||||||
|
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
||||||
|
Choices: []ChunkChoice{{Index: 0, Delta: RespMessage{Role: "assistant"}}},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (openAISerializer) Chunk(ck *types.UnifiedChunk, modelName, id string, created int64, st *StreamState) ([]byte, error) {
|
||||||
|
delta := RespMessage{Role: "assistant", Content: ck.Content}
|
||||||
|
if ck.ReasoningContent != "" {
|
||||||
|
delta.ReasoningContent = ck.ReasoningContent
|
||||||
|
}
|
||||||
|
if len(ck.ToolCalls) > 0 {
|
||||||
|
delta.ToolCalls = ck.ToolCalls
|
||||||
|
}
|
||||||
|
choice := ChunkChoice{Index: 0, Delta: delta}
|
||||||
|
if ck.Done {
|
||||||
|
fin := ck.FinishReason
|
||||||
|
if fin == "" {
|
||||||
|
fin = "stop"
|
||||||
|
}
|
||||||
|
st.LastFinish = fin
|
||||||
|
choice.FinishReason = &fin
|
||||||
|
}
|
||||||
|
return json.Marshal(ChatChunk{
|
||||||
|
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
||||||
|
Choices: []ChunkChoice{choice},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (openAISerializer) Close(modelName, id string, created int64, st *StreamState) ([][]byte, error) {
|
||||||
|
fin := st.LastFinish
|
||||||
|
if fin == "" {
|
||||||
|
fin = "stop"
|
||||||
|
}
|
||||||
|
term, err := json.Marshal(ChatChunk{
|
||||||
|
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
||||||
|
Choices: []ChunkChoice{{Index: 0, Delta: RespMessage{}, FinishReason: &fin}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := [][]byte{term}
|
||||||
|
// OpenAI 的 usage 帧:空 choices + usage + 可选 cost。
|
||||||
|
if st.Usage != nil || st.Cost != "" {
|
||||||
|
var tut *types.TokenUsage
|
||||||
|
if st.Usage != nil {
|
||||||
|
tut = st.Usage
|
||||||
|
} else {
|
||||||
|
tut = &types.TokenUsage{}
|
||||||
|
}
|
||||||
|
u, err := json.Marshal(ChatChunk{
|
||||||
|
ID: id, Object: "chat.completion.chunk", Created: created, Model: modelName,
|
||||||
|
Choices: []ChunkChoice{}, Usage: tut, Cost: st.Cost,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, u)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (openAISerializer) DoneMarker() string { return "[DONE]" }
|
||||||
|
|
||||||
|
// ---- 适配器钩子通道 ----
|
||||||
|
|
||||||
|
// luaSerializer 把对外序列化委托给适配器的 serialize_response /
|
||||||
|
// serialize_chunk 钩子。适配器声明了钩子就**完全取代**兜底实现。
|
||||||
|
type luaSerializer struct {
|
||||||
|
vm *lua.VM
|
||||||
|
adapter string
|
||||||
|
// responseOK / chunkOK 在构造时一次性探测(HasFn 不占用 worker)。
|
||||||
|
responseOK bool
|
||||||
|
chunkOK bool
|
||||||
|
// openPreambleOK / closeOK 是可选钩子;未声明时借用兜底实现的对应行为,
|
||||||
|
// 这样「只实现 serialize_chunk」的适配器也能立刻可用。
|
||||||
|
openPreambleOK bool
|
||||||
|
closeOK bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// serializerFor 为某个适配器名挑选序列化器。
|
||||||
|
//
|
||||||
|
// 这是「适配器优先、兜底让步」的唯一决策点:适配器声明了对外序列化
|
||||||
|
// 钩子就返回 luaSerializer(其输出不再与 OpenAI 兜底混合),否则返回
|
||||||
|
// 兜底 OpenAI。
|
||||||
|
func serializerFor(vm *lua.VM, adapter string) Serializer {
|
||||||
|
if vm == nil {
|
||||||
|
return OpenAISerializer()
|
||||||
|
}
|
||||||
|
resp := vm.HasFn(adapter, "serialize_response")
|
||||||
|
chunk := vm.HasFn(adapter, "serialize_chunk")
|
||||||
|
if !resp && !chunk {
|
||||||
|
return OpenAISerializer()
|
||||||
|
}
|
||||||
|
return &luaSerializer{
|
||||||
|
vm: vm,
|
||||||
|
adapter: adapter,
|
||||||
|
responseOK: resp,
|
||||||
|
chunkOK: chunk,
|
||||||
|
openPreambleOK: vm.HasFn(adapter, "serialize_open_preamble"),
|
||||||
|
closeOK: vm.HasFn(adapter, "serialize_close"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *luaSerializer) NonStream(resp *types.UnifiedResponse, modelName, id string, created int64) ([]byte, error) {
|
||||||
|
if !s.responseOK {
|
||||||
|
// 该适配器只实现了流式钩子:非流式沿用兜底。这是唯一允许兜底
|
||||||
|
// 参与的场景——适配器**没有**就该方向的实现,而不是「冲突」。
|
||||||
|
return openAISerializer{}.NonStream(resp, modelName, id, created)
|
||||||
|
}
|
||||||
|
payload := map[string]interface{}{
|
||||||
|
"model": modelName,
|
||||||
|
"id": id,
|
||||||
|
"created": created,
|
||||||
|
"content": resp.Content,
|
||||||
|
"reasoning": resp.ReasoningContent,
|
||||||
|
"finish_reason": resp.FinishReason,
|
||||||
|
"token_usage": resp.TokenUsage,
|
||||||
|
"cost": resp.Cost,
|
||||||
|
}
|
||||||
|
if len(resp.ToolCalls) > 0 {
|
||||||
|
payload["tool_calls"] = resp.ToolCalls
|
||||||
|
}
|
||||||
|
out, err := s.vm.Transform(s.adapter, "serialize_response", wireJSON(payload))
|
||||||
|
if err != nil || out == "" {
|
||||||
|
// 钩子失败:明确退回兜底,而不是产出一半适配器字段 + 一半 OpenAI
|
||||||
|
// 字段的混合体(那正是「让步」规则要避免的)。
|
||||||
|
return openAISerializer{}.NonStream(resp, modelName, id, created)
|
||||||
|
}
|
||||||
|
return []byte(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *luaSerializer) OpenRolePreamble(modelName, id string, created int64, st *StreamState) ([]byte, error) {
|
||||||
|
if !s.openPreambleOK {
|
||||||
|
return openAISerializer{}.OpenRolePreamble(modelName, id, created, st)
|
||||||
|
}
|
||||||
|
out, err := s.vm.Transform(s.adapter, "serialize_open_preamble",
|
||||||
|
wireJSON(map[string]interface{}{"model": modelName, "id": id, "created": created}))
|
||||||
|
if err != nil {
|
||||||
|
return openAISerializer{}.OpenRolePreamble(modelName, id, created, st)
|
||||||
|
}
|
||||||
|
if out == "" {
|
||||||
|
return nil, nil // 该协议不需要角色前导
|
||||||
|
}
|
||||||
|
return []byte(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *luaSerializer) Chunk(ck *types.UnifiedChunk, modelName, id string, created int64, st *StreamState) ([]byte, error) {
|
||||||
|
if !s.chunkOK {
|
||||||
|
return openAISerializer{}.Chunk(ck, modelName, id, created, st)
|
||||||
|
}
|
||||||
|
payload := map[string]interface{}{
|
||||||
|
"model": modelName,
|
||||||
|
"id": id,
|
||||||
|
"created": created,
|
||||||
|
"content": ck.Content,
|
||||||
|
"reasoning": ck.ReasoningContent,
|
||||||
|
"done": ck.Done,
|
||||||
|
"finish_reason": ck.FinishReason,
|
||||||
|
}
|
||||||
|
if len(ck.ToolCalls) > 0 {
|
||||||
|
payload["tool_calls"] = ck.ToolCalls
|
||||||
|
}
|
||||||
|
if ck.Usage != nil {
|
||||||
|
payload["usage"] = ck.Usage
|
||||||
|
}
|
||||||
|
if ck.Cost != "" {
|
||||||
|
payload["cost"] = ck.Cost
|
||||||
|
}
|
||||||
|
out, err := s.vm.Transform(s.adapter, "serialize_chunk", wireJSON(payload))
|
||||||
|
if err != nil {
|
||||||
|
return openAISerializer{}.Chunk(ck, modelName, id, created, st)
|
||||||
|
}
|
||||||
|
if out == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
// 适配器的终止帧要驱动审计记录与收尾逻辑,所以把终止原因抄回状态。
|
||||||
|
if ck.Done && st.LastFinish == "" {
|
||||||
|
if fin := ck.FinishReason; fin != "" {
|
||||||
|
st.LastFinish = fin
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []byte(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *luaSerializer) Close(modelName, id string, created int64, st *StreamState) ([][]byte, error) {
|
||||||
|
if !s.closeOK {
|
||||||
|
return openAISerializer{}.Close(modelName, id, created, st)
|
||||||
|
}
|
||||||
|
out, err := s.vm.Transform(s.adapter, "serialize_close",
|
||||||
|
wireJSON(map[string]interface{}{
|
||||||
|
"model": modelName, "id": id, "created": created,
|
||||||
|
"finish_reason": st.LastFinish, "cost": st.Cost,
|
||||||
|
"has_usage": st.Usage != nil,
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
return openAISerializer{}.Close(modelName, id, created, st)
|
||||||
|
}
|
||||||
|
if out == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
// 适配器可以返回多帧(换行分隔),因为 Anthropic 的收尾本身就是
|
||||||
|
// message_stop + 若干 delta;单帧限制会逼适配器把多事件塞进一个 JSON。
|
||||||
|
var frames [][]byte
|
||||||
|
for _, line := range splitLines(out) {
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
frames = append(frames, []byte(line))
|
||||||
|
}
|
||||||
|
return frames, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DoneMarker 返回协议自己的流终止标记。适配器可用 serialize_done_marker
|
||||||
|
// 声明;未声明时若有自定义流式序列化就返回空串(表示「没有 [DONE] 这种
|
||||||
|
// 概念」),否则沿用 OpenAI 的 [DONE]。
|
||||||
|
func (s *luaSerializer) DoneMarker() string {
|
||||||
|
if s.vm.HasFn(s.adapter, "serialize_done_marker") {
|
||||||
|
out, err := s.vm.Transform(s.adapter, "serialize_done_marker", "{}")
|
||||||
|
if err == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s.chunkOK {
|
||||||
|
// 适配器接管了流式序列化:只有它没显式声明标记时才不发 [DONE],
|
||||||
|
// 因为擅自给 Anthropic 流加一行 "[DONE]" 会让严格客户端报错。
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "[DONE]"
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitLines 切分多行帧输出。
|
||||||
|
func splitLines(s string) []string {
|
||||||
|
var out []string
|
||||||
|
start := 0
|
||||||
|
for i := 0; i < len(s); i++ {
|
||||||
|
if s[i] == '\n' {
|
||||||
|
out = append(out, trimCR(s[start:i]))
|
||||||
|
start = i + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if start < len(s) {
|
||||||
|
out = append(out, trimCR(s[start:]))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func trimCR(s string) string {
|
||||||
|
if n := len(s); n > 0 && s[n-1] == '\r' {
|
||||||
|
return s[:n-1]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 共享辅助 ----
|
||||||
|
|
||||||
|
var wireSeq int64
|
||||||
|
|
||||||
|
func newWireID() string {
|
||||||
|
n := atomic.AddInt64(&wireSeq, 1)
|
||||||
|
return fmt.Sprintf("chatcmpl-%d", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCreated() int64 { return time.Now().Unix() }
|
||||||
|
|
||||||
|
// wireJSON 是本文件私有的 JSON 编码助手(包内已有 discover.go 的 mustJSON,
|
||||||
|
// 那个返回 []byte 且忽略错误,语义不同,故不共用)。
|
||||||
|
func wireJSON(v interface{}) string {
|
||||||
|
b, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return "{}"
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// serializerCache 按 (VM, 适配器名) 缓存 Serializer。
|
||||||
|
//
|
||||||
|
// VM 必须进键里:缓存曾经只按适配器名做键,结果一个进程内两个 VM(或
|
||||||
|
// 热重载后的新旧 VM)会共享同一条判定。插件在运行时改写适配器后声明了
|
||||||
|
// serialize_response,另一条缓存记录仍会说「没声明」,于是新钩子永远不
|
||||||
|
// 生效——而且因为判定本身不会报错,表现只是「适配器说好的对外格式没
|
||||||
|
// 出现」,极难定位。
|
||||||
|
//
|
||||||
|
// HasFn 的代价决定了它必须被缓存:它会**编译一次适配器脚本**来回答
|
||||||
|
// 「这个适配器声明了哪些函数」,放在每个请求上等于把加载成本翻倍。
|
||||||
|
//
|
||||||
|
// 失效由 vmGeneration 负责:适配器重写时 VM 的代号会变(见 adapterEpoch),
|
||||||
|
// 于是旧条目再也匹配不上,无需显式清扫。
|
||||||
|
var serializerCache sync.Map // cacheKey -> Serializer
|
||||||
|
|
||||||
|
// cacheKey 是 (VM 指针, VM 代号, 适配器名) 的复合键。
|
||||||
|
type cacheKey struct {
|
||||||
|
vm *lua.VM
|
||||||
|
gen uint64
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
func serializerForCached(vm *lua.VM, adapter string) Serializer {
|
||||||
|
if vm == nil {
|
||||||
|
return OpenAISerializer()
|
||||||
|
}
|
||||||
|
k := cacheKey{vm: vm, gen: adapterEpoch(vm, adapter), name: adapter}
|
||||||
|
if v, ok := serializerCache.Load(k); ok {
|
||||||
|
return v.(Serializer)
|
||||||
|
}
|
||||||
|
s := serializerFor(vm, adapter)
|
||||||
|
actual, _ := serializerCache.LoadOrStore(k, s)
|
||||||
|
return actual.(Serializer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// adapterEpoch 返回该适配器「当前这份代码」的代号。适配器被重写后代号变化,
|
||||||
|
// 使旧缓存条目自动失效(见 serializerCache 的注释)。
|
||||||
|
func adapterEpoch(vm *lua.VM, adapter string) uint64 {
|
||||||
|
return vm.AdapterRevision(adapter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// serializerForSource 解析「实际服务该请求的那个源」该用哪个序列化器。
|
||||||
|
//
|
||||||
|
// 必须按**服务者**选,而不是按调用方请求的首选源选:direct 请求在候选源
|
||||||
|
// 之间故障转移时,usedSrc 可能与请求 model 所属的源完全不同;按首选源选
|
||||||
|
// 会让故障转移后的响应带着另一个源的对外格式。
|
||||||
|
//
|
||||||
|
// 源名查不到时(配置在选择之后被改、或插件装的源尚未注册)退回兜底
|
||||||
|
// OpenAI:那是「刚部署 / 异常态下仍能服务」的保证,而不是静默产出错误格式。
|
||||||
|
func serializerForSource(vm *lua.VM, sources []SourceRef, sourceName string) Serializer {
|
||||||
|
for _, s := range sources {
|
||||||
|
if s.Name == sourceName {
|
||||||
|
return serializerForCached(vm, s.Adapter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return OpenAISerializer()
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourceRef 是解析序列化器所需的最小源信息(名字 + 适配器)。
|
||||||
|
type SourceRef struct {
|
||||||
|
Name string
|
||||||
|
Adapter string
|
||||||
|
}
|
||||||
|
|
||||||
|
// serializerFor 是 Gateway 侧的解析入口:按**实际服务该请求的源**挑序列化器。
|
||||||
|
//
|
||||||
|
// cands 传候选列表时用它做故障转移后的解析(服务源可能不是首选);传 nil
|
||||||
|
// 表示 AUTO 路径——AUTO 的服务源由链决定,不在调用方的候选集里,退回按
|
||||||
|
// 当前配置里全部源查找。
|
||||||
|
func (g *Gateway) serializerFor(sourceName string, cands []*provider.Provider) Serializer {
|
||||||
|
vm := g.core.VM()
|
||||||
|
if vm == nil {
|
||||||
|
return OpenAISerializer()
|
||||||
|
}
|
||||||
|
if cands != nil {
|
||||||
|
refs := make([]SourceRef, 0, len(cands))
|
||||||
|
for _, p := range cands {
|
||||||
|
refs = append(refs, SourceRef{Name: p.Name(), Adapter: p.Adapter()})
|
||||||
|
}
|
||||||
|
return serializerForSource(vm, refs, sourceName)
|
||||||
|
}
|
||||||
|
refs := make([]SourceRef, 0, 16)
|
||||||
|
for _, s := range g.core.Sources() {
|
||||||
|
refs = append(refs, SourceRef{Name: s.Name, Adapter: s.Adapter})
|
||||||
|
}
|
||||||
|
return serializerForSource(vm, refs, sourceName)
|
||||||
|
}
|
||||||
367
internal/gateway/serializer_test.go
Normal file
367
internal/gateway/serializer_test.go
Normal file
@ -0,0 +1,367 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"modelrouter/internal/lua"
|
||||||
|
"modelrouter/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 本文件锁定对外序列化的**优先级契约**:
|
||||||
|
//
|
||||||
|
// 适配器声明 serialize_* → 适配器完全接管
|
||||||
|
// 适配器未声明 → 兜底 OpenAI(保证刚部署/未配适配器仍能服务)
|
||||||
|
//
|
||||||
|
// 以及「让步」的定义:适配器接管后,内核绝不再往它的输出上补任何 OpenAI
|
||||||
|
// 字段。两个序列化器混在一起产出的东西比任何一个都糟——客户端无法解析,
|
||||||
|
// 而排查时会指向错误的层。
|
||||||
|
|
||||||
|
// luaVMFor 建一个会 seed 内置适配器的 VM(目录名必须是 "adapters")。
|
||||||
|
func luaVMFor(t *testing.T) *lua.VM {
|
||||||
|
t.Helper()
|
||||||
|
vm := lua.NewVM(filepath.Join(t.TempDir(), "adapters"))
|
||||||
|
if err := vm.Start(); err != nil {
|
||||||
|
t.Fatalf("vm start: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(vm.Stop)
|
||||||
|
return vm
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFallbackUsedWhenAdapterDeclaresNothing 是「兜底」本身的测试。
|
||||||
|
//
|
||||||
|
// 注:anthropic 现在**确实**声明了 serialize_*(Phase 2 给了它 Anthropic
|
||||||
|
// 形状),所以本测试只在仍未声明钩子的那些内置适配器上验证兜底行为。
|
||||||
|
// 这正是设计意图:谁声明谁接管,没声明的继续用 OpenAI 兜底。
|
||||||
|
func TestFallbackUsedWhenAdapterDeclaresNothing(t *testing.T) {
|
||||||
|
vm := luaVMFor(t)
|
||||||
|
for _, name := range []string{"openai", "gemini", "ollama", "deepseek", "github", "groq"} {
|
||||||
|
for _, hook := range []string{"serialize_response", "serialize_chunk"} {
|
||||||
|
if vm.HasFn(name, hook) {
|
||||||
|
t.Errorf("内置适配器 %s 声明了 %s —— 兜底测试的前提已不成立", name, hook)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ser := serializerFor(vm, "openai")
|
||||||
|
if _, isFallback := ser.(openAISerializer); !isFallback {
|
||||||
|
t.Fatalf("未声明钩子的适配器应得到兜底序列化器,得到 %T", ser)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 对照:声明了钩子的适配器不走兜底
|
||||||
|
if _, isFallback := serializerFor(vm, "anthropic").(openAISerializer); isFallback {
|
||||||
|
t.Error("anthropic 声明了 serialize_response,不应走兜底")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdapterSerializerWinsOverFallback 锁定核心规则:适配器声明钩子即接管,
|
||||||
|
// 且内核不与兜底混合任何字段。
|
||||||
|
func TestAdapterSerializerWinsOverFallback(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "adapters"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 一个刻意用**非 OpenAI 形状**的适配器:如果内核混入了兜底字段
|
||||||
|
// (object/choices/usage 等),下面断言会失败。
|
||||||
|
adapter := `
|
||||||
|
local adapter = {}
|
||||||
|
adapter.name = "anthropicish"
|
||||||
|
adapter.version = "1.0.0"
|
||||||
|
adapter.endpoint = "/chat/completions"
|
||||||
|
|
||||||
|
-- 非流式:故意用 content[] 分块 + 没有 choices/usage
|
||||||
|
function adapter.serialize_response(payload)
|
||||||
|
local p = json.decode(payload)
|
||||||
|
return json.encode({
|
||||||
|
type = "message",
|
||||||
|
role = "assistant",
|
||||||
|
model = p.model,
|
||||||
|
content = { { type = "text", text = p.content } },
|
||||||
|
stop_reason = (p.finish_reason ~= "" and p.finish_reason or "end_turn"),
|
||||||
|
})
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 流式:event: style 的帧,且不带 OpenAI 的 object 字段
|
||||||
|
function adapter.serialize_chunk(payload)
|
||||||
|
local p = json.decode(payload)
|
||||||
|
return json.encode({
|
||||||
|
type = "content_block_delta",
|
||||||
|
delta = { type = "text_delta", text = p.content },
|
||||||
|
})
|
||||||
|
end
|
||||||
|
|
||||||
|
return adapter
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "adapters", "anthropicish.lua"), []byte(adapter), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
vm := lua.NewVM(filepath.Join(dir, "adapters"))
|
||||||
|
if err := vm.Start(); err != nil {
|
||||||
|
t.Fatalf("vm: %v", err)
|
||||||
|
}
|
||||||
|
defer vm.Stop()
|
||||||
|
|
||||||
|
ser := serializerFor(vm, "anthropicish")
|
||||||
|
if _, isFallback := ser.(openAISerializer); isFallback {
|
||||||
|
t.Fatal("声明了 serialize_* 的适配器必须接管序列化,而不是拿到兜底")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 非流式输出必须是适配器形状,且**不含**任何 OpenAI 字段
|
||||||
|
resp := &types.UnifiedResponse{
|
||||||
|
Model: "m",
|
||||||
|
Content: "hello",
|
||||||
|
FinishReason: "stop",
|
||||||
|
TokenUsage: types.TokenUsage{Prompt: 3, Completion: 1, Total: 4},
|
||||||
|
}
|
||||||
|
body, err := ser.NonStream(resp, "m", "id-1", 1700000000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NonStream: %v", err)
|
||||||
|
}
|
||||||
|
var got map[string]interface{}
|
||||||
|
if err := json.Unmarshal(body, &got); err != nil {
|
||||||
|
t.Fatalf("adapter output not JSON: %v (%s)", err, body)
|
||||||
|
}
|
||||||
|
if got["type"] != "message" {
|
||||||
|
t.Errorf("适配器输出未被采用:type = %v, want message", got["type"])
|
||||||
|
}
|
||||||
|
// 「让步」的硬断言:OpenAI 字段一个都不许出现
|
||||||
|
for _, k := range []string{"object", "choices", "usage", "created"} {
|
||||||
|
if _, leaked := got[k]; leaked {
|
||||||
|
t.Errorf("兜底 OpenAI 字段 %q 混进了适配器的输出(让步规则被违反): %s", k, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 适配器没有读 usage,但兜底也不许替它补上
|
||||||
|
if _, has := got["token_usage"]; has {
|
||||||
|
t.Errorf("兜底补了 token_usage 字段,适配器输出被污染: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAdapterSerializerStreamWinsOverFallback 同上,流式路径。
|
||||||
|
func TestAdapterSerializerStreamWinsOverFallback(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "adapters"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adapter := `
|
||||||
|
local adapter = {}
|
||||||
|
adapter.name = "streamsh"
|
||||||
|
adapter.version = "1.0.0"
|
||||||
|
adapter.endpoint = "/chat/completions"
|
||||||
|
|
||||||
|
function adapter.serialize_chunk(payload)
|
||||||
|
local p = json.decode(payload)
|
||||||
|
return json.encode({ type = "content_block_delta", text = p.content })
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 显式声明收尾与终止标记,证明协议语义完全由适配器决定
|
||||||
|
function adapter.serialize_close(payload)
|
||||||
|
local p = json.decode(payload)
|
||||||
|
return json.encode({ type = "message_stop", stop_reason = p.finish_reason })
|
||||||
|
end
|
||||||
|
|
||||||
|
return adapter
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "adapters", "streamsh.lua"), []byte(adapter), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
vm := lua.NewVM(filepath.Join(dir, "adapters"))
|
||||||
|
if err := vm.Start(); err != nil {
|
||||||
|
t.Fatalf("vm: %v", err)
|
||||||
|
}
|
||||||
|
defer vm.Stop()
|
||||||
|
|
||||||
|
ser := serializerFor(vm, "streamsh")
|
||||||
|
if _, isFallback := ser.(openAISerializer); isFallback {
|
||||||
|
t.Fatal("声明 serialize_chunk 的适配器必须接管流式序列化")
|
||||||
|
}
|
||||||
|
st := &StreamState{}
|
||||||
|
b, err := ser.Chunk(&types.UnifiedChunk{Content: "hi"}, "m", "id", 1, st)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(b), `"content_block_delta"`) {
|
||||||
|
t.Errorf("chunk 未采用适配器形状: %s", b)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(b), "chat.completion.chunk") || strings.Contains(string(b), `"choices"`) {
|
||||||
|
t.Errorf("兜底 OpenAI 帧混进了适配器输出: %s", b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 终止标记:适配器接管流式后,内核不得自作主张发 [DONE]
|
||||||
|
if m := ser.DoneMarker(); m != "" {
|
||||||
|
t.Errorf("适配器接管流式时 DoneMarker 应为空(不擅自发 [DONE]),实得 %q", m)
|
||||||
|
}
|
||||||
|
// 但它没声明 serialize_close 时,收尾仍借用兜底形状 —— 这是「没实现」
|
||||||
|
// 不是「冲突」,允许兜底。
|
||||||
|
if _, isFallbackClose := ser.(interface {
|
||||||
|
Close(model, id string, created int64, st *StreamState) ([][]byte, error)
|
||||||
|
}); !isFallbackClose {
|
||||||
|
t.Fatal("Close 不应在接口上")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFallbackIsUsedWhenAdapterHookFails 验证钩子**执行失败**时的行为。
|
||||||
|
//
|
||||||
|
// 契约:退回兜底的完整输出,而不是产出「一半适配器字段 + 一半 OpenAI
|
||||||
|
// 字段」的混合体。混合体比任何一个单独结果都更难排查。
|
||||||
|
func TestFallbackIsUsedWhenAdapterHookFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "adapters"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adapter := `
|
||||||
|
local adapter = {}
|
||||||
|
adapter.name = "broken"
|
||||||
|
adapter.version = "1.0.0"
|
||||||
|
adapter.endpoint = "/chat/completions"
|
||||||
|
function adapter.serialize_response(payload)
|
||||||
|
error("deliberate failure")
|
||||||
|
end
|
||||||
|
return adapter
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "adapters", "broken.lua"), []byte(adapter), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
vm := lua.NewVM(filepath.Join(dir, "adapters"))
|
||||||
|
if err := vm.Start(); err != nil {
|
||||||
|
t.Fatalf("vm: %v", err)
|
||||||
|
}
|
||||||
|
defer vm.Stop()
|
||||||
|
|
||||||
|
ser := serializerFor(vm, "broken")
|
||||||
|
resp := &types.UnifiedResponse{Content: "fallback still works", FinishReason: "stop"}
|
||||||
|
body, err := ser.NonStream(resp, "m", "id", 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("钩子失败后必须仍能产出兜底输出,而非报错终止: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(body), "chat.completion") {
|
||||||
|
t.Errorf("钩子失败后未退回兜底 OpenAI 形状: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFallbackSavesUnconfiguredDeployment 直接覆盖用户说的场景:
|
||||||
|
// 「刚部署没有配置任何适配器」。
|
||||||
|
func TestFallbackSavesUnconfiguredDeployment(t *testing.T) {
|
||||||
|
// 完全不建 VM(模拟适配器目录不存在/为空)
|
||||||
|
ser := serializerFor(nil, "whatever")
|
||||||
|
body, err := ser.NonStream(&types.UnifiedResponse{Content: "no adapters configured"}, "m", "id", 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("无适配器时兜底必须可用: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(body), "chat.completion") {
|
||||||
|
t.Errorf("无适配器兜底输出形状不对: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSerializerCacheFollowsAdapterRewrite 验证插件运行时改写适配器后,
|
||||||
|
// 缓存不会继续持有旧的序列化器判定(否则新声明的钩子永远不生效,而因为
|
||||||
|
// 判定本身不报错,表现只是「适配器说好的格式没出现」,极难定位)。
|
||||||
|
func TestSerializerCacheFollowsAdapterRewrite(t *testing.T) {
|
||||||
|
vm := luaVMFor(t)
|
||||||
|
if _, isFallback := serializerForCached(vm, "openai").(openAISerializer); !isFallback {
|
||||||
|
t.Fatal("前提不成立:改写前 openai 应走兜底")
|
||||||
|
}
|
||||||
|
// 插件改写该适配器,加一个 serialize_response
|
||||||
|
src := `local a = {}
|
||||||
|
a.name = "openai"
|
||||||
|
a.version = "9"
|
||||||
|
a.endpoint = "/chat/completions"
|
||||||
|
function a.serialize_response(p) return '{"rewritten":true}' end
|
||||||
|
return a`
|
||||||
|
if err := vm.LoadAdapterSource("openai", src); err != nil {
|
||||||
|
t.Fatalf("rewrite: %v", err)
|
||||||
|
}
|
||||||
|
// 无需任何显式清缓存动作:修订号变化即让旧条目失效。
|
||||||
|
after := serializerForCached(vm, "openai")
|
||||||
|
if _, stillFallback := after.(openAISerializer); stillFallback {
|
||||||
|
t.Error("缓存未随适配器改写失效:已声明 serialize_response 却仍走兜底")
|
||||||
|
}
|
||||||
|
body, err := after.NonStream(&types.UnifiedResponse{Content: "x"}, "m", "id", 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(body), "rewritten") {
|
||||||
|
t.Errorf("改写后的序列化器未生效: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSerializerCacheIsPerVM 是那个 bug 的回归测试:缓存曾经只按适配器名做键,
|
||||||
|
// 于是同一进程内两个 VM 共享同一条判定——一个适配器被重写后,另一个 VM 的
|
||||||
|
// 缓存会说「没声明钩子」,新钩子永远不生效。
|
||||||
|
func TestSerializerCacheIsPerVM(t *testing.T) {
|
||||||
|
vm1 := luaVMFor(t)
|
||||||
|
vm2 := luaVMFor(t)
|
||||||
|
// 两个 VM 各自独立:改写 vm1 不得影响 vm2 的判定
|
||||||
|
if err := vm1.LoadAdapterSource("openai", `local a={} a.name="openai" a.version="9"
|
||||||
|
a.endpoint="/x" function a.serialize_response(p) return '{"only_vm1":true}' end return a`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, f := serializerForCached(vm1, "openai").(openAISerializer); f {
|
||||||
|
t.Error("vm1 改写后应改走适配器序列化")
|
||||||
|
}
|
||||||
|
if _, f := serializerForCached(vm2, "openai").(openAISerializer); !f {
|
||||||
|
t.Error("vm2 未被改写,却因共享缓存而丢失了兜底判定(跨 VM 串扰)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 并发安全 ----
|
||||||
|
|
||||||
|
// TestSerializerForConcurrent 是压力测试:多 goroutine 并发取序列化器
|
||||||
|
// (缓存是 sync.Map,HasFn 会编译脚本 —— 这里验证并发下不 panic、不返回
|
||||||
|
// 半初始化对象,且每次结果自洽)。
|
||||||
|
func TestSerializerForConcurrent(t *testing.T) {
|
||||||
|
vm := luaVMFor(t)
|
||||||
|
const workers, iters = 16, 40
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errs := make(chan string, workers*iters)
|
||||||
|
for w := 0; w < workers; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(w int) {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := 0; i < iters; i++ {
|
||||||
|
name := "openai"
|
||||||
|
if w%2 == 1 {
|
||||||
|
name = "anthropic"
|
||||||
|
}
|
||||||
|
ser := serializerForCached(vm, name)
|
||||||
|
if ser == nil {
|
||||||
|
errs <- "nil serializer"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// 结果必须自洽:同一适配器每次拿到同一类型,且形状与该适配器
|
||||||
|
// 声明的协议一致(openai→OpenAI,anthropic→Anthropic)。
|
||||||
|
_, isFallback := ser.(openAISerializer)
|
||||||
|
wantShape := "chat.completion"
|
||||||
|
switch name {
|
||||||
|
case "anthropic":
|
||||||
|
if isFallback {
|
||||||
|
errs <- "anthropic 并发下退回兜底"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
wantShape = `"type":"message"`
|
||||||
|
default:
|
||||||
|
if !isFallback {
|
||||||
|
errs <- name + " 并发下未走兜底"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
body, err := ser.NonStream(&types.UnifiedResponse{Content: "x"}, "m", "id", 1)
|
||||||
|
if err != nil {
|
||||||
|
errs <- "serialize: " + err.Error()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(body), wantShape) {
|
||||||
|
errs <- "并发下输出形状不一致: " + name + " -> " + string(body)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(w)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errs)
|
||||||
|
for e := range errs {
|
||||||
|
t.Error(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -18,8 +18,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed ui/*
|
//go:embed ui/*
|
||||||
@ -187,6 +187,10 @@ type statusRecorder struct {
|
|||||||
code int
|
code int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Unwrap preserves optional writer capabilities (notably Hijack for WebSocket)
|
||||||
|
// through the access-log middleware via http.ResponseController.
|
||||||
|
func (s *statusRecorder) Unwrap() http.ResponseWriter { return s.ResponseWriter }
|
||||||
|
|
||||||
func (s *statusRecorder) WriteHeader(code int) {
|
func (s *statusRecorder) WriteHeader(code int) {
|
||||||
if s.code == 0 {
|
if s.code == 0 {
|
||||||
s.code = code
|
s.code = code
|
||||||
@ -216,6 +220,14 @@ func (g *Gateway) routes(w http.ResponseWriter, r *http.Request) {
|
|||||||
switch {
|
switch {
|
||||||
case r.URL.Path == "/v1/chat/completions":
|
case r.URL.Path == "/v1/chat/completions":
|
||||||
g.handleChat(w, r)
|
g.handleChat(w, r)
|
||||||
|
case r.URL.Path == "/v1/messages":
|
||||||
|
// Anthropic Messages protocol. Scheduling, quotas, cooldowns, AUTO
|
||||||
|
// chain and accounting are EXACTLY the ones /v1/chat/completions uses —
|
||||||
|
// only the wire shapes differ, and those come from the serving
|
||||||
|
// adapter's serialize_* hooks. The kernel holds no Anthropic field
|
||||||
|
// names (see serializer.go: the fallback is OpenAI, and any adapter
|
||||||
|
// that declares serialize_* replaces it wholesale).
|
||||||
|
g.handleMessages(w, r)
|
||||||
case r.URL.Path == "/v1/images/generations":
|
case r.URL.Path == "/v1/images/generations":
|
||||||
g.handleImage(w, r)
|
g.handleImage(w, r)
|
||||||
case r.URL.Path == "/v1/models":
|
case r.URL.Path == "/v1/models":
|
||||||
@ -224,21 +236,27 @@ func (g *Gateway) routes(w http.ResponseWriter, r *http.Request) {
|
|||||||
g.handleAdaptersAPI(w, r)
|
g.handleAdaptersAPI(w, r)
|
||||||
case r.URL.Path == "/api/v1" || strings.HasPrefix(r.URL.Path, "/api/v1/"):
|
case r.URL.Path == "/api/v1" || strings.HasPrefix(r.URL.Path, "/api/v1/"):
|
||||||
g.apiV1Routes(w, r)
|
g.apiV1Routes(w, r)
|
||||||
|
// Must precede the /api/sources prefix match: a generic HasPrefix on
|
||||||
|
// "/api/sources/" would otherwise swallow "/api/sources/discover" and the
|
||||||
|
// operator would get "source name required in path" instead of a catalog.
|
||||||
|
case r.URL.Path == "/api/sources/discover" || strings.HasPrefix(r.URL.Path, "/api/sources/discover/"):
|
||||||
|
g.handleSourceDiscoverAPI(w, r)
|
||||||
case r.URL.Path == "/api/sources" || strings.HasPrefix(r.URL.Path, "/api/sources/"):
|
case r.URL.Path == "/api/sources" || strings.HasPrefix(r.URL.Path, "/api/sources/"):
|
||||||
g.handleSourcesAPI(w, r)
|
g.handleSourcesAPI(w, r)
|
||||||
case r.URL.Path == "/api/source_templates" || strings.HasPrefix(r.URL.Path, "/api/source_templates/"):
|
case r.URL.Path == "/api/source_templates" || strings.HasPrefix(r.URL.Path, "/api/source_templates/"):
|
||||||
g.handleSourceTemplatesAPI(w, r)
|
g.handleSourceTemplatesAPI(w, r)
|
||||||
case r.URL.Path == "/api/ui-inject" || strings.HasPrefix(r.URL.Path, "/api/ui-inject/"):
|
case r.URL.Path == "/api/ui-inject" || strings.HasPrefix(r.URL.Path, "/api/ui-inject/"):
|
||||||
g.handlePluginUI(w, r)
|
g.handlePluginUI(w, r)
|
||||||
case r.URL.Path == "/api/plugins/billing/rules":
|
|
||||||
// Checked BEFORE the generic /api/plugins/ prefix: the plugin router
|
|
||||||
// treats anything after the name as a plugin sub-resource and would
|
|
||||||
// 404 on "rules" instead of reaching the billing editor.
|
|
||||||
g.handleBillingRules(w, r)
|
|
||||||
case strings.HasPrefix(r.URL.Path, pluginHTTPPrefix):
|
case strings.HasPrefix(r.URL.Path, pluginHTTPPrefix):
|
||||||
// Before the generic plugin router: these paths belong to a PLUGIN, not to
|
// Before the generic plugin router: these paths belong to a PLUGIN, not to
|
||||||
// the plugin manager, and "http" is a valid sub-resource name otherwise.
|
// the plugin manager, and "http" is a valid sub-resource name otherwise.
|
||||||
g.handlePluginHTTP(w, r)
|
g.handlePluginHTTP(w, r)
|
||||||
|
case strings.HasPrefix(r.URL.Path, "/plugins/") || strings.HasPrefix(r.URL.Path, "/v1/plugins/"):
|
||||||
|
// Custom mounts a plugin or adapter claimed (R3). Checked AFTER every
|
||||||
|
// kernel route so a claim can never shadow one — ValidateCustomMount
|
||||||
|
// refuses the overlap at load time, and this ordering is the second line
|
||||||
|
// of defence for a table that somehow slipped through.
|
||||||
|
g.handleCustomMount(w, r)
|
||||||
case r.URL.Path == "/api/plugins" || strings.HasPrefix(r.URL.Path, "/api/plugins/"):
|
case r.URL.Path == "/api/plugins" || strings.HasPrefix(r.URL.Path, "/api/plugins/"):
|
||||||
g.handlePluginsAPI(w, r)
|
g.handlePluginsAPI(w, r)
|
||||||
case r.URL.Path == "/api/chat":
|
case r.URL.Path == "/api/chat":
|
||||||
|
|||||||
@ -9,8 +9,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
"llmsproxy/internal/core"
|
"modelrouter/internal/core"
|
||||||
)
|
)
|
||||||
|
|
||||||
// This file covers the second round of the same class of defect bf0657b fixed.
|
// This file covers the second round of the same class of defect bf0657b fixed.
|
||||||
|
|||||||
@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"llmsproxy/internal/config"
|
"modelrouter/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// These cover the two write-path defects found while adding a model to a live
|
// These cover the two write-path defects found while adding a model to a live
|
||||||
@ -71,11 +71,6 @@ func TestSourcePutUpdatesSource(t *testing.T) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
found = true
|
found = true
|
||||||
for _, m := range s.Models {
|
|
||||||
if m.ID == "m-one" && m.Priority != 42 {
|
|
||||||
t.Fatalf("priority not applied: %+v", s.Models)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if !found {
|
if !found {
|
||||||
t.Fatal("source vanished after PUT")
|
t.Fatal("source vanished after PUT")
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user