mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-11-07 17:57:23 +00:00
Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6733534282 | |||
| 499f0cac2f | |||
| 7fb8f96b82 | |||
| 6a34cd6f86 | |||
| f0986751de | |||
| c6c3e0dcd7 | |||
| b9944332ff | |||
| 6bdb9fcc44 | |||
| 25eb30c15b | |||
| 131a42a169 | |||
| 3cdb16c906 | |||
| ef92ce4d82 | |||
| c524831616 | |||
| 868ac59692 | |||
| 4d197e4bd3 | |||
| 3806aaee03 | |||
| 5b628b4d6f | |||
| 2e3d5b79ad |
@ -490,6 +490,15 @@ internal/gateway # OpenAI 兼容 HTTP 服务 + 鉴权 + SDK/流式 + Web
|
||||
internal/types # 统一格式 & OpenAI wire 类型
|
||||
```
|
||||
|
||||
## 开发协作
|
||||
|
||||
- **Git 分支工作流**:主分支 / 特性分支 / 发布分支的规范见
|
||||
**[docs/git-workflow.md](docs/git-workflow.md)**([English](docs/git-workflow-en.md))。
|
||||
主线:`main` 永可部署 → 特性合入 `main` → 切 `release/vX.Y.Z` 打 tag 发布 →
|
||||
hotfix 提交发布分支并 cherry-pick 回 `main`。
|
||||
- **Lua 适配器协议**:见 **[docs/lua-adapters.md](docs/lua-adapters.md)**
|
||||
([English](docs/lua-adapters-en.md))。
|
||||
|
||||
## 测试
|
||||
|
||||
```bash
|
||||
|
||||
11
README_EN.md
11
README_EN.md
@ -507,6 +507,17 @@ internal/gateway # OpenAI-compatible HTTP + auth + SDK/streaming + WebUI
|
||||
internal/types # unified format & OpenAI wire types
|
||||
```
|
||||
|
||||
## Development & collaboration
|
||||
|
||||
- **Git branching workflow**: main / feature / release-branch conventions in
|
||||
**[docs/git-workflow-en.md](docs/git-workflow-en.md)**
|
||||
([中文](docs/git-workflow.md)). The line: `main` is always deployable →
|
||||
features merge into `main` → cut `release/vX.Y.Z` and tag → release → hotfixes
|
||||
land on the release branch and are cherry-picked back into `main`.
|
||||
- **Lua adapter protocol**: see
|
||||
**[docs/lua-adapters-en.md](docs/lua-adapters-en.md)**
|
||||
([中文](docs/lua-adapters.md)).
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
|
||||
4
cmd/gui/package-lock.json
generated
4
cmd/gui/package-lock.json
generated
@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "modelrouter-gui",
|
||||
"version": "1.4.1",
|
||||
"version": "1.5.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "modelrouter-gui",
|
||||
"version": "1.4.1",
|
||||
"version": "1.5.2",
|
||||
"license": "Proprietary",
|
||||
"devDependencies": {
|
||||
"electron": "^33.0.0",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "modelrouter-gui",
|
||||
"version": "1.4.1",
|
||||
"version": "1.5.2",
|
||||
"description": "ModelRouter Desktop — embedded ModelRouter core with tray",
|
||||
"author": "ModelRouter",
|
||||
"main": "main.js",
|
||||
|
||||
@ -23,8 +23,24 @@ import (
|
||||
|
||||
func main() {
|
||||
cfgPath := flag.String("config", "config.yaml", "path to gateway config file")
|
||||
checkOnly := flag.Bool("check", false, "validate the config file and exit (0 = valid, 1 = invalid); nothing is started and no file is written")
|
||||
flag.Parse()
|
||||
|
||||
// -check is the deploy-time preflight: parse and validate the config
|
||||
// without starting the Lua VM, touching runtime.json, or binding a port.
|
||||
// It deliberately does NOT call EnsureDefault, so a missing file is an
|
||||
// error here instead of being silently created.
|
||||
if *checkOnly {
|
||||
if _, err := os.Stat(*cfgPath); err != nil {
|
||||
log.Fatalf("[llmsproxy] check: %v", err)
|
||||
}
|
||||
if _, err := config.Load(*cfgPath); err != nil {
|
||||
log.Fatalf("[llmsproxy] check: %v", err)
|
||||
}
|
||||
log.Printf("[llmsproxy] check: %s is valid", *cfgPath)
|
||||
return
|
||||
}
|
||||
|
||||
created, err := config.EnsureDefault(*cfgPath)
|
||||
if err != nil {
|
||||
log.Fatalf("[llmsproxy] config: %v", err)
|
||||
|
||||
173
deploy.sh
173
deploy.sh
@ -10,6 +10,16 @@
|
||||
# 4. systemctl restart → 服务中断仅在 Go 进程重启瞬间 (<500ms)
|
||||
# 5. 健康检查确认新进程正常
|
||||
#
|
||||
# 配置文件(2026-09-05 起):
|
||||
# 本脚本不再假设 config.yaml 与部署无关。实测教训:给源加 headers 时
|
||||
# 漏删原有的 `headers: {}` → YAML 重复键 → 进程启动即退出 → 健康检查
|
||||
# 失败 → 旧的 rollback 只恢复二进制,配置错误仍在,于是旧二进制也起不来,
|
||||
# 服务陷入 systemd 重启循环(当时累计 446 次)。
|
||||
# 因此:
|
||||
# - 支持 --config <file> 投放一份待部署配置(原子 rename 到 TARGET_CONFIG)
|
||||
# - 重启前先用新二进制 `-check` 预检配置,不合法就地失败,绝不重启服务
|
||||
# - rollback 同时恢复二进制与配置文件,回滚后再次 -check 才重启
|
||||
#
|
||||
# 注意事项:
|
||||
# - 本服务依赖自身(推理依赖),restart 前后会短暂中断,
|
||||
# 但重试机制(systemd RestartSec=5 + 客户端 retry)可自愈
|
||||
@ -19,6 +29,7 @@ set -euo pipefail
|
||||
|
||||
# ---------- 配置 ----------
|
||||
readonly TARGET_BIN="/usr/local/bin/llmsproxy"
|
||||
readonly TARGET_CONFIG="/etc/llmsproxy/config.yaml"
|
||||
readonly TARGET_ADAPTERS="/etc/llmsproxy/adapters"
|
||||
readonly REPO_DIR="/home/program/llmsproxy"
|
||||
readonly HEALTH_URL="http://127.0.0.1:8081/v1/models"
|
||||
@ -28,8 +39,40 @@ readonly BIN_NAME="llmsproxy"
|
||||
readonly TMP_DIR="/tmp/llmsproxy-deploy"
|
||||
# 同文件系统暂存文件(用于原子 rename)
|
||||
readonly STAGING_BIN="${TARGET_BIN}.staging"
|
||||
# 回滚备份:健康检查失败时从这里恢复旧二进制
|
||||
readonly STAGING_CONFIG="${TARGET_CONFIG}.staging"
|
||||
# 回滚备份:健康检查失败时从这里恢复旧二进制 / 旧配置
|
||||
readonly BACKUP_BIN="${TARGET_BIN}.bak"
|
||||
readonly BACKUP_CONFIG="${TARGET_CONFIG}.rollback.bak"
|
||||
|
||||
# 待部署配置(--config <file> 指定;为空 = 不改动线上配置)
|
||||
NEW_CONFIG_SRC=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法: ./deploy.sh [--config <file>]
|
||||
|
||||
--config <file> 投放这份配置到 /etc/llmsproxy/config.yaml(原子替换)。
|
||||
省略时不改动线上配置,只部署二进制与适配器。
|
||||
无论是否指定,重启前都会用新二进制 -check 预检线上配置。
|
||||
|
||||
流程: 构建 → 适配器同步 → 备份(二进制+配置) → 原子替换 → 配置预检
|
||||
→ 重启 → 健康检查 → 失败则回滚(二进制+配置)并重启
|
||||
EOF
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while (( $# > 0 )); do
|
||||
case "$1" in
|
||||
--config)
|
||||
[[ $# -ge 2 ]] || { echo "--config 需要一个文件参数" >&2; exit 2; }
|
||||
NEW_CONFIG_SRC="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "未知参数: $1" >&2; usage; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# ---------- 日志 ----------
|
||||
log() { printf '\033[1;34m[%s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*"; }
|
||||
@ -40,7 +83,7 @@ fail() { err "$@"; exit 1; }
|
||||
# ---------- 清理 ----------
|
||||
cleanup() {
|
||||
log "清理临时文件"
|
||||
rm -rf "$TMP_DIR" "$STAGING_BIN"
|
||||
rm -rf "$TMP_DIR" "$STAGING_BIN" "$STAGING_CONFIG"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
@ -52,6 +95,18 @@ precheck() {
|
||||
command -v systemctl &>/dev/null || fail "未找到 systemctl"
|
||||
[[ $(id -u) -eq 0 ]] || fail "需要 root 权限"
|
||||
|
||||
if [[ -n "$NEW_CONFIG_SRC" ]]; then
|
||||
[[ -f "$NEW_CONFIG_SRC" ]] || fail "待部署配置不存在: $NEW_CONFIG_SRC"
|
||||
# 同一文件比对:--config 指向线上文件本身时没有"投放"动作可做,
|
||||
# 但仍然会走重启前的 -check 预检。
|
||||
if [[ "$(readlink -f "$NEW_CONFIG_SRC")" == "$(readlink -f "$TARGET_CONFIG" 2>/dev/null)" ]]; then
|
||||
warn "--config 指向线上配置本身,跳过投放(仍会预检)"
|
||||
NEW_CONFIG_SRC=""
|
||||
else
|
||||
log "待部署配置: $NEW_CONFIG_SRC → $TARGET_CONFIG"
|
||||
fi
|
||||
fi
|
||||
|
||||
OLD_SIZE=$(stat -c%s "$TARGET_BIN" 2>/dev/null || echo 0)
|
||||
OLD_SHA256=$(sha256sum "$TARGET_BIN" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
@ -59,8 +114,10 @@ precheck() {
|
||||
log "go: $(go version)"
|
||||
}
|
||||
|
||||
# ---------- 备份旧二进制(用于回滚)----------
|
||||
backup_old_binary() {
|
||||
# ---------- 备份旧二进制 + 旧配置(用于回滚)----------
|
||||
# 配置也必须备份:启动失败常常是配置问题,只回滚二进制会让旧二进制
|
||||
# 继续吃坏配置,服务照样起不来。
|
||||
backup_old_state() {
|
||||
log "备份旧二进制到 $BACKUP_BIN"
|
||||
if [[ -f "$TARGET_BIN" ]]; then
|
||||
cp -f "$TARGET_BIN" "$BACKUP_BIN"
|
||||
@ -68,6 +125,13 @@ backup_old_binary() {
|
||||
else
|
||||
warn "旧二进制不存在,跳过备份(无回滚点)"
|
||||
fi
|
||||
|
||||
if [[ -f "$TARGET_CONFIG" ]]; then
|
||||
cp -f "$TARGET_CONFIG" "$BACKUP_CONFIG"
|
||||
log "✓ 旧配置已备份到 $BACKUP_CONFIG (sha256=$(sha256sum "$BACKUP_CONFIG" | awk '{print $1}' | cut -c1-16))"
|
||||
else
|
||||
warn "线上配置不存在: $TARGET_CONFIG(首次部署?无配置回滚点)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------- 构建 ----------
|
||||
@ -89,8 +153,13 @@ build() {
|
||||
if [[ "$NEW_SIZE" -eq "$OLD_SIZE" ]]; then
|
||||
NEW_SHA256=$(sha256sum "$NEW_BIN" | awk '{print $1}')
|
||||
if [[ "$NEW_SHA256" == "$OLD_SHA256" ]]; then
|
||||
warn "新二进制与旧版本完全一致(sha256 相同),跳过部署"
|
||||
exit 0
|
||||
# 二进制无变化。只有在同时也没有待部署配置时才能直接退出;
|
||||
# 否则会造成“--config 指定了新配置却被静默丢弃”。
|
||||
if [[ -z "$NEW_CONFIG_SRC" ]]; then
|
||||
warn "新二进制与旧版本完全一致(sha256 相同)且无待部署配置,跳过部署"
|
||||
exit 0
|
||||
fi
|
||||
log "二进制无变化(sha256 相同),但有待部署配置,继续"
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -140,6 +209,41 @@ atomic_replace_binary() {
|
||||
log "✓ 二进制已原子替换 (sha256=${NEW_SHA256:0:16})"
|
||||
}
|
||||
|
||||
# ---------- 原子替换配置 ----------
|
||||
# 与二进制同一套 copy→rename 手法:先落到 /etc 上的 .staging,再 rename(2)。
|
||||
# 只有 --config 指定了来源时才动线上配置。
|
||||
atomic_replace_config() {
|
||||
[[ -n "$NEW_CONFIG_SRC" ]] || return 0
|
||||
|
||||
log "原子替换配置"
|
||||
log " copy → $STAGING_CONFIG (同文件系统暂存)"
|
||||
cp -f "$NEW_CONFIG_SRC" "$STAGING_CONFIG"
|
||||
chmod 0644 "$STAGING_CONFIG"
|
||||
|
||||
log " rename → $TARGET_CONFIG (原子)"
|
||||
mv -f "$STAGING_CONFIG" "$TARGET_CONFIG"
|
||||
log "✓ 配置已原子替换 (sha256=$(sha256sum "$TARGET_CONFIG" | awk '{print $1}' | cut -c1-16))"
|
||||
}
|
||||
|
||||
# ---------- 配置预检 ----------
|
||||
# 用刚部署的新二进制解析线上配置。`-check` 只做 parse + ApplyDefaults,
|
||||
# 不启动 Lua VM、不碰 runtime.json、不绑端口,因此可以安全地在服务仍在
|
||||
# 运行时执行。不合法就在重启前失败——这正是上次事故缺的那一步。
|
||||
verify_config() {
|
||||
log "配置预检(新二进制 -check)"
|
||||
[[ -f "$TARGET_CONFIG" ]] || fail "线上配置不存在: $TARGET_CONFIG"
|
||||
|
||||
local out
|
||||
if out=$("$TARGET_BIN" -check -config "$TARGET_CONFIG" 2>&1); then
|
||||
log "✓ 配置合法"
|
||||
return 0
|
||||
fi
|
||||
|
||||
err "配置预检失败,未重启服务(线上进程仍在跑旧配置):"
|
||||
printf '%s\n' "$out" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# ---------- 同步适配器 ----------
|
||||
sync_adapters() {
|
||||
log "同步适配器"
|
||||
@ -206,20 +310,49 @@ healthcheck() {
|
||||
}
|
||||
|
||||
# ---------- 回滚 ----------
|
||||
# 健康检查失败时恢复旧二进制并重启,保证推理服务可用。
|
||||
# 健康检查失败时恢复旧二进制**和旧配置**并重启,保证推理服务可用。
|
||||
# 这是最后一道保险:即使新版本启动失败(编译错误、配置不兼容、
|
||||
# panic on start 等),也能让旧版本继续服务,避免推理能力丢失。
|
||||
#
|
||||
# 为什么必须一并回滚配置:启动失败最常见的原因就是配置本身不合法
|
||||
# (实测:YAML 重复键)。只恢复二进制的话,旧二进制照样解析不了坏配置,
|
||||
# 服务继续在 systemd 重启循环里空转。
|
||||
rollback() {
|
||||
err "健康检查失败,启动回滚流程"
|
||||
if [[ ! -f "$BACKUP_BIN" ]]; then
|
||||
err "无回滚备份 ($BACKUP_BIN 不存在),无法自动恢复"
|
||||
|
||||
local restored_any=0
|
||||
if [[ -f "$BACKUP_BIN" ]]; then
|
||||
log "恢复旧二进制"
|
||||
cp -f "$BACKUP_BIN" "$TARGET_BIN"
|
||||
chmod 0755 "$TARGET_BIN"
|
||||
restored_any=1
|
||||
else
|
||||
err "无二进制回滚备份 ($BACKUP_BIN 不存在)"
|
||||
fi
|
||||
|
||||
# 只在本次真的换过配置时才回滚配置,避免把用户在部署期间
|
||||
# 通过 WebUI 做的合法改动一起抹掉。
|
||||
if [[ -n "$NEW_CONFIG_SRC" && -f "$BACKUP_CONFIG" ]]; then
|
||||
log "恢复旧配置"
|
||||
cp -f "$BACKUP_CONFIG" "$TARGET_CONFIG"
|
||||
chmod 0644 "$TARGET_CONFIG"
|
||||
restored_any=1
|
||||
fi
|
||||
|
||||
if (( restored_any == 0 )); then
|
||||
err "无任何回滚点,无法自动恢复"
|
||||
err "请手动检查: systemctl status llmsproxy.service && journalctl -u llmsproxy -n 50"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "恢复旧二进制"
|
||||
cp -f "$BACKUP_BIN" "$TARGET_BIN"
|
||||
chmod 0755 "$TARGET_BIN"
|
||||
# 回滚后先验证恢复出来的组合真的能解析,再重启——否则只是把
|
||||
# 重启循环换个二进制继续跑。
|
||||
if ! "$TARGET_BIN" -check -config "$TARGET_CONFIG" >/dev/null 2>&1; then
|
||||
err "回滚后的配置仍不合法,拒绝重启(避免 systemd 重启风暴)"
|
||||
err " 手动修复 $TARGET_CONFIG 后执行: $TARGET_BIN -check -config $TARGET_CONFIG"
|
||||
err " 备份可用: 二进制=$BACKUP_BIN 配置=$BACKUP_CONFIG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "重启服务(旧版本)"
|
||||
systemctl restart llmsproxy.service || {
|
||||
@ -248,18 +381,32 @@ print_summary() {
|
||||
log "═════════════════════════════════════════════"
|
||||
log "部署完成"
|
||||
log " 二进制: $TARGET_BIN"
|
||||
log " 配置: $TARGET_CONFIG$([[ -n "$NEW_CONFIG_SRC" ]] && echo " (本次已替换)" || echo " (未改动)")"
|
||||
log " 适配器: $TARGET_ADAPTERS"
|
||||
log "═════════════════════════════════════════════"
|
||||
}
|
||||
|
||||
# ---------- 主流程 ----------
|
||||
# 顺序要点:配置投放与预检都排在 restart_service **之前**,
|
||||
# 于是坏配置在服务仍然健康时就被拦下,根本不会触发重启。
|
||||
main() {
|
||||
parse_args "$@"
|
||||
log "═══ llmsproxy 原子部署开始 ═══"
|
||||
precheck
|
||||
build
|
||||
sync_adapters
|
||||
backup_old_binary
|
||||
backup_old_state
|
||||
atomic_replace_binary
|
||||
atomic_replace_config
|
||||
if ! verify_config; then
|
||||
# 配置不合法:服务还没重启,只需把二进制/配置退回原状。
|
||||
err "配置预检失败,回退本次改动(服务未受影响)"
|
||||
[[ -f "$BACKUP_BIN" ]] && cp -f "$BACKUP_BIN" "$TARGET_BIN" && chmod 0755 "$TARGET_BIN"
|
||||
if [[ -n "$NEW_CONFIG_SRC" && -f "$BACKUP_CONFIG" ]]; then
|
||||
cp -f "$BACKUP_CONFIG" "$TARGET_CONFIG" && chmod 0644 "$TARGET_CONFIG"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
restart_service
|
||||
if ! healthcheck; then
|
||||
rollback || true
|
||||
|
||||
141
docs/git-workflow-en.md
Normal file
141
docs/git-workflow-en.md
Normal file
@ -0,0 +1,141 @@
|
||||
# Git Branching Workflow
|
||||
|
||||
ModelRouter uses a **GitHub Flow + release-branch** model: `main` is the only
|
||||
long-lived branch and is always deployable; new work branches off it as a
|
||||
feature branch; each version gets its own release branch with a tag; hotfixes
|
||||
land on the release branch and MUST flow back to `main`.
|
||||
|
||||
Goal: make "patching an already-released version during its support window" a
|
||||
first-class operation, while `main` always contains every fix and stays
|
||||
deployable.
|
||||
|
||||
## Branch roles
|
||||
|
||||
| Branch | Name | Lifetime | Purpose |
|
||||
|---|---|---|---|
|
||||
| Main | `main` | permanent | Only long-lived branch. Always deployable. Accumulates the next version. |
|
||||
| Feature | `feature/<desc>` | short (dev → merge → delete) | New features / ordinary fixes. Born from `main`, merged back into `main`. |
|
||||
| Release | `release/vX.Y.Z` | one version cycle | Cut from `main`, tagged for release. Version-specific hotfixes land here. |
|
||||
|
||||
## Change flow (important)
|
||||
|
||||
```
|
||||
feature/foo ───────┐
|
||||
▼
|
||||
main ──────────────── next version ──────►
|
||||
│ │
|
||||
│ cut │ cut
|
||||
▼ ▼
|
||||
release/v1.4.2 release/v1.4.3
|
||||
│ │
|
||||
tag: v1.4.2 tag: v1.4.3
|
||||
│ │
|
||||
hotfix ◄─────┘ hotfix ◄─────┘
|
||||
│ │
|
||||
└── cherry-pick back ──────────┘
|
||||
```
|
||||
|
||||
### Key rules
|
||||
|
||||
1. **main is always deployable**: never leave half-done work on `main`.
|
||||
Unfinished work lives on a feature branch.
|
||||
2. **Features start from main and merge back**: `git checkout -b feature/xxx main`,
|
||||
then `git merge --no-ff feature/xxx` (or squash) when done.
|
||||
3. **Release = cut a release branch from main + tag**:
|
||||
```bash
|
||||
git checkout -b release/v1.4.2 main
|
||||
git tag -a v1.4.2 -m "ModelRouter v1.4.2"
|
||||
git push origin release/v1.4.2 v1.4.2
|
||||
```
|
||||
Build installers and upload the GitCode Release from this tag so the
|
||||
published state is exactly reproducible.
|
||||
4. **Only version-specific hotfixes go into a release branch during its
|
||||
window**: new features always go to `main` for the next version, never into
|
||||
an already-released branch (unless you deliberately ship a minor revision).
|
||||
5. **Hotfixes MUST flow back to main**:
|
||||
```bash
|
||||
git checkout release/v1.4.2 # fix in the release branch
|
||||
git commit -m "fix: ..."
|
||||
git checkout main
|
||||
git cherry-pick <hotfix-commit> # and into main
|
||||
```
|
||||
Most important rule: **main only moves forward and never loses fixes**. If a
|
||||
hotfix never reaches main, the next release ships with the old bug.
|
||||
|
||||
### End of a version lifecycle
|
||||
|
||||
When the next version ships, the previous release branch retires:
|
||||
|
||||
- **Default: delete the remote release branch**
|
||||
(`git push origin :release/v1.4.2`). All hotfixes were already
|
||||
cherry-picked into main, so main contains everything; no merge needed.
|
||||
- **Long-term maintenance** (e.g. an enterprise client pinned to an old
|
||||
version): keep the branch, accept only security fixes, keep the
|
||||
commit-then-cherry-pick loop.
|
||||
|
||||
## Explicit non-goals
|
||||
|
||||
- **Never rebase main**: main's history stays append-only; anyone pulling gets
|
||||
directly usable history.
|
||||
- **Release branches are not merged back wholesale**: hotfixes were already
|
||||
cherry-picked; a whole-branch merge only creates conflicts with no benefit.
|
||||
- **No "quick edit" directly on main**, even single-person: at minimum use
|
||||
`feature/xxx → merge main` so history keeps its why-boundaries.
|
||||
|
||||
## Single vs multi person
|
||||
|
||||
- **Single** (this repo today): skip PRs — `git checkout -b feature/xxx` →
|
||||
finish → merge back to main. Release branch + tag flow unchanged.
|
||||
- **Multi / open source**: feature branches go through PR + Code Review,
|
||||
maintainers merge; resolve conflicts on the feature branch, never on main.
|
||||
|
||||
## Commit message conventions
|
||||
|
||||
- `feat(...)`: new feature
|
||||
- `fix(...)`: bug fix
|
||||
- `chore(...)`: build / version / deps / non-code changes
|
||||
- `docs(...)`: documentation
|
||||
- `refactor(...)`: refactoring
|
||||
- Scope in parens, e.g. `fix(adapters)`, `feat(webui)`, `chore(version)`
|
||||
- One-line summary; expand with problem/root-cause/fix/verification when needed
|
||||
|
||||
## Release checklist (companion)
|
||||
|
||||
```bash
|
||||
# 1. main is ready
|
||||
git checkout main && git pull
|
||||
|
||||
# 2. cut release branch + tag
|
||||
git checkout -b release/vX.Y.Z main
|
||||
git tag -a vX.Y.Z -m "ModelRouter vX.Y.Z"
|
||||
|
||||
# 3. push branch + tag
|
||||
git push origin release/vX.Y.Z
|
||||
git push origin vX.Y.Z
|
||||
|
||||
# 4. build installers (core + GUI platforms)
|
||||
make core-dist VERSION=X.Y.Z
|
||||
|
||||
# 5. create GitCode Release + upload
|
||||
# https://gitcode.com/JianFeeeee/ModelRouter/releases
|
||||
# upload MUST use PUT --http1.1, otherwise the OBS callback fails and the
|
||||
# file never lands in the release; replacing artifacts = delete the tag
|
||||
# (release disappears with it) then recreate.
|
||||
```
|
||||
|
||||
## Why this model (background)
|
||||
|
||||
Before 2026-08 everything went straight to `main` plus a tag, which caused two
|
||||
pain points:
|
||||
1. After 1.4.2 shipped, a scheduler-scoring fix had to be added to the same
|
||||
version — the only option was deleting the remote tag to clear the release
|
||||
and re-uploading all installers. The published state was detached from code
|
||||
history; "what exactly shipped" could not be traced.
|
||||
2. No feature branches meant two independent efforts could not proceed in
|
||||
parallel without colliding.
|
||||
|
||||
With release branches: the published state = `release/vX.Y.Z` branch +
|
||||
`vX.Y.Z` tag, exactly reproducible; hotfixes have a clear landing spot; main
|
||||
stays "latest + all fixes + deployable".
|
||||
|
||||
> 中文版见 [docs/git-workflow.md](git-workflow.md)。
|
||||
128
docs/git-workflow.md
Normal file
128
docs/git-workflow.md
Normal file
@ -0,0 +1,128 @@
|
||||
# Git 分支工作流(Branching Workflow)
|
||||
|
||||
ModelRouter 采用 **GitHub Flow + 发布分支** 模型:`main` 是唯一长命分支且永远可部署,
|
||||
新工作从它开出特性分支;每个版本从 `main` 分出独立的发布分支并打 tag;hotfix 提交到
|
||||
发布分支且必须回流 `main`。
|
||||
|
||||
目的:让「已发版的版本在生命周期内可被单独修补」成为一等操作,同时 `main` 始终包含
|
||||
全部修复、永远可部署。
|
||||
|
||||
## 分支角色
|
||||
|
||||
| 分支 | 命名 | 生命周期 | 用途 |
|
||||
|---|---|---|---|
|
||||
| 主分支 | `main` | 永久 | 唯一长命分支。永远可部署。积攒下一个版本的功能。 |
|
||||
| 特性分支 | `feature/<描述>` | 短命(开发→合并即删) | 新特性 / 一般 bug 修复。从 `main` 开出,完成后合回 `main`。 |
|
||||
| 发布分支 | `release/vX.Y.Z` | 一个版本周期 | 从 `main` 分出,打 tag 发布。该版本生命周期内的 hotfix 都提交在此分支。 |
|
||||
|
||||
## 变更流向(重要)
|
||||
|
||||
```
|
||||
feature/foo ───────┐
|
||||
▼
|
||||
main ──────────────── 下一个版本 ──────►
|
||||
│ │
|
||||
│ 切出 │ 切出
|
||||
▼ ▼
|
||||
release/v1.4.2 release/v1.4.3
|
||||
│ │
|
||||
tag: v1.4.2 tag: v1.4.3
|
||||
│ │
|
||||
hotfix ◄─────┘ hotfix ◄─────┘
|
||||
│ │
|
||||
└── cherry-pick 回 main ───────┘
|
||||
```
|
||||
|
||||
### 关键规则
|
||||
|
||||
1. **main 永远可部署**:不在 main 上留半成品。任何未完成的工作必须在特性分支上。
|
||||
2. **特性从 main 开,完成后合回 main**:`git checkout -b feature/xxx main`,
|
||||
开发完 `git merge --no-ff feature/xxx` 或 squash 合回。
|
||||
3. **发布 = 从 main 切 release 分支 + 打 tag**:
|
||||
```bash
|
||||
git checkout -b release/v1.4.2 main
|
||||
git tag -a v1.4.2 -m "ModelRouter v1.4.2"
|
||||
git push origin release/v1.4.2 v1.4.2
|
||||
```
|
||||
构建安装包、上传 GitCode Release 都基于这个 tag,保证可精确回溯发布态。
|
||||
4. **版本生命周期内只收该版本的 hotfix**:新特性一律并入 `main` 等下一个版本,
|
||||
绝不塞进已发布的 release 分支(除非主动选择在该版本内发次要版)。
|
||||
5. **hotfix 必须回流 main**:
|
||||
```bash
|
||||
git checkout release/v1.4.2 # 在发布分支提交修复
|
||||
git commit -m "fix: ..."
|
||||
git checkout main
|
||||
git cherry-pick <hotfix-commit> # 回主分支
|
||||
```
|
||||
这是本模型最重要的一条:**main 只前进、不丢修复**。如果 hotfix 不回 main,
|
||||
下一个版本就会带着旧 bug 发布。
|
||||
|
||||
### 版本生命周期结束
|
||||
|
||||
下一个版本发布时,上一个 release 分支退役:
|
||||
|
||||
- **默认:直接删除远端 release 分支**(`git push origin :release/v1.4.2`)。
|
||||
因为 hotfix 都已逐个 cherry-pick 回 main,main 已包含全部修复,无需再合并。
|
||||
- **如需要长期维护旧版**(例如企业大客户卡在旧版本):保留分支,仅 stopship 接受
|
||||
该版本的安全修复,继续走「提交 + cherry-pick 回 main」循环。
|
||||
|
||||
## 明确不做的事
|
||||
|
||||
- **不 rebase main**:`main` 的历史保持追加式,任何人拉取后 `git pull` 都得到直接可用的历史。
|
||||
- **发布分支不整体 merge 回 main**:hotfix 已逐个 cherry-pick,整体合并只会制造冲突且无收益。
|
||||
- **不在 main 直接写"临时改一下"**:即使是单人项目,也至少走
|
||||
`feature/xxx → merge main` 的形式,让历史保留"为什么改"的边界。
|
||||
|
||||
## 单人 vs 多人
|
||||
|
||||
- **单人**(本项目当前):特性分支可省去 PR,`git checkout -b feature/xxx` →
|
||||
完成 → 合并回 main。发布分支与 tag 流程完全一致。
|
||||
- **多人 / 开源协作**:特性分支走 PR(pull request)+ Code Review,
|
||||
由维护者 merge;冲突在特性分支上解决,不在 main 上解决。
|
||||
|
||||
## 提交信息规范(与分支配套)
|
||||
|
||||
- `feat(...)`: 新功能
|
||||
- `fix(...)`: 修复
|
||||
- `chore(...)`: 构建 / 版本号 / 依赖 / 文档无关改动
|
||||
- `docs(...)`: 文档
|
||||
- `refactor(...)`: 重构
|
||||
- 括号内为影响域,如 `fix(adapters)`、`feat(webui)`、`chore(version)`
|
||||
- 一句话总结,必要时正文展开「问题 / 根因 / 修复 / 验证」
|
||||
|
||||
## 发布操作速查(配套)
|
||||
|
||||
每次发布严格按此顺序:
|
||||
|
||||
```bash
|
||||
# 1. 确认 main 就绪
|
||||
git checkout main && git pull
|
||||
|
||||
# 2. 切发布分支并打 tag
|
||||
git checkout -b release/vX.Y.Z main
|
||||
git tag -a vX.Y.Z -m "ModelRouter vX.Y.Z"
|
||||
|
||||
# 3. 推送分支 + tag
|
||||
git push origin release/vX.Y.Z
|
||||
git push origin vX.Y.Z
|
||||
|
||||
# 4. 构建安装包(核心包 + GUI 各平台)
|
||||
make core-dist VERSION=X.Y.Z
|
||||
|
||||
# 5. 创建 GitCode Release + 上传安装包
|
||||
# 见 https://gitcode.com/JianFeeeee/ModelRouter/releases
|
||||
# 上传注意:必须 PUT --http1.1,否则 OBS 回调失败文件不进列表;
|
||||
# 替换旧版附件 = 删 tag(release 随 tag 一起消失)后重建。
|
||||
```
|
||||
|
||||
## 为什么是这套(背景)
|
||||
|
||||
2026-08 之前本项目直接在所有改动提交到 `main` + 打 tag,出现两个痛点:
|
||||
1. 1.4.2 已发版后又追了一个调度打分修复,只能删远端 tag 清空 release 再重传安装包
|
||||
—— 版本发布态与代码历史脱节,无法精确回溯"当时发的是什么"。
|
||||
2. 没有特性分支,无法并行开发互不干扰的两件事。
|
||||
|
||||
引入发布分支后:版本发布态 = `release/vX.Y.Z` 分支 + `vX.Y.Z` tag,可精确回溯;
|
||||
hotfix 有明确落点;main 保持"最新 + 全部修复 + 可部署"。
|
||||
|
||||
> 英文版见 [docs/git-workflow-en.md](git-workflow-en.md)。
|
||||
@ -47,15 +47,19 @@ type Model struct {
|
||||
|
||||
// Source describes a single upstream LLM provider.
|
||||
type Source struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
BaseURL string `yaml:"base_url" json:"base_url"`
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
APIKeyEnv string `yaml:"api_key_env,omitempty" json:"-"` // reference to an env var holding the key (overrides api_key)
|
||||
Adapter string `yaml:"adapter" json:"adapter"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"` // chat endpoint override
|
||||
ImageEndpoint string `yaml:"image_endpoint" json:"image_endpoint,omitempty"` // image endpoint override
|
||||
Models []Model `yaml:"models" json:"models"`
|
||||
Headers map[string]string `yaml:"headers" json:"headers,omitempty"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
BaseURL string `yaml:"base_url" json:"base_url"`
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
APIKeyEnv string `yaml:"api_key_env,omitempty" json:"-"` // reference to an env var holding the key (overrides api_key)
|
||||
Adapter string `yaml:"adapter" json:"adapter"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"` // chat endpoint override
|
||||
ImageEndpoint string `yaml:"image_endpoint" json:"image_endpoint,omitempty"` // image endpoint override
|
||||
Models []Model `yaml:"models" json:"models"`
|
||||
Headers map[string]string `yaml:"headers" json:"headers,omitempty"`
|
||||
// ProxyURL routes this source's HTTP(S) traffic through an explicit proxy
|
||||
// (e.g. http://127.0.0.1:7890). Empty = direct connection. Needed for
|
||||
// upstreams behind geo/IP blocks that only respond through a proxy.
|
||||
ProxyURL string `yaml:"proxy_url" json:"proxy_url,omitempty"`
|
||||
Meta map[string]interface{} `yaml:"meta" json:"meta,omitempty"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature,omitempty"`
|
||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens,omitempty"`
|
||||
@ -328,13 +332,15 @@ type RuntimeConfig struct {
|
||||
// name and api_key) so the WebUI can spin up multiple key-bearing sources
|
||||
// from one shared template.
|
||||
type SourceTemplate struct {
|
||||
Name string `json:"name"`
|
||||
BaseURL string `json:"base_url"`
|
||||
Adapter string `json:"adapter"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
ImageEndpoint string `json:"image_endpoint,omitempty"`
|
||||
Models []Model `json:"models"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Name string `json:"name"`
|
||||
BaseURL string `json:"base_url"`
|
||||
Adapter string `json:"adapter"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
ImageEndpoint string `json:"image_endpoint,omitempty"`
|
||||
Models []Model `json:"models"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
// ProxyURL routes this source's HTTP(S) traffic through an explicit proxy.
|
||||
ProxyURL string `json:"proxy_url,omitempty"`
|
||||
Meta map[string]interface{} `json:"meta,omitempty"`
|
||||
Temperature float64 `json:"temperature,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
|
||||
@ -115,13 +115,15 @@ func (g *Gateway) resolveCands(ctx context.Context, req *chatRequest) ([]*provid
|
||||
}
|
||||
|
||||
// filterCandsByModels keeps only providers exposing at least one model of the
|
||||
// scope (used for user keys with a restricted model scope). An "AUTO" scope
|
||||
// entry means the key is allowed to use any model. Scope entries with a
|
||||
// Source pinned to a specific upstream narrow the candidates to that source
|
||||
// scope (used for user keys with a restricted model scope). Scope entries with
|
||||
// a Source pinned to a specific upstream narrow the candidates to that source
|
||||
// for the matching model.
|
||||
//
|
||||
// An "AUTO" scope entry only allows the AUTO routing mode; it does NOT grant
|
||||
// access to specific models.
|
||||
func filterCandsByModels(cands []*provider.Provider, allow []config.ModelScope) []*provider.Provider {
|
||||
for _, m := range allow {
|
||||
if m.Model == "" || strings.EqualFold(m.Model, "AUTO") {
|
||||
if m.Model == "" {
|
||||
return cands
|
||||
}
|
||||
}
|
||||
@ -152,20 +154,18 @@ func filterCandsByModels(cands []*provider.Provider, allow []config.ModelScope)
|
||||
return out
|
||||
}
|
||||
|
||||
// intersectModels restricts a model list to the scope (preserving order). An
|
||||
// "AUTO" scope entry grants every model.
|
||||
// intersectModels restricts a model list to the scope (preserving order).
|
||||
// An "AUTO" scope entry only allows the AUTO routing mode; it does NOT grant
|
||||
// every model.
|
||||
func intersectModels(models []string, allow []config.ModelScope) []string {
|
||||
allowed := make(map[string]bool, len(allow))
|
||||
any := false
|
||||
for _, m := range allow {
|
||||
if m.Model == "" || strings.EqualFold(m.Model, "AUTO") {
|
||||
any = true
|
||||
break
|
||||
if m.Model == "" {
|
||||
return models
|
||||
}
|
||||
if !strings.EqualFold(m.Model, "AUTO") {
|
||||
allowed[m.Model] = true
|
||||
}
|
||||
allowed[m.Model] = true
|
||||
}
|
||||
if any {
|
||||
return models
|
||||
}
|
||||
out := make([]string, 0, len(models))
|
||||
seen := map[string]bool{}
|
||||
@ -179,24 +179,16 @@ func intersectModels(models []string, allow []config.ModelScope) []string {
|
||||
}
|
||||
|
||||
// checkModelScope validates the effective model against the key's model scope
|
||||
// and token quota. Returns an error message when rejected. A scope entry with
|
||||
// model "AUTO" grants all models; its quota caps the key's total tokens.
|
||||
// and token quota. Returns an error message when rejected.
|
||||
//
|
||||
// A scope entry with model "AUTO" only allows requests where the effective
|
||||
// model is AUTO (the routing mode). It does NOT grant access to specific model
|
||||
// ids — that requires an explicit scope entry for the model.
|
||||
func (g *Gateway) checkModelScope(ctx context.Context, model string) string {
|
||||
allow := g.allowedModels(ctx)
|
||||
if allow == nil {
|
||||
return ""
|
||||
}
|
||||
for _, sc := range allow {
|
||||
if sc.Model != "" && strings.EqualFold(sc.Model, "AUTO") {
|
||||
if sc.TokenQuota > 0 {
|
||||
used := g.scopeTokens(ctx, sc)
|
||||
if used >= sc.TokenQuota {
|
||||
return fmt.Sprintf("token quota exceeded (%d/%d)", used, sc.TokenQuota)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
for _, sc := range allow {
|
||||
if sc.Model != model {
|
||||
continue
|
||||
@ -229,12 +221,15 @@ func (g *Gateway) scopeTokens(ctx context.Context, sc config.ModelScope) int64 {
|
||||
// source that actually serves the bare model (via Registry.EffectiveModel), so
|
||||
// model ids that themselves contain separators (e.g. "deepseek-v4-flash-free")
|
||||
// are never corrupted (P10-2).
|
||||
//
|
||||
// A scope entry with model "AUTO" only matches the literal AUTO routing mode;
|
||||
// it does NOT grant access to specific model ids.
|
||||
func (g *Gateway) hasScopeModel(list []config.ModelScope, s string) bool {
|
||||
if r := g.core.Registry(); r != nil {
|
||||
s = r.EffectiveModel(s)
|
||||
}
|
||||
for _, x := range list {
|
||||
if x.Model == s || (x.Model != "" && strings.EqualFold(x.Model, "AUTO")) {
|
||||
if x.Model == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@ -670,8 +670,11 @@ func TestHasScopeModelWithSourcePrefix(t *testing.T) {
|
||||
if g.hasScopeModel(scope, "deepseek-v4-flash-free-extra") {
|
||||
t.Error("hasScopeModel returned true for unrelated model")
|
||||
}
|
||||
if !g.hasScopeModel([]config.ModelScope{{Model: "AUTO"}}, "zen:anything") {
|
||||
t.Error("AUTO scope should allow any prefixed model")
|
||||
if g.hasScopeModel([]config.ModelScope{{Model: "AUTO"}}, "zen:deepseek-v4-flash-free") {
|
||||
t.Error("AUTO scope must NOT grant specific model ids even with prefix")
|
||||
}
|
||||
if g.hasScopeModel([]config.ModelScope{{Model: "AUTO"}}, "deepseek-v4-pro") {
|
||||
t.Error("AUTO scope must NOT grant specific model ids")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -5,12 +5,34 @@ adapter.version = "1.0.0"
|
||||
adapter.endpoint = "/chat/completions"
|
||||
adapter.headers = {}
|
||||
|
||||
-- AgentRouter 的 WAF 会按客户端指纹白名单校验,只放行官方客户端。
|
||||
-- 这里通过 build_headers 注入官方客户端 User-Agent 以通过校验。
|
||||
-- 默认 QwenCode 指纹(实测可通);可在 source.meta.user_agent 覆盖成 Claude Code 指纹。
|
||||
|
||||
-- AgentRouter 的阿里云 WAF 按三重维度校验:出口 IP(必须海外)、
|
||||
-- TLS 指纹(JS/Go HTTP 层与 curl 不同)、浏览器指纹 header 集。
|
||||
-- 纯 UA 已不够:WAF 拦 12 种 agent CLI UA 中的 10 种(全 405 HTML),
|
||||
-- 只有同时带 Origin: qwen.ai + Referer + X-Requested-With 才放行,
|
||||
-- 且同 IP 短时间连续 3+ 次请求会被速率封禁,靠 adapter 侧低频调用 +
|
||||
-- llmsproxy 的 prefFailStep 冷却自然限制,来源无法控制速率。
|
||||
local default_ua = "QwenCode/0.2.0 (linux; x64)"
|
||||
|
||||
-- AgentRouter fronts Claude models (claude-opus-4-8), and Claude upstreams
|
||||
-- enforce Anthropic's tool id rule ^[a-zA-Z0-9_-]{1,64}$ by rejecting the WHOLE
|
||||
-- request. Plain OpenAI does not, so an id minted by a permissive model (e.g.
|
||||
-- "bash:0" from moonshotai/kimi-k3) is replayed here by the client and kills
|
||||
-- every turn. See anthropic.lua / openai.lua for the same helper: Lua adapters
|
||||
-- have no shared prelude, so each carries its own copy.
|
||||
local TOOL_ID_MAX = 64
|
||||
|
||||
local function safe_tool_id(id)
|
||||
if type(id) ~= "string" or id == "" then return id end
|
||||
local clean = string.gsub(id, "[^A-Za-z0-9_-]", "_")
|
||||
if clean == id and #clean <= TOOL_ID_MAX then
|
||||
return clean
|
||||
end
|
||||
local digest = string.sub(sha256_hex(id), 1, 8)
|
||||
local keep = TOOL_ID_MAX - #digest - 1
|
||||
if #clean > keep then clean = string.sub(clean, 1, keep) end
|
||||
return clean .. "_" .. digest
|
||||
end
|
||||
|
||||
function adapter.transform_request(raw_body)
|
||||
local ok, req = pcall(json.decode, raw_body)
|
||||
if not ok then return raw_body end
|
||||
@ -19,6 +41,16 @@ function adapter.transform_request(raw_body)
|
||||
if req.messages then
|
||||
for _, msg in ipairs(req.messages) do
|
||||
msg.reasoning_content = nil
|
||||
if msg.tool_call_id ~= nil then
|
||||
msg.tool_call_id = safe_tool_id(msg.tool_call_id)
|
||||
end
|
||||
if type(msg.tool_calls) == "table" then
|
||||
for _, tc in ipairs(msg.tool_calls) do
|
||||
if type(tc) == "table" and tc.id ~= nil then
|
||||
tc.id = safe_tool_id(tc.id)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return json.encode(req)
|
||||
@ -35,6 +67,13 @@ function adapter.build_headers(meta)
|
||||
["Content-Type"] = "application/json",
|
||||
["Authorization"] = "Bearer " .. meta.api_key,
|
||||
["User-Agent"] = ua,
|
||||
-- QwenCode client fingerprint: WAF rejects bare UA without these.
|
||||
["Accept"] = "application/json, text/plain, */*",
|
||||
["Accept-Language"] = "zh-CN,zh;q=0.9",
|
||||
["Origin"] = "https://qwen.ai",
|
||||
["Referer"] = "https://qwen.ai/",
|
||||
["X-Requested-With"] = "XMLHttpRequest",
|
||||
["Connection"] = "keep-alive",
|
||||
}
|
||||
end
|
||||
|
||||
@ -79,7 +118,7 @@ function adapter.transform_response(raw_body)
|
||||
local args_ok, args = pcall(json.decode, tc["function"].arguments)
|
||||
if not args_ok then args = {} end
|
||||
table.insert(tcs, {
|
||||
id = tc.id,
|
||||
id = safe_tool_id(tc.id),
|
||||
type = tc.type or "function",
|
||||
name = tc["function"].name,
|
||||
arguments = args
|
||||
@ -138,6 +177,15 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
unified.reasoning_content = delta.reasoning_content
|
||||
end
|
||||
if delta.tool_calls then
|
||||
-- Sanitize outbound too: a dirty id must never enter a client session,
|
||||
-- because the client replays it to every other source. Only the first
|
||||
-- fragment of a streamed call carries an id; later argument fragments
|
||||
-- have none and must stay id-less for index-based accumulation.
|
||||
for _, tc in ipairs(delta.tool_calls) do
|
||||
if type(tc) == "table" and tc.id ~= nil then
|
||||
tc.id = safe_tool_id(tc.id)
|
||||
end
|
||||
end
|
||||
unified.tool_calls = delta.tool_calls
|
||||
end
|
||||
if uses ~= nil then
|
||||
|
||||
@ -14,6 +14,70 @@ local ROLE_WHITELIST = {
|
||||
system = true, user = true, assistant = true, tool = true,
|
||||
}
|
||||
|
||||
-- Anthropic requires tool_use.id / tool_result.tool_use_id to match
|
||||
-- ^[a-zA-Z0-9_-]{1,64}$ and rejects the WHOLE request otherwise with
|
||||
-- REQUEST_BODY_INVALID / "Invalid tool use format". OpenAI imposes no such
|
||||
-- rule, so an OpenAI-compatible upstream can hand a client an id like
|
||||
-- "bash:0" (observed from moonshotai/kimi-k3); once that lands in the
|
||||
-- client's history, every later replay kills every Claude slot at once and
|
||||
-- an AUTO chain falls through all tiers.
|
||||
--
|
||||
-- safe_tool_id is pure and deterministic, so a tool_use block and its
|
||||
-- matching tool_result are rewritten identically within one request. A
|
||||
-- rewritten id keeps an 8-hex digest of the ORIGINAL id: without it two
|
||||
-- distinct ids could collapse into one ("a:b" and "a_b"), which Anthropic
|
||||
-- rejects as a duplicate/unpaired tool_use. Already-legal ids pass through
|
||||
-- untouched so upstreams that round-trip their own ids are unaffected.
|
||||
local TOOL_ID_MAX = 64
|
||||
|
||||
local function safe_tool_id(id)
|
||||
if type(id) ~= "string" or id == "" then return "" end
|
||||
local clean = string.gsub(id, "[^A-Za-z0-9_-]", "_")
|
||||
if clean == id and #clean <= TOOL_ID_MAX then
|
||||
return clean
|
||||
end
|
||||
local digest = string.sub(sha256_hex(id), 1, 8)
|
||||
local keep = TOOL_ID_MAX - #digest - 1
|
||||
if #clean > keep then clean = string.sub(clean, 1, keep) end
|
||||
return clean .. "_" .. digest
|
||||
end
|
||||
|
||||
-- map_usage converts one Anthropic usage object to the gateway's TokenUsage
|
||||
-- shape. The two APIs disagree on what the prompt count MEANS:
|
||||
--
|
||||
-- Anthropic: input_tokens EXCLUDES cached blocks. cache_read_input_tokens and
|
||||
-- cache_creation_input_tokens are separate, additive, billed input.
|
||||
-- OpenAI: prompt_tokens INCLUDES its cached_tokens subset.
|
||||
--
|
||||
-- Mapping input_tokens straight onto prompt therefore did two wrong things at
|
||||
-- once: it undercounted the billed prompt by the entire cache portion, and it
|
||||
-- could report cached_tokens > prompt_tokens — a cache hit rate above 100%.
|
||||
-- cache_creation_input_tokens was not read at all, so a cache-write turn lost
|
||||
-- those billed tokens outright. Summing all three restores OpenAI semantics,
|
||||
-- which is what clients (and the gateway's own audit trail) assume.
|
||||
--
|
||||
-- Returns nil when the object carries no counts, so callers can keep treating
|
||||
-- "no usage in this chunk" as "say nothing" rather than reporting zeros.
|
||||
local function map_usage(u)
|
||||
if type(u) ~= "table" then return nil end
|
||||
local fresh = u.input_tokens or 0
|
||||
local read = u.cache_read_input_tokens or 0
|
||||
local create = u.cache_creation_input_tokens or 0
|
||||
local p = fresh + read + create
|
||||
local c = u.output_tokens or 0
|
||||
if p <= 0 and c <= 0 then return nil end
|
||||
local uses = { prompt = p, completion = c, total = p + c }
|
||||
-- Emit the split whenever Anthropic reports either cache field, even at 0:
|
||||
-- a reported zero-hit ("cache missed") must stay distinguishable from "this
|
||||
-- upstream never reports cache info at all".
|
||||
if u.cache_read_input_tokens ~= nil or u.cache_creation_input_tokens ~= nil then
|
||||
uses.prompt_tokens_details = { cached_tokens = read }
|
||||
uses.prompt_cache_hit_tokens = read
|
||||
uses.prompt_cache_miss_tokens = fresh + create
|
||||
end
|
||||
return uses
|
||||
end
|
||||
|
||||
local function collect_blocks(content)
|
||||
if type(content) == "string" then
|
||||
if content == "" then return {} end
|
||||
@ -104,7 +168,7 @@ function adapter.transform_request(raw_body)
|
||||
end
|
||||
table.insert(blocks, {
|
||||
type = "tool_use",
|
||||
id = tc.id or "",
|
||||
id = safe_tool_id(tc.id),
|
||||
name = fn.name or "",
|
||||
input = input,
|
||||
})
|
||||
@ -119,7 +183,7 @@ function adapter.transform_request(raw_body)
|
||||
-- user message with tool_result content blocks.
|
||||
table.insert(pending_tool, {
|
||||
type = "tool_result",
|
||||
tool_use_id = m.tool_call_id or "",
|
||||
tool_use_id = safe_tool_id(m.tool_call_id),
|
||||
content = text_of(m.content),
|
||||
})
|
||||
|
||||
@ -220,17 +284,9 @@ function adapter.transform_response(raw_body)
|
||||
token_usage = { prompt = 0, completion = 0, total = 0 },
|
||||
}
|
||||
|
||||
if resp.usage then
|
||||
unified.token_usage.prompt = resp.usage.input_tokens or 0
|
||||
unified.token_usage.completion = resp.usage.output_tokens or 0
|
||||
unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0)
|
||||
-- Emit details whenever Anthropic reports the field, even at 0, so a
|
||||
-- reported cache miss stays distinguishable from "not reported".
|
||||
if resp.usage.cache_read_input_tokens ~= nil then
|
||||
unified.token_usage.prompt_tokens_details = {
|
||||
cached_tokens = resp.usage.cache_read_input_tokens
|
||||
}
|
||||
end
|
||||
local mapped = map_usage(resp.usage)
|
||||
if mapped ~= nil then
|
||||
unified.token_usage = mapped
|
||||
end
|
||||
|
||||
if resp.content and #resp.content > 0 then
|
||||
@ -242,7 +298,7 @@ function adapter.transform_response(raw_body)
|
||||
unified.reasoning_content = (unified.reasoning_content or "") .. block.thinking
|
||||
elseif block.type == "tool_use" then
|
||||
table.insert(tcs, {
|
||||
id = block.id or "",
|
||||
id = safe_tool_id(block.id),
|
||||
type = "function",
|
||||
name = block.name or "",
|
||||
arguments = block.input or {},
|
||||
@ -275,17 +331,13 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
|
||||
-- ── message_start: initial usage ─────────────────────────
|
||||
if chunk.type == "message_start" then
|
||||
if chunk.message and type(chunk.message.usage) == "table" then
|
||||
local u = chunk.message.usage
|
||||
local p = u.input_tokens or 0
|
||||
local c = u.output_tokens or 0
|
||||
if p > 0 or c > 0 then
|
||||
local uses = { prompt = p, completion = c, total = p + c }
|
||||
if u.cache_read_input_tokens ~= nil then
|
||||
uses.prompt_tokens_details = {
|
||||
cached_tokens = u.cache_read_input_tokens
|
||||
}
|
||||
end
|
||||
if chunk.message then
|
||||
-- Some Anthropic-compatible upstreams report a placeholder here and
|
||||
-- only send the true prompt count in message_delta (justwoker:
|
||||
-- 160 at message_start vs 6931 at message_delta). The gateway's
|
||||
-- mergeUsage lets a later non-zero value win, so both are emitted.
|
||||
local uses = map_usage(chunk.message.usage)
|
||||
if uses ~= nil then
|
||||
return json.encode({ usage = uses, done = false })
|
||||
end
|
||||
end
|
||||
@ -305,15 +357,10 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
finish = "stop"
|
||||
end
|
||||
end
|
||||
local uses = nil
|
||||
if type(chunk.usage) == "table" then
|
||||
local u = chunk.usage
|
||||
local p = u.input_tokens or 0
|
||||
local c = u.output_tokens or 0
|
||||
if p > 0 or c > 0 then
|
||||
uses = { prompt = p, completion = c, total = p + c }
|
||||
end
|
||||
end
|
||||
-- message_delta carries the FINAL usage, so it must map the cache
|
||||
-- fields too; the old code dropped them here, which silently lost the
|
||||
-- whole cache split on every streamed response.
|
||||
local uses = map_usage(chunk.usage)
|
||||
if uses ~= nil then
|
||||
return json.encode({ content = "", done = (finish ~= nil), finish_reason = finish, usage = uses })
|
||||
end
|
||||
@ -330,7 +377,7 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
content = "", done = false,
|
||||
tool_calls = { {
|
||||
index = chunk.index or 0,
|
||||
id = cb.id or "",
|
||||
id = safe_tool_id(cb.id),
|
||||
type = "function",
|
||||
["function"] = { name = cb.name or "", arguments = "" },
|
||||
} },
|
||||
|
||||
@ -81,16 +81,42 @@ function adapter.transform_response(raw_body)
|
||||
|
||||
if resp.candidates and #resp.candidates > 0 then
|
||||
local cand = resp.candidates[1]
|
||||
local tools = {}
|
||||
if cand.content and cand.content.parts then
|
||||
for _, part in ipairs(cand.content.parts) do
|
||||
if part.text then
|
||||
unified.content = unified.content .. part.text
|
||||
elseif part.functionCall then
|
||||
-- Non-streaming tool calls used to be dropped here while
|
||||
-- transform_stream_chunk handled them, so a non-streaming
|
||||
-- agent turn looked like a plain text answer and the tool
|
||||
-- loop died. Gemini's args are already an object.
|
||||
local args = part.functionCall.args
|
||||
if type(args) == "string" then
|
||||
local aok, decoded = pcall(json.decode, args)
|
||||
args = aok and decoded or {}
|
||||
elseif type(args) ~= "table" then
|
||||
args = {}
|
||||
end
|
||||
table.insert(tools, {
|
||||
id = part.functionCall.id or ("call_" .. #tools),
|
||||
type = "function",
|
||||
name = part.functionCall.name or "",
|
||||
arguments = args
|
||||
})
|
||||
end
|
||||
end
|
||||
end
|
||||
if cand.finishReason then
|
||||
unified.finish_reason = cand.finishReason
|
||||
end
|
||||
if #tools > 0 then
|
||||
unified.tool_calls = tools
|
||||
-- Gemini reports finishReason "STOP" even when it emitted a
|
||||
-- functionCall; clients keyed on finish_reason would treat that as
|
||||
-- a completed answer and never run the tool.
|
||||
unified.finish_reason = "tool_calls"
|
||||
end
|
||||
end
|
||||
|
||||
return json.encode(unified)
|
||||
|
||||
@ -58,13 +58,39 @@ function adapter.transform_response(raw_body)
|
||||
local unified = {
|
||||
content = "",
|
||||
finish_reason = resp.done_reason or "",
|
||||
tool_calls = {},
|
||||
-- key must be token_usage to match Go's UnifiedResponse json tag
|
||||
token_usage = { prompt = p, completion = c, total = p + c }
|
||||
}
|
||||
|
||||
if resp.message then
|
||||
unified.content = resp.message.content or ""
|
||||
-- Non-streaming tool calls were previously dropped: the field was
|
||||
-- initialized to an empty table and never filled, while
|
||||
-- transform_stream_chunk handled them. A non-streaming agent turn thus
|
||||
-- looked like a plain answer and the tool loop stopped.
|
||||
if type(resp.message.tool_calls) == "table" and #resp.message.tool_calls > 0 then
|
||||
local tcs = {}
|
||||
for _, tc in ipairs(resp.message.tool_calls) do
|
||||
local fn = tc["function"] or {}
|
||||
-- Ollama sends arguments as an object already
|
||||
local args = fn.arguments
|
||||
if type(args) == "string" then
|
||||
local aok, decoded = pcall(json.decode, args)
|
||||
args = aok and decoded or {}
|
||||
elseif type(args) ~= "table" then
|
||||
args = {}
|
||||
end
|
||||
table.insert(tcs, {
|
||||
id = tc.id or ("call_" .. #tcs),
|
||||
type = tc.type or "function",
|
||||
name = fn.name or "",
|
||||
arguments = args
|
||||
})
|
||||
end
|
||||
unified.tool_calls = tcs
|
||||
-- Ollama reports done_reason "stop" alongside tool calls
|
||||
unified.finish_reason = "tool_calls"
|
||||
end
|
||||
end
|
||||
|
||||
return json.encode(unified)
|
||||
|
||||
@ -5,6 +5,36 @@ adapter.version = "2.0.0"
|
||||
adapter.endpoint = "/chat/completions"
|
||||
adapter.headers = {}
|
||||
|
||||
-- Claude-behind-OpenAI upstreams (tabitoken, 扇贝, …) convert /chat/completions
|
||||
-- to the Anthropic Messages API internally, so they inherit Anthropic's
|
||||
-- tool id rule: ^[a-zA-Z0-9_-]{1,64}$, enforced by rejecting the WHOLE request
|
||||
-- with "Invalid tool use format" / "tool_use.id: String should match pattern".
|
||||
-- Plain OpenAI has no such rule, so an OpenAI-compatible model can mint an id
|
||||
-- like "bash:0" (observed from moonshotai/kimi-k3). In a fan-out router that id
|
||||
-- is replayed to every other source, so one such id kills every Claude slot at
|
||||
-- once and an AUTO request falls through all tiers.
|
||||
--
|
||||
-- safe_tool_id is pure and deterministic, so a tool_calls entry and its
|
||||
-- matching tool_call_id are rewritten identically within one request. A
|
||||
-- rewritten id keeps an 8-hex digest of the ORIGINAL id, without which two
|
||||
-- distinct ids could collapse into one ("a:b" and "a_b") and become an
|
||||
-- unpaired/duplicate tool call. Already-legal ids are returned untouched, so
|
||||
-- well-behaved traffic is byte-identical to before.
|
||||
-- (anthropic.lua carries the same helper; Lua adapters have no shared prelude.)
|
||||
local TOOL_ID_MAX = 64
|
||||
|
||||
local function safe_tool_id(id)
|
||||
if type(id) ~= "string" or id == "" then return id end
|
||||
local clean = string.gsub(id, "[^A-Za-z0-9_-]", "_")
|
||||
if clean == id and #clean <= TOOL_ID_MAX then
|
||||
return clean
|
||||
end
|
||||
local digest = string.sub(sha256_hex(id), 1, 8)
|
||||
local keep = TOOL_ID_MAX - #digest - 1
|
||||
if #clean > keep then clean = string.sub(clean, 1, keep) end
|
||||
return clean .. "_" .. digest
|
||||
end
|
||||
|
||||
-- OpenAI /chat/completions format (pass-through, strip provider-specific fields)
|
||||
function adapter.transform_request(raw_body)
|
||||
local ok, req = pcall(json.decode, raw_body)
|
||||
@ -14,6 +44,16 @@ function adapter.transform_request(raw_body)
|
||||
if req.messages then
|
||||
for _, msg in ipairs(req.messages) do
|
||||
msg.reasoning_content = nil
|
||||
if msg.tool_call_id ~= nil then
|
||||
msg.tool_call_id = safe_tool_id(msg.tool_call_id)
|
||||
end
|
||||
if type(msg.tool_calls) == "table" then
|
||||
for _, tc in ipairs(msg.tool_calls) do
|
||||
if type(tc) == "table" and tc.id ~= nil then
|
||||
tc.id = safe_tool_id(tc.id)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return json.encode(req)
|
||||
@ -64,7 +104,7 @@ function adapter.transform_response(raw_body)
|
||||
local args_ok, args = pcall(json.decode, tc["function"].arguments)
|
||||
if not args_ok then args = {} end
|
||||
table.insert(tcs, {
|
||||
id = tc.id,
|
||||
id = safe_tool_id(tc.id),
|
||||
type = tc.type or "function",
|
||||
name = tc["function"].name,
|
||||
arguments = args
|
||||
@ -129,6 +169,15 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
unified.reasoning_content = delta.reasoning_content
|
||||
end
|
||||
if delta.tool_calls then
|
||||
-- Sanitize on the way OUT too: an id this upstream happily minted (it
|
||||
-- does not validate them) becomes a landmine once the client replays
|
||||
-- it to a Claude upstream. Only the first fragment of a streamed call
|
||||
-- carries an id; later argument fragments have none and are untouched.
|
||||
for _, tc in ipairs(delta.tool_calls) do
|
||||
if type(tc) == "table" and tc.id ~= nil then
|
||||
tc.id = safe_tool_id(tc.id)
|
||||
end
|
||||
end
|
||||
unified.tool_calls = delta.tool_calls
|
||||
end
|
||||
return json.encode(unified)
|
||||
|
||||
@ -34,8 +34,10 @@ end
|
||||
|
||||
-- OpenAI /chat/completions format (pass-through, strip provider-specific fields)
|
||||
-- zen 上游 schema 只接受 text content part(无视觉/音频能力):多模态 part
|
||||
-- (image_url / input_audio / file 等)一律剥离;因此失去全部 content 的
|
||||
-- 消息整条丢弃,避免上游 "unknown variant `image_url`, expected `text`"。
|
||||
-- (image_url / input_audio / file 等)一律剥离。剥离后 content 变空的消息
|
||||
-- 若不再携带 tool_calls / tool_call_id 才整条丢弃(避免上游
|
||||
-- "unknown variant `image_url`, expected `text`");带工具调用的必须保留,
|
||||
-- 否则会把紧随其后的 tool 结果变成孤儿,模型会反复重发同一个调用。
|
||||
-- zen 上游角色白名单只有 system / user / assistant / tool / latest_reminder:
|
||||
-- OpenAI 的 developer(及 function 等)不在其中,直接透传会触发上游
|
||||
-- "unknown variant `developer`, expected one of ..." 错误;统一归一化为 system。
|
||||
@ -72,7 +74,24 @@ function adapter.transform_request(raw_body)
|
||||
end
|
||||
end
|
||||
if #parts == 0 then
|
||||
drop = true
|
||||
-- Content collapsed to nothing after stripping unsupported
|
||||
-- parts. A message that still carries a tool call must
|
||||
-- NEVER be dropped: the very next message is its tool
|
||||
-- result, and dropping the call orphans that result. The
|
||||
-- model then sees a result for a call it never made and
|
||||
-- re-issues the same tool call on every turn (observed as
|
||||
-- an infinite "repeated tool call" loop).
|
||||
--
|
||||
-- This is the common shape for agent clients: an assistant
|
||||
-- turn whose content is only [thinking, toolCall] serialises
|
||||
-- to content:[] with tool_calls — exactly the case that used
|
||||
-- to vanish here. Emit "" instead, which zen accepts.
|
||||
if (type(msg.tool_calls) == "table" and #msg.tool_calls > 0)
|
||||
or msg.tool_call_id ~= nil then
|
||||
msg.content = ""
|
||||
else
|
||||
drop = true
|
||||
end
|
||||
else
|
||||
msg.content = parts
|
||||
end
|
||||
|
||||
@ -61,6 +61,34 @@ function adapter.transform_response(raw_body)
|
||||
-- sensenova-6.8-flash-lite 用 reasoning 字段而不是 reasoning_content
|
||||
unified.reasoning_content = ch.message.reasoning
|
||||
end
|
||||
-- Tool calls MUST be forwarded. Dropping them while keeping
|
||||
-- finish_reason="tool_calls" makes the client replay an assistant
|
||||
-- message whose function name/arguments are empty, and sensenova
|
||||
-- then rejects the next turn with
|
||||
-- 400 invalid tool_call function, function/name/arguments cannot be empty
|
||||
-- i.e. a tool-using conversation dies on its second request.
|
||||
if type(ch.message.tool_calls) == "table" and #ch.message.tool_calls > 0 then
|
||||
local tcs = {}
|
||||
for _, tc in ipairs(ch.message.tool_calls) do
|
||||
local fn = tc["function"] or {}
|
||||
-- arguments arrives as a JSON *string* on the wire; the
|
||||
-- unified shape expects a decoded object.
|
||||
local args = fn.arguments
|
||||
if type(args) == "string" then
|
||||
local args_ok, decoded = pcall(json.decode, args)
|
||||
args = args_ok and decoded or {}
|
||||
elseif type(args) ~= "table" then
|
||||
args = {}
|
||||
end
|
||||
table.insert(tcs, {
|
||||
id = tc.id,
|
||||
type = tc.type or "function",
|
||||
name = fn.name,
|
||||
arguments = args
|
||||
})
|
||||
end
|
||||
unified.tool_calls = tcs
|
||||
end
|
||||
end
|
||||
unified.finish_reason = ch.finish_reason or ""
|
||||
end
|
||||
|
||||
@ -25,6 +25,95 @@ function adapter.transform_request(raw_body)
|
||||
return json.encode(req)
|
||||
end
|
||||
|
||||
-- parse_text_tool_calls extracts tool calls that an upstream emitted as PLAIN
|
||||
-- TEXT instead of using the OpenAI tool_calls field.
|
||||
--
|
||||
-- trae-local-api's OpenAI endpoint (/v1/chat/completions) does not read the
|
||||
-- request's `tools` array at all, so the relayed model is never told the tool
|
||||
-- schema; it falls back to printing
|
||||
-- <tool_call>
|
||||
-- {"name": "get_weather", "arguments": {"location": "北京"}}
|
||||
-- </tool_call>
|
||||
-- into message.content, leaves message.tool_calls null, and reports
|
||||
-- finish_reason="stop". A client following the OpenAI contract therefore never
|
||||
-- sees a tool call: the agent loop terminates unexpectedly mid-conversation
|
||||
-- (and a hand-written replay produces an empty function name next turn).
|
||||
--
|
||||
-- Both tag spellings are accepted: the same codebase's Anthropic endpoint
|
||||
-- instructs models to emit <toolcall>, and models mix the two. Key names vary
|
||||
-- too (arguments / params / input), so all are tried.
|
||||
--
|
||||
-- Returns (tool_calls_array_or_nil, content_with_blocks_removed).
|
||||
local function parse_text_tool_calls(content)
|
||||
if type(content) ~= "string" or content == "" then return nil, content end
|
||||
|
||||
local tcs = {}
|
||||
local idx = 0
|
||||
|
||||
local function collect(pattern)
|
||||
for payload in content:gmatch(pattern) do
|
||||
local ok, obj = pcall(json.decode, payload)
|
||||
if ok and type(obj) == "table" then
|
||||
-- some models wrap it as {"function":{"name":..,"arguments":..}}
|
||||
local fn = obj["function"]
|
||||
local name = obj.name or (type(fn) == "table" and fn.name) or nil
|
||||
if name then
|
||||
local args = obj.arguments or obj.params or obj.input
|
||||
if args == nil and type(fn) == "table" then
|
||||
args = fn.arguments or fn.params
|
||||
end
|
||||
if type(args) == "string" then
|
||||
local aok, decoded = pcall(json.decode, args)
|
||||
args = aok and decoded or {}
|
||||
elseif type(args) ~= "table" then
|
||||
args = {}
|
||||
end
|
||||
idx = idx + 1
|
||||
table.insert(tcs, {
|
||||
id = obj.id or ("call_text_" .. idx),
|
||||
type = "function",
|
||||
name = name,
|
||||
arguments = args
|
||||
})
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- `<tag ...>` allows attributes; %s* handles the "</tool_call >" spacing
|
||||
-- that trae-local-api's own prompt example uses.
|
||||
collect("<tool_call[^>]*>%s*(.-)%s*</tool_call%s*>")
|
||||
collect("<toolcall[^>]*>%s*(.-)%s*</toolcall%s*>")
|
||||
|
||||
-- Legacy / mimicked form: models that saw past assistant turns folded as
|
||||
-- [Called tool: name({...})]
|
||||
-- tend to emit the same shape instead of a real block. Recover it so
|
||||
-- the agent loop keeps working; otherwise the client receives plain text
|
||||
-- where a structured tool call should be. Capture the name and the JSON
|
||||
-- object literal independently (the JSON is the only {...} run here).
|
||||
if #tcs == 0 and content:find("%[Called tool") then
|
||||
for name, argsraw in content:gmatch("%[Called tool%s*:%s*([%w_%-%.]+)%s*%((%b{})%s*%)%s*%]") do
|
||||
local aok, decoded = pcall(json.decode, argsraw)
|
||||
idx = idx + 1
|
||||
table.insert(tcs, {
|
||||
id = "call_legacy_" .. idx,
|
||||
type = "function",
|
||||
name = name,
|
||||
arguments = aok and decoded or {}
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
if #tcs == 0 then return nil, content end
|
||||
|
||||
-- drop the blocks from user-visible content; keep any surrounding prose
|
||||
local stripped = content:gsub("<tool_call[^>]*>%s*.-%s*</tool_call%s*>", "")
|
||||
stripped = stripped:gsub("<toolcall[^>]*>%s*.-%s*</toolcall%s*>", "")
|
||||
stripped = stripped:gsub("%[Called tool%s*:%s*[%w_%-%.]+%s*%(%b{}%s*%)%s*%]", "")
|
||||
stripped = stripped:gsub("^%s+", ""):gsub("%s+$", "")
|
||||
return tcs, stripped
|
||||
end
|
||||
|
||||
function adapter.transform_response(raw_body)
|
||||
-- trae-local-api 偶发在非流式请求中返回 SSE 格式数据,
|
||||
-- 表现为多个 data: {...} 行或混合了 reasoning_chunk 等。
|
||||
@ -71,15 +160,21 @@ function adapter.transform_response(raw_body)
|
||||
if ch.message.reasoning_content then
|
||||
unified.reasoning_content = ch.message.reasoning_content
|
||||
end
|
||||
if type(ch.message.tool_calls) == "table" then
|
||||
if type(ch.message.tool_calls) == "table" and #ch.message.tool_calls > 0 then
|
||||
local tcs = {}
|
||||
for _, tc in ipairs(ch.message.tool_calls) do
|
||||
local args_ok, args = pcall(json.decode, tc["function"].arguments)
|
||||
if not args_ok then args = {} end
|
||||
local fn = tc["function"] or {}
|
||||
local args = fn.arguments
|
||||
if type(args) == "string" then
|
||||
local args_ok, decoded = pcall(json.decode, args)
|
||||
args = args_ok and decoded or {}
|
||||
elseif type(args) ~= "table" then
|
||||
args = {}
|
||||
end
|
||||
table.insert(tcs, {
|
||||
id = tc.id,
|
||||
type = tc.type or "function",
|
||||
name = tc["function"].name,
|
||||
name = fn.name,
|
||||
arguments = args
|
||||
})
|
||||
end
|
||||
@ -87,6 +182,22 @@ function adapter.transform_response(raw_body)
|
||||
end
|
||||
end
|
||||
unified.finish_reason = ch.finish_reason or ""
|
||||
|
||||
-- Fallback: trae-local-api's OpenAI endpoint drops the request's
|
||||
-- `tools` array entirely, so the relayed model is never told the tool
|
||||
-- schema and instead PRINTS a <tool_call>{...}</tool_call> block into
|
||||
-- content, leaving message.tool_calls null and finish_reason="stop".
|
||||
-- A client following the OpenAI contract then sees a normal completion
|
||||
-- and its agent loop terminates mid-conversation. Recover the
|
||||
-- structured call so the loop can continue.
|
||||
if unified.tool_calls == nil then
|
||||
local recovered, cleaned = parse_text_tool_calls(unified.content)
|
||||
if recovered then
|
||||
unified.tool_calls = recovered
|
||||
unified.content = cleaned
|
||||
unified.finish_reason = "tool_calls"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return json.encode(unified)
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@ -220,6 +221,70 @@ func TestOpenCodeStripsMultiModalParts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenCodeKeepsToolCallWithEmptyContent pins the fix for an infinite
|
||||
// "repeated tool call" loop: an assistant turn whose content serialises to an
|
||||
// empty ARRAY while carrying tool_calls must survive the multimodal strip.
|
||||
// Dropping it orphans the following tool result, so the model never sees the
|
||||
// call it already made and re-issues it on every turn.
|
||||
func TestOpenCodeKeepsToolCallWithEmptyContent(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
body := `{"model":"x","messages":[
|
||||
{"role":"user","content":"read /tmp/x"},
|
||||
{"role":"assistant","content":[],"tool_calls":[{"id":"call_abc","type":"function","function":{"name":"read","arguments":"{\"path\":\"/tmp/x\"}"}}]},
|
||||
{"role":"tool","tool_call_id":"call_abc","content":"hello"},
|
||||
{"role":"user","content":"now summarize"}
|
||||
]}`
|
||||
out, err := vm.Transform("opencode", "transform_request", body)
|
||||
if err != nil {
|
||||
t.Fatalf("transform_request: %v", err)
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tool_calls"`
|
||||
ToolCallID string `json:"tool_call_id"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &req); err != nil {
|
||||
t.Fatalf("unmarshal: %v (%s)", err, out)
|
||||
}
|
||||
if len(req.Messages) != 4 {
|
||||
t.Fatalf("expected 4 messages (tool call must not be dropped), got %d: %s", len(req.Messages), out)
|
||||
}
|
||||
asst := req.Messages[1]
|
||||
if asst.Role != "assistant" || len(asst.ToolCalls) != 1 || asst.ToolCalls[0].ID != "call_abc" {
|
||||
t.Fatalf("assistant tool_calls lost: %s", out)
|
||||
}
|
||||
if string(asst.Content) != `""` {
|
||||
t.Fatalf("empty content must be rewritten to \"\", got %s", asst.Content)
|
||||
}
|
||||
if req.Messages[2].Role != "tool" || req.Messages[2].ToolCallID != "call_abc" {
|
||||
t.Fatalf("tool result must stay paired with its call: %s", out)
|
||||
}
|
||||
|
||||
// Negative control: a message with neither text nor tool calls is still dropped.
|
||||
only := `{"model":"x","messages":[
|
||||
{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://ex.com/a.png"}}]},
|
||||
{"role":"user","content":"keep me"}
|
||||
]}`
|
||||
out2, err := vm.Transform("opencode", "transform_request", only)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Count(out2, `"role"`); got != 1 {
|
||||
t.Fatalf("image-only message without tool calls must still be dropped, got %d: %s", got, out2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHeadersCustomHook(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
@ -1373,3 +1438,650 @@ func TestBurstLeftoverIsReclaimedUnderSteadyTraffic(t *testing.T) {
|
||||
created, idle)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNonStreamToolCallsPreserved is the regression for the bug that killed
|
||||
// tool-using conversations on their SECOND request: several adapters handled
|
||||
// tool_calls in transform_stream_chunk but dropped them in
|
||||
// transform_response. A client then received finish_reason:"tool_calls" with no
|
||||
// tool_calls payload, replayed an assistant message whose function
|
||||
// name/arguments were empty, and the upstream rejected the next turn with
|
||||
//
|
||||
// 400 invalid tool_call function, function/name/arguments cannot be empty
|
||||
//
|
||||
// Every OpenAI-shaped adapter must forward non-streaming tool calls.
|
||||
func TestNonStreamToolCallsPreserved(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
// upstream shape: arguments is a JSON *string*, as OpenAI sends it
|
||||
body := `{"choices":[{"index":0,"message":{"role":"assistant","content":"",` +
|
||||
`"tool_calls":[{"index":0,"id":"call_abc","type":"function",` +
|
||||
`"function":{"name":"get_weather","arguments":"{\"city\":\"Beijing\"}"}}]},` +
|
||||
`"finish_reason":"tool_calls"}],` +
|
||||
`"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`
|
||||
|
||||
for _, name := range []string{
|
||||
"openai", "sensenova", "trae", "deepseek", "github",
|
||||
"groq", "kimicode", "mistral", "opencode", "agentrouter",
|
||||
} {
|
||||
out, err := vm.Transform(name, "transform_response", body)
|
||||
if err != nil {
|
||||
t.Fatalf("%s transform_response: %v", name, err)
|
||||
}
|
||||
var got struct {
|
||||
FinishReason string `json:"finish_reason"`
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Arguments map[string]interface{} `json:"arguments"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("%s: unmarshal %v (%s)", name, err, out)
|
||||
}
|
||||
if len(got.ToolCalls) != 1 {
|
||||
t.Fatalf("%s: dropped non-streaming tool_calls (client would replay an empty function): %s", name, out)
|
||||
}
|
||||
tc := got.ToolCalls[0]
|
||||
if tc.Name != "get_weather" {
|
||||
t.Errorf("%s: tool name lost: %s", name, out)
|
||||
}
|
||||
if tc.ID != "call_abc" {
|
||||
t.Errorf("%s: tool id lost: %s", name, out)
|
||||
}
|
||||
// arguments must be decoded into an object, not left as a string
|
||||
if tc.Arguments == nil || tc.Arguments["city"] != "Beijing" {
|
||||
t.Errorf("%s: arguments not decoded: %s", name, out)
|
||||
}
|
||||
if got.FinishReason != "tool_calls" {
|
||||
t.Errorf("%s: finish_reason lost: %s", name, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeminiNonStreamToolCalls covers Gemini's native shape (functionCall parts
|
||||
// under candidates[].content.parts). The streaming path handled these; the
|
||||
// non-streaming path silently returned text only.
|
||||
func TestGeminiNonStreamToolCalls(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
body := `{"candidates":[{"content":{"parts":[` +
|
||||
`{"text":"checking"},` +
|
||||
`{"functionCall":{"name":"get_weather","args":{"city":"Beijing"}}}` +
|
||||
`]},"finishReason":"STOP"}],` +
|
||||
`"usageMetadata":{"promptTokenCount":8,"candidatesTokenCount":4,"totalTokenCount":12}}`
|
||||
|
||||
out, err := vm.Transform("gemini", "transform_response", body)
|
||||
if err != nil {
|
||||
t.Fatalf("gemini transform_response: %v", err)
|
||||
}
|
||||
var got struct {
|
||||
Content string `json:"content"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
ToolCalls []struct {
|
||||
Name string `json:"name"`
|
||||
Arguments map[string]interface{} `json:"arguments"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("unmarshal: %v (%s)", err, out)
|
||||
}
|
||||
if len(got.ToolCalls) != 1 || got.ToolCalls[0].Name != "get_weather" {
|
||||
t.Fatalf("gemini dropped non-streaming functionCall: %s", out)
|
||||
}
|
||||
if got.ToolCalls[0].Arguments["city"] != "Beijing" {
|
||||
t.Errorf("gemini args lost: %s", out)
|
||||
}
|
||||
// a tool call must not be reported as a plain stop
|
||||
if got.FinishReason != "tool_calls" {
|
||||
t.Errorf("gemini finish_reason should become tool_calls: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOllamaNonStreamToolCalls covers Ollama's non-streaming shape
|
||||
// (message.tool_calls with an already-decoded arguments object). The adapter
|
||||
// initialized tool_calls to an empty table and never filled it.
|
||||
func TestOllamaNonStreamToolCalls(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
body := `{"message":{"role":"assistant","content":"",` +
|
||||
`"tool_calls":[{"function":{"name":"get_weather","arguments":{"city":"Beijing"}}}]},` +
|
||||
`"done_reason":"stop","prompt_eval_count":7,"eval_count":3}`
|
||||
|
||||
out, err := vm.Transform("ollama", "transform_response", body)
|
||||
if err != nil {
|
||||
t.Fatalf("ollama transform_response: %v", err)
|
||||
}
|
||||
var got struct {
|
||||
FinishReason string `json:"finish_reason"`
|
||||
ToolCalls []struct {
|
||||
Name string `json:"name"`
|
||||
Arguments map[string]interface{} `json:"arguments"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("unmarshal: %v (%s)", err, out)
|
||||
}
|
||||
if len(got.ToolCalls) != 1 || got.ToolCalls[0].Name != "get_weather" {
|
||||
t.Fatalf("ollama dropped non-streaming tool_calls: %s", out)
|
||||
}
|
||||
if got.ToolCalls[0].Arguments["city"] != "Beijing" {
|
||||
t.Errorf("ollama args lost: %s", out)
|
||||
}
|
||||
if got.FinishReason != "tool_calls" {
|
||||
t.Errorf("ollama finish_reason should become tool_calls: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTraeTextToolCallRecovery covers the trae-specific failure: its upstream
|
||||
// (trae-local-api's OpenAI endpoint) never reads the request's `tools` array, so
|
||||
// the relayed model prints a <tool_call> block as TEXT, leaves
|
||||
// message.tool_calls null and reports finish_reason:"stop". An OpenAI client
|
||||
// then sees an ordinary completion and its agent loop ends mid-conversation.
|
||||
// The adapter must recover the structured call, strip the block from content,
|
||||
// and correct finish_reason.
|
||||
func TestTraeTextToolCallRecovery(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{
|
||||
"tool_call tag with arguments",
|
||||
`{"choices":[{"message":{"role":"assistant","content":"<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Beijing\"}}\n</tool_call>"},"finish_reason":"stop"}]}`,
|
||||
},
|
||||
{
|
||||
// the same codebase's Anthropic endpoint prompts for <toolcall> and "params"
|
||||
"toolcall tag with params",
|
||||
`{"choices":[{"message":{"role":"assistant","content":"<toolcall>{\"name\": \"get_weather\", \"params\": {\"city\": \"Beijing\"}}</toolcall>"},"finish_reason":"stop"}]}`,
|
||||
},
|
||||
{
|
||||
"prose around the block is preserved",
|
||||
`{"choices":[{"message":{"role":"assistant","content":"Let me check.\n<tool_call>{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Beijing\"}}</tool_call>"},"finish_reason":"stop"}]}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
out, err := vm.Transform("trae", "transform_response", c.body)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: transform: %v", c.name, err)
|
||||
}
|
||||
var got struct {
|
||||
Content string `json:"content"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
ToolCalls []struct {
|
||||
Name string `json:"name"`
|
||||
Arguments map[string]interface{} `json:"arguments"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("%s: unmarshal %v (%s)", c.name, err, out)
|
||||
}
|
||||
if len(got.ToolCalls) != 1 {
|
||||
t.Fatalf("%s: text tool call not recovered: %s", c.name, out)
|
||||
}
|
||||
if got.ToolCalls[0].Name != "get_weather" {
|
||||
t.Errorf("%s: name wrong: %s", c.name, out)
|
||||
}
|
||||
if got.ToolCalls[0].Arguments["city"] != "Beijing" {
|
||||
t.Errorf("%s: args wrong: %s", c.name, out)
|
||||
}
|
||||
if got.FinishReason != "tool_calls" {
|
||||
t.Errorf("%s: finish_reason must be corrected to tool_calls: %s", c.name, out)
|
||||
}
|
||||
if strings.Contains(got.Content, "<tool_call") || strings.Contains(got.Content, "<toolcall") {
|
||||
t.Errorf("%s: raw block left in user-visible content: %s", c.name, out)
|
||||
}
|
||||
}
|
||||
|
||||
// a plain text answer must be untouched
|
||||
plain := `{"choices":[{"message":{"role":"assistant","content":"just text"},"finish_reason":"stop"}]}`
|
||||
out, err := vm.Transform("trae", "transform_response", plain)
|
||||
if err != nil {
|
||||
t.Fatalf("plain: %v", err)
|
||||
}
|
||||
if strings.Contains(out, "tool_calls") {
|
||||
t.Errorf("plain answer must not gain tool_calls: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `"finish_reason":"stop"`) {
|
||||
t.Errorf("plain answer finish_reason changed: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToolIDSanitize locks the tool-call id sanitizer in both adapters that
|
||||
// talk to Claude upstreams. Anthropic requires tool_use.id /
|
||||
// tool_result.tool_use_id to match ^[a-zA-Z0-9_-]{1,64}$ and rejects the WHOLE
|
||||
// request otherwise (REQUEST_BODY_INVALID / "Invalid tool use format"), while
|
||||
// OpenAI has no such rule — so an OpenAI-compatible model can mint "bash:0"
|
||||
// (observed from moonshotai/kimi-k3) and poison a client's history, killing
|
||||
// every Claude slot on replay. tabitoken/扇贝 are Claude-behind-OpenAI, so
|
||||
// openai.lua needs the same treatment as anthropic.lua.
|
||||
func TestToolIDSanitize(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
legal := regexp.MustCompile(`^[a-zA-Z0-9_-]{1,64}$`)
|
||||
long := strings.Repeat("a", 200)
|
||||
|
||||
body := func(id string) string {
|
||||
b, err := json.Marshal(map[string]interface{}{
|
||||
"model": "x",
|
||||
"messages": []interface{}{
|
||||
map[string]interface{}{"role": "user", "content": "run whoami"},
|
||||
map[string]interface{}{"role": "assistant", "content": "", "tool_calls": []interface{}{
|
||||
map[string]interface{}{"id": id, "type": "function",
|
||||
"function": map[string]interface{}{"name": "bash", "arguments": `{"command":"whoami"}`}},
|
||||
}},
|
||||
map[string]interface{}{"role": "tool", "tool_call_id": id, "content": "root"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// ---- anthropic: tool_use.id and tool_result.tool_use_id must agree ----
|
||||
for _, id := range []string{"bash:0", "call_ok_123", long, "toolu~sig1:AB+/=="} {
|
||||
out, err := vm.Transform("anthropic", "transform_request", body(id))
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic transform %q: %v", id, err)
|
||||
}
|
||||
var r struct {
|
||||
Messages []struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &r); err != nil {
|
||||
t.Fatalf("anthropic unmarshal %q: %v (%s)", id, err, out)
|
||||
}
|
||||
var useID, resID string
|
||||
for _, m := range r.Messages {
|
||||
for _, c := range m.Content {
|
||||
switch c.Type {
|
||||
case "tool_use":
|
||||
useID = c.ID
|
||||
case "tool_result":
|
||||
resID = c.ToolUseID
|
||||
}
|
||||
}
|
||||
}
|
||||
if !legal.MatchString(useID) {
|
||||
t.Errorf("anthropic %q: tool_use.id %q violates Anthropic's id pattern", id, useID)
|
||||
}
|
||||
if useID != resID {
|
||||
t.Errorf("anthropic %q: tool_use.id %q != tool_result.tool_use_id %q (unpaired call)", id, useID, resID)
|
||||
}
|
||||
if legal.MatchString(id) && useID != id {
|
||||
t.Errorf("anthropic %q: already-legal id must pass through untouched, got %q", id, useID)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- openai + agentrouter: same rewrite, and both sites must stay in sync ----
|
||||
// agentrouter fronts claude-opus-4-8, so it inherits the same id rule.
|
||||
for _, adapterName := range []string{"openai", "agentrouter"} {
|
||||
for _, id := range []string{"bash:0", "call_ok_123", long} {
|
||||
out, err := vm.Transform(adapterName, "transform_request", body(id))
|
||||
if err != nil {
|
||||
t.Fatalf("%s transform %q: %v", adapterName, id, err)
|
||||
}
|
||||
var r struct {
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tool_calls"`
|
||||
ToolCallID string `json:"tool_call_id"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &r); err != nil {
|
||||
t.Fatalf("%s unmarshal %q: %v (%s)", adapterName, id, err, out)
|
||||
}
|
||||
var callID, resID string
|
||||
for _, m := range r.Messages {
|
||||
if len(m.ToolCalls) > 0 {
|
||||
callID = m.ToolCalls[0].ID
|
||||
}
|
||||
if m.Role == "tool" {
|
||||
resID = m.ToolCallID
|
||||
}
|
||||
}
|
||||
if !legal.MatchString(callID) {
|
||||
t.Errorf("%s %q: tool_calls[0].id %q still illegal for Claude-behind-OpenAI upstreams", adapterName, id, callID)
|
||||
}
|
||||
if callID != resID {
|
||||
t.Errorf("%s %q: tool_calls id %q != tool_call_id %q (unpaired call)", adapterName, id, callID, resID)
|
||||
}
|
||||
if legal.MatchString(id) && callID != id {
|
||||
t.Errorf("%s %q: already-legal id must pass through untouched, got %q", adapterName, id, callID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- distinct dirty ids must NOT collapse into one (digest suffix) ----
|
||||
seen := map[string]string{}
|
||||
for _, id := range []string{"a:b", "a_b", "a-b", "a b", "a/b"} {
|
||||
out, err := vm.Transform("anthropic", "transform_request", body(id))
|
||||
if err != nil {
|
||||
t.Fatalf("collision probe %q: %v", id, err)
|
||||
}
|
||||
var r struct {
|
||||
Messages []struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := ""
|
||||
for _, m := range r.Messages {
|
||||
for _, c := range m.Content {
|
||||
if c.Type == "tool_use" {
|
||||
got = c.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
if prev, dup := seen[got]; dup {
|
||||
t.Errorf("ids %q and %q both map to %q — a collision makes a duplicate tool_use", prev, id, got)
|
||||
}
|
||||
seen[got] = id
|
||||
}
|
||||
|
||||
// ---- determinism: same input, same output ----
|
||||
a, _ := vm.Transform("anthropic", "transform_request", body("bash:0"))
|
||||
b, _ := vm.Transform("anthropic", "transform_request", body("bash:0"))
|
||||
if a != b {
|
||||
t.Errorf("sanitizer is not deterministic:\n%s\n%s", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToolIDSanitizeResponse locks the OUTBOUND half of the sanitizer. The
|
||||
// request-side rewrite alone is not enough: an id minted by a permissive
|
||||
// OpenAI-compatible upstream (moonshotai/kimi-k3 returns "bash:0") is handed to
|
||||
// the client, stored in its history, and replayed forever after. Sanitizing on
|
||||
// the way out means a poisoned id never enters a client session in the first
|
||||
// place — both for non-streaming responses and for the first fragment of a
|
||||
// streamed tool call (later argument fragments carry no id).
|
||||
func TestToolIDSanitizeResponse(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
legal := regexp.MustCompile(`^[a-zA-Z0-9_-]{1,64}$`)
|
||||
|
||||
// ---- openai + agentrouter non-streaming ----
|
||||
resp := `{"choices":[{"message":{"role":"assistant","content":null,"tool_calls":[
|
||||
{"id":"bash:0","type":"function","function":{"name":"bash","arguments":"{\"command\":\"whoami\"}"}}]},
|
||||
"finish_reason":"tool_calls"}]}`
|
||||
var nonStreamID string
|
||||
for _, adapterName := range []string{"openai", "agentrouter"} {
|
||||
out, err := vm.Transform(adapterName, "transform_response", resp)
|
||||
if err != nil {
|
||||
t.Fatalf("%s transform_response: %v", adapterName, err)
|
||||
}
|
||||
var r struct {
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &r); err != nil {
|
||||
t.Fatalf("%s unmarshal: %v (%s)", adapterName, err, out)
|
||||
}
|
||||
if len(r.ToolCalls) != 1 {
|
||||
t.Fatalf("%s tool_calls lost: %s", adapterName, out)
|
||||
}
|
||||
if !legal.MatchString(r.ToolCalls[0].ID) {
|
||||
t.Errorf("%s non-stream response leaks illegal id %q to the client", adapterName, r.ToolCalls[0].ID)
|
||||
}
|
||||
if nonStreamID != "" && r.ToolCalls[0].ID != nonStreamID {
|
||||
t.Errorf("%s rewrote %q differently than another adapter (%q) — a client mixing sources gets unpaired calls",
|
||||
adapterName, r.ToolCalls[0].ID, nonStreamID)
|
||||
}
|
||||
nonStreamID = r.ToolCalls[0].ID
|
||||
}
|
||||
|
||||
// ---- streaming: first fragment carries the id (both adapters) ----
|
||||
chunk := `{"choices":[{"delta":{"tool_calls":[
|
||||
{"index":0,"id":"bash:0","type":"function","function":{"name":"bash","arguments":""}}]},
|
||||
"finish_reason":null}]}`
|
||||
var c struct {
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
for _, adapterName := range []string{"openai", "agentrouter"} {
|
||||
out, err := vm.Transform(adapterName, "transform_stream_chunk", chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("%s transform_stream_chunk: %v", adapterName, err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &c); err != nil {
|
||||
t.Fatalf("%s unmarshal chunk: %v (%s)", adapterName, err, out)
|
||||
}
|
||||
if len(c.ToolCalls) != 1 {
|
||||
t.Fatalf("%s stream tool_calls lost: %s", adapterName, out)
|
||||
}
|
||||
if !legal.MatchString(c.ToolCalls[0].ID) {
|
||||
t.Errorf("%s stream response leaks illegal id %q to the client", adapterName, c.ToolCalls[0].ID)
|
||||
}
|
||||
// Streaming and non-streaming must agree, otherwise a client that mixes
|
||||
// modes within one session produces unpaired tool calls.
|
||||
if c.ToolCalls[0].ID != nonStreamID {
|
||||
t.Errorf("%s stream id %q != non-stream id %q for the same input", adapterName, c.ToolCalls[0].ID, nonStreamID)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- a later argument fragment has no id and must stay id-less ----
|
||||
argChunk := `{"choices":[{"delta":{"tool_calls":[
|
||||
{"index":0,"function":{"arguments":"{\"command\":\"whoami\"}"}}]},"finish_reason":null}]}`
|
||||
for _, adapterName := range []string{"openai", "agentrouter"} {
|
||||
out, err := vm.Transform(adapterName, "transform_stream_chunk", argChunk)
|
||||
if err != nil {
|
||||
t.Fatalf("%s arg fragment: %v", adapterName, err)
|
||||
}
|
||||
if strings.Contains(out, `"id"`) {
|
||||
t.Errorf("%s argument fragment gained an id (breaks index-based accumulation): %s", adapterName, out)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- anthropic response side: tool_use id and stream start ----
|
||||
var r struct {
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
aResp := `{"content":[{"type":"tool_use","id":"toolu~sig1:AB+/==","name":"bash","input":{"command":"whoami"}}],"stop_reason":"tool_use"}`
|
||||
out, err := vm.Transform("anthropic", "transform_response", aResp)
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic transform_response: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &r); err != nil {
|
||||
t.Fatalf("unmarshal anthropic: %v (%s)", err, out)
|
||||
}
|
||||
if len(r.ToolCalls) != 1 || !legal.MatchString(r.ToolCalls[0].ID) {
|
||||
t.Errorf("anthropic response leaks illegal id: %s", out)
|
||||
}
|
||||
|
||||
aChunk := `{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu~sig1:AB+/==","name":"bash"}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", aChunk)
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic stream chunk: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &c); err != nil {
|
||||
t.Fatalf("unmarshal anthropic chunk: %v (%s)", err, out)
|
||||
}
|
||||
if len(c.ToolCalls) != 1 || !legal.MatchString(c.ToolCalls[0].ID) {
|
||||
t.Errorf("anthropic stream leaks illegal id: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnthropicUsageCacheMapping pins the Anthropic→OpenAI usage conversion.
|
||||
// The two APIs disagree on what the prompt count means: Anthropic's
|
||||
// input_tokens EXCLUDES cached blocks (cache_read_input_tokens and
|
||||
// cache_creation_input_tokens are separate, additive, billed input), while
|
||||
// OpenAI's prompt_tokens INCLUDES its cached_tokens subset. Mapping
|
||||
// input_tokens straight onto prompt undercounted the billed prompt by the whole
|
||||
// cache portion and could report cached_tokens > prompt_tokens (a hit rate over
|
||||
// 100%); cache_creation_input_tokens was dropped entirely.
|
||||
func TestAnthropicUsageCacheMapping(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
type usage struct {
|
||||
Prompt int `json:"prompt"`
|
||||
Completion int `json:"completion"`
|
||||
Total int `json:"total"`
|
||||
Details *struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
Hit int `json:"prompt_cache_hit_tokens"`
|
||||
Miss int `json:"prompt_cache_miss_tokens"`
|
||||
}
|
||||
|
||||
// ---- non-streaming, cache read + cache write reported ----
|
||||
resp := `{"content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn",
|
||||
"usage":{"input_tokens":1200,"cache_read_input_tokens":40000,
|
||||
"cache_creation_input_tokens":500,"output_tokens":80}}`
|
||||
out, err := vm.Transform("anthropic", "transform_response", resp)
|
||||
if err != nil {
|
||||
t.Fatalf("transform_response: %v", err)
|
||||
}
|
||||
// Each case decodes into a FRESH value: json.Unmarshal leaves fields absent
|
||||
// from the payload untouched, so a reused struct would carry the previous
|
||||
// case's prompt_tokens_details into a response that has none.
|
||||
decode := func(out string) usage {
|
||||
t.Helper()
|
||||
var r struct {
|
||||
TokenUsage usage `json:"token_usage"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &r); err != nil {
|
||||
t.Fatalf("unmarshal: %v (%s)", err, out)
|
||||
}
|
||||
return r.TokenUsage
|
||||
}
|
||||
decodeChunk := func(out string) (usage, string) {
|
||||
t.Helper()
|
||||
var c struct {
|
||||
Usage usage `json:"usage"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &c); err != nil {
|
||||
t.Fatalf("unmarshal chunk: %v (%s)", err, out)
|
||||
}
|
||||
return c.Usage, c.FinishReason
|
||||
}
|
||||
|
||||
u := decode(out)
|
||||
if u.Prompt != 1200+40000+500 {
|
||||
t.Errorf("prompt must sum fresh+cache_read+cache_creation (41700), got %d", u.Prompt)
|
||||
}
|
||||
if u.Total != u.Prompt+u.Completion {
|
||||
t.Errorf("total %d != prompt %d + completion %d", u.Total, u.Prompt, u.Completion)
|
||||
}
|
||||
if u.Details == nil || u.Details.CachedTokens != 40000 {
|
||||
t.Errorf("cached_tokens must carry cache_read (40000): %s", out)
|
||||
}
|
||||
if u.Details != nil && u.Details.CachedTokens > u.Prompt {
|
||||
t.Errorf("cached_tokens %d > prompt %d implies a hit rate above 100%%",
|
||||
u.Details.CachedTokens, u.Prompt)
|
||||
}
|
||||
if u.Hit != 40000 || u.Miss != 1200+500 {
|
||||
t.Errorf("hit/miss split wrong: hit=%d miss=%d (want 40000/1700)", u.Hit, u.Miss)
|
||||
}
|
||||
|
||||
// ---- a reported zero hit must stay distinguishable from "not reported" ----
|
||||
zero := `{"content":[],"stop_reason":"end_turn","usage":{"input_tokens":100,"cache_read_input_tokens":0,"output_tokens":5}}`
|
||||
out, err = vm.Transform("anthropic", "transform_response", zero)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decode(out).Details == nil {
|
||||
t.Errorf("a reported cache_read of 0 must still emit prompt_tokens_details: %s", out)
|
||||
}
|
||||
|
||||
// An upstream that never mentions caching must not gain a fabricated split
|
||||
// (justwoker reports only input_tokens/output_tokens plus its own cost fields).
|
||||
none := `{"content":[],"stop_reason":"end_turn","usage":{"input_tokens":6931,"output_tokens":1}}`
|
||||
out, err = vm.Transform("anthropic", "transform_response", none)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plain := decode(out)
|
||||
if plain.Details != nil {
|
||||
t.Errorf("upstream reported no cache fields; details must be absent: %s", out)
|
||||
}
|
||||
if plain.Prompt != 6931 || plain.Total != 6932 {
|
||||
t.Errorf("plain usage mismapped: %s", out)
|
||||
}
|
||||
|
||||
// ---- streaming: message_delta carries the FINAL usage and must map cache ----
|
||||
delta := `{"type":"message_delta","delta":{"stop_reason":"end_turn"},
|
||||
"usage":{"input_tokens":1200,"cache_read_input_tokens":40000,"output_tokens":80}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", delta)
|
||||
if err != nil {
|
||||
t.Fatalf("stream delta: %v", err)
|
||||
}
|
||||
du, fin := decodeChunk(out)
|
||||
if du.Prompt != 41200 {
|
||||
t.Errorf("stream final prompt must include cache_read: got %d", du.Prompt)
|
||||
}
|
||||
if du.Details == nil || du.Details.CachedTokens != 40000 {
|
||||
t.Errorf("stream final usage dropped the cache split: %s", out)
|
||||
}
|
||||
if fin != "stop" {
|
||||
t.Errorf("finish_reason regressed: %s", out)
|
||||
}
|
||||
|
||||
// message_start's placeholder count must not suppress the later real one:
|
||||
// justwoker reports input_tokens=160 at message_start and 6931 at
|
||||
// message_delta, and the gateway's mergeUsage lets the later value win.
|
||||
start := `{"type":"message_start","message":{"usage":{"input_tokens":160,"output_tokens":1}}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", start)
|
||||
if err != nil {
|
||||
t.Fatalf("message_start: %v", err)
|
||||
}
|
||||
if su, _ := decodeChunk(out); su.Prompt != 160 {
|
||||
t.Errorf("message_start usage lost: %s", out)
|
||||
}
|
||||
|
||||
// A usage-less chunk must stay silent rather than reporting zeros.
|
||||
quiet := `{"type":"message_start","message":{}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", quiet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(out, "usage") {
|
||||
t.Errorf("usage-less chunk must not emit a usage object: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,6 +13,7 @@ import (
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@ -24,6 +25,20 @@ import (
|
||||
"llmsproxy/internal/types"
|
||||
)
|
||||
|
||||
// proxyFor picks the transport proxy for one source. An explicit per-source
|
||||
// proxy_url wins (e.g. clash on 127.0.0.1:7890 for a geo-blocked upstream);
|
||||
// otherwise fall back to the process-wide env proxy, which is direct by
|
||||
// default.
|
||||
func proxyFor(cfg config.Source) func(*http.Request) (*url.URL, error) {
|
||||
if cfg.ProxyURL != "" {
|
||||
proxyURL, err := url.Parse(cfg.ProxyURL)
|
||||
if err == nil {
|
||||
return http.ProxyURL(proxyURL)
|
||||
}
|
||||
}
|
||||
return http.ProxyFromEnvironment
|
||||
}
|
||||
|
||||
// ---- per-(source,model) scheduling state ----
|
||||
|
||||
// ModelState is the scheduling state of one (source, model) pair: a soft
|
||||
@ -34,9 +49,14 @@ import (
|
||||
// cooldown always expires and any success resets the state, so a fixed
|
||||
// upstream recovers on its own.
|
||||
type ModelState struct {
|
||||
pref atomic.Int64 // +1 per success / -5 per failure, clamped
|
||||
pref atomic.Int64 // +prefSuccessStep per success / -prefFailStep per failure, clamped
|
||||
failCount atomic.Int64
|
||||
cooldownUntil atomic.Int64 // unix seconds; 0 = schedulable
|
||||
// prefTouched is the unix second pref was last moved by an outcome. It is
|
||||
// the reference point for decay: a negative score with no traffic since
|
||||
// prefDecayAfter drifts one step toward 0 per elapsed interval, so a slot
|
||||
// cannot stay written off forever just because nothing selected it.
|
||||
prefTouched atomic.Int64
|
||||
// cooldownFrom is the unix second the current cooldown window opened. It
|
||||
// is the reference point for the half-cooldown probe gate: probing is
|
||||
// allowed only in the second half of [cooldownFrom, cooldownUntil), so a
|
||||
@ -50,10 +70,35 @@ type ModelState struct {
|
||||
}
|
||||
|
||||
const (
|
||||
// prefFailStep is the penalty for a REAL failure (5xx, transport error,
|
||||
// unusable adapter output): something is wrong with the upstream or the
|
||||
// model, so two of them are enough to push a slot to the back of its tier.
|
||||
prefFailStep = 5
|
||||
prefMin = -20
|
||||
prefMax = 20
|
||||
backoffBase = 5 * time.Second
|
||||
// prefQuotaStep is the penalty for running out of allowance (quota/TPM
|
||||
// exhausted). Deliberately much smaller than prefFailStep: an exhausted
|
||||
// quota means "no budget right now", not "this model is bad", and for
|
||||
// allowance-metered sources it is an everyday event rather than a fault.
|
||||
// Charging it the full failure penalty made such sources drift to a deeply
|
||||
// negative score with ZERO visible failures - RecordQuotaExhausted
|
||||
// intentionally does not touch failCount, so the WebUI showed fails=0 and
|
||||
// cooling=false while the score sat at -10.
|
||||
prefQuotaStep = 1
|
||||
// prefSuccessStep is the reward for a successful request. Recovery used to
|
||||
// be +1 against a -5 penalty, a 5:1 asymmetry that made a slot at -10 need
|
||||
// ten consecutive successes just to reach neutral. Rewarding +2 keeps
|
||||
// failures meaningful (a real failure still costs more than one success
|
||||
// earns) while letting a recovered slot return to normal rotation quickly.
|
||||
prefSuccessStep = 2
|
||||
prefMin = -20
|
||||
prefMax = 20
|
||||
// prefDecayAfter is how long a slot may sit at a negative score without any
|
||||
// traffic before the score drifts back toward neutral. Without this a slot
|
||||
// that was penalised and then simply not selected again (the score itself
|
||||
// makes it unattractive) stays penalised indefinitely: it has no way to earn
|
||||
// the successes that would rehabilitate it. Applied lazily on read, so there
|
||||
// are no timers.
|
||||
prefDecayAfter = 2 * time.Minute
|
||||
backoffBase = 5 * time.Second
|
||||
// backoffCap bounds the exponential schedule. It used to be 30min, which
|
||||
// in practice meant "a slot whose upstream blipped is unusable for half an
|
||||
// hour even though its quota recovered in seconds". Combined with the
|
||||
@ -142,8 +187,55 @@ func (s *ModelState) RecordFailure(auth bool) {
|
||||
if auth {
|
||||
pen *= 2
|
||||
}
|
||||
s.pref.Add(-pen)
|
||||
s.addPref(-pen)
|
||||
}
|
||||
|
||||
// addPref moves the preference score and stamps prefTouched, which restarts the
|
||||
// decay clock. Every outcome-driven score change must go through here so decay
|
||||
// never fires on a slot that is actively being scored.
|
||||
func (s *ModelState) addPref(delta int64) {
|
||||
s.pref.Add(delta)
|
||||
clampPref(&s.pref, prefMin, prefMax)
|
||||
s.prefTouched.Store(time.Now().Unix())
|
||||
}
|
||||
|
||||
// decayPref drifts a negative score back toward neutral once a slot has been
|
||||
// idle for prefDecayAfter. This closes a starvation loop: a penalised slot is
|
||||
// unattractive, so it stops being selected, so it never earns the successes that
|
||||
// would rehabilitate it, so it stays penalised. One step per elapsed interval,
|
||||
// never past 0, and positive scores are left alone (a proven-good slot should
|
||||
// not be dragged down for being idle).
|
||||
//
|
||||
// Called lazily from Pref(); no timers and no goroutines.
|
||||
func (s *ModelState) decayPref() {
|
||||
cur := s.pref.Load()
|
||||
if cur >= 0 {
|
||||
return
|
||||
}
|
||||
last := s.prefTouched.Load()
|
||||
if last <= 0 {
|
||||
// no reference point yet: start the clock instead of decaying blindly
|
||||
s.prefTouched.CompareAndSwap(0, time.Now().Unix())
|
||||
return
|
||||
}
|
||||
step := int64(prefDecayAfter / time.Second)
|
||||
if step <= 0 {
|
||||
return
|
||||
}
|
||||
elapsed := time.Now().Unix() - last
|
||||
if elapsed < step {
|
||||
return
|
||||
}
|
||||
steps := elapsed / step
|
||||
target := cur + steps
|
||||
if target > 0 {
|
||||
target = 0
|
||||
}
|
||||
// Advance prefTouched by exactly the consumed intervals so leftover time
|
||||
// still counts toward the next step.
|
||||
if s.pref.CompareAndSwap(cur, target) {
|
||||
s.prefTouched.Store(last + steps*step)
|
||||
}
|
||||
}
|
||||
|
||||
// openCooldown starts a fresh cooldown window of length cd. Recording both
|
||||
@ -164,8 +256,10 @@ func (s *ModelState) openCooldown(cd time.Duration) {
|
||||
func (s *ModelState) RecordRateLimit() {
|
||||
s.failCount.Add(1)
|
||||
s.openCooldown(rateLimitCooldown)
|
||||
s.pref.Add(-prefFailStep)
|
||||
clampPref(&s.pref, prefMin, prefMax)
|
||||
// A 429 is "too fast", not "broken": charge it like an allowance event, not
|
||||
// like a fault, or a throttled-but-healthy source sinks purely for being
|
||||
// popular.
|
||||
s.addPref(-prefQuotaStep)
|
||||
}
|
||||
|
||||
// RecordQuotaExhausted marks the model as out of allowance until its quota
|
||||
@ -185,20 +279,24 @@ func (s *ModelState) RecordQuotaExhausted(resetIn time.Duration) {
|
||||
// No failCount bump: an exhausted quota is not an error streak, and
|
||||
// letting it inflate failCount would make the NEXT real failure jump
|
||||
// straight to the capped backoff.
|
||||
s.pref.Add(-prefFailStep)
|
||||
clampPref(&s.pref, prefMin, prefMax)
|
||||
//
|
||||
// The score penalty is prefQuotaStep, not prefFailStep, for the same
|
||||
// reason: cooldown alone already keeps the slot out of rotation until the
|
||||
// window resets, so the score only needs to express a mild preference for
|
||||
// slots with budget left. Charging the full failure penalty here is what
|
||||
// drove allowance-metered sources to -10 and beyond with fails=0.
|
||||
s.addPref(-prefQuotaStep)
|
||||
}
|
||||
|
||||
// RecordSuccess resets the failure counter and cooldown and bumps the
|
||||
// preference score by one. A single success is enough to lift a slot off the
|
||||
// prefMin floor (prefMin+1 > prefMin), which is what lets one successful probe
|
||||
// return a written-off slot to normal rotation.
|
||||
// RecordSuccess resets the failure counter and cooldown and rewards the
|
||||
// preference score. A single success lifts a slot off the prefMin floor, which
|
||||
// is what lets one successful probe return a written-off slot to normal
|
||||
// rotation.
|
||||
func (s *ModelState) RecordSuccess() {
|
||||
s.failCount.Store(0)
|
||||
s.cooldownUntil.Store(0)
|
||||
s.cooldownFrom.Store(0)
|
||||
s.pref.Add(1)
|
||||
clampPref(&s.pref, prefMin, prefMax)
|
||||
s.addPref(prefSuccessStep)
|
||||
}
|
||||
|
||||
func (s *ModelState) reset() {
|
||||
@ -207,6 +305,7 @@ func (s *ModelState) reset() {
|
||||
s.cooldownFrom.Store(0)
|
||||
s.probeInFlight.Store(false)
|
||||
s.pref.Store(0)
|
||||
s.prefTouched.Store(0)
|
||||
}
|
||||
|
||||
// TryProbe attempts to claim the single probe permit for a model that normal
|
||||
@ -225,6 +324,7 @@ func (s *ModelState) TryProbe() bool {
|
||||
now := time.Now().Unix()
|
||||
until := s.cooldownUntil.Load()
|
||||
if until <= now {
|
||||
s.decayPref() // a naturally-recovered slot needs no probe
|
||||
if s.pref.Load() > prefMin {
|
||||
return false // schedulable normally, no probe needed
|
||||
}
|
||||
@ -251,8 +351,13 @@ func (s *ModelState) Probing() bool { return s.probeInFlight.Load() }
|
||||
// when the model is not cooling.
|
||||
func (s *ModelState) CooldownFrom() int64 { return s.cooldownFrom.Load() }
|
||||
|
||||
// Pref is the current preference score (higher = preferred).
|
||||
func (s *ModelState) Pref() int64 { return s.pref.Load() }
|
||||
// Pref is the current preference score (higher = preferred). Reading it also
|
||||
// applies idle decay, so a slot penalised long ago and never selected since
|
||||
// drifts back toward neutral instead of being written off permanently.
|
||||
func (s *ModelState) Pref() int64 {
|
||||
s.decayPref()
|
||||
return s.pref.Load()
|
||||
}
|
||||
|
||||
// FailCount is the number of consecutive failures.
|
||||
func (s *ModelState) FailCount() int64 { return s.failCount.Load() }
|
||||
@ -298,8 +403,11 @@ func New(cfg config.Source, vm *lua.VM) *Provider {
|
||||
// Shared transport: ResponseHeaderTimeout bounds how long we wait for the
|
||||
// first response byte (applies to both paths); the stream client has no
|
||||
// client-level Timeout so the SSE body can run past the header timeout.
|
||||
// A per-source ProxyURL (e.g. clash on 127.0.0.1:7890) overrides the
|
||||
// process-wide env proxy for upstreams that are geo/IP-blocked; sources
|
||||
// without one keep http.ProxyFromEnvironment (direct by default).
|
||||
tr := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
Proxy: proxyFor(cfg),
|
||||
DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 100,
|
||||
|
||||
@ -242,8 +242,10 @@ func TestModelStateCooldownAndRecovery(t *testing.T) {
|
||||
if st.FailCount() != 0 {
|
||||
t.Fatalf("fail count after success = %d", st.FailCount())
|
||||
}
|
||||
if st.Pref() != 1-int64(prefFailStep) {
|
||||
t.Fatalf("pref after one failure (-5) then success (+1) = %d, want %d", st.Pref(), 1-int64(prefFailStep))
|
||||
// failure penalties prefFailStep, success rewards prefSuccessStep:
|
||||
// -5 then +2 = -3
|
||||
if st.Pref() != int64(prefSuccessStep)-int64(prefFailStep) {
|
||||
t.Fatalf("pref after one failure then success = %d, want %d", st.Pref(), int64(prefSuccessStep)-int64(prefFailStep))
|
||||
}
|
||||
}
|
||||
|
||||
@ -265,8 +267,8 @@ func TestModelStateRateLimitShortCooldown(t *testing.T) {
|
||||
if st.FailCount() != 15 {
|
||||
t.Fatalf("fail count = %d, want 15 (counted but not escalating)", st.FailCount())
|
||||
}
|
||||
if st.Pref() != -15*int64(prefFailStep) && st.Pref() > int64(prefMin) {
|
||||
t.Fatalf("pref = %d", st.Pref())
|
||||
if st.Pref() != -15*int64(prefQuotaStep) {
|
||||
t.Fatalf("pref after 15 x 429 = %d, want %d (a 429 is a quota-class event, not a fault)", st.Pref(), -15*int64(prefQuotaStep))
|
||||
}
|
||||
if p.ModelAvailable("m") {
|
||||
t.Fatal("model must be cooling right after a 429")
|
||||
@ -1044,3 +1046,140 @@ func TestRecoveryProbeBudgetIsBounded(t *testing.T) {
|
||||
t.Fatalf("3 probe windows must cost exactly 3 upstream requests, got %d", extra)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQuotaPenaltyIsLighterThanFailure is the regression for the report
|
||||
// "今天一次失败没看到,分数目前已经 -10 了" on an allowance-metered source
|
||||
// (sensenova). RecordQuotaExhausted deliberately does not touch failCount — the
|
||||
// WebUI therefore showed fails=0 and cooling=false — yet it used to charge the
|
||||
// SAME score penalty as a real failure, so two ordinary quota resets dragged the
|
||||
// slot to -10 with no visible fault. An exhausted allowance means "no budget
|
||||
// right now", not "this model is broken": the cooldown already keeps it out of
|
||||
// rotation, so the score penalty must be mild.
|
||||
func TestQuotaPenaltyIsLighterThanFailure(t *testing.T) {
|
||||
quota := &ModelState{}
|
||||
quota.RecordQuotaExhausted(time.Minute)
|
||||
fail := &ModelState{}
|
||||
fail.RecordFailure(false)
|
||||
|
||||
qp, fp := quota.Pref(), fail.Pref()
|
||||
if qp <= fp {
|
||||
t.Fatalf("a quota reset must cost less score than a real failure: quota=%d failure=%d", qp, fp)
|
||||
}
|
||||
if qp != -prefQuotaStep {
|
||||
t.Errorf("quota penalty = %d, want -%d", qp, prefQuotaStep)
|
||||
}
|
||||
if fp != -prefFailStep {
|
||||
t.Errorf("failure penalty = %d, want -%d", fp, prefFailStep)
|
||||
}
|
||||
|
||||
// The reported symptom: several quota resets in a day must NOT approach the
|
||||
// prefMin floor, since each one is a normal event for a metered source.
|
||||
s := &ModelState{}
|
||||
for i := 0; i < 5; i++ {
|
||||
s.RecordQuotaExhausted(time.Minute)
|
||||
}
|
||||
if got := s.Pref(); got <= prefMin/2 {
|
||||
t.Errorf("5 quota resets sank the score to %d (floor %d); a metered source would be written off", got, prefMin)
|
||||
}
|
||||
if s.FailCount() != 0 {
|
||||
t.Errorf("quota exhaustion must not inflate failCount, got %d", s.FailCount())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRateLimitPenaltyIsMild covers the sibling case: a 429 means "too fast",
|
||||
// not "broken", so a popular-but-healthy slot must not sink for being throttled.
|
||||
func TestRateLimitPenaltyIsMild(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
s.RecordRateLimit()
|
||||
if got := s.Pref(); got != -prefQuotaStep {
|
||||
t.Errorf("429 penalty = %d, want -%d (same class as a quota event)", got, prefQuotaStep)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuccessRecoversFasterThanBefore pins the penalty/reward asymmetry. Reward
|
||||
// used to be +1 against a -5 penalty, so a slot at -10 needed TEN consecutive
|
||||
// successes just to reach neutral — in practice it never got there, because a
|
||||
// low score makes the slot unattractive to the scheduler in the first place.
|
||||
func TestSuccessRecoversFasterThanBefore(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
s.RecordFailure(false)
|
||||
s.RecordFailure(false) // -10, the score from the bug report
|
||||
|
||||
n := 0
|
||||
for s.Pref() < 0 {
|
||||
s.RecordSuccess()
|
||||
n++
|
||||
if n > 50 {
|
||||
t.Fatal("score never recovered")
|
||||
}
|
||||
}
|
||||
// with +2 per success, -10 needs 5; the old +1 needed 10
|
||||
if n > 5 {
|
||||
t.Errorf("recovery from -10 took %d successes, want <= 5", n)
|
||||
}
|
||||
// a real failure must still outweigh a single success, or failures stop
|
||||
// meaning anything
|
||||
if prefSuccessStep >= prefFailStep {
|
||||
t.Errorf("success reward %d must stay below the failure penalty %d", prefSuccessStep, prefFailStep)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefDecayRehabilitatesIdleSlot closes the starvation loop: a penalised
|
||||
// slot is unattractive, so the scheduler stops picking it, so it never earns the
|
||||
// successes that would rehabilitate it. Without decay the score stays negative
|
||||
// forever even though the upstream may be perfectly healthy.
|
||||
func TestPrefDecayRehabilitatesIdleSlot(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
s.RecordFailure(false)
|
||||
s.RecordFailure(false)
|
||||
if s.Pref() != -2*prefFailStep {
|
||||
t.Fatalf("setup: pref=%d", s.Pref())
|
||||
}
|
||||
|
||||
// backdate the last touch by three decay intervals
|
||||
step := int64(prefDecayAfter / time.Second)
|
||||
s.prefTouched.Store(time.Now().Unix() - 3*step)
|
||||
|
||||
got := s.Pref()
|
||||
if got != -2*prefFailStep+3 {
|
||||
t.Errorf("after 3 idle intervals pref=%d, want %d", got, -2*prefFailStep+3)
|
||||
}
|
||||
|
||||
// decay must never overshoot into positive territory
|
||||
s.prefTouched.Store(time.Now().Unix() - 1000*step)
|
||||
if got := s.Pref(); got != 0 {
|
||||
t.Errorf("decay overshot to %d, want to stop at 0", got)
|
||||
}
|
||||
|
||||
// a positive score must not be dragged down for being idle
|
||||
good := &ModelState{}
|
||||
good.RecordSuccess()
|
||||
good.RecordSuccess()
|
||||
before := good.Pref()
|
||||
good.prefTouched.Store(time.Now().Unix() - 1000*step)
|
||||
if after := good.Pref(); after != before {
|
||||
t.Errorf("idle decay must not touch a positive score: %d -> %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecayLiftsSlotOffTheFloor verifies decay actually restores schedulability:
|
||||
// a slot at prefMin is refused by ModelAvailable, and before decay existed only
|
||||
// a probe could rescue it.
|
||||
func TestDecayLiftsSlotOffTheFloor(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
for i := 0; i < 10; i++ {
|
||||
s.RecordFailure(false)
|
||||
}
|
||||
if s.Pref() != prefMin {
|
||||
t.Fatalf("setup: want floor %d, got %d", prefMin, s.Pref())
|
||||
}
|
||||
// clear the cooldown so only the score gates the slot
|
||||
s.cooldownUntil.Store(0)
|
||||
s.cooldownFrom.Store(0)
|
||||
|
||||
step := int64(prefDecayAfter / time.Second)
|
||||
s.prefTouched.Store(time.Now().Unix() - 2*step)
|
||||
if got := s.Pref(); got <= prefMin {
|
||||
t.Errorf("decay failed to lift the slot off the floor: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user