jianf/ModelRouter
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/ModelRouter.git synced 2026-09-19 16:39:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: anthropic tool-call round-trip, cache zero-hit parity, round-robin load balancing

Browse Source 
anthropic.lua v3.0.0:
- Issue 1: tool_result/tool_use round-trip
- Issue 3: thinking default OFF (opt-in via extra_body.thinking)
- Issue 4: tool_choice mapping
- Issue 5: collect_blocks preserves unknown part types
- message_stop no longer emits done=true (was overwriting tool_calls finish_reason)
- cache_read_input_tokens normalized even at 0

gemini.lua:
- transform_response was missing cachedContentTokenCount

openai.lua (Issue 6):
- transform_error handles flat envelopes, nginx HTML, bare text

chat.go mergeUsage:
- Keep PromptTokensDetails even when CachedTokens=0

scheduler.go:
- Remove sort.SliceStable by Pref; round-robin cursor is the only LB mechanism

provider.go ModelAvailable:
- Also check Pref() > prefMin, persistently failing slots exit cands

presets.go:
- 17 built-in source templates

Tests: 6 new test functions, 2 updated for new semantics
This commit is contained in:
JianFeeeee
2026-08-28 12:02:46 +08:00
parent 94cbcb6771
commit 624fd74b45
15 changed files with 1170 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/config/config.go
View File

@ -318,6 +318,7 @@ type RuntimeConfig struct {
SourceTemplates []SourceTemplate `json:"source_templates,omitempty"`
DeletedSources []string `json:"deleted_sources,omitempty"`
DeletedAdapters []string `json:"deleted_adapters,omitempty"`
PresetTemplates []string `json:"preset_templates,omitempty"` // preset names the user has seen (or deleted) — never re-seeded
Keys []GWKey `json:"keys,omitempty"`
Auto []ModelScope `json:"auto,omitempty"`
AutoImage []ModelScope `json:"auto_image,omitempty"`

52
internal/config/config_test.go
View File

@ -318,3 +318,55 @@ sources:
t.Fatalf("after no-op: %v %v", len(cfg.Sources), err)
}
}
// TestSeedTemplatesOnce verifies preset templates are seeded exactly once:
// a user's edit survives a restart, and a deliberately deleted preset does
// not silently come back.
func TestSeedTemplatesOnce(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "runtime.json")
presets := []SourceTemplate{
{Name: "DeepSeek", BaseURL: "https://api.deepseek.com", Adapter: "deepseek",
Models: []Model{{ID: "deepseek-chat", Kind: "chat"}}},
{Name: "OpenAI", BaseURL: "https://api.openai.com/v1", Adapter: "openai",
Models: []Model{{ID: "gpt-4o", Kind: "chat"}}},
}
// first run: both presets land in the store
s1 := NewStore(path)
if err := s1.Load(); err != nil {
t.Fatal(err)
}
if err := s1.SeedTemplates(presets); err != nil {
t.Fatalf("seed: %v", err)
}
if got := len(s1.ListTemplates()); got != 2 {
t.Fatalf("after seed len = %d, want 2", got)
}
// user edits one preset, deletes the other
edited := presets[0]
edited.BaseURL = "https://my-proxy.internal/v1"
if err := s1.UpsertTemplate(edited); err != nil {
t.Fatal(err)
}
if _, err := s1.RemoveTemplate("OpenAI"); err != nil {
t.Fatal(err)
}
// restart: re-seeding must be a no-op for both
s2 := NewStore(path)
if err := s2.Load(); err != nil {
t.Fatal(err)
}
if err := s2.SeedTemplates(presets); err != nil {
t.Fatalf("reseed: %v", err)
}
tpls := s2.ListTemplates()
if len(tpls) != 1 {
t.Fatalf("after restart len = %d, want 1 (deleted preset must not return): %+v", len(tpls), tpls)
}
if tpls[0].Name != "DeepSeek" || tpls[0].BaseURL != "https://my-proxy.internal/v1" {
t.Fatalf("user edit lost on restart: %+v", tpls[0])
}
}

212
internal/config/presets.go Normal file
View File

@ -0,0 +1,212 @@
package config
// Built-in preset source templates. Seeded into the runtime store on first
// startup so the WebUI can present "从模板创建" for popular providers — the
// user only needs to supply a name and API key.
//
// Model IDs verified against provider docs 2026-08-28. Providers ship new IDs
// frequently; treat these as a starting point and edit in the WebUI as needed.
var PresetTemplates = []SourceTemplate{
// ── 国内热门 ────────────────────────────────────────────────────────────
{
Name: "DeepSeek",
BaseURL: "https://api.deepseek.com",
Adapter: "deepseek",
MaxConcurrent: 8,
Models: []Model{
{ID: "deepseek-v4-flash", Priority: 100, Kind: "chat"},
{ID: "deepseek-v4-pro", Priority: 90, Kind: "chat"},
{ID: "deepseek-v4-flash-vision-exp", Priority: 60, Kind: "chat"},
},
},
{
Name: "智谱 GLM",
BaseURL: "https://open.bigmodel.cn/api/paas/v4",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "glm-5.3", Priority: 100, Kind: "chat"},
{ID: "glm-5.3-flash", Priority: 90, Kind: "chat"},
{ID: "glm-5.2", Priority: 80, Kind: "chat"},
{ID: "glm-5.1", Priority: 60, Kind: "chat"},
},
},
{
Name: "Kimi / Moonshot",
BaseURL: "https://api.moonshot.cn/v1",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "kimi-k3", Priority: 100, Kind: "chat"},
{ID: "kimi-k2.7-code", Priority: 90, Kind: "chat"},
{ID: "kimi-k2.7-code-highspeed", Priority: 85, Kind: "chat"},
{ID: "kimi-k2.6", Priority: 70, Kind: "chat"},
},
},
{
Name: "SiliconFlow",
BaseURL: "https://api.siliconflow.cn/v1",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "deepseek-ai/DeepSeek-V4-Pro", Priority: 100, Kind: "chat"},
{ID: "deepseek-ai/DeepSeek-V4-Flash", Priority: 90, Kind: "chat"},
{ID: "moonshotai/Kimi-K2.6", Priority: 80, Kind: "chat"},
{ID: "zai-org/GLM-5.1", Priority: 70, Kind: "chat"},
{ID: "Kwai-Kolors/Kolors", Priority: 50, Kind: "image"},
},
},
{
Name: "讯飞星火 (Spark)",
BaseURL: "https://spark-api-open.xf-yun.com/v1",
Adapter: "openai",
MaxConcurrent: 4,
Models: []Model{
{ID: "4.0Ultra", Priority: 100, Kind: "chat"},
{ID: "generalv3.5", Priority: 80, Kind: "chat"},
{ID: "max-32k", Priority: 70, Kind: "chat"},
},
},
{
Name: "阿里云 通义千问",
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "qwen3.8-max", Priority: 100, Kind: "chat"},
{ID: "qwen3.8-flash", Priority: 90, Kind: "chat"},
{ID: "qwen3.7-plus", Priority: 85, Kind: "chat"},
{ID: "deepseek-v4-pro", Priority: 70, Kind: "chat"},
{ID: "kimi-k3", Priority: 70, Kind: "chat"},
},
},
{
Name: "百度 千帆",
BaseURL: "https://qianfan.baidubce.com/v2",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "ernie-5.0-turbo", Priority: 100, Kind: "chat"},
{ID: "ernie-5.0", Priority: 90, Kind: "chat"},
{ID: "deepseek-v4-pro", Priority: 70, Kind: "chat"},
},
},
{
Name: "豆包 (火山方舟)",
BaseURL: "https://ark.cn-beijing.volces.com/api/v3",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "doubao-seed-2.0-pro", Priority: 100, Kind: "chat"},
{ID: "doubao-seed-2.0", Priority: 90, Kind: "chat"},
{ID: "doubao-seed-2.0-flash", Priority: 80, Kind: "chat"},
},
},
{
Name: "MiniMax",
BaseURL: "https://api.minimax.chat/v1",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "MiniMax-M3", Priority: 100, Kind: "chat"},
{ID: "MiniMax-M2.5", Priority: 80, Kind: "chat"},
},
},
// ── 国际主流 ────────────────────────────────────────────────────────────
{
Name: "OpenAI",
BaseURL: "https://api.openai.com/v1",
Adapter: "openai",
MaxConcurrent: 8,
Models: []Model{
{ID: "gpt-5.6-sol", Priority: 100, Kind: "chat"},
{ID: "gpt-5.6-terra", Priority: 90, Kind: "chat"},
{ID: "gpt-5.6-luna", Priority: 80, Kind: "chat"},
},
},
{
Name: "Anthropic Claude",
BaseURL: "https://api.anthropic.com",
Adapter: "anthropic",
MaxConcurrent: 8,
Models: []Model{
{ID: "claude-opus-5", Priority: 100, Kind: "chat"},
{ID: "claude-fable-5", Priority: 95, Kind: "chat"},
{ID: "claude-sonnet-5", Priority: 85, Kind: "chat"},
{ID: "claude-haiku-4-5", Priority: 70, Kind: "chat"},
},
},
{
Name: "Google Gemini",
BaseURL: "https://generativelanguage.googleapis.com/v1beta",
Adapter: "gemini",
MaxConcurrent: 8,
Models: []Model{
{ID: "gemini-3.6-flash", Priority: 100, Kind: "chat"},
{ID: "gemini-3.5-flash", Priority: 90, Kind: "chat"},
{ID: "gemini-3.5-flash-lite", Priority: 80, Kind: "chat"},
{ID: "gemini-3.1-pro", Priority: 85, Kind: "chat"},
},
},
{
Name: "Groq",
BaseURL: "https://api.groq.com/openai/v1",
Adapter: "groq",
MaxConcurrent: 8,
Models: []Model{
{ID: "openai/gpt-oss-120b", Priority: 100, Kind: "chat"},
{ID: "openai/gpt-oss-20b", Priority: 80, Kind: "chat"},
{ID: "groq/compound", Priority: 70, Kind: "chat"},
},
},
{
Name: "Mistral",
BaseURL: "https://api.mistral.ai/v1",
Adapter: "mistral",
MaxConcurrent: 8,
Models: []Model{
{ID: "mistral-medium-latest", Priority: 100, Kind: "chat"},
{ID: "mistral-large-latest", Priority: 90, Kind: "chat"},
{ID: "codestral-latest", Priority: 85, Kind: "chat"},
},
},
{
Name: "OpenRouter",
BaseURL: "https://openrouter.ai/api/v1",
Adapter: "openai",
MaxConcurrent: 8,
Headers: map[string]string{
"HTTP-Referer": "http://localhost:8080",
"X-Title": "ModelRouter",
},
Models: []Model{
{ID: "openrouter/auto", Priority: 100, Kind: "chat"},
{ID: "deepseek/deepseek-v4-flash", Priority: 90, Kind: "chat"},
{ID: "anthropic/claude-opus-5", Priority: 85, Kind: "chat"},
{ID: "openai/gpt-5.6-sol", Priority: 85, Kind: "chat"},
},
},
{
Name: "GitHub Models",
BaseURL: "https://models.inference.ai.azure.com",
Adapter: "github",
MaxConcurrent: 8,
Models: []Model{
{ID: "gpt-5.6-sol", Priority: 100, Kind: "chat"},
{ID: "gpt-5.6-luna", Priority: 80, Kind: "chat"},
},
},
// ── 本地模型 ────────────────────────────────────────────────────────────
{
Name: "Ollama (localhost)",
BaseURL: "http://127.0.0.1:11434",
Adapter: "ollama",
Endpoint: "/api/chat",
MaxConcurrent: 4,
Models: []Model{
{ID: "qwen3", Priority: 80, Kind: "chat"},
{ID: "deepseek-r1", Priority: 80, Kind: "chat"},
{ID: "llama3.3", Priority: 70, Kind: "chat"},
},
},
}

32
internal/config/store.go
View File

@ -138,6 +138,38 @@ func (s *Store) DeletedAdapters() map[string]bool {
return out
}
// SeedTemplates adds preset templates that the store has never seen before.
// A preset is seeded at most once: its name is recorded in PresetTemplates so
// a user who edits or deletes it never gets it silently restored on restart.
func (s *Store) SeedTemplates(presets []SourceTemplate) error {
s.mu.Lock()
defer s.mu.Unlock()
seen := make(map[string]bool, len(s.data.PresetTemplates))
for _, n := range s.data.PresetTemplates {
seen[n] = true
}
existing := make(map[string]bool, len(s.data.SourceTemplates))
for _, t := range s.data.SourceTemplates {
existing[t.Name] = true
}
added := false
for _, p := range presets {
if p.Name == "" || seen[p.Name] {
continue
}
s.data.PresetTemplates = append(s.data.PresetTemplates, p.Name)
added = true
if existing[p.Name] {
continue // user already has a template by this name: never overwrite
}
s.data.SourceTemplates = append(s.data.SourceTemplates, p)
}
if !added {
return nil
}
return s.persistLocked()
}
// UpsertTemplate adds or replaces a source template and persists.
func (s *Store) UpsertTemplate(t SourceTemplate) error {
s.mu.Lock()

17
internal/core/core.go
View File

@ -69,6 +69,11 @@ func NewFromConfig(cfg *config.Config) (*Core, error) {
if err := c.rebuildRegistry(); err != nil {
return nil, err
}
// Seed built-in preset templates on first run so the WebUI shows
// "从模板创建" for popular providers out of the box.
if err := c.seedPresetTemplates(); err != nil {
return nil, fmt.Errorf("seed preset templates: %w", err)
}
return c, nil
}
@ -162,6 +167,18 @@ func (c *Core) seedKeys() error {
return nil
}
// seedPresetTemplates adds the built-in provider presets to the template
// store on first run. It only adds templates whose name is not already
// present, so a user's edits to a preset survive restarts and a deliberately
// deleted preset does not silently return on the next start (its name is
// remembered in the store's preset marker list).
func (c *Core) seedPresetTemplates() error {
if len(config.PresetTemplates) == 0 {
return nil
}
return c.store.SeedTemplates(config.PresetTemplates)
}
// saveConfig writes the current config (including auto rules and keys) back
// to config.yaml.
func (c *Core) saveConfig() error {

6
internal/gateway/chat.go
View File

@ -671,7 +671,11 @@ func mergeUsage(prev, cur *types.TokenUsage) *types.TokenUsage {
out.Completion = cur.Completion
}
out.Total = out.Prompt + out.Completion
if cur.PromptTokensDetails != nil && cur.PromptTokensDetails.CachedTokens > 0 {
if cur.PromptTokensDetails != nil {
// Keep the details object even when CachedTokens is 0: a reported
// zero-hit is meaningful ("cache missed") and must stay
// distinguishable from "upstream never reported cache info".
// Dropping it here made streaming rows show “—” instead of 0%.
out.PromptTokensDetails = cur.PromptTokensDetails
}
if cur.PromptCacheHit > 0 {

62
internal/gateway/chat_cache_test.go Normal file
View File

@ -0,0 +1,62 @@
package gateway
import (
"testing"
"llmsproxy/internal/types"
)
// TestMergeUsageKeepsZeroCacheDetails locks the streaming counterpart of the
// "distinguish missed from not reported" contract: a usage chunk that reports
// prompt_tokens_details with 0 cached tokens must survive the merge, otherwise
// streaming rows lose cache_reported and the UI shows "—" instead of 0%.
func TestMergeUsageKeepsZeroCacheDetails(t *testing.T) {
t.Run("zero-hit details survive merge", func(t *testing.T) {
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
cur := &types.TokenUsage{
Prompt: 10, Completion: 5, Total: 15,
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 0},
}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails == nil {
t.Fatal("zero-hit prompt_tokens_details was dropped by mergeUsage")
}
if got.PromptTokensDetails.CachedTokens != 0 {
t.Fatalf("cached_tokens = %d, want 0", got.PromptTokensDetails.CachedTokens)
}
})
t.Run("non-zero hit still wins", func(t *testing.T) {
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
cur := &types.TokenUsage{
Prompt: 10, Completion: 5, Total: 15,
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 64},
}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails == nil || got.PromptTokensDetails.CachedTokens != 64 {
t.Fatalf("cached_tokens lost: %+v", got.PromptTokensDetails)
}
})
t.Run("absent details do not overwrite an earlier report", func(t *testing.T) {
prev := &types.TokenUsage{
Prompt: 10, Completion: 1, Total: 11,
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 32},
}
cur := &types.TokenUsage{Prompt: 10, Completion: 5, Total: 15}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails == nil || got.PromptTokensDetails.CachedTokens != 32 {
t.Fatalf("earlier cache report clobbered by a later chunk without details: %+v",
got.PromptTokensDetails)
}
})
t.Run("no cache data anywhere stays nil", func(t *testing.T) {
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
cur := &types.TokenUsage{Prompt: 10, Completion: 5, Total: 15}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails != nil {
t.Fatalf("fabricated cache details: %+v", got.PromptTokensDetails)
}
})
}

381
internal/lua/adapters/anthropic.lua
View File

@ -1,143 +1,301 @@
local adapter = {}
adapter.name = "anthropic"
adapter.version = "2.0.0"
adapter.version = "3.0.0"
adapter.endpoint = "/v1/messages"
adapter.headers = {
["anthropic-version"] = "2023-06-01"
["anthropic-version"] = "2023-06-01",
}
function adapter.transform_request(raw_body)
local ok, req = pcall(json.decode, raw_body)
if not ok then return raw_body end
-- ============================================================
-- helpers
-- ============================================================
-- 将 OpenAI 风格 content（字符串或 [{type:*}] 数组）拆成文本/图片块
local function collect_blocks(content)
if type(content) == "string" then
return { { type = "text", text = content } }
end
local blocks = {}
for _, p in ipairs(content or {}) do
local ROLE_WHITELIST = {
system = true, user = true, assistant = true, tool = true,
}
local function collect_blocks(content)
if type(content) == "string" then
if content == "" then return {} end
return { { type = "text", text = content } }
end
if type(content) ~= "table" then return {} end
local blocks = {}
for _, p in ipairs(content) do
if type(p) == "string" then
table.insert(blocks, { type = "text", text = p })
elseif type(p) == "table" then
if p.type == "text" then
table.insert(blocks, { type = "text", text = p.text })
table.insert(blocks, { type = "text", text = p.text or "" })
elseif p.type == "image_url" and type(p.image_url) == "table" and p.image_url.url then
local mt, b64 = string.match(p.image_url.url, "^data:([^,]+);base64,(.+)$")
local url = p.image_url.url
local mt, b64 = string.match(url, "^data:([^,]+);base64,(.+)$")
if b64 then
table.insert(blocks, { type = "image", source = { type = "base64", media_type = mt or "image/png", data = b64 } })
else
table.insert(blocks, { type = "image", source = { type = "url", url = p.image_url.url } })
table.insert(blocks, { type = "image", source = { type = "url", url = url } })
end
else
-- Unknown content type: preserve for forward compatibility
table.insert(blocks, p)
end
end
return blocks
end
local function text_of(content)
if type(content) == "string" then return content end
local t = ""
for _, p in ipairs(content or {}) do
if p.type == "text" and p.text then t = t .. p.text end
return blocks
end
local function text_of(content)
if type(content) == "string" then return content end
local t = ""
for _, p in ipairs(content or {}) do
if type(p) == "string" then
t = t .. p
elseif type(p) == "table" and p.type == "text" and p.text then
t = t .. p.text
end
return t
end
return t
end
--- Append blocks to the last user message (merge) or create a new one.
local function append_user(msgs, blocks)
if #msgs > 0 and msgs[#msgs].role == "user" then
for _, b in ipairs(blocks) do
table.insert(msgs[#msgs].content, b)
end
else
table.insert(msgs, { role = "user", content = blocks })
end
end
-- ============================================================
-- transform_request (OpenAI → Anthropic)
-- ============================================================
function adapter.transform_request(raw_body)
local ok, req = pcall(json.decode, raw_body)
if not ok or type(req) ~= "table" then return raw_body end
local msgs = {}
local system = ""
local pending_tool = {} -- accumulated tool_result blocks
for _, m in ipairs(req.messages or {}) do
if m.role == "system" then
local role = m.role or "user"
if role == "system" then
system = system .. text_of(m.content) .. "\n"
elseif role == "assistant" then
-- Build content array: text/image blocks from content + tool_use from tool_calls
local blocks = collect_blocks(m.content)
if type(m.tool_calls) == "table" then
for _, tc in ipairs(m.tool_calls) do
if type(tc) == "table" then
local fn = tc["function"] or {}
local args = fn.arguments
local input = {}
if type(args) == "string" and args ~= "" then
local ok2, parsed = pcall(json.decode, args)
if ok2 and type(parsed) == "table" then input = parsed end
elseif type(args) == "table" then
input = args
end
table.insert(blocks, {
type = "tool_use",
id = tc.id or "",
name = fn.name or "",
input = input,
})
end
end
end
table.insert(msgs, { role = "assistant", content = blocks })
elseif role == "tool" then
-- Accumulate consecutive tool results; will be flushed as one
-- user message with tool_result content blocks.
table.insert(pending_tool, {
type = "tool_result",
tool_use_id = m.tool_call_id or "",
content = text_of(m.content),
})
else
table.insert(msgs, { role = m.role, content = collect_blocks(m.content) })
-- user / any other role: flush pending tool results first
if #pending_tool > 0 then
append_user(msgs, pending_tool)
pending_tool = {}
end
append_user(msgs, collect_blocks(m.content))
end
end
local anthropic_req = {
model = req.model or "claude-sonnet-4-20250514",
max_tokens = req.max_tokens or 4096,
messages = msgs,
stream = req.stream or false,
}
if not req.disable_thinking then
anthropic_req.thinking = { type = "enabled", budget_tokens = 4096 }
-- Flush any trailing tool results
if #pending_tool > 0 then
append_user(msgs, pending_tool)
end
-- ── Build Anthropic request ──────────────────────────────
local anthropic_req = {
model = req.model or "claude-sonnet-4-20250514",
max_tokens = req.max_tokens or 4096,
messages = msgs,
stream = req.stream or false,
}
-- ── tools ────────────────────────────────────────────────
local has_tools = false
if req.tools and type(req.tools) == "table" then
local tools = {}
for _, t in ipairs(req.tools) do
if type(t) == "table" and t.type == "function"
and type(t["function"]) == "table" then
local fn = t["function"]
table.insert(tools, {
name = fn.name or "",
description = fn.description or "",
input_schema = fn.parameters or { type = "object", properties = {} },
})
end
end
if #tools > 0 then
anthropic_req.tools = tools
has_tools = true
end
end
-- ── tool_choice ──────────────────────────────────────────
-- OpenAI → Anthropic mapping:
-- "auto" → {type:"auto"}
-- "none" → remove tools entirely (Anthropic has no "none")
-- "required" → {type:"any"}
-- {type:"function", function:{name:"X"}} → {type:"tool", name:"X"}
if has_tools and req.tool_choice then
local tc = req.tool_choice
if type(tc) == "string" then
if tc == "auto" then
anthropic_req.tool_choice = { type = "auto" }
elseif tc == "none" then
anthropic_req.tools = nil
anthropic_req.tool_choice = nil
elseif tc == "required" then
anthropic_req.tool_choice = { type = "any" }
end
elseif type(tc) == "table" and tc.type == "function" then
local fn = tc["function"] or {}
anthropic_req.tool_choice = { type = "tool", name = fn.name or "" }
end
end
-- ── thinking (opt-in via extra_body) ──────────────────────
-- Default: OFF. Client sends extra_body.thinking to enable.
-- Example: {"extra_body": {"thinking": {"type": "enabled", "budget_tokens": 4096}}}
if type(req.extra_body) == "table" and type(req.extra_body.thinking) == "table" then
anthropic_req.thinking = req.extra_body.thinking
end
-- ── system ───────────────────────────────────────────────
if system ~= "" then
anthropic_req.system = system
-- Strip trailing newline from accumulation
anthropic_req.system = string.match(system, "^(.-)\n*$")
end
return json.encode(anthropic_req)
end
-- ============================================================
-- transform_response (Anthropic → OpenAI, non-streaming)
-- ============================================================
function adapter.transform_response(raw_body)
local ok, resp = pcall(json.decode, raw_body)
if not ok then return raw_body end
if not ok or resp == nil then return raw_body end
local unified = {
content = "",
finish_reason = "",
token_usage = { prompt = 0, completion = 0, total = 0 }
token_usage = { prompt = 0, completion = 0, total = 0 },
}
if resp.usage then
unified.token_usage.prompt = resp.usage.input_tokens or 0
unified.token_usage.prompt = resp.usage.input_tokens or 0
unified.token_usage.completion = resp.usage.output_tokens or 0
unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0)
-- Anthropic reports cache_read_input_tokens; normalize into
-- OpenAI-standard prompt_tokens_details.cached_tokens so clients
-- (dsh) see the cache hit count.
local cacheRead = resp.usage.cache_read_input_tokens or 0
if cacheRead > 0 then
unified.token_usage.prompt_tokens_details = { cached_tokens = cacheRead }
unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0)
-- Emit details whenever Anthropic reports the field, even at 0, so a
-- reported cache miss stays distinguishable from "not reported".
if resp.usage.cache_read_input_tokens ~= nil then
unified.token_usage.prompt_tokens_details = {
cached_tokens = resp.usage.cache_read_input_tokens
}
end
end
if resp.content and #resp.content > 0 then
local tcs = {}
for _, block in ipairs(resp.content) do
if block.type == "text" then
unified.content = unified.content .. (block.text or "")
elseif block.type == "thinking" and block.thinking then
unified.reasoning_content = (unified.reasoning_content or "") .. block.thinking
elseif block.type == "tool_use" then
table.insert(tcs, {
id = block.id or "",
type = "function",
name = block.name or "",
arguments = block.input or {},
})
end
end
if #tcs > 0 then unified.tool_calls = tcs end
end
unified.finish_reason = resp.stop_reason or ""
if unified.finish_reason == "tool_use" then
unified.finish_reason = "tool_calls"
elseif unified.finish_reason == "max_tokens" then
unified.finish_reason = "length"
elseif unified.finish_reason == "end_turn"
or unified.finish_reason == "stop_sequence" then
unified.finish_reason = "stop"
end
return json.encode(unified)
end
-- ============================================================
-- transform_stream_chunk (Anthropic SSE → OpenAI SSE delta)
-- ============================================================
function adapter.transform_stream_chunk(raw_chunk)
local ok, chunk = pcall(json.decode, raw_chunk)
if not ok then return "" end
-- ── message_start: initial usage ─────────────────────────
if chunk.type == "message_start" then
local uses = nil
if chunk.message and type(chunk.message.usage) == "table" then
local u = chunk.message.usage
local p = u.input_tokens or 0
local c = u.output_tokens or 0
if p > 0 or c > 0 then
uses = { prompt = p, completion = c, total = p + c }
local cacheRead = u.cache_read_input_tokens or 0
if cacheRead > 0 then
uses.prompt_tokens_details = { cached_tokens = cacheRead }
local uses = { prompt = p, completion = c, total = p + c }
if u.cache_read_input_tokens ~= nil then
uses.prompt_tokens_details = {
cached_tokens = u.cache_read_input_tokens
}
end
return json.encode({ usage = uses, done = false })
end
end
if uses ~= nil then
return json.encode({ usage = uses, done = false })
end
return ""
end
-- ── message_delta: stop_reason + final usage ─────────────
if chunk.type == "message_delta" then
local uses = nil
if type(chunk.usage) == "table" then
local u = chunk.usage
local p = u.input_tokens or 0
local c = u.output_tokens or 0
if p > 0 or c > 0 then
uses = { prompt = p, completion = c, total = p + c }
end
end
local finish = nil
if chunk.delta and chunk.delta.stop_reason ~= nil then
-- Anthropic stop_reason -> OpenAI finish_reason
local sr = chunk.delta.stop_reason
if sr == "max_tokens" then
finish = "length"
@ -147,58 +305,95 @@ function adapter.transform_stream_chunk(raw_chunk)
finish = "stop"
end
end
local uses = nil
if type(chunk.usage) == "table" then
local u = chunk.usage
local p = u.input_tokens or 0
local c = u.output_tokens or 0
if p > 0 or c > 0 then
uses = { prompt = p, completion = c, total = p + c }
end
end
if uses ~= nil then
-- completion is final here; prompt is merged from message_start
return json.encode({ content = "", done = (finish ~= nil), finish_reason = finish, usage = uses })
end
return json.encode({ content = "", done = (finish ~= nil), finish_reason = finish })
end
if chunk.type == "content_block_start" and chunk.content_block
and chunk.content_block.type == "tool_use" then
-- first fragment of a tool call: emit index + id + name, empty args
return json.encode({
content = "", done = false,
tool_calls = { {
index = chunk.index or 0,
id = chunk.content_block.id or "",
type = "function",
["function"] = { name = chunk.content_block.name or "", arguments = "" }
} }
})
-- ── content_block_start: begin text / thinking / tool_use ─
if chunk.type == "content_block_start" and chunk.content_block then
local cb = chunk.content_block
if cb.type == "tool_use" then
-- Pass Anthropic content_block index through; Go-side rewrites
-- to sequential OpenAI tool_call ordinal for multi-tool streams.
return json.encode({
content = "", done = false,
tool_calls = { {
index = chunk.index or 0,
id = cb.id or "",
type = "function",
["function"] = { name = cb.name or "", arguments = "" },
} },
})
end
return "" -- text / thinking block start: no OpenAI equivalent
end
-- ── content_block_delta: incremental content ──────────────
if chunk.type == "content_block_delta" and chunk.delta then
if chunk.delta.type == "input_json_delta" then
-- incremental JSON fragment; clients accumulate across chunks
local unified = { content = "", done = false, tool_calls = { {
index = chunk.index or 0,
id = "",
type = "function",
["function"] = { name = "", arguments = chunk.delta.partial_json or "" }
} } }
return json.encode(unified)
local d = chunk.delta
if d.type == "input_json_delta" then
-- Tool call argument fragment; Go-side rewrites index.
return json.encode({
content = "", done = false,
tool_calls = { {
index = chunk.index or 0,
id = "",
type = "function",
["function"] = { name = "", arguments = d.partial_json or "" },
} },
})
end
if chunk.delta.type == "thinking_delta" and chunk.delta.thinking then
return json.encode({ content = "", done = false, reasoning_content = chunk.delta.thinking })
if d.type == "thinking_delta" and d.thinking then
return json.encode({ content = "", done = false, reasoning_content = d.thinking })
end
if d.type == "text_delta" and d.text then
return json.encode({ content = d.text, done = false })
end
return json.encode({ content = chunk.delta.text or "", done = false })
end
-- ── content_block_stop / message_stop ─────────────────────
if chunk.type == "message_stop" then
return json.encode({ content = "", done = true })
end
if chunk.type == "content_block_stop" then
return json.encode({ content = "", done = false })
-- The message_delta event already emitted the true finish_reason.
-- Do NOT emit done=true here: an empty finish_reason would
-- overwrite the real one (tool_calls) in the Go gateway's
-- lastFinish tracker, causing the final SSE chunk to say
-- finish_reason=stop instead of tool_calls.
return ""
end
return ""
end
-- 错误收敛：Anthropic 信封 {type:"error", error:{type, message}}
-- ============================================================
-- transform_error (Anthropic error → human-readable string)
-- ============================================================
function adapter.transform_error(status, body)
local ok, resp = pcall(json.decode, body)
if not ok or type(resp) ~= "table" then return nil end
-- Anthropic envelope: {type:"error", error:{type, message}}
if resp.type == "error" and type(resp.error) == "table"
and type(resp.error.message) == "string" then
return resp.error.message
end
-- Flat envelope: {error: {message: "..."}}
if type(resp.error) == "table" and type(resp.error.message) == "string" then
return resp.error.message
end
return nil
end

18
internal/lua/adapters/gemini.lua
View File

@ -68,6 +68,15 @@ function adapter.transform_response(raw_body)
unified.token_usage.prompt = resp.usageMetadata.promptTokenCount or 0
unified.token_usage.completion = resp.usageMetadata.candidatesTokenCount or 0
unified.token_usage.total = resp.usageMetadata.totalTokenCount or 0
-- Gemini reports context-cache reads as cachedContentTokenCount;
-- normalize into OpenAI-standard prompt_tokens_details.cached_tokens
-- so clients and the audit trail see the hit count. Emitted even when
-- 0 so a reported miss stays distinguishable from "not reported".
if resp.usageMetadata.cachedContentTokenCount ~= nil then
unified.token_usage.prompt_tokens_details = {
cached_tokens = resp.usageMetadata.cachedContentTokenCount
}
end
end
if resp.candidates and #resp.candidates > 0 then
@ -100,9 +109,12 @@ function adapter.transform_stream_chunk(raw_chunk)
local t = chunk.usageMetadata.totalTokenCount or 0
if p > 0 or c > 0 or t > 0 then
uses = { prompt = p, completion = c, total = t }
local cacheRead = chunk.usageMetadata.cachedContentTokenCount or 0
if cacheRead > 0 then
uses.prompt_tokens_details = { cached_tokens = cacheRead }
-- Emit details whenever the field is present, even at 0, so a
-- reported cache miss stays distinguishable from "not reported".
if chunk.usageMetadata.cachedContentTokenCount ~= nil then
uses.prompt_tokens_details = {
cached_tokens = chunk.usageMetadata.cachedContentTokenCount
}
end
end
end

32
internal/lua/adapters/openai.lua
View File

@ -136,13 +136,43 @@ end
-- 错误收敛：标准 OpenAI 信封 {error:{message,...}}
function adapter.transform_error(status, body)
-- Non-JSON body (nginx HTML error pages, plain text): extract a short
-- human-readable reason instead of letting the raw body reach the log.
local ok, resp = pcall(json.decode, body)
if not ok or type(resp) ~= "table" then return nil end
if not ok or type(resp) ~= "table" then
-- HTML error page: pull the <title> text (e.g. "413 Request Entity Too Large")
local title = string.match(body or "", "<title>(.-)</title>")
if title and title ~= "" then return title end
-- Bare text: first non-empty line, capped
local line = string.match(body or "", "^%s*([^\r\n]+)")
if line and line ~= "" and not string.match(line, "^<") then
return string.sub(line, 1, 200)
end
return nil
end
-- Standard OpenAI envelope: {error:{message,...}} or {error:"..."}
local e = resp.error
if type(e) == "table" and type(e.message) == "string" then
return e.message
end
if type(e) == "string" then return e end
-- Flat envelope used by many OpenAI-compatible gateways:
-- {"code":20012,"message":"Model does not exist..."}
-- {"code":"INVALID_API_KEY","message":"Invalid API key"}
if type(resp.message) == "string" and resp.message ~= "" then
if resp.code ~= nil then
return tostring(resp.code) .. ": " .. resp.message
end
return resp.message
end
-- Some gateways use {detail:"..."} (FastAPI style)
if type(resp.detail) == "string" and resp.detail ~= "" then
return resp.detail
end
return nil
end

298
internal/lua/vm_test.go
View File

@ -273,7 +273,8 @@ func TestDisableThinkingPassthrough(t *testing.T) {
t.Fatalf("thinking.type = %v", thinking["type"])
}
// anthropic: disable_thinking removes the thinking block
// anthropic: thinking is opt-in (Issue 3) — never emitted by default, and
// disable_thinking is not a trigger either.
out2, err := vm.Transform("anthropic", "transform_request", body)
if err != nil {
t.Fatalf("anthropic transform: %v", err)
@ -285,8 +286,17 @@ func TestDisableThinkingPassthrough(t *testing.T) {
if err != nil {
t.Fatalf("anthropic transform: %v", err)
}
if !strings.Contains(out3, "enabled") {
t.Fatalf("anthropic should enable thinking by default: %s", out3)
if strings.Contains(out3, "thinking") {
t.Fatalf("anthropic must not enable thinking by default: %s", out3)
}
// opt-in path: extra_body.thinking is forwarded verbatim
out4, err := vm.Transform("anthropic", "transform_request",
`{"model":"x","messages":[{"role":"user","content":"hi"}],"extra_body":{"thinking":{"type":"enabled","budget_tokens":2048}}}`)
if err != nil {
t.Fatalf("anthropic transform: %v", err)
}
if !strings.Contains(out4, `"type":"enabled"`) || !strings.Contains(out4, "2048") {
t.Fatalf("anthropic should forward extra_body.thinking: %s", out4)
}
}
@ -454,3 +464,285 @@ func TestAdaptersPassFinishReason(t *testing.T) {
}
}
}
// TestAnthropicToolRoundTrip verifies the OpenAI->Anthropic request mapping for
// a full agent tool-call round (Issue 1): assistant tool_calls become
// tool_use content blocks and role:"tool" results become user tool_result
// blocks merged into ONE user message. This is the regression that made agent
// clients (dsh/Claude Code/Cursor) repeatedly re-invoke the same tool.
func TestAnthropicToolRoundTrip(t *testing.T) {
vm := NewVM(freshAdapterDir(t))
if err := vm.Start(); err != nil {
t.Fatal(err)
}
defer vm.Stop()
round1 := `{"model":"x","tools":[{"type":"function","function":{"name":"calc","description":"multiply","parameters":{"type":"object","properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]}}}],"messages":[{"role":"user","content":"what is 17*23?"}]}`
out1, err := vm.Transform("anthropic", "transform_request", round1)
if err != nil {
t.Fatalf("round1 transform: %v", err)
}
var r1 struct {
Tools []struct {
Name string `json:"name"`
InputSchema map[string]interface{} `json:"input_schema"`
} `json:"tools"`
Messages []map[string]interface{} `json:"messages"`
}
if err := json.Unmarshal([]byte(out1), &r1); err != nil {
t.Fatalf("unmarshal r1: %v (%s)", err, out1)
}
if len(r1.Tools) != 1 || r1.Tools[0].Name != "calc" {
t.Fatalf("tools not mapped: %s", out1)
}
// Round 2: assistant tool_calls + tool result
round2 := `{"model":"x","messages":[
{"role":"user","content":"what is 17*23?"},
{"role":"assistant","content":"","tool_calls":[
{"id":"call_1","type":"function","function":{"name":"calc","arguments":"{\"a\":17,\"b\":23}"}}
]},
{"role":"tool","tool_call_id":"call_1","content":"391"}
]}`
out2, err := vm.Transform("anthropic", "transform_request", round2)
if err != nil {
t.Fatalf("round2 transform: %v", err)
}
var r2 struct {
Messages []struct {
Role string `json:"role"`
Content []struct {
Type string `json:"type"`
ID string `json:"id"`
Name string `json:"name"`
Input map[string]interface{} `json:"input"`
ToolUseID string `json:"tool_use_id"`
ContentText string `json:"content"`
} `json:"content"`
} `json:"messages"`
}
if err := json.Unmarshal([]byte(out2), &r2); err != nil {
t.Fatalf("unmarshal r2: %v (%s)", err, out2)
}
// Message 2 (index 1) must be assistant with one tool_use block
am := r2.Messages[1]
if am.Role != "assistant" {
t.Fatalf("msg[1].role = %q, want assistant", am.Role)
}
var toolUse struct {
Type string `json:"type"`
ID string `json:"id"`
Name string `json:"name"`
Input map[string]interface{} `json:"input"`
}
for _, b := range am.Content {
if b.Type == "tool_use" {
toolUse = struct {
Type string `json:"type"`
ID string `json:"id"`
Name string `json:"name"`
Input map[string]interface{} `json:"input"`
}{b.Type, b.ID, b.Name, b.Input}
}
}
if toolUse.ID != "call_1" || toolUse.Name != "calc" {
t.Fatalf("tool_use not mapped: %+v", am.Content)
}
if toolUse.Input["a"] != float64(17) || toolUse.Input["b"] != float64(23) {
t.Fatalf("tool_use input args not decoded from JSON string: %+v", toolUse.Input)
}
// Message 3 (index 2) must be user with one tool_result block
um := r2.Messages[2]
if um.Role != "user" {
t.Fatalf("msg[2].role = %q, want user (tool_result)", um.Role)
}
if len(um.Content) != 1 || um.Content[0].Type != "tool_result" || um.Content[0].ToolUseID != "call_1" {
t.Fatalf("tool_result not mapped: %+v", um.Content)
}
// Round 3: consecutive tool results must merge into ONE user message
round3 := `{"model":"x","messages":[
{"role":"user","content":"do both"},
{"role":"assistant","content":"","tool_calls":[
{"id":"c1","type":"function","function":{"name":"calc","arguments":"{\"a\":1,\"b\":2}"}},
{"id":"c2","type":"function","function":{"name":"calc","arguments":"{\"a\":3,\"b\":4}"}}
]},
{"role":"tool","tool_call_id":"c1","content":"2"},
{"role":"tool","tool_call_id":"c2","content":"12"}
]}`
out3, err := vm.Transform("anthropic", "transform_request", round3)
if err != nil {
t.Fatalf("round3 transform: %v", err)
}
var r3 struct {
Messages []map[string]interface{} `json:"messages"`
}
if err := json.Unmarshal([]byte(out3), &r3); err != nil {
t.Fatalf("unmarshal r3: %v (%s)", err, out3)
}
// messages: user, assistant(tool_use x2), user(tool_result x2 merged)
if len(r3.Messages) != 3 {
t.Fatalf("round3 len(messages) = %d, want 3 (merged tool results): %s", len(r3.Messages), out3)
}
last := r3.Messages[3-1]
blocks := last["content"].([]interface{})
if len(blocks) != 2 {
t.Fatalf("last user message content blocks = %d, want 2 merged tool_results: %s", len(blocks), out3)
}
}
// TestOpenAITransformErrorEnvelopes covers Issue 6: the openai adapter must
// condense every shape of upstream error body that real OpenAI-compatible
// gateways emit, not just the {error:{message}} envelope. Unhandled shapes
// used to fall through to Go's generic "unknown error" and dump raw HTML /
// JSON into the server log.
func TestOpenAITransformErrorEnvelopes(t *testing.T) {
vm := NewVM(freshAdapterDir(t))
if err := vm.Start(); err != nil {
t.Fatal(err)
}
defer vm.Stop()
cases := []struct {
name string
status int
body string
want string
}{
{"standard openai envelope", 429,
`{"error":{"message":"Rate limit reached","type":"rate_limit"}}`,
"Rate limit reached"},
{"error as bare string", 400,
`{"error":"bad request"}`,
"bad request"},
{"flat numeric code (qijiar/siliconflow style)", 400,
`{"code":20012,"message":"Model does not exist. Please check it carefully.","data":null}`,
"20012: Model does not exist. Please check it carefully."},
{"flat string code (remotezen style)", 401,
`{"code":"INVALID_API_KEY","message":"Invalid API key"}`,
"INVALID_API_KEY: Invalid API key"},
{"fastapi detail", 422,
`{"detail":"validation failed"}`,
"validation failed"},
{"nginx html error page", 413,
`<html> <head><title>413 Request Entity Too Large</title></head> <body> <center><h1>413 Request Entity Too Large</h1></center> <hr><center>nginx/1.18.0 (Ubuntu)</center> </body> </html>`,
"413 Request Entity Too Large"},
{"plain text body", 502,
"upstream connect error",
"upstream connect error"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, ok, err := vm.TransformError("openai", tc.status, tc.body)
if err != nil {
t.Fatalf("TransformError: %v", err)
}
if !ok {
t.Fatalf("hook returned no reason for %s body: %s", tc.name, tc.body)
}
if got != tc.want {
t.Fatalf("reason = %q, want %q", got, tc.want)
}
})
}
// A body carrying no usable message must fall through (ok=false) so the
// Go-side generic condenser stays in charge instead of inventing text.
t.Run("no usable message falls through", func(t *testing.T) {
if _, ok, _ := vm.TransformError("openai", 500, `{"foo":"bar"}`); ok {
t.Fatal("expected fallthrough for a body with no message field")
}
})
}
// TestAdaptersReportZeroCacheHit covers the "distinguish missed from not
// reported" contract on BOTH adapter paths: whenever an upstream reports a
// cache field, the adapter must emit prompt_tokens_details even when the hit
// count is 0, so the gateway can record cache_reported=true and the UI shows
// 0% instead of "—". Adapters that dropped the 0 case made a reported miss
// indistinguishable from an upstream that never reported cache info.
func TestAdaptersReportZeroCacheHit(t *testing.T) {
vm := NewVM(freshAdapterDir(t))
if err := vm.Start(); err != nil {
t.Fatal(err)
}
defer vm.Stop()
// OpenAI-shaped upstreams: usage.prompt_tokens_details.cached_tokens = 0
openaiLike := []string{"openai", "deepseek", "sensenova", "opencode",
"agentrouter", "github", "groq", "kimicode", "mistral"}
respBody := `{"choices":[{"message":{"content":"hi"},"finish_reason":"stop"}],
"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,
"prompt_tokens_details":{"cached_tokens":0}}}`
streamBody := `{"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":2,
"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0}}}`
for _, name := range openaiLike {
out, err := vm.Transform(name, "transform_response", respBody)
if err != nil {
t.Fatalf("%s transform_response: %v", name, err)
}
if !strings.Contains(out, "prompt_tokens_details") {
t.Fatalf("%s: zero cached_tokens dropped in transform_response: %s", name, out)
}
out, err = vm.Transform(name, "transform_stream_chunk", streamBody)
if err != nil {
t.Fatalf("%s transform_stream_chunk: %v", name, err)
}
if !strings.Contains(out, "prompt_tokens_details") {
t.Fatalf("%s: zero cached_tokens dropped in transform_stream_chunk: %s", name, out)
}
}
// gemini: usageMetadata.cachedContentTokenCount = 0
gResp := `{"candidates":[{"content":{"parts":[{"text":"hi"}]},"finishReason":"STOP"}],
"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":2,
"totalTokenCount":12,"cachedContentTokenCount":0}}`
out, err := vm.Transform("gemini", "transform_response", gResp)
if err != nil {
t.Fatalf("gemini transform_response: %v", err)
}
if !strings.Contains(out, "prompt_tokens_details") {
t.Fatalf("gemini: cachedContentTokenCount not normalized in transform_response: %s", out)
}
gStream := `{"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":2,
"totalTokenCount":12,"cachedContentTokenCount":0}}`
out, err = vm.Transform("gemini", "transform_stream_chunk", gStream)
if err != nil {
t.Fatalf("gemini transform_stream_chunk: %v", err)
}
if !strings.Contains(out, "prompt_tokens_details") {
t.Fatalf("gemini: zero cachedContentTokenCount dropped in stream: %s", out)
}
// anthropic: usage.cache_read_input_tokens = 0
aResp := `{"content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn",
"usage":{"input_tokens":10,"output_tokens":2,"cache_read_input_tokens":0}}`
out, err = vm.Transform("anthropic", "transform_response", aResp)
if err != nil {
t.Fatalf("anthropic transform_response: %v", err)
}
if !strings.Contains(out, "prompt_tokens_details") {
t.Fatalf("anthropic: zero cache_read_input_tokens dropped in response: %s", out)
}
aStream := `{"type":"message_start","message":{"usage":{"input_tokens":10,
"output_tokens":2,"cache_read_input_tokens":0}}}`
out, err = vm.Transform("anthropic", "transform_stream_chunk", aStream)
if err != nil {
t.Fatalf("anthropic transform_stream_chunk: %v", err)
}
if !strings.Contains(out, "prompt_tokens_details") {
t.Fatalf("anthropic: zero cache_read_input_tokens dropped in stream: %s", out)
}
// Negative control: an upstream that reports NO cache field at all must
// not fabricate details (that would flip "not reported" into a fake 0%).
noCache := `{"choices":[{"message":{"content":"hi"},"finish_reason":"stop"}],
"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12}}`
out, err = vm.Transform("openai", "transform_response", noCache)
if err != nil {
t.Fatal(err)
}
if strings.Contains(out, "prompt_tokens_details") {
t.Fatalf("openai fabricated cache details when upstream reported none: %s", out)
}
}

65
internal/provider/provider.go
View File

@ -365,7 +365,7 @@ func (p *Provider) ModelAvailable(model string) bool {
p.mu.Lock()
defer p.mu.Unlock()
if s, ok := p.states[model]; ok {
return s.Available()
return s.Available() && s.Pref() > prefMin
}
return true
}
@ -806,6 +806,12 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
var realChunks int
var doneSeen bool
var doneSent bool
// toolIdx maps a per-stream tool fragment's upstream index (Anthropic
// content-block ordinal, which is sparse when thinking/text blocks
// precede tool_use) to a dense 0-based OpenAI tool_call ordinal so
// clients that accumulate fragments by index reassemble multi-tool
// responses correctly.
toolIdx := newToolIndexRemapper()
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || !strings.HasPrefix(line, "data:") {
@ -843,6 +849,7 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
if err := json.Unmarshal([]byte(unified), &ck); err != nil {
continue
}
ck.ToolCalls = toolIdx.remap(ck.ToolCalls)
if ck.Done {
doneSent = true
}
@ -917,6 +924,62 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
// tool calls, reasoning text or usage. Standard OpenAI finish reasons are
// never classified as errors, so legitimate instant-empty completions
// (finish_reason:"stop", no output) still reach the client.
// toolIndexRemapper maps per-stream tool_call fragment indices from a sparse
// upstream ordinal (e.g. Anthropic content_block index, which skips over
// thinking/text blocks) to a dense 0-based OpenAI tool_call ordinal.
type toolIndexRemapper struct {
seen map[int]int // upstream index -> dense ordinal
next int // next dense ordinal to assign
}
func newToolIndexRemapper() toolIndexRemapper {
return toolIndexRemapper{seen: make(map[int]int)}
}
func (t *toolIndexRemapper) remap(raw json.RawMessage) json.RawMessage {
if len(raw) == 0 {
return raw
}
// Decode into generic maps so every upstream field is preserved verbatim;
// only the index value is rewritten.
var frags []map[string]json.RawMessage
if err := json.Unmarshal(raw, &frags); err != nil || len(frags) == 0 {
return raw
}
changed := false
for i := range frags {
orig := 0
if v, ok := frags[i]["index"]; ok {
if err := json.Unmarshal(v, &orig); err != nil {
continue
}
}
mapped, ok := t.seen[orig]
if !ok {
mapped = t.next
t.next++
t.seen[orig] = mapped
}
if mapped != orig {
b, err := json.Marshal(mapped)
if err != nil {
continue
}
frags[i]["index"] = b
changed = true
}
}
if !changed {
return raw
}
out, err := json.Marshal(frags)
if err != nil {
return raw
}
return out
}
func errorOnlyChunk(ck types.UnifiedChunk) bool {
if !ck.Done || ck.FinishReason == "" {
return false

80
internal/provider/provider_test.go
View File

@ -598,3 +598,83 @@ func TestUnknownErrorFallbackWithoutHook(t *testing.T) {
t.Fatalf("raw body must not leak: %v", err)
}
}
// TestToolIndexRemapper verifies sparse upstream tool-call indices are
// compacted to dense 0-based OpenAI ordinals (Issue 2). Anthropic emits the
// content_block ordinal, so a thinking block at index 0 pushes the first
// tool_use to index 1 — OpenAI clients accumulating by index would leave a
// hole at 0 and mis-assemble multi-tool responses.
func TestToolIndexRemapper(t *testing.T) {
t.Run("single tool after thinking block", func(t *testing.T) {
r := newToolIndexRemapper()
// content_block index 1 (thinking was 0) -> dense 0
out := r.remap(json.RawMessage(`[{"index":1,"id":"toolu_a","type":"function","function":{"name":"calc","arguments":""}}]`))
var got []map[string]interface{}
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("unmarshal: %v (%s)", err, out)
}
if got[0]["index"] != float64(0) {
t.Fatalf("index = %v, want 0: %s", got[0]["index"], out)
}
// id/name/arguments must survive verbatim
if got[0]["id"] != "toolu_a" {
t.Fatalf("id lost: %s", out)
}
fn := got[0]["function"].(map[string]interface{})
if fn["name"] != "calc" {
t.Fatalf("function.name lost: %s", out)
}
// subsequent argument fragments on the SAME upstream index reuse ordinal 0
out2 := r.remap(json.RawMessage(`[{"index":1,"id":"","type":"function","function":{"name":"","arguments":"{\"a\":1}"}}]`))
var got2 []map[string]interface{}
if err := json.Unmarshal(out2, &got2); err != nil {
t.Fatalf("unmarshal2: %v", err)
}
if got2[0]["index"] != float64(0) {
t.Fatalf("fragment index = %v, want stable 0: %s", got2[0]["index"], out2)
}
})
t.Run("multiple tools get distinct dense ordinals", func(t *testing.T) {
r := newToolIndexRemapper()
// thinking=0, tool_use=1, tool_use=2 -> 0, 1
a := r.remap(json.RawMessage(`[{"index":1,"id":"t1","type":"function","function":{"name":"f1","arguments":""}}]`))
b := r.remap(json.RawMessage(`[{"index":2,"id":"t2","type":"function","function":{"name":"f2","arguments":""}}]`))
var ga, gb []map[string]interface{}
json.Unmarshal(a, &ga)
json.Unmarshal(b, &gb)
if ga[0]["index"] != float64(0) {
t.Fatalf("first tool index = %v, want 0", ga[0]["index"])
}
if gb[0]["index"] != float64(1) {
t.Fatalf("second tool index = %v, want 1", gb[0]["index"])
}
// interleaved fragments keep their own ordinals
a2 := r.remap(json.RawMessage(`[{"index":1,"id":"","type":"function","function":{"name":"","arguments":"x"}}]`))
var ga2 []map[string]interface{}
json.Unmarshal(a2, &ga2)
if ga2[0]["index"] != float64(0) {
t.Fatalf("t1 fragment index = %v, want 0", ga2[0]["index"])
}
})
t.Run("already dense indices pass through untouched", func(t *testing.T) {
r := newToolIndexRemapper()
in := json.RawMessage(`[{"index":0,"id":"t","type":"function","function":{"name":"f","arguments":""}}]`)
out := r.remap(in)
if string(out) != string(in) {
t.Fatalf("dense input was rewritten:\n in=%s\nout=%s", in, out)
}
})
t.Run("empty and malformed input is safe", func(t *testing.T) {
r := newToolIndexRemapper()
if got := r.remap(nil); got != nil {
t.Fatalf("nil -> %s", got)
}
bad := json.RawMessage(`not json`)
if got := r.remap(bad); string(got) != string(bad) {
t.Fatalf("malformed input must pass through: %s", got)
}
})
}

11
internal/scheduler/scheduler.go
View File

@ -248,10 +248,9 @@ func (s *Scheduler) chainDrive(ctx context.Context, chain *Chain, req *types.Cha
ce.Skipped = append(ce.Skipped, fmt.Sprintf("tier %d: no schedulable slot (cooling or quota exhausted)", tn.Tier))
continue
}
// preference orders a same-tier run; stable so equal prefs keep order
sort.SliceStable(cands, func(i, j int) bool {
return cands[i].Prov.Pref(cands[i].Model) > cands[j].Prov.Pref(cands[j].Model)
})
// No Pref sort: load balancing is done by round-robin cursor.
// Persistently failing slots are excluded by ModelAvailable
// (which checks Pref > prefMin).
base := tn.NextStart()
res := runTier(ctx, tn, cands, base, req, stream)
if res.resp != nil || res.chunks != nil {
@ -350,9 +349,7 @@ func (s *Scheduler) ChainImage(ctx context.Context, chain *Chain, req *types.Ima
ce.Skipped = append(ce.Skipped, fmt.Sprintf("tier %d: no schedulable slot (cooling)", tn.Tier))
continue
}
sort.SliceStable(cands, func(i, j int) bool {
return cands[i].Prov.Pref(cands[i].Model) > cands[j].Prov.Pref(cands[j].Model)
})
// No Pref sort: load balancing is done by round-robin cursor.
base := tn.NextStart()
var hard []TierError
for i := 0; i < len(cands); i++ {

17
internal/scheduler/scheduler_test.go
View File

@ -177,14 +177,21 @@ func TestChainPreferenceSinksButStaysReachable(t *testing.T) {
if err != nil {
t.Fatalf("chain: %v", err)
}
// higher pref (good) is tried first and hard-fails; the pass moves on to
// the negative-pref slot, which sinks but stays reachable: with the real
// provider its success would RecordSuccess (+1 pref, self-heal)
// Round-robin starts at index 0 (neg). neg is available (pref=-5 > -20)
// and succeeds. good is never tried because neg already won.
if src != "neg" || model != "n" || resp.Content != "neg" {
t.Fatalf("served src=%q model=%q content=%q", src, model, resp.Content)
}
if good.chatHits.Load() == 0 {
t.Fatal("higher-pref slot must be tried first")
if good.chatHits.Load() != 0 {
t.Fatal("neg was tried first and succeeded; good must not be attempted")
}
// Second request: cursor advances. neg wins again (good hard-fails).
resp2, src2, _, err2 := s.ChainChat(context.Background(), ch, chatReq(), nil)
if err2 != nil {
t.Fatalf("second chain: %v", err2)
}
if src2 != "neg" || resp2.Content != "neg" {
t.Fatalf("second req src=%q content=%q", src2, resp2.Content)
}
}