mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 00:48:00 +00:00
fix: anthropic tool-call round-trip, cache zero-hit parity, round-robin load balancing
anthropic.lua v3.0.0: - Issue 1: tool_result/tool_use round-trip - Issue 3: thinking default OFF (opt-in via extra_body.thinking) - Issue 4: tool_choice mapping - Issue 5: collect_blocks preserves unknown part types - message_stop no longer emits done=true (was overwriting tool_calls finish_reason) - cache_read_input_tokens normalized even at 0 gemini.lua: - transform_response was missing cachedContentTokenCount openai.lua (Issue 6): - transform_error handles flat envelopes, nginx HTML, bare text chat.go mergeUsage: - Keep PromptTokensDetails even when CachedTokens=0 scheduler.go: - Remove sort.SliceStable by Pref; round-robin cursor is the only LB mechanism provider.go ModelAvailable: - Also check Pref() > prefMin, persistently failing slots exit cands presets.go: - 17 built-in source templates Tests: 6 new test functions, 2 updated for new semantics
This commit is contained in:
JianFeeeee
@ -273,7 +273,8 @@ func TestDisableThinkingPassthrough(t *testing.T) {
|
||||
t.Fatalf("thinking.type = %v", thinking["type"])
|
||||
}
|
||||
|
||||
// anthropic: disable_thinking removes the thinking block
|
||||
// anthropic: thinking is opt-in (Issue 3) — never emitted by default, and
|
||||
// disable_thinking is not a trigger either.
|
||||
out2, err := vm.Transform("anthropic", "transform_request", body)
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic transform: %v", err)
|
||||
@ -285,8 +286,17 @@ func TestDisableThinkingPassthrough(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic transform: %v", err)
|
||||
}
|
||||
if !strings.Contains(out3, "enabled") {
|
||||
t.Fatalf("anthropic should enable thinking by default: %s", out3)
|
||||
if strings.Contains(out3, "thinking") {
|
||||
t.Fatalf("anthropic must not enable thinking by default: %s", out3)
|
||||
}
|
||||
// opt-in path: extra_body.thinking is forwarded verbatim
|
||||
out4, err := vm.Transform("anthropic", "transform_request",
|
||||
`{"model":"x","messages":[{"role":"user","content":"hi"}],"extra_body":{"thinking":{"type":"enabled","budget_tokens":2048}}}`)
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic transform: %v", err)
|
||||
}
|
||||
if !strings.Contains(out4, `"type":"enabled"`) || !strings.Contains(out4, "2048") {
|
||||
t.Fatalf("anthropic should forward extra_body.thinking: %s", out4)
|
||||
}
|
||||
}
|
||||
|
||||
@ -454,3 +464,285 @@ func TestAdaptersPassFinishReason(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnthropicToolRoundTrip verifies the OpenAI->Anthropic request mapping for
|
||||
// a full agent tool-call round (Issue 1): assistant tool_calls become
|
||||
// tool_use content blocks and role:"tool" results become user tool_result
|
||||
// blocks merged into ONE user message. This is the regression that made agent
|
||||
// clients (dsh/Claude Code/Cursor) repeatedly re-invoke the same tool.
|
||||
func TestAnthropicToolRoundTrip(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
round1 := `{"model":"x","tools":[{"type":"function","function":{"name":"calc","description":"multiply","parameters":{"type":"object","properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]}}}],"messages":[{"role":"user","content":"what is 17*23?"}]}`
|
||||
out1, err := vm.Transform("anthropic", "transform_request", round1)
|
||||
if err != nil {
|
||||
t.Fatalf("round1 transform: %v", err)
|
||||
}
|
||||
var r1 struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
InputSchema map[string]interface{} `json:"input_schema"`
|
||||
} `json:"tools"`
|
||||
Messages []map[string]interface{} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out1), &r1); err != nil {
|
||||
t.Fatalf("unmarshal r1: %v (%s)", err, out1)
|
||||
}
|
||||
if len(r1.Tools) != 1 || r1.Tools[0].Name != "calc" {
|
||||
t.Fatalf("tools not mapped: %s", out1)
|
||||
}
|
||||
|
||||
// Round 2: assistant tool_calls + tool result
|
||||
round2 := `{"model":"x","messages":[
|
||||
{"role":"user","content":"what is 17*23?"},
|
||||
{"role":"assistant","content":"","tool_calls":[
|
||||
{"id":"call_1","type":"function","function":{"name":"calc","arguments":"{\"a\":17,\"b\":23}"}}
|
||||
]},
|
||||
{"role":"tool","tool_call_id":"call_1","content":"391"}
|
||||
]}`
|
||||
out2, err := vm.Transform("anthropic", "transform_request", round2)
|
||||
if err != nil {
|
||||
t.Fatalf("round2 transform: %v", err)
|
||||
}
|
||||
var r2 struct {
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input map[string]interface{} `json:"input"`
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
ContentText string `json:"content"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out2), &r2); err != nil {
|
||||
t.Fatalf("unmarshal r2: %v (%s)", err, out2)
|
||||
}
|
||||
// Message 2 (index 1) must be assistant with one tool_use block
|
||||
am := r2.Messages[1]
|
||||
if am.Role != "assistant" {
|
||||
t.Fatalf("msg[1].role = %q, want assistant", am.Role)
|
||||
}
|
||||
var toolUse struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input map[string]interface{} `json:"input"`
|
||||
}
|
||||
for _, b := range am.Content {
|
||||
if b.Type == "tool_use" {
|
||||
toolUse = struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input map[string]interface{} `json:"input"`
|
||||
}{b.Type, b.ID, b.Name, b.Input}
|
||||
}
|
||||
}
|
||||
if toolUse.ID != "call_1" || toolUse.Name != "calc" {
|
||||
t.Fatalf("tool_use not mapped: %+v", am.Content)
|
||||
}
|
||||
if toolUse.Input["a"] != float64(17) || toolUse.Input["b"] != float64(23) {
|
||||
t.Fatalf("tool_use input args not decoded from JSON string: %+v", toolUse.Input)
|
||||
}
|
||||
// Message 3 (index 2) must be user with one tool_result block
|
||||
um := r2.Messages[2]
|
||||
if um.Role != "user" {
|
||||
t.Fatalf("msg[2].role = %q, want user (tool_result)", um.Role)
|
||||
}
|
||||
if len(um.Content) != 1 || um.Content[0].Type != "tool_result" || um.Content[0].ToolUseID != "call_1" {
|
||||
t.Fatalf("tool_result not mapped: %+v", um.Content)
|
||||
}
|
||||
|
||||
// Round 3: consecutive tool results must merge into ONE user message
|
||||
round3 := `{"model":"x","messages":[
|
||||
{"role":"user","content":"do both"},
|
||||
{"role":"assistant","content":"","tool_calls":[
|
||||
{"id":"c1","type":"function","function":{"name":"calc","arguments":"{\"a\":1,\"b\":2}"}},
|
||||
{"id":"c2","type":"function","function":{"name":"calc","arguments":"{\"a\":3,\"b\":4}"}}
|
||||
]},
|
||||
{"role":"tool","tool_call_id":"c1","content":"2"},
|
||||
{"role":"tool","tool_call_id":"c2","content":"12"}
|
||||
]}`
|
||||
out3, err := vm.Transform("anthropic", "transform_request", round3)
|
||||
if err != nil {
|
||||
t.Fatalf("round3 transform: %v", err)
|
||||
}
|
||||
var r3 struct {
|
||||
Messages []map[string]interface{} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out3), &r3); err != nil {
|
||||
t.Fatalf("unmarshal r3: %v (%s)", err, out3)
|
||||
}
|
||||
// messages: user, assistant(tool_use x2), user(tool_result x2 merged)
|
||||
if len(r3.Messages) != 3 {
|
||||
t.Fatalf("round3 len(messages) = %d, want 3 (merged tool results): %s", len(r3.Messages), out3)
|
||||
}
|
||||
last := r3.Messages[3-1]
|
||||
blocks := last["content"].([]interface{})
|
||||
if len(blocks) != 2 {
|
||||
t.Fatalf("last user message content blocks = %d, want 2 merged tool_results: %s", len(blocks), out3)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAITransformErrorEnvelopes covers Issue 6: the openai adapter must
|
||||
// condense every shape of upstream error body that real OpenAI-compatible
|
||||
// gateways emit, not just the {error:{message}} envelope. Unhandled shapes
|
||||
// used to fall through to Go's generic "unknown error" and dump raw HTML /
|
||||
// JSON into the server log.
|
||||
func TestOpenAITransformErrorEnvelopes(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
want string
|
||||
}{
|
||||
{"standard openai envelope", 429,
|
||||
`{"error":{"message":"Rate limit reached","type":"rate_limit"}}`,
|
||||
"Rate limit reached"},
|
||||
{"error as bare string", 400,
|
||||
`{"error":"bad request"}`,
|
||||
"bad request"},
|
||||
{"flat numeric code (qijiar/siliconflow style)", 400,
|
||||
`{"code":20012,"message":"Model does not exist. Please check it carefully.","data":null}`,
|
||||
"20012: Model does not exist. Please check it carefully."},
|
||||
{"flat string code (remotezen style)", 401,
|
||||
`{"code":"INVALID_API_KEY","message":"Invalid API key"}`,
|
||||
"INVALID_API_KEY: Invalid API key"},
|
||||
{"fastapi detail", 422,
|
||||
`{"detail":"validation failed"}`,
|
||||
"validation failed"},
|
||||
{"nginx html error page", 413,
|
||||
`<html> <head><title>413 Request Entity Too Large</title></head> <body> <center><h1>413 Request Entity Too Large</h1></center> <hr><center>nginx/1.18.0 (Ubuntu)</center> </body> </html>`,
|
||||
"413 Request Entity Too Large"},
|
||||
{"plain text body", 502,
|
||||
"upstream connect error",
|
||||
"upstream connect error"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, ok, err := vm.TransformError("openai", tc.status, tc.body)
|
||||
if err != nil {
|
||||
t.Fatalf("TransformError: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatalf("hook returned no reason for %s body: %s", tc.name, tc.body)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("reason = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A body carrying no usable message must fall through (ok=false) so the
|
||||
// Go-side generic condenser stays in charge instead of inventing text.
|
||||
t.Run("no usable message falls through", func(t *testing.T) {
|
||||
if _, ok, _ := vm.TransformError("openai", 500, `{"foo":"bar"}`); ok {
|
||||
t.Fatal("expected fallthrough for a body with no message field")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestAdaptersReportZeroCacheHit covers the "distinguish missed from not
|
||||
// reported" contract on BOTH adapter paths: whenever an upstream reports a
|
||||
// cache field, the adapter must emit prompt_tokens_details even when the hit
|
||||
// count is 0, so the gateway can record cache_reported=true and the UI shows
|
||||
// 0% instead of "—". Adapters that dropped the 0 case made a reported miss
|
||||
// indistinguishable from an upstream that never reported cache info.
|
||||
func TestAdaptersReportZeroCacheHit(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
// OpenAI-shaped upstreams: usage.prompt_tokens_details.cached_tokens = 0
|
||||
openaiLike := []string{"openai", "deepseek", "sensenova", "opencode",
|
||||
"agentrouter", "github", "groq", "kimicode", "mistral"}
|
||||
respBody := `{"choices":[{"message":{"content":"hi"},"finish_reason":"stop"}],
|
||||
"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,
|
||||
"prompt_tokens_details":{"cached_tokens":0}}}`
|
||||
streamBody := `{"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":2,
|
||||
"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0}}}`
|
||||
for _, name := range openaiLike {
|
||||
out, err := vm.Transform(name, "transform_response", respBody)
|
||||
if err != nil {
|
||||
t.Fatalf("%s transform_response: %v", name, err)
|
||||
}
|
||||
if !strings.Contains(out, "prompt_tokens_details") {
|
||||
t.Fatalf("%s: zero cached_tokens dropped in transform_response: %s", name, out)
|
||||
}
|
||||
out, err = vm.Transform(name, "transform_stream_chunk", streamBody)
|
||||
if err != nil {
|
||||
t.Fatalf("%s transform_stream_chunk: %v", name, err)
|
||||
}
|
||||
if !strings.Contains(out, "prompt_tokens_details") {
|
||||
t.Fatalf("%s: zero cached_tokens dropped in transform_stream_chunk: %s", name, out)
|
||||
}
|
||||
}
|
||||
|
||||
// gemini: usageMetadata.cachedContentTokenCount = 0
|
||||
gResp := `{"candidates":[{"content":{"parts":[{"text":"hi"}]},"finishReason":"STOP"}],
|
||||
"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":2,
|
||||
"totalTokenCount":12,"cachedContentTokenCount":0}}`
|
||||
out, err := vm.Transform("gemini", "transform_response", gResp)
|
||||
if err != nil {
|
||||
t.Fatalf("gemini transform_response: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "prompt_tokens_details") {
|
||||
t.Fatalf("gemini: cachedContentTokenCount not normalized in transform_response: %s", out)
|
||||
}
|
||||
gStream := `{"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":2,
|
||||
"totalTokenCount":12,"cachedContentTokenCount":0}}`
|
||||
out, err = vm.Transform("gemini", "transform_stream_chunk", gStream)
|
||||
if err != nil {
|
||||
t.Fatalf("gemini transform_stream_chunk: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "prompt_tokens_details") {
|
||||
t.Fatalf("gemini: zero cachedContentTokenCount dropped in stream: %s", out)
|
||||
}
|
||||
|
||||
// anthropic: usage.cache_read_input_tokens = 0
|
||||
aResp := `{"content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn",
|
||||
"usage":{"input_tokens":10,"output_tokens":2,"cache_read_input_tokens":0}}`
|
||||
out, err = vm.Transform("anthropic", "transform_response", aResp)
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic transform_response: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "prompt_tokens_details") {
|
||||
t.Fatalf("anthropic: zero cache_read_input_tokens dropped in response: %s", out)
|
||||
}
|
||||
aStream := `{"type":"message_start","message":{"usage":{"input_tokens":10,
|
||||
"output_tokens":2,"cache_read_input_tokens":0}}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", aStream)
|
||||
if err != nil {
|
||||
t.Fatalf("anthropic transform_stream_chunk: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "prompt_tokens_details") {
|
||||
t.Fatalf("anthropic: zero cache_read_input_tokens dropped in stream: %s", out)
|
||||
}
|
||||
|
||||
// Negative control: an upstream that reports NO cache field at all must
|
||||
// not fabricate details (that would flip "not reported" into a fake 0%).
|
||||
noCache := `{"choices":[{"message":{"content":"hi"},"finish_reason":"stop"}],
|
||||
"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12}}`
|
||||
out, err = vm.Transform("openai", "transform_response", noCache)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(out, "prompt_tokens_details") {
|
||||
t.Fatalf("openai fabricated cache details when upstream reported none: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user