mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 08:57:57 +00:00
fix: anthropic tool-call round-trip, cache zero-hit parity, round-robin load balancing
anthropic.lua v3.0.0: - Issue 1: tool_result/tool_use round-trip - Issue 3: thinking default OFF (opt-in via extra_body.thinking) - Issue 4: tool_choice mapping - Issue 5: collect_blocks preserves unknown part types - message_stop no longer emits done=true (was overwriting tool_calls finish_reason) - cache_read_input_tokens normalized even at 0 gemini.lua: - transform_response was missing cachedContentTokenCount openai.lua (Issue 6): - transform_error handles flat envelopes, nginx HTML, bare text chat.go mergeUsage: - Keep PromptTokensDetails even when CachedTokens=0 scheduler.go: - Remove sort.SliceStable by Pref; round-robin cursor is the only LB mechanism provider.go ModelAvailable: - Also check Pref() > prefMin, persistently failing slots exit cands presets.go: - 17 built-in source templates Tests: 6 new test functions, 2 updated for new semantics
This commit is contained in:
@ -671,7 +671,11 @@ func mergeUsage(prev, cur *types.TokenUsage) *types.TokenUsage {
|
||||
out.Completion = cur.Completion
|
||||
}
|
||||
out.Total = out.Prompt + out.Completion
|
||||
if cur.PromptTokensDetails != nil && cur.PromptTokensDetails.CachedTokens > 0 {
|
||||
if cur.PromptTokensDetails != nil {
|
||||
// Keep the details object even when CachedTokens is 0: a reported
|
||||
// zero-hit is meaningful ("cache missed") and must stay
|
||||
// distinguishable from "upstream never reported cache info".
|
||||
// Dropping it here made streaming rows show “—” instead of 0%.
|
||||
out.PromptTokensDetails = cur.PromptTokensDetails
|
||||
}
|
||||
if cur.PromptCacheHit > 0 {
|
||||
|
||||
62
internal/gateway/chat_cache_test.go
Normal file
62
internal/gateway/chat_cache_test.go
Normal file
@ -0,0 +1,62 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"llmsproxy/internal/types"
|
||||
)
|
||||
|
||||
// TestMergeUsageKeepsZeroCacheDetails locks the streaming counterpart of the
|
||||
// "distinguish missed from not reported" contract: a usage chunk that reports
|
||||
// prompt_tokens_details with 0 cached tokens must survive the merge, otherwise
|
||||
// streaming rows lose cache_reported and the UI shows "—" instead of 0%.
|
||||
func TestMergeUsageKeepsZeroCacheDetails(t *testing.T) {
|
||||
t.Run("zero-hit details survive merge", func(t *testing.T) {
|
||||
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
|
||||
cur := &types.TokenUsage{
|
||||
Prompt: 10, Completion: 5, Total: 15,
|
||||
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 0},
|
||||
}
|
||||
got := mergeUsage(prev, cur)
|
||||
if got.PromptTokensDetails == nil {
|
||||
t.Fatal("zero-hit prompt_tokens_details was dropped by mergeUsage")
|
||||
}
|
||||
if got.PromptTokensDetails.CachedTokens != 0 {
|
||||
t.Fatalf("cached_tokens = %d, want 0", got.PromptTokensDetails.CachedTokens)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-zero hit still wins", func(t *testing.T) {
|
||||
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
|
||||
cur := &types.TokenUsage{
|
||||
Prompt: 10, Completion: 5, Total: 15,
|
||||
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 64},
|
||||
}
|
||||
got := mergeUsage(prev, cur)
|
||||
if got.PromptTokensDetails == nil || got.PromptTokensDetails.CachedTokens != 64 {
|
||||
t.Fatalf("cached_tokens lost: %+v", got.PromptTokensDetails)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("absent details do not overwrite an earlier report", func(t *testing.T) {
|
||||
prev := &types.TokenUsage{
|
||||
Prompt: 10, Completion: 1, Total: 11,
|
||||
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 32},
|
||||
}
|
||||
cur := &types.TokenUsage{Prompt: 10, Completion: 5, Total: 15}
|
||||
got := mergeUsage(prev, cur)
|
||||
if got.PromptTokensDetails == nil || got.PromptTokensDetails.CachedTokens != 32 {
|
||||
t.Fatalf("earlier cache report clobbered by a later chunk without details: %+v",
|
||||
got.PromptTokensDetails)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no cache data anywhere stays nil", func(t *testing.T) {
|
||||
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
|
||||
cur := &types.TokenUsage{Prompt: 10, Completion: 5, Total: 15}
|
||||
got := mergeUsage(prev, cur)
|
||||
if got.PromptTokensDetails != nil {
|
||||
t.Fatalf("fabricated cache details: %+v", got.PromptTokensDetails)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user