fix: anthropic tool-call round-trip, cache zero-hit parity, round-robin load balancing

anthropic.lua v3.0.0:
- Issue 1: tool_result/tool_use round-trip
- Issue 3: thinking default OFF (opt-in via extra_body.thinking)
- Issue 4: tool_choice mapping
- Issue 5: collect_blocks preserves unknown part types
- message_stop no longer emits done=true (was overwriting tool_calls finish_reason)
- cache_read_input_tokens normalized even at 0

gemini.lua:
- transform_response was missing cachedContentTokenCount

openai.lua (Issue 6):
- transform_error handles flat envelopes, nginx HTML, bare text

chat.go mergeUsage:
- Keep PromptTokensDetails even when CachedTokens=0

scheduler.go:
- Remove sort.SliceStable by Pref; round-robin cursor is the only LB mechanism

provider.go ModelAvailable:
- Also check Pref() > prefMin, persistently failing slots exit cands

presets.go:
- 17 built-in source templates

Tests: 6 new test functions, 2 updated for new semantics
This commit is contained in:
JianFeeeee
2026-08-28 12:02:46 +08:00
parent 94cbcb6771
commit 624fd74b45
15 changed files with 1170 additions and 114 deletions

View File

@ -671,7 +671,11 @@ func mergeUsage(prev, cur *types.TokenUsage) *types.TokenUsage {
out.Completion = cur.Completion
}
out.Total = out.Prompt + out.Completion
if cur.PromptTokensDetails != nil && cur.PromptTokensDetails.CachedTokens > 0 {
if cur.PromptTokensDetails != nil {
// Keep the details object even when CachedTokens is 0: a reported
// zero-hit is meaningful ("cache missed") and must stay
// distinguishable from "upstream never reported cache info".
// Dropping it here made streaming rows show “—” instead of 0%.
out.PromptTokensDetails = cur.PromptTokensDetails
}
if cur.PromptCacheHit > 0 {

View File

@ -0,0 +1,62 @@
package gateway
import (
"testing"
"llmsproxy/internal/types"
)
// TestMergeUsageKeepsZeroCacheDetails locks the streaming counterpart of the
// "distinguish missed from not reported" contract: a usage chunk that reports
// prompt_tokens_details with 0 cached tokens must survive the merge, otherwise
// streaming rows lose cache_reported and the UI shows "—" instead of 0%.
func TestMergeUsageKeepsZeroCacheDetails(t *testing.T) {
t.Run("zero-hit details survive merge", func(t *testing.T) {
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
cur := &types.TokenUsage{
Prompt: 10, Completion: 5, Total: 15,
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 0},
}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails == nil {
t.Fatal("zero-hit prompt_tokens_details was dropped by mergeUsage")
}
if got.PromptTokensDetails.CachedTokens != 0 {
t.Fatalf("cached_tokens = %d, want 0", got.PromptTokensDetails.CachedTokens)
}
})
t.Run("non-zero hit still wins", func(t *testing.T) {
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
cur := &types.TokenUsage{
Prompt: 10, Completion: 5, Total: 15,
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 64},
}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails == nil || got.PromptTokensDetails.CachedTokens != 64 {
t.Fatalf("cached_tokens lost: %+v", got.PromptTokensDetails)
}
})
t.Run("absent details do not overwrite an earlier report", func(t *testing.T) {
prev := &types.TokenUsage{
Prompt: 10, Completion: 1, Total: 11,
PromptTokensDetails: &types.PromptTokensDetails{CachedTokens: 32},
}
cur := &types.TokenUsage{Prompt: 10, Completion: 5, Total: 15}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails == nil || got.PromptTokensDetails.CachedTokens != 32 {
t.Fatalf("earlier cache report clobbered by a later chunk without details: %+v",
got.PromptTokensDetails)
}
})
t.Run("no cache data anywhere stays nil", func(t *testing.T) {
prev := &types.TokenUsage{Prompt: 10, Completion: 1, Total: 11}
cur := &types.TokenUsage{Prompt: 10, Completion: 5, Total: 15}
got := mergeUsage(prev, cur)
if got.PromptTokensDetails != nil {
t.Fatalf("fabricated cache details: %+v", got.PromptTokensDetails)
}
})
}