mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 00:48:00 +00:00
fix: anthropic tool-call round-trip, cache zero-hit parity, round-robin load balancing
anthropic.lua v3.0.0: - Issue 1: tool_result/tool_use round-trip - Issue 3: thinking default OFF (opt-in via extra_body.thinking) - Issue 4: tool_choice mapping - Issue 5: collect_blocks preserves unknown part types - message_stop no longer emits done=true (was overwriting tool_calls finish_reason) - cache_read_input_tokens normalized even at 0 gemini.lua: - transform_response was missing cachedContentTokenCount openai.lua (Issue 6): - transform_error handles flat envelopes, nginx HTML, bare text chat.go mergeUsage: - Keep PromptTokensDetails even when CachedTokens=0 scheduler.go: - Remove sort.SliceStable by Pref; round-robin cursor is the only LB mechanism provider.go ModelAvailable: - Also check Pref() > prefMin, persistently failing slots exit cands presets.go: - 17 built-in source templates Tests: 6 new test functions, 2 updated for new semantics
This commit is contained in:
@ -365,7 +365,7 @@ func (p *Provider) ModelAvailable(model string) bool {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if s, ok := p.states[model]; ok {
|
||||
return s.Available()
|
||||
return s.Available() && s.Pref() > prefMin
|
||||
}
|
||||
return true
|
||||
}
|
||||
@ -806,6 +806,12 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
|
||||
var realChunks int
|
||||
var doneSeen bool
|
||||
var doneSent bool
|
||||
// toolIdx maps a per-stream tool fragment's upstream index (Anthropic
|
||||
// content-block ordinal, which is sparse when thinking/text blocks
|
||||
// precede tool_use) to a dense 0-based OpenAI tool_call ordinal so
|
||||
// clients that accumulate fragments by index reassemble multi-tool
|
||||
// responses correctly.
|
||||
toolIdx := newToolIndexRemapper()
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || !strings.HasPrefix(line, "data:") {
|
||||
@ -843,6 +849,7 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
|
||||
if err := json.Unmarshal([]byte(unified), &ck); err != nil {
|
||||
continue
|
||||
}
|
||||
ck.ToolCalls = toolIdx.remap(ck.ToolCalls)
|
||||
if ck.Done {
|
||||
doneSent = true
|
||||
}
|
||||
@ -917,6 +924,62 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
|
||||
// tool calls, reasoning text or usage. Standard OpenAI finish reasons are
|
||||
// never classified as errors, so legitimate instant-empty completions
|
||||
// (finish_reason:"stop", no output) still reach the client.
|
||||
|
||||
// toolIndexRemapper maps per-stream tool_call fragment indices from a sparse
|
||||
// upstream ordinal (e.g. Anthropic content_block index, which skips over
|
||||
// thinking/text blocks) to a dense 0-based OpenAI tool_call ordinal.
|
||||
type toolIndexRemapper struct {
|
||||
seen map[int]int // upstream index -> dense ordinal
|
||||
next int // next dense ordinal to assign
|
||||
}
|
||||
|
||||
func newToolIndexRemapper() toolIndexRemapper {
|
||||
return toolIndexRemapper{seen: make(map[int]int)}
|
||||
}
|
||||
|
||||
func (t *toolIndexRemapper) remap(raw json.RawMessage) json.RawMessage {
|
||||
if len(raw) == 0 {
|
||||
return raw
|
||||
}
|
||||
// Decode into generic maps so every upstream field is preserved verbatim;
|
||||
// only the index value is rewritten.
|
||||
var frags []map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &frags); err != nil || len(frags) == 0 {
|
||||
return raw
|
||||
}
|
||||
changed := false
|
||||
for i := range frags {
|
||||
orig := 0
|
||||
if v, ok := frags[i]["index"]; ok {
|
||||
if err := json.Unmarshal(v, &orig); err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
mapped, ok := t.seen[orig]
|
||||
if !ok {
|
||||
mapped = t.next
|
||||
t.next++
|
||||
t.seen[orig] = mapped
|
||||
}
|
||||
if mapped != orig {
|
||||
b, err := json.Marshal(mapped)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
frags[i]["index"] = b
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return raw
|
||||
}
|
||||
out, err := json.Marshal(frags)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func errorOnlyChunk(ck types.UnifiedChunk) bool {
|
||||
if !ck.Done || ck.FinishReason == "" {
|
||||
return false
|
||||
|
||||
@ -598,3 +598,83 @@ func TestUnknownErrorFallbackWithoutHook(t *testing.T) {
|
||||
t.Fatalf("raw body must not leak: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToolIndexRemapper verifies sparse upstream tool-call indices are
|
||||
// compacted to dense 0-based OpenAI ordinals (Issue 2). Anthropic emits the
|
||||
// content_block ordinal, so a thinking block at index 0 pushes the first
|
||||
// tool_use to index 1 — OpenAI clients accumulating by index would leave a
|
||||
// hole at 0 and mis-assemble multi-tool responses.
|
||||
func TestToolIndexRemapper(t *testing.T) {
|
||||
t.Run("single tool after thinking block", func(t *testing.T) {
|
||||
r := newToolIndexRemapper()
|
||||
// content_block index 1 (thinking was 0) -> dense 0
|
||||
out := r.remap(json.RawMessage(`[{"index":1,"id":"toolu_a","type":"function","function":{"name":"calc","arguments":""}}]`))
|
||||
var got []map[string]interface{}
|
||||
if err := json.Unmarshal(out, &got); err != nil {
|
||||
t.Fatalf("unmarshal: %v (%s)", err, out)
|
||||
}
|
||||
if got[0]["index"] != float64(0) {
|
||||
t.Fatalf("index = %v, want 0: %s", got[0]["index"], out)
|
||||
}
|
||||
// id/name/arguments must survive verbatim
|
||||
if got[0]["id"] != "toolu_a" {
|
||||
t.Fatalf("id lost: %s", out)
|
||||
}
|
||||
fn := got[0]["function"].(map[string]interface{})
|
||||
if fn["name"] != "calc" {
|
||||
t.Fatalf("function.name lost: %s", out)
|
||||
}
|
||||
// subsequent argument fragments on the SAME upstream index reuse ordinal 0
|
||||
out2 := r.remap(json.RawMessage(`[{"index":1,"id":"","type":"function","function":{"name":"","arguments":"{\"a\":1}"}}]`))
|
||||
var got2 []map[string]interface{}
|
||||
if err := json.Unmarshal(out2, &got2); err != nil {
|
||||
t.Fatalf("unmarshal2: %v", err)
|
||||
}
|
||||
if got2[0]["index"] != float64(0) {
|
||||
t.Fatalf("fragment index = %v, want stable 0: %s", got2[0]["index"], out2)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple tools get distinct dense ordinals", func(t *testing.T) {
|
||||
r := newToolIndexRemapper()
|
||||
// thinking=0, tool_use=1, tool_use=2 -> 0, 1
|
||||
a := r.remap(json.RawMessage(`[{"index":1,"id":"t1","type":"function","function":{"name":"f1","arguments":""}}]`))
|
||||
b := r.remap(json.RawMessage(`[{"index":2,"id":"t2","type":"function","function":{"name":"f2","arguments":""}}]`))
|
||||
var ga, gb []map[string]interface{}
|
||||
json.Unmarshal(a, &ga)
|
||||
json.Unmarshal(b, &gb)
|
||||
if ga[0]["index"] != float64(0) {
|
||||
t.Fatalf("first tool index = %v, want 0", ga[0]["index"])
|
||||
}
|
||||
if gb[0]["index"] != float64(1) {
|
||||
t.Fatalf("second tool index = %v, want 1", gb[0]["index"])
|
||||
}
|
||||
// interleaved fragments keep their own ordinals
|
||||
a2 := r.remap(json.RawMessage(`[{"index":1,"id":"","type":"function","function":{"name":"","arguments":"x"}}]`))
|
||||
var ga2 []map[string]interface{}
|
||||
json.Unmarshal(a2, &ga2)
|
||||
if ga2[0]["index"] != float64(0) {
|
||||
t.Fatalf("t1 fragment index = %v, want 0", ga2[0]["index"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("already dense indices pass through untouched", func(t *testing.T) {
|
||||
r := newToolIndexRemapper()
|
||||
in := json.RawMessage(`[{"index":0,"id":"t","type":"function","function":{"name":"f","arguments":""}}]`)
|
||||
out := r.remap(in)
|
||||
if string(out) != string(in) {
|
||||
t.Fatalf("dense input was rewritten:\n in=%s\nout=%s", in, out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty and malformed input is safe", func(t *testing.T) {
|
||||
r := newToolIndexRemapper()
|
||||
if got := r.remap(nil); got != nil {
|
||||
t.Fatalf("nil -> %s", got)
|
||||
}
|
||||
bad := json.RawMessage(`not json`)
|
||||
if got := r.remap(bad); string(got) != string(bad) {
|
||||
t.Fatalf("malformed input must pass through: %s", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user