mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-10-03 15:44:05 +00:00
fix(provider): fail the candidate on error-only streams instead of serving empty replies
Some upstreams (zen free pool) answer HTTP 200 with a single-chunk stream whose only payload is finish_reason:"network_error" and an empty delta. ChatStream used to hand that channel up as a success, so clients received a laundered empty reply (agents ended their turn mid-loop) and the broken slot kept its scheduling preference. ChatStream now holds back the first chunk: an error-only done chunk fails the candidate before any byte reaches the gateway, letting the scheduler fall through to the next source. Standard finish reasons (including instant-empty "stop") are never classified as errors.
This commit is contained in:
@ -417,7 +417,8 @@ func TestChatClientCancelNotRecorded(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestChatStreamEmptyBodyNotSuccess: a 200 that yields zero chunks and no
|
||||
// [DONE] is a failure before the first chunk — the slot must back off.
|
||||
// [DONE] must fail the candidate (error return, no delivered chunks) so the
|
||||
// scheduler falls through to the next source, and record the failure.
|
||||
func TestChatStreamEmptyBodyNotSuccess(t *testing.T) {
|
||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
@ -429,15 +430,12 @@ func TestChatStreamEmptyBodyNotSuccess(t *testing.T) {
|
||||
Model: "m",
|
||||
Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("stream: %v", err)
|
||||
}
|
||||
got := 0
|
||||
for range ch {
|
||||
got++
|
||||
}
|
||||
if got != 0 {
|
||||
t.Fatalf("want empty stream, got %d chunks", got)
|
||||
if err == nil {
|
||||
if ch != nil {
|
||||
for range ch {
|
||||
}
|
||||
}
|
||||
t.Fatal("empty stream must fail the candidate for scheduler fallback")
|
||||
}
|
||||
eventually(t, 2*time.Second, func() bool {
|
||||
_, fc, _ := p.ModelHealthInfo("m")
|
||||
@ -445,6 +443,62 @@ func TestChatStreamEmptyBodyNotSuccess(t *testing.T) {
|
||||
}, "empty stream must record a failure")
|
||||
}
|
||||
|
||||
// TestChatStreamErrorFinishFailsCandidate: zen free pool answers HTTP 200
|
||||
// with a single-chunk stream whose only payload is finish_reason:"network_error"
|
||||
// and an empty delta. That must fail the candidate (not deliver a laundered
|
||||
// empty reply), so the scheduler can fall through to the next source.
|
||||
func TestChatStreamErrorFinishFailsCandidate(t *testing.T) {
|
||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprint(w, `data: {"choices":[{"index":0,"finish_reason":"network_error","delta":{"role":"assistant","content":""}}]}`+"\n\n")
|
||||
fmt.Fprint(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
defer up.Close()
|
||||
p := newTestProvider(t, src("mock", up.URL, "opencode", "m"))
|
||||
_, err := p.ChatStream(context.Background(), &types.ChatRequest{
|
||||
Model: "m",
|
||||
Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("network_error-only stream must fail the candidate")
|
||||
}
|
||||
eventually(t, 2*time.Second, func() bool {
|
||||
_, fc, _ := p.ModelHealthInfo("m")
|
||||
return fc >= 1
|
||||
}, "error-only stream must record a failure")
|
||||
}
|
||||
|
||||
// TestChatStreamInstantEmptyStopStillDelivered: a legitimate completion that
|
||||
// ends immediately with the standard finish_reason:"stop" and zero content is
|
||||
// NOT an upstream error and must still reach the client.
|
||||
func TestChatStreamInstantEmptyStopStillDelivered(t *testing.T) {
|
||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprint(w, `data: {"choices":[{"index":0,"finish_reason":"stop","delta":{}}]}`+"\n\n")
|
||||
fmt.Fprint(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
defer up.Close()
|
||||
p := newTestProvider(t, src("mock", up.URL, "openai", "m"))
|
||||
ch, err := p.ChatStream(context.Background(), &types.ChatRequest{
|
||||
Model: "m",
|
||||
Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("standard stop stream must be delivered: %v", err)
|
||||
}
|
||||
n := 0
|
||||
for range ch {
|
||||
n++
|
||||
}
|
||||
if n == 0 {
|
||||
t.Fatal("expected at least the terminating chunk")
|
||||
}
|
||||
eventually(t, 2*time.Second, func() bool {
|
||||
_, fc, _ := p.ModelHealthInfo("m")
|
||||
return fc == 0
|
||||
}, "standard stop must count as success")
|
||||
}
|
||||
|
||||
// TestImageAutoUsesImageModel: AUTO image generation on a mixed source must
|
||||
// send the image-kind model id, never the best chat model.
|
||||
func TestImageAutoUsesImageModel(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user