fix(provider): fail the candidate on error-only streams instead of serving empty replies

Some upstreams (zen free pool) answer HTTP 200 with a single-chunk stream
whose only payload is finish_reason:"network_error" and an empty delta.
ChatStream used to hand that channel up as a success, so clients received
a laundered empty reply (agents ended their turn mid-loop) and the broken
slot kept its scheduling preference. ChatStream now holds back the first
chunk: an error-only done chunk fails the candidate before any byte
reaches the gateway, letting the scheduler fall through to the next
source. Standard finish reasons (including instant-empty "stop") are never
classified as errors.
This commit is contained in:
JianFeeeee
2026-08-24 15:41:45 +08:00
parent bb3af3bdb3
commit e19cff248f
2 changed files with 130 additions and 16 deletions

View File

@ -777,7 +777,7 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
rc <- respOrErr{resp, err}
}()
ch := make(chan types.UnifiedChunk, 64)
inner := make(chan types.UnifiedChunk, 64)
sel := <-rc
if sel.err != nil {
// client disconnect/cancel before the first byte: not a scheduling
@ -797,11 +797,12 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
}
go func() {
defer p.Release()
defer close(ch)
defer close(inner)
defer sel.resp.Body.Close()
scanner := bufio.NewScanner(sel.resp.Body)
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
var chunks int
var realChunks int
var doneSeen bool
var doneSent bool
for scanner.Scan() {
@ -821,7 +822,7 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
// finish_reason downstream.
if !doneSent {
select {
case ch <- types.UnifiedChunk{Done: true}:
case inner <- types.UnifiedChunk{Done: true}:
case <-ctx.Done():
}
}
@ -844,9 +845,12 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
if ck.Done {
doneSent = true
}
if !errorOnlyChunk(ck) {
realChunks++
}
chunks++
select {
case ch <- ck:
case inner <- ck:
case <-ctx.Done():
return
}
@ -857,15 +861,71 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
// success nor failure for scheduling purposes. A 200 that produced
// zero chunks and no [DONE] is an empty stream, i.e. a failure
// before the first chunk — record it so the slot can fall back.
// A stream whose only payload was error-only done chunks (e.g.
// finish_reason:"network_error") is likewise a failure, not a success.
if ctx.Err() == nil && scanner.Err() == nil {
if doneSeen || chunks > 0 {
if realChunks > 0 || (doneSeen && chunks == 0) {
p.RecordSuccess(model)
} else {
p.RecordFailure(model, 0)
}
}
}()
return ch, nil
// Hold back the first chunk to validate the stream actually carries
// content: some upstreams answer HTTP 200 with a degenerate stream whose
// only payload is an error finish reason (zen free pool sends
// finish_reason:"network_error" with empty delta). Failing the candidate
// here — before any byte reaches the gateway — lets the scheduler fall
// through to the next source instead of serving the client an empty reply.
type heldChunk struct {
ck types.UnifiedChunk
ok bool
}
var first heldChunk
select {
case ck, ok := <-inner:
first = heldChunk{ck, ok}
case <-ctx.Done():
return nil, ctx.Err()
}
if !first.ok {
return nil, fmt.Errorf("provider %s: empty stream", p.Name())
}
if errorOnlyChunk(first.ck) {
go func() {
for range inner {
}
}()
return nil, fmt.Errorf("provider %s: upstream returned %q stream",
p.Name(), first.ck.FinishReason)
}
out := make(chan types.UnifiedChunk, 64)
go func() {
defer close(out)
out <- first.ck
for ck := range inner {
out <- ck
}
}()
return out, nil
}
// errorOnlyChunk reports whether ck carries nothing but an upstream error
// signal: a done chunk with a non-standard finish reason and zero content,
// tool calls, reasoning text or usage. Standard OpenAI finish reasons are
// never classified as errors, so legitimate instant-empty completions
// (finish_reason:"stop", no output) still reach the client.
func errorOnlyChunk(ck types.UnifiedChunk) bool {
if !ck.Done || ck.FinishReason == "" {
return false
}
switch ck.FinishReason {
case "stop", "length", "tool_calls", "function_call", "content_filter":
return false
}
return ck.Content == "" && len(ck.ToolCalls) == 0 &&
ck.ReasoningContent == "" && ck.Usage == nil
}
// Image generates images via /v1/images/generations. Same scheduling-state