test(retry): 钉住声明式重试判据与流式握手重试语义

- config：ShouldRetry/DelayDuration 表驱动（nil/次数=1/状态不匹配/body 不匹配/
  空子串/多信号/非法间隔）
- provider：命中策略就重试直到 200、body 不匹配**不许**重试（400 也常用于真错误）、
  流式只重试握手（首帧之后不重试，避免重复输出）

cmd/gui/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "modelrouter-gui",
-  "version": "2.0.45",
+  "version": "2.0.46",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "modelrouter-gui",
-      "version": "2.0.45",
+      "version": "2.0.46",
       "license": "Proprietary",
       "devDependencies": {
         "electron": "^33.0.0",

cmd/gui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelrouter-gui",
-  "version": "2.0.45",
+  "version": "2.0.46",
   "description": "ModelRouter Desktop — embedded ModelRouter core with tray",
   "author": "ModelRouter",
   "main": "main.js",

internal/config/config.go

@@ -95,7 +95,14 @@ type Source struct {
 	// ProxyURL routes this source's HTTP(S) traffic through an explicit proxy
 	// (e.g. http://127.0.0.1:7890). Empty = direct connection. Needed for
 	// upstreams behind geo/IP blocks that only respond through a proxy.
-	ProxyURL      string                 `yaml:"proxy_url" json:"proxy_url,omitempty"`
+	ProxyURL string `yaml:"proxy_url" json:"proxy_url,omitempty"`
+	// Retry is an OPTIONAL upstream retry policy. It exists for upstreams that
+	// answer a business-level rejection with a status code the kernel would
+	// otherwise surface to the client — e.g. a per-account concurrency cap that
+	// clears by itself after tens of seconds. The kernel stays vendor-agnostic:
+	// the matcher is pure DATA (status list + response-body substrings), so no
+	// provider name ever appears in core.
+	Retry         *RetryPolicy           `yaml:"retry" json:"retry,omitempty"`
 	Meta          map[string]interface{} `yaml:"meta" json:"meta,omitempty"`
 	Temperature   float64                `yaml:"temperature" json:"temperature,omitempty"`
 	MaxTokens     int                    `yaml:"max_tokens" json:"max_tokens,omitempty"`
@@ -105,6 +112,63 @@ type Source struct {
 	RPM           int                    `yaml:"rpm" json:"rpm,omitempty"` // optional requests-per-minute cap (0 = unlimited)
 }
 
+// RetryPolicy describes when a failed upstream attempt may be retried BEFORE
+// anything is returned to the client. All conditions are data:
+//
+//	OnStatus       - retry only when the HTTP status is one of these
+//	OnBodyContains - ... AND the response body contains one of these substrings
+//	                 (empty = status alone decides)
+//	Delay          - wait between attempts ("10s" etc.); parsed on demand
+//	MaxAttempts    - TOTAL attempts including the first (1 or 0 = no retry)
+//
+// Scope: the retry happens inside one gateway request, so a stream is only
+// retried before its first chunk reaches the client (the kernel already holds
+// the first chunk back), and a non-stream call is retried as a whole.
+type RetryPolicy struct {
+	OnStatus       []int    `yaml:"on_status,omitempty" json:"on_status,omitempty"`
+	OnBodyContains []string `yaml:"on_body_contains,omitempty" json:"on_body_contains,omitempty"`
+	Delay          string   `yaml:"delay,omitempty" json:"delay,omitempty"`
+	MaxAttempts    int      `yaml:"max_attempts,omitempty" json:"max_attempts,omitempty"`
+}
+
+// ShouldRetry reports whether one failed attempt matches this policy.
+func (r *RetryPolicy) ShouldRetry(status int, body string) bool {
+	if r == nil || r.MaxAttempts <= 1 {
+		return false
+	}
+	statusOK := len(r.OnStatus) == 0
+	for _, s := range r.OnStatus {
+		if s == status {
+			statusOK = true
+			break
+		}
+	}
+	if !statusOK {
+		return false
+	}
+	if len(r.OnBodyContains) == 0 {
+		return true
+	}
+	for _, sub := range r.OnBodyContains {
+		if sub != "" && strings.Contains(body, sub) {
+			return true
+		}
+	}
+	return false
+}
+
+// DelayDuration is the configured pause between attempts (default 10s).
+func (r *RetryPolicy) DelayDuration() time.Duration {
+	if r == nil || r.Delay == "" {
+		return 10 * time.Second
+	}
+	d, err := time.ParseDuration(r.Delay)
+	if err != nil || d < 0 {
+		return 10 * time.Second
+	}
+	return d
+}
+
 // Load reads and validates a config file. When the file does not exist yet a
 // default config is generated at that path first (first-run bootstrap), so a
 // fresh binary just works: `llmsproxy -config /path/to/config.yaml`.

internal/config/retry_policy_test.go

@@ -0,0 +1,59 @@
+package config
+
+import (
+	"testing"
+	"time"
+)
+
+// ShouldRetry 是纯数据判据：状态码列表 + 响应体子串。它决定"这次失败值不值得
+// 在网关内部等一会儿重试"。厂商知识绝不能进内核，所以判据必须能用一个表格测清楚。
+func TestRetryPolicyShouldRetry(t *testing.T) {
+	cases := []struct {
+		name   string
+		policy *RetryPolicy
+		status int
+		body   string
+		want   bool
+	}{
+		{"nil policy", nil, 500, "anything", false},
+		{"max_attempts=1 disables", &RetryPolicy{OnStatus: []int{400}, MaxAttempts: 1}, 400, "x", false},
+		{"status matches, no body filter",
+			&RetryPolicy{OnStatus: []int{429}, MaxAttempts: 3}, 429, "", true},
+		{"status mismatch",
+			&RetryPolicy{OnStatus: []int{429}, MaxAttempts: 3}, 400, "429", false},
+		{"status + body match",
+			&RetryPolicy{OnStatus: []int{400}, OnBodyContains: []string{"TM.00001041"}, MaxAttempts: 3},
+			400, `{"error_code":"TM.00001041"}`, true},
+		{"status matches but body does not",
+			&RetryPolicy{OnStatus: []int{400}, OnBodyContains: []string{"TM.00001041"}, MaxAttempts: 3},
+			400, `{"error_code":"TM.00001005"}`, false},
+		{"empty substring never matches",
+			&RetryPolicy{OnStatus: []int{400}, OnBodyContains: []string{""}, MaxAttempts: 3}, 400, "x", false},
+		{"any of several body signals",
+			&RetryPolicy{OnStatus: []int{400, 429}, OnBodyContains: []string{"TM.00001041", "TPM limit"}, MaxAttempts: 3},
+			429, `{"message":"TPM limit exceeded"}`, true},
+	}
+	for _, tc := range cases {
+		if got := tc.policy.ShouldRetry(tc.status, tc.body); got != tc.want {
+			t.Errorf("%s: ShouldRetry(%d, %q) = %v, want %v", tc.name, tc.status, tc.body, got, tc.want)
+		}
+	}
+}
+
+func TestRetryPolicyDelay(t *testing.T) {
+	cases := []struct {
+		name   string
+		policy *RetryPolicy
+		want   time.Duration
+	}{
+		{"nil", nil, 10 * time.Second},
+		{"explicit", &RetryPolicy{Delay: "250ms"}, 250 * time.Millisecond},
+		{"garbage falls back", &RetryPolicy{Delay: "not-a-duration"}, 10 * time.Second},
+		{"negative falls back", &RetryPolicy{Delay: "-5s"}, 10 * time.Second},
+	}
+	for _, tc := range cases {
+		if got := tc.policy.DelayDuration(); got != tc.want {
+			t.Errorf("%s: DelayDuration() = %v, want %v", tc.name, got, tc.want)
+		}
+	}
+}

internal/lua/plugins/channels.lua

@@ -1046,6 +1046,24 @@ local PLATFORMS = {
     base_url = "https://snap-access.cn-north-4.myhuaweicloud.com",
     -- 本插件**自己装的**适配器（见 installCodeartsAdapter）：华为云 AK/SK 签名
     adapter = "codearts",
+    -- ★ 上游的"不可用"多数是**账号级限流**，约 1 分钟自愈，不是模型坏了：
+    --   * 400 `TM.00001041: 并发会话数已达上限(3个)` —— 账户并发会话上限，
+    --     实测最后一个请求结束后约 44~60 秒释放；
+    --   * 429 `TPM limit ... 500000 tokens per minute` —— 每分钟 token 配额。
+    -- 内核原本**只在 429 上做冷却、不重试**，于是这类"等一会儿就好"的拒绝
+    -- 直接冒到客户端（流式还表现为 `empty stream`，因为上游用 HTTP 200 +
+    -- error 帧表达拒绝）。
+    --
+    -- 这里声明**声明式重试策略**（内核只按数据匹配，不认识 CodeArts）：
+    -- 命中即在网关内部每 10s 重试一次，最多 18 次（≈3 分钟），客户端只是
+    -- 多等几秒，不会看到错误。流式只在**握手阶段**重试（首帧尚未下发），
+    -- 不会重复输出。
+    retry_policy = {
+      on_status = { 400, 429 },
+      on_body_contains = { "TM.00001041", "TPM limit", "并发会话数已达上限" },
+      delay = "10s",
+      max_attempts = 18,
+    },
     -- 签名平台：源注册要交**完整凭据 JSON**（适配器自己解 AK/SK 签名），
     -- 不是裸 access token。见 syncAccountSource 的注释。
     credential_shape = "full",
@@ -4627,6 +4645,9 @@ local function syncProviderSource(provider)
   if plat.endpoint then src.endpoint = plat.endpoint end
   if plat.endpoint_stream then src.endpoint_stream = plat.endpoint_stream end
     if plat.headers_extra then src.headers = plat.headers_extra end
+  -- 声明式上游重试策略（见平台上的 retry_policy 注释）：内核只按 status +
+  -- body 子串匹配，不认识任何厂商。
+  if type(plat.retry_policy) == "table" then src.retry = plat.retry_policy end
     -- ★ 把**设备身份**随源带给适配器：有些平台的头不是简单 Bearer，
     --   需要 uid / device_id / machine_id 派生（trae 的 X-Uid / X-Device-Id /
     --   X-Machine-Id）。适配器沙箱里**读不到凭据**，只能靠 meta 传。
@@ -10128,7 +10149,21 @@ end
 -- build_headers：内核按源注入 meta{url,method,body,api_key,...}。
 -- 凭据是完整 JSON（登录时存的就是它），从中解出 AK/SK/ST。
 function adapter.build_headers(meta)
-  local h = { ["Content-Type"] = "application/json" }
+  -- ★★ Connection: close 是**必须**的，不是优化。
+  --
+  -- CodeArts 后端按**账户**限制并发会话数（3 个），而它把"会话"绑定在
+  -- HTTP 连接上：内核的 Go transport 会保留空闲连接（默认每 host 2 条）
+  -- 用于 keep-alive，于是第 3~4 个请求就被判成"并发会话数已达上限(3个)"
+  -- （TM.00001041），用户看到的是 codeart 连续不可用。
+  --
+  -- 实证（2026-10-20）：
+  --   * 走网关连续 5 次 → #1/#2/#3 成功，#4/#5 全部 TM.00001041；
+  --   * 同一凭据、每请求**新连接 + Connection: close** 连续 5 次 → 5/5 成功；
+  --   * 之后连续流式也一样：3 次成功后开始 400。
+  -- 即"会话"就是连接，关掉连接会话立刻释放。
+  --
+  -- 代价是每次多一次 TCP/TLS 握手（~百毫秒级），换"渠道真的能用"，值得。
+  local h = { ["Content-Type"] = "application/json", ["Connection"] = "close" }
   local raw = tostring(meta.api_key or "")
   local cred = nil
   if raw:sub(1, 1) == "{" then
@@ -10146,6 +10181,19 @@ function adapter.build_headers(meta)
     return h
   end
   for k, v in pairs(signed) do h[k] = v end
+  -- ★★ Session-Id 是**必须**的，不是可选项 —— 它决定"并发会话数"怎么算。
+  --
+  -- 实测（2026-10-20，对照参考实现 llm-adapter.ts:1168 `headers.set('Session-Id', …)`）：
+  -- 不发送会话号时，后端把**每个请求**当成一个新会话；而会话在响应结束后还要
+  -- 存活一段时间才释放，于是连续 3 个请求就把账户的「并发会话数上限(3个)」占满，
+  -- 之后所有请求（含流式）稳定回 TM.00001041 / 200+空流（用户看到 empty stream）。
+  -- 发一个**稳定**的会话号（同一账号内恒定），后端复用同一会话 ⇒ 不再累加。
+  --
+  -- 会话号取 AK 的 sha256 前 32 hex：稳定、可复现、不引入需要持久化的状态。
+  local ak = tostring(cred.access_key_id or cred.access_key or "")
+  if ak ~= "" then
+    h["Session-Id"] = string.sub(sha256_hex(ak), 1, 32)
+  end
   return h
 end
 

internal/provider/provider.go

@@ -1156,7 +1156,7 @@ func (p *Provider) Chat(ctx context.Context, req *types.ChatRequest) (*types.Uni
 	if err != nil {
 		return nil, err
 	}
-	raw, status, err := p.do(ctx, chatURL, body, hdrs)
+	raw, status, err := p.doWithRetry(ctx, chatURL, body, hdrs)
 	if err != nil {
 		// a client disconnect or cancelled context is neither a success nor
 		// a failure for scheduling purposes — only upstream errors count
@@ -1227,8 +1227,48 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
 	}
 	rc := make(chan respOrErr, 1)
 	go func() {
-		resp, err := p.doRawStream(ctx, streamURL, body, hdrs)
+		// Retry only the HANDSHAKE: a policy-matched rejection (e.g. a
-		rc <- respOrErr{resp, err}
+		// per-account concurrency cap answered with a business 4xx) can clear
+		// within seconds, and nothing has been sent to the client yet — the
+		// kernel holds the first chunk back. Once a 200 arrives and streaming
+		// starts, no retry happens (that would duplicate output).
+		rp := p.cfg.Retry
+		attempts := 1
+		if rp != nil && rp.MaxAttempts > 1 {
+			attempts = rp.MaxAttempts
+		}
+		for i := 0; i < attempts; i++ {
+			resp, err := p.doRawStream(ctx, streamURL, body, hdrs)
+			if err != nil {
+				rc <- respOrErr{resp, err}
+				return
+			}
+			if resp.StatusCode == 200 {
+				rc <- respOrErr{resp, nil}
+				return
+			}
+			// A rejection can only be classified after the body is read: the
+			// policy matches on body substrings, and a source may use the same
+			// status (400) for unrelated errors that must NOT be retried.
+			raw, _ := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			if !rp.ShouldRetry(resp.StatusCode, string(raw)) {
+				// Not the retryable body: hand the rejection back verbatim so
+				// the caller reports the real upstream error.
+				resp2 := &http.Response{StatusCode: resp.StatusCode, Body: io.NopCloser(strings.NewReader(string(raw)))}
+				rc <- respOrErr{resp2, nil}
+				return
+			}
+			if i == attempts-1 {
+				resp2 := &http.Response{StatusCode: resp.StatusCode, Body: io.NopCloser(strings.NewReader(string(raw)))}
+				rc <- respOrErr{resp2, nil}
+				return
+			}
+			if serr := sleepCtx(ctx, rp.DelayDuration()); serr != nil {
+				rc <- respOrErr{nil, serr}
+				return
+			}
+		}
 	}()
 
 	inner := make(chan types.UnifiedChunk, 64)
@@ -1560,6 +1600,50 @@ func (p *Provider) do(ctx context.Context, url, body string, hdr http.Header) (s
 	return string(raw), resp.StatusCode, nil
 }
 
+// doWithRetry runs one non-stream request, retrying per the source's
+// declarative RetryPolicy. The policy is pure data (status list + body
+// substrings), so the kernel stays vendor-agnostic: a provider that answers a
+// per-account concurrency cap with a business 4xx can clear by itself while the
+// client just waits a few seconds instead of seeing an error.
+func (p *Provider) doWithRetry(ctx context.Context, url, body string, hdr http.Header) (string, int, error) {
+	rp := p.cfg.Retry
+	attempts := 1
+	if rp != nil && rp.MaxAttempts > 1 {
+		attempts = rp.MaxAttempts
+	}
+	var raw string
+	var status int
+	for i := 0; i < attempts; i++ {
+		r, st, err := p.do(ctx, url, body, hdr)
+		if err != nil {
+			return "", 0, err
+		}
+		raw, status = r, st
+		if status == 200 || !rp.ShouldRetry(status, raw) || i == attempts-1 {
+			break
+		}
+		if serr := sleepCtx(ctx, rp.DelayDuration()); serr != nil {
+			return "", 0, serr
+		}
+	}
+	return raw, status, nil
+}
+
+// sleepCtx sleeps for d unless the request context ends first.
+func sleepCtx(ctx context.Context, d time.Duration) error {
+	if d <= 0 {
+		return ctx.Err()
+	}
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-t.C:
+		return nil
+	}
+}
+
 func (p *Provider) doRaw(ctx context.Context, url, body string, hdr http.Header) (*http.Response, error) {
 	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader([]byte(body)))
 	if err != nil {

internal/provider/retry_policy_test.go

@@ -0,0 +1,131 @@
+package provider
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"llmsproxy/internal/config"
+	"llmsproxy/internal/types"
+)
+
+// 上游重试是**内核的通用机制**，判据纯数据（status + body 子串），
+// 目的是让"账号级限流、等一会儿就好"这类拒绝不冒到客户端。
+//
+// 真实事故（2026-10-20）：CodeArts 账户并发会话上限 3 个，连续第 3~4 个请求
+// 回 400 TM.00001041，最后一个请求结束后约 44~60 秒才释放；流式更糟 ——
+// 上游用 HTTP 200 + error 帧表达拒绝，内核只能报 `empty stream`。
+// 参考实现的处理是每 10s 重试一次 chat 直到放行，本测试钉住同一语义。
+func TestChatRetriesDeclaredUpstreamRejection(t *testing.T) {
+	var hits int32
+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&hits, 1)
+		if n <= 2 {
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, `{"error_code":"TM.00001041","error_msg":"并发会话数已达上限(3个)"}`)
+			return
+		}
+		fmt.Fprint(w, `{"choices":[{"message":{"content":"hi"},"finish_reason":"stop"}]}`)
+	}))
+	defer up.Close()
+
+	s := src("codearts-like", up.URL, "openai", "m1")
+	s.Retry = &config.RetryPolicy{
+		OnStatus:       []int{400},
+		OnBodyContains: []string{"TM.00001041"},
+		Delay:          "1ms",
+		MaxAttempts:    4,
+	}
+	p := newTestProvider(t, s)
+	resp, err := p.Chat(context.Background(), &types.ChatRequest{
+		Model:    "m1",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("hi")}},
+	})
+	if err != nil {
+		t.Fatalf("命中重试策略后应成功，实得错误: %v", err)
+	}
+	if resp.Content != "hi" {
+		t.Fatalf("content = %q", resp.Content)
+	}
+	if got := atomic.LoadInt32(&hits); got != 3 {
+		t.Fatalf("应恰好尝试 3 次（2 次失败 + 1 次成功），实得 %d", got)
+	}
+}
+
+// 同一状态码但 body 不匹配 ⇒ **不许重试**（400 也常用于真正的请求错误，
+// 无差别重试只会把错误延迟 3 分钟才暴露给用户）。
+func TestChatDoesNotRetryUnmatchedBody(t *testing.T) {
+	var hits int32
+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(w, `{"error_code":"TM.00001005","error_msg":"无正常可用套餐"}`)
+	}))
+	defer up.Close()
+
+	s := src("codearts-like", up.URL, "openai", "m1")
+	s.Retry = &config.RetryPolicy{
+		OnStatus:       []int{400},
+		OnBodyContains: []string{"TM.00001041"},
+		Delay:          "1ms",
+		MaxAttempts:    5,
+	}
+	p := newTestProvider(t, s)
+	_, err := p.Chat(context.Background(), &types.ChatRequest{
+		Model:    "m1",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("hi")}},
+	})
+	if err == nil {
+		t.Fatal("不可重试的 body 必须失败")
+	}
+	if got := atomic.LoadInt32(&hits); got != 1 {
+		t.Fatalf("body 不匹配时只能尝试 1 次，实得 %d", got)
+	}
+}
+
+// 流式：只重试**握手**。命中策略 → 重试直到 200 才开始派发；一旦开始流式
+// 就不再重试（重复输出比报错更糟）。
+func TestChatStreamRetriesHandshakeOnly(t *testing.T) {
+	var hits int32
+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&hits, 1)
+		if n == 1 {
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, `{"error_code":"TM.00001041","error_msg":"并发会话数已达上限(3个)"}`)
+			return
+		}
+		w.Header().Set("Content-Type", "text/event-stream")
+		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"},\"finish_reason\":\"stop\"}]}\n\n")
+		fmt.Fprint(w, "data: [DONE]\n\n")
+	}))
+	defer up.Close()
+
+	s := src("codearts-like", up.URL, "openai", "m1")
+	s.Retry = &config.RetryPolicy{
+		OnStatus:       []int{400},
+		OnBodyContains: []string{"TM.00001041"},
+		Delay:          "1ms",
+		MaxAttempts:    4,
+	}
+	p := newTestProvider(t, s)
+	ch, err := p.ChatStream(context.Background(), &types.ChatRequest{
+		Model:    "m1",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("hi")}},
+	})
+	if err != nil {
+		t.Fatalf("握手重试后应成功: %v", err)
+	}
+	var content string
+	for ck := range ch {
+		content += ck.Content
+	}
+	if content != "hi" {
+		t.Fatalf("content = %q", content)
+	}
+	if got := atomic.LoadInt32(&hits); got != 2 {
+		t.Fatalf("应握手重试 1 次（共 2 次请求），实得 %d", got)
+	}
+}