mirror of
https://gitcode.com/JianFeeeee/webui4frpc.git
synced 2026-10-15 12:14:37 +00:00
feat(cluster): 负载均衡迁移 —— 定向投递 + 两阶段提交
承接 875e580（只落决策逻辑）。本提交补齐运行时：周期检查、迟滞判定、
两阶段迁移，以及让迁移任务**定向**到指定节点。
## 为什么必须是定向的（核心安全点）
loadSnapshot() 的 Forwards 来自 ForwardsOwnedBy，所以旧 owner 一旦被摘掉条目，
它立刻变成最闲节点。而普通创建任务是"任何最闲者认领"—— 于是它会把刚迁走的
转发**当场抢回**，迁移变成"只重启一次 worker"的空操作。
故 Task 新增 Migrate + Target 字段，runCommands 的选择循环照 Restart 的范式
处理：Target == 自己才认领，否则 `continue` 让任务随环前进。
## 为什么必须两阶段（不能一次发 revoke + submit）
1. 旧 owner 变最闲后会把未定向的任务抢回（上面那条）；
2. PendingList 按 `t<ID>` **字符串**排序（`t10 < t5`），同轮内两个任务的先后
不可控。
Balancer 因此是显式状态机：phase1 发布 revoke 并等待旧条目真正消失，确认后才
发布 phase2 的定向 migrate。两个超时兜底：revoke 超时则放弃；migrate 超时
（target 没认领）则退回**无定向** SubmitTask，让"任何最闲者"接手 —— 转发不会
被丢，只是延迟。这与环本身的行为一致（实测：只要还有更闲的节点，pending 任务
最终会被认领；只有"全员都更忙"时才滞留）。
## 两个被测试逼出来的设计修正
- **绝对下限 minForwards=2**：只有"超均值倍数"在小集群失真——3 节点 1 条转发时
owner Score≈105 而两台空闲≈5，20 倍于均值会触发迁移，但那是完全正常的分布，
迁它只会断一条活跃连接。倍数负责看形状，下限负责"有没有东西值得搬"。
- **Migrate 也要绕过 duplicate-claim 守卫**，否则旧条目还在时目标节点会拒绝
认领（这条守卫正是为了防止同一 remotePort 注册两次 → frps 报
"proxy already exists"）。
## 测试（本项目第六次假绿的教训）
新增 6 条，全绿。做双向验证时发现
TestMigrateTaskIsAddressedNotClaimedByLightest **删掉定向分支后仍然 PASS**，
查因：`loadSnapshot()每次 OnToken 都用 Handler.RuntimeLoad() 刷新 mem/net，
我手工设的 State.Nodes[].Load 被静默覆盖 ⇒ n2/n3 始终平手 ⇒ "最闲者"恰好
就是 target，定向性从未被验证。
改为**用三个 Handler 的不同 RuntimeLoad 制造真实差异**（n3 最闲、n2 是较重的
target）后，删掉定向分支如期变红：
n3 (the LIGHTEST node) claimed 1 migrate task(s); the task is addressed
to n2 and must ignore load entirely
⇒ 这条纪律再次生效：测试必须制造出能让错误实现暴露的**不对称**，否则"恰好通过"
比"失败"更危险。
go build / go vet / go test ./... 全绿，gofmt 干净。
This commit is contained in:
JianFeeeee
@ -1,6 +1,7 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
@ -144,3 +145,212 @@ func TestPlanBalanceIgnoresDeadNodes(t *testing.T) {
|
||||
t.Fatal("chose an OFFLINE node as the migration destination")
|
||||
}
|
||||
}
|
||||
|
||||
// migrateRing builds a 3-node ring where n1 owns every forward, plus n2/n3 idle.
|
||||
func migrateRing(t *testing.T, ownCount int) (*Engine, *Engine, *Engine) {
|
||||
t.Helper()
|
||||
mk := func(id string, leader bool) *Engine {
|
||||
h := &fakeHandler{load: Load{MemPct: 5, NetPct: 5}}
|
||||
return NewEngine(id, id+":7500", "u", "p", "0.1.3", nil, h,
|
||||
func(ctx context.Context, next string, tk *Token) error { return nil },
|
||||
id+":7500", leader, "")
|
||||
}
|
||||
n1, n2, n3 := mk("n1", true), mk("n2", false), mk("n3", false)
|
||||
|
||||
seed := State{
|
||||
LeaderID: "n1",
|
||||
Nodes: []Node{
|
||||
{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true, Load: Load{Forwards: ownCount, MemPct: 5}},
|
||||
{ID: "n2", Addr: "n2:7500", Alive: true, Load: Load{MemPct: 5}},
|
||||
{ID: "n3", Addr: "n3:7500", Alive: true, Load: Load{MemPct: 5}},
|
||||
},
|
||||
Topology: map[string]*TopoEntry{},
|
||||
PendingTasks: map[string]*Task{},
|
||||
}
|
||||
for _, e := range []*Engine{n1, n2, n3} {
|
||||
var own State
|
||||
b, _ := json.Marshal(seed)
|
||||
if err := json.Unmarshal(b, &own); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e.AdoptState(own)
|
||||
}
|
||||
// n1 claims `ownCount` forwards.
|
||||
for i := 0; i < ownCount; i++ {
|
||||
n1.state.AddTopology(&Task{
|
||||
ID: "f" + string(rune('a'+i)),
|
||||
Local: store.Local{Name: "web"}, Remote: store.Remote{Name: "frps1"},
|
||||
Link: store.Link{RemotePort: 100 + i},
|
||||
}, "n1")
|
||||
}
|
||||
var snap State
|
||||
b, _ := json.Marshal(n1.state)
|
||||
_ = json.Unmarshal(b, &snap)
|
||||
for _, e := range []*Engine{n1, n2, n3} {
|
||||
e.AdoptState(snap)
|
||||
}
|
||||
return n1, n2, n3
|
||||
}
|
||||
|
||||
// TestMigrateTaskIsAddressedNotClaimedByLightest is the core safety test.
|
||||
//
|
||||
// After the revoke lands, the node we took the forward from becomes the
|
||||
// lightest — an untargeted creation task would be claimed straight back, making
|
||||
// the migration a no-op that merely restarts the worker. Migrate tasks must be
|
||||
// delivered to their named target only.
|
||||
//
|
||||
// The forward is removed from the topology first (as the balancer's phase 1
|
||||
// does), so this exercises the realistic post-revoke state.
|
||||
func TestMigrateTaskIsAddressedNotClaimedByLightest(t *testing.T) {
|
||||
n1, n2, n3 := migrateRing(t, 3)
|
||||
n1.State().RemoveTopology("web", "frps1", 100) // phase 1 has landed
|
||||
|
||||
n1.submitMigrate(store.Local{Name: "web"}, store.Remote{Name: "frps1"},
|
||||
store.Link{RemotePort: 100}, "n2")
|
||||
|
||||
// Give each node a DISTINCT load via its handler. loadSnapshot() refreshes
|
||||
// mem/net from Handler.RuntimeLoad() on every OnToken, so seeding
|
||||
// State.Nodes[].Load by hand is silently overwritten — which is why an
|
||||
// earlier version of this test had n2 and n3 tied and therefore passed even
|
||||
// with the targeted delivery removed (the "lightest" node happened to be the
|
||||
// target). n3 is now strictly the lightest, so an untargeted rule would hand
|
||||
// the task to n3 and fail here.
|
||||
claimedBy := map[string]int{}
|
||||
mk := func(id string, mem float64) Handler {
|
||||
return &fakeHandler{load: Load{MemPct: mem, NetPct: 0},
|
||||
claim: func(ctx context.Context, tk *Task) error {
|
||||
claimedBy[id]++
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
n1.Handler = mk("n1", 50)
|
||||
n2.Handler = mk("n2", 40) // the TARGET — heavier than n3
|
||||
n3.Handler = mk("n3", 1) // strictly the lightest
|
||||
var snap State
|
||||
b, _ := json.Marshal(n1.State())
|
||||
_ = json.Unmarshal(b, &snap)
|
||||
for _, e := range []*Engine{n1, n2, n3} {
|
||||
e.AdoptState(snap)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
for _, e := range []*Engine{n1, n2, n3} {
|
||||
if _, err := e.OnToken(context.Background(), &Token{
|
||||
Cycle: int64(i + 1), State: *e.State(), SentAt: int64(i + 1),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if claimedBy["n1"] != 0 {
|
||||
t.Errorf("n1 claimed %d migrate task(s); the old owner must never take it back",
|
||||
claimedBy["n1"])
|
||||
}
|
||||
if claimedBy["n3"] != 0 {
|
||||
t.Errorf("n3 (the LIGHTEST node) claimed %d migrate task(s); the task is addressed "+
|
||||
"to n2 and must ignore load entirely", claimedBy["n3"])
|
||||
}
|
||||
if claimedBy["n2"] != 1 {
|
||||
t.Errorf("the target n2 claimed %d time(s), want exactly 1", claimedBy["n2"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestMigrateRefusedWhileEntryStillOwned pins the duplicate guard: a migrate that
|
||||
// arrives before the revoke completed must NOT create a second topology entry
|
||||
// for the same forward. This is the failure that would otherwise leave two nodes
|
||||
// running workers for one remotePort — which frps rejects with
|
||||
// "proxy already exists" (observed 14k times before that was fixed).
|
||||
func TestMigrateRefusedWhileEntryStillOwned(t *testing.T) {
|
||||
n1, _, _ := migrateRing(t, 3)
|
||||
claimed := 0
|
||||
n1.Handler = &fakeHandler{load: Load{MemPct: 5, NetPct: 5},
|
||||
claim: func(ctx context.Context, tk *Task) error { claimed++; return nil }}
|
||||
|
||||
// Entry still owned by n1 — the revoke has NOT landed.
|
||||
n1.submitMigrate(store.Local{Name: "web"}, store.Remote{Name: "frps1"},
|
||||
store.Link{RemotePort: 100}, "n1")
|
||||
|
||||
if _, err := n1.OnToken(context.Background(), &Token{Cycle: 1, State: *n1.State(), SentAt: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claimed != 0 {
|
||||
t.Errorf("Claim ran %d time(s) while the forward was still owned; "+
|
||||
"this would create a duplicate proxy", claimed)
|
||||
}
|
||||
if n := len(n1.State().TopologyList()); n != 3 {
|
||||
t.Errorf("topology has %d entries, want 3 (no duplicate created)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBalancerWaitsForRevokeBeforePublishing covers the two-phase guarantee: no
|
||||
// migrate task may exist while the old entry is still present, because that is
|
||||
// what would let the old owner race the new destination.
|
||||
func TestBalancerWaitsForRevokeBeforePublishing(t *testing.T) {
|
||||
n1, _, _ := migrateRing(t, 3)
|
||||
b := NewBalancer(n1, 2, 0)
|
||||
|
||||
// First step starts phase 1 (revoke) and publishes nothing else.
|
||||
_ = b.Step(1000)
|
||||
if b.revoking == nil {
|
||||
t.Fatal("first step should have started a revoke")
|
||||
}
|
||||
for _, tk := range n1.State().PendingList() {
|
||||
if tk.Migrate {
|
||||
t.Fatal("a migrate task was published before the revoke completed")
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 1 completes (the owner drops the entry) → phase 2 publishes.
|
||||
n1.State().RemoveTopology("web", "frps1", 100)
|
||||
_ = b.Step(1010)
|
||||
if b.revoking != nil {
|
||||
t.Error("revoke phase should have finished")
|
||||
}
|
||||
found := false
|
||||
for _, tk := range n1.State().PendingList() {
|
||||
if tk.Migrate && tk.Target != "" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("after the revoke completed, a targeted migrate task should exist")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBalancerHysteresis: an imbalance must persist before anything moves, so a
|
||||
// momentary CPU spike never restarts a worker.
|
||||
func TestBalancerHysteresis(t *testing.T) {
|
||||
n1, _, _ := migrateRing(t, 3)
|
||||
b := NewBalancer(n1, 2, 600) // must be over for 10 minutes
|
||||
|
||||
_ = b.Step(1000) // first over-threshold observation: only starts the timer
|
||||
for _, tk := range n1.State().PendingList() {
|
||||
if tk.Revoke || tk.Migrate {
|
||||
t.Fatal("acted before the sustained window elapsed")
|
||||
}
|
||||
}
|
||||
// Still inside the window.
|
||||
_ = b.Step(1300)
|
||||
if b.revoking != nil {
|
||||
t.Fatal("acted 5 minutes in; the window is 10")
|
||||
}
|
||||
// Past the window.
|
||||
_ = b.Step(1700)
|
||||
if b.revoking == nil {
|
||||
t.Fatal("did not act after the imbalance persisted past the window")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBalancerQuietWhenBalanced: the threshold means a healthy ring is never
|
||||
// touched, which is the whole reason for not running this every hour blindly.
|
||||
func TestBalancerQuietWhenBalanced(t *testing.T) {
|
||||
n1, _, _ := migrateRing(t, 1)
|
||||
b := NewBalancer(n1, 2, 0)
|
||||
for i := 0; i < 5; i++ {
|
||||
if msg := b.Step(int64(1000 + i*600)); msg != "" {
|
||||
t.Fatalf("a balanced ring produced action: %q", msg)
|
||||
}
|
||||
}
|
||||
if b.InFlight() {
|
||||
t.Error("balancer started work on a balanced ring")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user