mirror of
https://gitcode.com/JianFeeeee/webui4frpc.git
synced 2026-10-15 12:14:37 +00:00
feat(cluster,httpapi): GET /cluster/balance 只读负载视图
## 为什么需要它
balancer 的判断原本只能从日志看,而且**只在"确实失衡 且 迟滞窗口已满"时才输出**。
运维问"这个集群均衡吗"——在健康状态下得到的是**沉默**,与"负载均衡没开"
完全无法区分。这个端点把它的判断做成可主动查询的。
## 关键性质:Report() 不扰动迟滞计时器
Report 只读,**不碰 overSince**。否则好奇的运维反复轮询这个端点就会不断重置
"失衡已持续多久"的计时,导致真正的迁移永远不会触发——一个"看一看就把它看没了"
的只读接口。测试 TestReportDoesNotDisturbHysteresis 正是钉住这点:轮询 5 次后
计时器必须原封不动,且随后满足窗口时迁移仍要发生。
## 只 leader 报计划
`wouldMigrate`/`overloadedSince` 仅在 `IsLeader()` 时给出。非 leader 仍返回各节点
Score(那是信息性的),但不报"会迁什么"——否则运维随便查一台节点会拿到与集群
实际行为不符的答案。TestReportOnFollowerHasNoPlan 钉住。
## 返回内容
nodes[] 各节点 Forwards/MemPct/NetPct/Score
isLeader 只有 leader 会行动
overloadedSince 超载已持续多久(迟滞进度)
wouldMigrate 窗口满足后会迁哪条、从哪到哪
inFlight 两阶段迁移进行中(revoke 慢 vs 卡死可区分)
dryRun / enabled 当前是观察还是真迁
## 接线
Balancer 在 NewBalancer 时就创建(HTTP 层需要),W4F_BALANCE_* 解析完再用
SetRatioAndSustain 应用;新增 SetDryRun / DryRun / SetRatioAndSustain,
避免初始化顺序耦合。Handler.Balancer 为 nil 时端点返回 404。
## 测试
新增 4 条:Report 不扰动迟滞(轮询后计时器不变、且迁移仍能触发)、
follower 无计划、窗口满足前后计划的有无、in-flight 可见。
go clean -testcache 后逐包实跑 7/7 全绿。
This commit is contained in:
@ -389,6 +389,11 @@ func main() {
|
||||
}
|
||||
})
|
||||
|
||||
// Load balancer is created here (before the Handler) so the HTTP layer can
|
||||
// expose its read-only report. Mode/parameters are applied further down, once
|
||||
// the W4F_BALANCE_* environment has been parsed.
|
||||
balancer := cluster.NewBalancer(ring, 2, 1800)
|
||||
|
||||
h := &httpapi.Handler{
|
||||
Store: st,
|
||||
Process: pm,
|
||||
@ -580,7 +585,7 @@ func main() {
|
||||
if balInterval < time.Second {
|
||||
balInterval = 10 * time.Second
|
||||
}
|
||||
balancer := cluster.NewBalancer(ring, balRatio, balSustain)
|
||||
balancer.SetRatioAndSustain(balRatio, balSustain)
|
||||
if balMode == "on" {
|
||||
log.Printf("ring[%s] load balancing ENABLED (ratio=%.2f sustain=%ds interval=%s)",
|
||||
selfID, balRatio, balSustain, balInterval)
|
||||
|
||||
@ -417,3 +417,107 @@ func TestOnlyLeaderBalances(t *testing.T) {
|
||||
}
|
||||
_ = before
|
||||
}
|
||||
|
||||
// TestReportDoesNotDisturbHysteresis: Report is a read-only view that an operator
|
||||
// can call at will. If it touched the overSince timer, a curious operator
|
||||
// polling the endpoint would keep resetting the sustained-imbalance clock and
|
||||
// balancing would never fire. It must observe without perturbing.
|
||||
func TestReportDoesNotDisturbHysteresis(t *testing.T) {
|
||||
n1, _, _ := migrateRing(t, 3)
|
||||
b := NewBalancer(n1, 2, 1800)
|
||||
|
||||
// First Step only starts the timer (no action yet).
|
||||
_ = b.Step(1000)
|
||||
started, seen := b.overSince["n1"]
|
||||
if !seen {
|
||||
t.Fatal("precondition: the overload timer should have been started")
|
||||
}
|
||||
|
||||
// Poll the report repeatedly, as an operator would.
|
||||
for i := 0; i < 5; i++ {
|
||||
rep := b.Report(int64(2000 + i*600))
|
||||
if len(rep.Nodes) != 3 {
|
||||
t.Fatalf("report listed %d nodes, want 3", len(rep.Nodes))
|
||||
}
|
||||
}
|
||||
if got := b.overSince["n1"]; got != started {
|
||||
t.Errorf("Report reset the overload timer: %d -> %d", started, got)
|
||||
}
|
||||
|
||||
// And the real path must still work: past the window it acts.
|
||||
_ = b.Step(1000 + 1801)
|
||||
if b.revoking == nil {
|
||||
t.Error("polling the report suppressed the migration it should have allowed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReportOnFollowerHasNoPlan: only the leader acts, so a follower must not
|
||||
// claim it would migrate anything — otherwise an operator querying a random node
|
||||
// gets an answer that does not reflect what the cluster will actually do.
|
||||
func TestReportOnFollowerHasNoPlan(t *testing.T) {
|
||||
_, n2, _ := migrateRing(t, 3) // n2 is not the leader
|
||||
b := NewBalancer(n2, 2, 0)
|
||||
b.overSince = map[string]int64{"n1": 1000}
|
||||
|
||||
rep := b.Report(9999)
|
||||
if rep.IsLeader {
|
||||
t.Fatal("precondition: n2 must not be the leader")
|
||||
}
|
||||
if rep.WouldMigrate != nil {
|
||||
t.Errorf("a follower reported a migration plan: %+v", rep.WouldMigrate)
|
||||
}
|
||||
if len(rep.Nodes) == 0 {
|
||||
t.Error("a follower should still report the node scores (that part is informational)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReportShowsPlanOnceHysteresisSatisfied pins the operator-facing contract:
|
||||
// before the window elapses there is no plan; after it, the plan appears with
|
||||
// the concrete forward and both endpoints.
|
||||
func TestReportShowsPlanOnceHysteresisSatisfied(t *testing.T) {
|
||||
n1, _, _ := migrateRing(t, 3)
|
||||
b := NewBalancer(n1, 2, 1800)
|
||||
|
||||
// Start the timer but do not satisfy the window.
|
||||
b.overSince = map[string]int64{"n1": 1000}
|
||||
if rep := b.Report(1500); rep.WouldMigrate != nil {
|
||||
t.Errorf("plan reported before the sustained window elapsed: %+v", rep.WouldMigrate)
|
||||
}
|
||||
|
||||
rep := b.Report(1000 + 1801)
|
||||
if rep.WouldMigrate == nil {
|
||||
t.Fatal("no plan reported even though the imbalance is past the window")
|
||||
}
|
||||
m := rep.WouldMigrate
|
||||
if m.From == "" || m.To == "" || m.Local == "" || m.Port == 0 {
|
||||
t.Fatalf("plan is incomplete: %+v", m)
|
||||
}
|
||||
if m.From == m.To {
|
||||
t.Fatalf("plan moves a forward to the node it already lives on: %+v", m)
|
||||
}
|
||||
if !rep.Enabled == false && rep.DryRun {
|
||||
// sanity: default is observe-only
|
||||
if !rep.DryRun {
|
||||
t.Error("balancer should default to dry-run")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestReportSurfacesInFlightMigration: the two-phase machine must be visible while
|
||||
// it runs, otherwise an operator sees a stalled plan and cannot tell a slow revoke
|
||||
// from a stuck one.
|
||||
func TestReportSurfacesInFlightMigration(t *testing.T) {
|
||||
n1, _, _ := migrateRing(t, 3)
|
||||
b := NewBalancer(n1, 2, 0)
|
||||
_ = b.Step(1000) // starts phase 1
|
||||
if !b.InFlight() {
|
||||
t.Fatal("precondition: a migration should be in flight")
|
||||
}
|
||||
rep := b.Report(1001)
|
||||
if rep.InFlight == nil {
|
||||
t.Fatal("report did not surface the in-flight migration")
|
||||
}
|
||||
if rep.InFlight.Local == "" || rep.InFlight.To == "" {
|
||||
t.Fatalf("in-flight plan is incomplete: %+v", rep.InFlight)
|
||||
}
|
||||
}
|
||||
|
||||
@ -154,6 +154,9 @@ type Balancer struct {
|
||||
// what stops a transient CPU/network spike from restarting workers.
|
||||
sustainedSec int64
|
||||
|
||||
// dryRun records the mode set at construction so Report can surface it.
|
||||
// SetDryRun updates it; main.go sets it once from W4F_BALANCE_MODE.
|
||||
dryRun bool
|
||||
// overSince records when the currently-overloaded node first crossed the
|
||||
// threshold. Zero means "not currently over".
|
||||
overSince map[string]int64
|
||||
@ -183,10 +186,17 @@ func NewBalancer(e *Engine, ratio float64, sustainedSec int64) *Balancer {
|
||||
}
|
||||
return &Balancer{
|
||||
engine: e, ratio: ratio, sustainedSec: sustainedSec,
|
||||
dryRun: true, // safe default: report, never move
|
||||
overSince: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
// SetDryRun selects the mode. Dry-run (the default) only reports.
|
||||
func (b *Balancer) SetDryRun(v bool) { b.dryRun = v }
|
||||
|
||||
// DryRun reports whether the balancer is in observe-only mode.
|
||||
func (b *Balancer) DryRun() bool { return b.dryRun }
|
||||
|
||||
// InFlight reports whether a migration is currently between phases.
|
||||
func (b *Balancer) InFlight() bool {
|
||||
return b.revoking != nil || b.publishing != nil
|
||||
@ -357,3 +367,107 @@ func (b *Balancer) stepDryRun(now int64) string {
|
||||
return "would migrate " + pick.Local.Name + ":" + strconv.Itoa(pick.Link.RemotePort) +
|
||||
" from " + pick.OwnerID + " to " + dest
|
||||
}
|
||||
|
||||
// BalanceReport is a read-only view of what the balancer sees and would do.
|
||||
//
|
||||
// It exists because the balancer's decision is otherwise only visible in the
|
||||
// log, and only when an imbalance exists AND the hysteresis window has elapsed.
|
||||
// An operator asking "is this cluster balanced?" gets silence in exactly the
|
||||
// healthy case, which is indistinguishable from "balancing is switched off".
|
||||
type BalanceReport struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DryRun bool `json:"dryRun"`
|
||||
Ratio float64 `json:"ratio"`
|
||||
SustainSec int64 `json:"sustainSec"`
|
||||
// Only the leader acts; on a follower these stay false/empty.
|
||||
IsLeader bool `json:"isLeader"`
|
||||
// Current per-node scores.
|
||||
Nodes []BalanceNodeReport `json:"nodes"`
|
||||
// Overloaded (above ratio x mean) and sustained past the window, if any.
|
||||
OverloadedSince int64 `json:"overloadedSince,omitempty"`
|
||||
// The migration that would happen on the next Step, if any.
|
||||
WouldMigrate *BalanceMigrationPlan `json:"wouldMigrate,omitempty"`
|
||||
// Set when a migration is between phases.
|
||||
InFlight *BalanceMigrationPlan `json:"inFlight,omitempty"`
|
||||
}
|
||||
|
||||
type BalanceNodeReport struct {
|
||||
ID string `json:"id"`
|
||||
Alive bool `json:"alive"`
|
||||
IsLeader bool `json:"isLeader"`
|
||||
Forwards int `json:"forwards"`
|
||||
MemPct float64 `json:"memPct"`
|
||||
NetPct float64 `json:"netPct"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
type BalanceMigrationPlan struct {
|
||||
Local string `json:"local"`
|
||||
Remote string `json:"remote"`
|
||||
Port int `json:"remotePort"`
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
}
|
||||
|
||||
// Report renders the balancer's current view. It never mutates anything —
|
||||
// note that it does NOT touch overSince, so calling it cannot start or reset
|
||||
// the hysteresis timer that the real Step path uses.
|
||||
func (b *Balancer) Report(now int64) BalanceReport {
|
||||
rep := BalanceReport{
|
||||
Enabled: !b.dryRun,
|
||||
DryRun: b.dryRun,
|
||||
Ratio: b.ratio,
|
||||
SustainSec: int64(b.sustainedSec),
|
||||
IsLeader: b.engine.IsLeader(),
|
||||
}
|
||||
for i := range b.engine.state.Nodes {
|
||||
n := &b.engine.state.Nodes[i]
|
||||
rep.Nodes = append(rep.Nodes, BalanceNodeReport{
|
||||
ID: n.ID, Alive: n.Alive, IsLeader: n.IsLeader,
|
||||
Forwards: n.Load.Forwards, MemPct: n.Load.MemPct, NetPct: n.Load.NetPct,
|
||||
Score: n.Load.Score(),
|
||||
})
|
||||
}
|
||||
if m := b.revoking; m != nil {
|
||||
rep.InFlight = &BalanceMigrationPlan{
|
||||
Local: m.local, Remote: m.remote, Port: m.port, From: m.fromOwner, To: m.toOwner,
|
||||
}
|
||||
}
|
||||
if m := b.publishing; m != nil {
|
||||
rep.InFlight = &BalanceMigrationPlan{
|
||||
Local: m.local, Remote: m.remote, Port: m.port, From: m.fromOwner, To: m.toOwner,
|
||||
}
|
||||
}
|
||||
// Only a leader would act, so only report a plan there.
|
||||
if rep.IsLeader {
|
||||
pick, dest, hot := b.engine.findBalanceCandidate(b.ratio)
|
||||
if hot != "" {
|
||||
rep.OverloadedSince = b.overSince[hot]
|
||||
}
|
||||
if pick != nil && dest != "" {
|
||||
plan := &BalanceMigrationPlan{
|
||||
Local: pick.Local.Name, Remote: pick.Remote.Name,
|
||||
Port: pick.Link.RemotePort, From: pick.OwnerID, To: dest,
|
||||
}
|
||||
// surface it as "would migrate" only once the window is satisfied
|
||||
if since, ok := b.overSince[pick.OwnerID]; ok && now-since >= int64(b.sustainedSec) {
|
||||
rep.WouldMigrate = plan
|
||||
}
|
||||
}
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
// SetRatioAndSustain updates the thresholds after construction, so main.go can
|
||||
// create the balancer early (the HTTP handler needs it) and apply the environment
|
||||
// configuration once it has parsed it.
|
||||
func (b *Balancer) SetRatioAndSustain(ratio float64, sustainedSec int64) {
|
||||
if ratio <= 1 {
|
||||
ratio = 2
|
||||
}
|
||||
if sustainedSec < 0 {
|
||||
sustainedSec = 0
|
||||
}
|
||||
b.ratio = ratio
|
||||
b.sustainedSec = sustainedSec
|
||||
}
|
||||
|
||||
@ -48,6 +48,8 @@ type Handler struct {
|
||||
Cluster *cluster.Registry
|
||||
// Ring is the token-ring engine (M6). Nil disables ring routes.
|
||||
Ring *cluster.Engine
|
||||
// Balancer is optional: when nil the balance endpoint reports 404.
|
||||
Balancer *cluster.Balancer
|
||||
// SelfAddr is this node's reachable listen address (from -addr), used for
|
||||
// cluster discovery so peers can reach back for binary exchange.
|
||||
SelfAddr string
|
||||
@ -114,6 +116,7 @@ func NewServeMux(h *Handler) (http.Handler, error) {
|
||||
mux.HandleFunc(apiPrefix+"/cluster/nodes", h.auth("read")(h.handleClusterNodes))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/cache", h.auth("read")(h.handleClusterCache)) // GET read; POST write-checked inside
|
||||
mux.HandleFunc(apiPrefix+"/cluster/ring", h.auth("read")(h.handleClusterRing))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/balance", h.auth("read")(h.handleClusterBalance))
|
||||
mux.HandleFunc(apiPrefix+"/node/logs", h.auth("read")(h.handleNodeLogs))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/logs/export", h.auth("read")(h.handleClusterLogsExport))
|
||||
|
||||
@ -634,6 +637,27 @@ func (h *Handler) handleClusterRing(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, h.Ring.Snapshot())
|
||||
}
|
||||
|
||||
// handleClusterBalance reports the load balancer's read-only view: per-node
|
||||
// scores, whether anything is over the threshold, and the migration it would
|
||||
// perform. Publishing nothing is the point — an operator asking "is this cluster
|
||||
// balanced?" currently gets silence from the log in the healthy case, which is
|
||||
// indistinguishable from balancing being switched off.
|
||||
func (h *Handler) handleClusterBalance(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w)
|
||||
return
|
||||
}
|
||||
if h.Ring == nil {
|
||||
http.Error(w, "ring engine not enabled", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if h.Balancer == nil {
|
||||
http.Error(w, "load balancer not enabled", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, h.Balancer.Report(time.Now().Unix()))
|
||||
}
|
||||
|
||||
// handleClusterJoin accepts a newcomer join request: the target node inserts
|
||||
// the newcomer after itself in the ring and returns the updated ring state
|
||||
// for the newcomer to adopt.
|
||||
|
||||
Reference in New Issue
Block a user