mirror of
https://gitcode.com/JianFeeeee/webui4frpc.git
synced 2026-09-20 00:47:57 +00:00
feat: Phase C 完成 — ModelRouter 风格 UI 重设计 + 模拟 frps 测试 + lastSync 修复 + 集群作坊搭建
- 主题: sakura×frost 玻璃拟态 (theme.css) + SCSS 变量重映射 - 侧栏: 玻璃侧栏 246px + 渐变品牌区 + 面包屑导航 - 状态页: 玻璃 KPI 卡 + 远程节点/本地服务卡片网格 - 集群页: 英雄玻璃卡 + 横向环拓扑链 + 待办命令/活跃拓扑/日志区 - 令牌环: 新增 lastSync 上次同步时间替代周期计数 - 模拟 frps: frps2/frps3 容器 + test-forward.sh 全链路验证脚本 - 修复: BinaryPath 空导致 worker 不启动, 撤销仅撤第一个 link, 任务复活风暴 (published 追踪)
This commit is contained in:
@ -53,6 +53,9 @@ type Task struct {
|
||||
// owning node cancels it (stop worker, drop from topology). Reuses the
|
||||
// same publish channel as creation (round-1 inject, round-2 apply).
|
||||
Revoke bool `json:"revoke,omitempty"`
|
||||
// RemoveNode: node ID to remove from the ring; the node self-removes when
|
||||
// the command reaches it via the token.
|
||||
RemoveNode string `json:"removeNode,omitempty"`
|
||||
}
|
||||
|
||||
// TopoEntry is an ESTABLISHED forward in the cluster topology: who (OwnerID)
|
||||
@ -72,7 +75,11 @@ type State struct {
|
||||
LeaderID string `json:"leaderId"`
|
||||
Nodes []Node `json:"nodes"`
|
||||
Cycle int64 `json:"cycle"`
|
||||
RoundDelay time.Duration `json:"-"`
|
||||
// RoundDelay rides the token so every node converges on the same ring
|
||||
// cadence (serialized as nanoseconds; the leader refreshes it each round
|
||||
// via tkDelaySince). MUST be serialized — otherwise a token round-trip
|
||||
// zeroed it and LossTimeout collapsed to the floor.
|
||||
RoundDelay time.Duration `json:"roundDelay,omitempty"`
|
||||
// PendingTasks: not yet claimed (disappear when claimed).
|
||||
PendingTasks map[string]*Task `json:"pendingTasks,omitempty"`
|
||||
// Topology: active forwards owned by members (full cluster view).
|
||||
@ -80,13 +87,12 @@ type State struct {
|
||||
Seq int64 `json:"seq"`
|
||||
}
|
||||
|
||||
// Token is the circulating message: one physical token, two phases per cycle.
|
||||
// Token is the circulating message: one physical token per cycle (single
|
||||
// round — adopt + append + execute in one pass, no phase split).
|
||||
type Token struct {
|
||||
Cycle int64 `json:"cycle"`
|
||||
Phase int `json:"phase"`
|
||||
State State `json:"state"`
|
||||
Log []LogEntry `json:"logDelta,omitempty"`
|
||||
Passed []string `json:"passed,omitempty"`
|
||||
SentAt int64 `json:"sentAt,omitempty"`
|
||||
}
|
||||
|
||||
@ -205,6 +211,20 @@ func (s *State) NextTaskID() string {
|
||||
return fmt.Sprintf("t%d", s.Seq)
|
||||
}
|
||||
|
||||
// AddRemoveNode publishes a node-removal command via the token; the target
|
||||
// node self-removes when it receives the command (re-queue forwards, drop
|
||||
// ring position, pass token to former successor).
|
||||
func (s *State) AddRemoveNode(nodeID string) *Task {
|
||||
t := &Task{
|
||||
ID: s.NextTaskID(), Created: time.Now().Unix(), RemoveNode: nodeID,
|
||||
}
|
||||
if s.PendingTasks == nil {
|
||||
s.PendingTasks = map[string]*Task{}
|
||||
}
|
||||
s.PendingTasks[t.ID] = t
|
||||
return t
|
||||
}
|
||||
|
||||
// AddRevoke publishes a REVOCATION task (same channel as creation): when a
|
||||
// node wants to cancel an established forward, it injects a Revoke task;
|
||||
// the owning node cancels the worker and drops it from topology.
|
||||
@ -280,6 +300,21 @@ func (s *State) AddTopology(t *Task, ownerID string) *TopoEntry {
|
||||
return e
|
||||
}
|
||||
|
||||
// SelfRemove removes this node from the ring: re-queues its own forwards
|
||||
// as pending (so another member claims them), drops its topology entries
|
||||
// and removes its ring position. Returns the re-queued task ids.
|
||||
func (s *State) SelfRemove(nodeID string) []string {
|
||||
reattached := s.OfflineReassign(nodeID)
|
||||
i := s.Find(nodeID)
|
||||
if i >= 0 {
|
||||
s.Nodes = append(s.Nodes[:i], s.Nodes[i+1:]...)
|
||||
}
|
||||
if s.LeaderID == nodeID {
|
||||
s.LeaderID = ""
|
||||
}
|
||||
return reattached
|
||||
}
|
||||
|
||||
// RemoveTopology drops the active forward for the given local/remote/port
|
||||
// (returns true if removed — the owning node revokes it).
|
||||
func (s *State) RemoveTopology(local, remote string, port int) bool {
|
||||
@ -292,6 +327,19 @@ func (s *State) RemoveTopology(local, remote string, port int) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// TopologyOwner returns the node that owns the active forward matching the
|
||||
// given task (by local/remote/port), or "" if no such entry exists. Used to
|
||||
// route a revocation to the OWNING node (plan §任务撤销: "持有该转发的节点
|
||||
// 收到撤销任务后取消") instead of letting any lowest-load node claim it.
|
||||
func (s *State) TopologyOwner(t *Task) string {
|
||||
for _, e := range s.Topology {
|
||||
if e.Local.Name == t.Local.Name && e.Remote.Name == t.Remote.Name && e.Link.RemotePort == t.Link.RemotePort {
|
||||
return e.OwnerID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TopologyList returns active forwards, stable by task id.
|
||||
func (s *State) TopologyList() []*TopoEntry {
|
||||
out := make([]*TopoEntry, 0, len(s.Topology))
|
||||
|
||||
@ -4,18 +4,14 @@ package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"webui4frpc/internal/store"
|
||||
)
|
||||
|
||||
// Phase constants for the two-round cycle.
|
||||
const (
|
||||
PhaseCollect = 1 // round 1: append own info to token
|
||||
PhaseSync = 2 // round 2: sync cluster state, claim tasks
|
||||
)
|
||||
|
||||
// Handler is what the engine calls when the node must act on a task
|
||||
// (create the frpc worker / forward). Injected to avoid import cycle.
|
||||
type Handler interface {
|
||||
@ -47,8 +43,40 @@ type Engine struct {
|
||||
lastLogSent int64
|
||||
// lastRingStart time of the previous cycle launch (leader throttle).
|
||||
lastRingStart time.Time
|
||||
// curPhase tracks the most recently processed token phase (frontend).
|
||||
curPhase int
|
||||
// lastTokenAt: leader-stamped token timestamp of the newest valid token
|
||||
// this node accepted. Older tokens (multi-token conflict leftovers) are
|
||||
// dropped so only one token effectively circulates.
|
||||
lastTokenAt int64
|
||||
// lastSyncAt unix-seconds when this node last completed a sync round.
|
||||
lastSyncAt int64
|
||||
// failCount counts consecutive send failures per neighbor; a node is
|
||||
// declared offline only after repeated failures (transient jitter must
|
||||
// not break the ring).
|
||||
failCount map[string]int
|
||||
// failMu guards failCount (HTTP handlers run concurrently).
|
||||
failMu sync.Mutex
|
||||
// pendingJoin: newcomers accepted via JoinNode but NOT YET injected into
|
||||
// the token. Per the authoritative design (plan §新节点加入): a sponsor
|
||||
// injects the newcomer into the token only when the token reaches it
|
||||
// ("令牌发到自己时,先转发给新节点,并把新节点加入令牌中的集群信息").
|
||||
// Until then the newcomer is NOT in e.state.Nodes — so the OnToken
|
||||
// state-merge (incoming authoritative) cannot wash it back out.
|
||||
pendingJoin []JoinInfo
|
||||
// selfRemoved is set when a node-remove command targeted THIS node and
|
||||
// it has run SelfRemove; the OnToken "append own node info" step is then
|
||||
// skipped (otherwise UpsertNode(self) would re-add the removed node).
|
||||
selfRemoved bool
|
||||
// removedNext: the original successor captured right before SelfRemove,
|
||||
// so Forward can hand the token to it even though this node is no longer
|
||||
// in state.Nodes (plan §移除节点 step 3: "令牌传递给自身原本的下一家").
|
||||
removedNext string
|
||||
// published records task IDs this node placed into the token it last
|
||||
// forwarded. When the next token returns WITHOUT one of those IDs, the
|
||||
// task was consumed downstream (claimed/revoked) — the localPending
|
||||
// re-merge must NOT resurrect it, or the task rides forever (a re-claim /
|
||||
// re-revoke storm centered on the submitter). Reset each round to the
|
||||
// tasks actually leaving on this token.
|
||||
published map[string]struct{}
|
||||
}
|
||||
|
||||
// NewEngine builds the engine; state holds this node as initial leader unless
|
||||
@ -62,11 +90,13 @@ func NewEngine(id, addr, user, pass, version string, cache []string, h Handler,
|
||||
Cycle: 0,
|
||||
PendingTasks: map[string]*Task{},
|
||||
Topology: map[string]*TopoEntry{},
|
||||
RoundDelay: 200 * time.Millisecond,
|
||||
RoundDelay: 2 * time.Second,
|
||||
},
|
||||
myAddr: selfAddr,
|
||||
send: send,
|
||||
Log: NewClusterLog(),
|
||||
myAddr: selfAddr,
|
||||
send: send,
|
||||
Log: NewClusterLog(),
|
||||
failCount: map[string]int{},
|
||||
published: map[string]struct{}{},
|
||||
}
|
||||
n := Node{ID: id, Addr: selfAddr, Alive: true, IsLeader: isLeader,
|
||||
Load: Load{MemPct: 10, NetPct: 10}, Version: version, Cache: cache}
|
||||
@ -99,17 +129,108 @@ func (e *Engine) loadSnapshot() Load {
|
||||
return Load{MemPct: 20, NetPct: 20}
|
||||
}
|
||||
|
||||
// phase1 appends this node's info to the token (round 1).
|
||||
func (e *Engine) phase1(tk *Token) {
|
||||
e.state.UpsertNode(Node{
|
||||
ID: e.ID, Addr: e.myAddr, Alive: true,
|
||||
IsLeader: e.state.LeaderID == e.ID || tk.State.LeaderID == e.ID,
|
||||
Load: e.loadSnapshot(),
|
||||
Version: e.Version, Cache: e.Cache,
|
||||
})
|
||||
// Attach our own log entries not yet seen by the ring (incremental sync):
|
||||
// entries after the last forwarded watermark ride the token for others.
|
||||
if e.Log != nil {
|
||||
// OnToken is the SINGLE-ROUND token handler. Per the authoritative design
|
||||
// (plan §令牌环协议): on receiving the token a node simultaneously
|
||||
// (a) ADOPTS the carried cluster picture — incoming state is authoritative
|
||||
// for membership + per-node fields. This is safe because structural
|
||||
// changes (join/remove) are NOT kept in local state waiting to survive
|
||||
// a merge: joins ride a separate pendingJoin channel injected INTO the
|
||||
// token here, and a self-remove writes the node out of state so the
|
||||
// downstream merge naturally drops it.
|
||||
// (b) APPLIES the incremental log delta, then re-attaches own fresh entries.
|
||||
// (c) APPENDS own node info (load/version) — skipped if we self-removed.
|
||||
// (d) INJECTS pending newcomers right after ourselves + forwards to them.
|
||||
// (e) Paces via a parallel rhythm timer (max(ops, timer)).
|
||||
func (e *Engine) OnToken(ctx context.Context, tk *Token) (*Token, error) {
|
||||
// Multi-token guard: only the newest leader-stamped token is kept.
|
||||
if tk.SentAt > 0 && e.lastTokenAt > 0 && tk.SentAt < e.lastTokenAt {
|
||||
log.Printf("ring[%s] drop stale token sentAt=%d (last=%d)", e.ID, tk.SentAt, e.lastTokenAt)
|
||||
return nil, nil
|
||||
}
|
||||
if tk.SentAt > e.lastTokenAt {
|
||||
e.lastTokenAt = tk.SentAt
|
||||
}
|
||||
log.Printf("ring[%s] OnToken cycle=%d", e.ID, tk.Cycle)
|
||||
|
||||
// Parallel rhythm timer: operations run while the pace clock ticks.
|
||||
rhythm := time.NewTimer(ringHopDelay)
|
||||
defer rhythm.Stop()
|
||||
|
||||
// (a) ADOPT the cluster picture. Incoming state is authoritative: joins
|
||||
// ride pendingJoin (injected below), removes write the node out of state
|
||||
// so a downstream merge drops it — so a plain assignment is correct and
|
||||
// does NOT wash out local structural changes.
|
||||
// EXCEPTION: pending COMMANDS (SubmitTask / RemoveNode / AddRevoke) are
|
||||
// injected into e.state.PendingTasks by HTTP handlers OUTSIDE OnToken, so
|
||||
// a blanket overwrite would drop them before the token carries them. Keep
|
||||
// local-only pending commands and re-merge them after the adoption — BUT
|
||||
// only those never yet published into a departing token. A task we already
|
||||
// published that is now absent from the incoming token was consumed
|
||||
// downstream (claimed/revoked); resurrecting it would make it ride forever
|
||||
// (a re-claim / re-revoke storm centered on the submitter node).
|
||||
localPending := map[string]*Task{}
|
||||
for id, t := range e.state.PendingTasks {
|
||||
if _, sent := e.published[id]; sent {
|
||||
continue
|
||||
}
|
||||
if _, inToken := tk.State.PendingTasks[id]; inToken {
|
||||
continue
|
||||
}
|
||||
localPending[id] = t
|
||||
}
|
||||
rd := e.state.RoundDelay // preserve if incoming carries none (zero-guard)
|
||||
e.state = tk.State
|
||||
if e.state.RoundDelay == 0 {
|
||||
e.state.RoundDelay = rd
|
||||
}
|
||||
if e.state.PendingTasks == nil {
|
||||
e.state.PendingTasks = map[string]*Task{}
|
||||
}
|
||||
for id, t := range localPending {
|
||||
e.state.PendingTasks[id] = t
|
||||
}
|
||||
|
||||
// (b) apply incremental log delta; trim consumed entries off the token.
|
||||
if e.Log != nil && len(tk.Log) > 0 {
|
||||
wm, err := e.Log.ApplyDelta(tk.Log)
|
||||
if err != nil {
|
||||
log.Printf("ring[%s] log delta gap: %v (request full sync later)", e.ID, err)
|
||||
}
|
||||
keep := tk.Log[:0]
|
||||
for _, en := range tk.Log {
|
||||
if en.Seq > wm {
|
||||
keep = append(keep, en)
|
||||
}
|
||||
}
|
||||
tk.Log = keep
|
||||
}
|
||||
|
||||
// execute pending commands addressed to us or claimable by lowest load.
|
||||
// runCommands may set e.selfRemoved + e.removedNext on a self-remove.
|
||||
if err := e.runCommands(ctx, tk); err != nil {
|
||||
return tk, err
|
||||
}
|
||||
|
||||
// (c) append own node info (refresh load/lastSeen) — skipped when we
|
||||
// just self-removed, otherwise UpsertNode(self) would resurrect us and
|
||||
// undo the removal the command just performed.
|
||||
if !e.selfRemoved {
|
||||
e.state.UpsertNode(Node{
|
||||
ID: e.ID, Addr: e.myAddr, Alive: true,
|
||||
IsLeader: e.state.LeaderID == e.ID,
|
||||
Load: e.loadSnapshot(),
|
||||
Version: e.Version, Cache: e.Cache,
|
||||
})
|
||||
}
|
||||
|
||||
// (d) inject pending newcomers right after ourselves + log node.join.
|
||||
// This is the plan's "令牌发到自己时把新节点加入令牌" step: the
|
||||
// sponsor writes the newcomer into the token's state and the token then
|
||||
// flows to the newcomer (its successor) so it can participate.
|
||||
e.injectPendingJoin()
|
||||
|
||||
// re-attach own fresh log entries so peers converge.
|
||||
if !e.selfRemoved && e.Log != nil {
|
||||
mine := e.Log.EntriesAfter(e.lastLogSent)
|
||||
if len(mine) > 0 {
|
||||
tk.Log = append(tk.Log, mine...)
|
||||
@ -118,38 +239,90 @@ func (e *Engine) phase1(tk *Token) {
|
||||
}
|
||||
}
|
||||
}
|
||||
tk.Passed = append(tk.Passed, e.ID)
|
||||
|
||||
// Publish our consolidated state back into the token.
|
||||
tk.State = e.state
|
||||
e.lastSyncAt = time.Now().Unix()
|
||||
// Record the tasks leaving on this token so their absence from the next
|
||||
// incoming token is recognized as "consumed downstream" rather than
|
||||
// "never sent" — otherwise the localPending re-merge above would
|
||||
// resurrect them and they would ride the ring forever.
|
||||
e.published = make(map[string]struct{}, len(e.state.PendingTasks))
|
||||
for id := range e.state.PendingTasks {
|
||||
e.published[id] = struct{}{}
|
||||
}
|
||||
// Forward only after BOTH the operations and the rhythm timer are done.
|
||||
select {
|
||||
case <-rhythm.C:
|
||||
case <-ctx.Done():
|
||||
return tk, ctx.Err()
|
||||
}
|
||||
return tk, nil
|
||||
}
|
||||
|
||||
// phase2 syncs cluster info from the token and claims pending tasks if we are
|
||||
// the lowest-load node. A claimed task DISAPPEARS from pending and is
|
||||
// written into the active topology so every member knows who runs what.
|
||||
func (e *Engine) phase2(ctx context.Context, tk *Token) error {
|
||||
e.state = tk.State
|
||||
e.state.LeaderID = tk.State.LeaderID
|
||||
// Incremental log sync: adopt deltas carried by the token, then attach
|
||||
// our own new entries so peers can converge.
|
||||
if e.Log != nil && len(tk.Log) > 0 {
|
||||
if _, err := e.Log.ApplyDelta(tk.Log); err != nil {
|
||||
log.Printf("ring[%s] log delta gap: %v (request full sync later)", e.ID, err)
|
||||
}
|
||||
}
|
||||
// runCommands executes pending commands carried by the token that this node
|
||||
// must handle. Per the authoritative design (plan §M6), commands are DIRECTED
|
||||
// to their executor — they are NOT blindly claimed by the lowest-load node:
|
||||
// - node-removal: rides the token until it reaches the TARGET node, which
|
||||
// self-removes (plan §移除节点: "令牌传递到被移除节点自身时,该节点执行
|
||||
// 自移除"). A remove command for ANOTHER node is left in pending so it keeps
|
||||
// riding; if its target is already gone/offline, the lowest node consumes
|
||||
// it as a no-op so it cannot ride forever.
|
||||
// - revocation: rides to the OWNING node (plan §任务撤销: "持有该转发的节点
|
||||
// 收到撤销任务后取消"); a revoke whose forward is already absent from the
|
||||
// topology is an idempotent no-op, consumed by the lowest-load node.
|
||||
// - forward creation: claimed by the lowest-load node (plan §负载摘取).
|
||||
func (e *Engine) runCommands(ctx context.Context, tk *Token) error {
|
||||
for {
|
||||
pending := e.state.PendingList()
|
||||
if len(pending) == 0 {
|
||||
break
|
||||
}
|
||||
low := e.state.LowestAlive()
|
||||
if low == nil || low.ID != e.ID {
|
||||
selfIsLowest := false
|
||||
if low := e.state.LowestAlive(); low != nil && low.ID == e.ID {
|
||||
selfIsLowest = true
|
||||
}
|
||||
// Pick the first command this node may act on this pass.
|
||||
var target *Task
|
||||
for _, t := range pending {
|
||||
if t.RemoveNode == e.ID {
|
||||
target = t // directed at us — execute regardless of load
|
||||
break
|
||||
}
|
||||
if t.RemoveNode != "" {
|
||||
// directed at another node; let it ride unless the target is
|
||||
// already gone (not in ring / offline) — then the lowest node
|
||||
// consumes the stale command as a no-op so it cannot loop.
|
||||
if selfIsLowest {
|
||||
if idx := e.state.Find(t.RemoveNode); idx < 0 || !e.state.Nodes[idx].Alive {
|
||||
target = t
|
||||
break
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if t.Revoke {
|
||||
if owner := e.state.TopologyOwner(t); owner == e.ID {
|
||||
target = t // we own the forward — execute the revoke
|
||||
break
|
||||
} else if owner == "" && selfIsLowest {
|
||||
target = t // forward already gone — idempotent no-op
|
||||
break
|
||||
}
|
||||
continue // owned elsewhere — ride to the owner
|
||||
}
|
||||
if selfIsLowest {
|
||||
target = t // generic forward create — lowest-load claim
|
||||
break
|
||||
}
|
||||
}
|
||||
if target == nil {
|
||||
break
|
||||
}
|
||||
tk0 := pending[0]
|
||||
claimed := e.state.ClaimPending(tk0.ID)
|
||||
claimed := e.state.ClaimPending(target.ID)
|
||||
if claimed == nil {
|
||||
break
|
||||
}
|
||||
// Revocation task: the owning node cancels the forward (stop worker,
|
||||
// drop from topology, log forward.remove). Idempotent if missing.
|
||||
if claimed.Revoke {
|
||||
if e.state.RemoveTopology(claimed.Local.Name, claimed.Remote.Name, claimed.Link.RemotePort) {
|
||||
if e.Handler != nil {
|
||||
@ -165,6 +338,44 @@ func (e *Engine) phase2(ctx context.Context, tk *Token) error {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if claimed.RemoveNode != "" && claimed.RemoveNode == e.ID {
|
||||
// Plan §移除节点: self-remove re-queues own forwards, drops ring
|
||||
// position, and the token continues to the original successor.
|
||||
// Stop the local workers for every forward we own FIRST (plan:
|
||||
// "完全取消一切集群远程转发,停其 worker"), then SelfRemove
|
||||
// moves them back to pending for another member to claim.
|
||||
if e.Handler != nil {
|
||||
for _, te := range e.state.ForwardsOwnedBy(e.ID) {
|
||||
t := &Task{ID: te.TaskID, Local: te.Local, Remote: te.Remote, Link: te.Link}
|
||||
if err := e.Handler.Revoke(ctx, t); err != nil {
|
||||
log.Printf("ring[%s] self-remove revoke %s: %v", e.ID, te.TaskID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if succ, ok := e.state.AliveSuccessor(e.ID); ok && succ != e.ID {
|
||||
e.removedNext = succ
|
||||
}
|
||||
e.state.SelfRemove(e.ID)
|
||||
e.selfRemoved = true
|
||||
if e.Log != nil {
|
||||
_, _ = e.Log.Append(e.ID, LogNodeLeave, map[string]string{"node": e.ID})
|
||||
}
|
||||
log.Printf("ring[%s] self-removed from cluster (token command %s); next=%s",
|
||||
e.ID, claimed.ID, e.removedNext)
|
||||
continue
|
||||
}
|
||||
if claimed.RemoveNode != "" {
|
||||
// A remove command whose target is no longer in the ring: the
|
||||
// target already self-removed (or was never a member). Drop it
|
||||
// here — it MUST NOT fall through to the forward-create path,
|
||||
// which would spawn a phantom empty forward (the remove task
|
||||
// carries no local/remote/link). This branch also absorbs the
|
||||
// command after the injector's localPending merge resurrects a
|
||||
// copy once the original has been consumed downstream.
|
||||
log.Printf("ring[%s] drop fulfilled remove %s (target %s gone)",
|
||||
e.ID, claimed.ID, claimed.RemoveNode)
|
||||
continue
|
||||
}
|
||||
if e.Handler != nil {
|
||||
if err := e.Handler.Claim(ctx, claimed); err != nil {
|
||||
e.state.PendingTasks[claimed.ID] = claimed
|
||||
@ -172,8 +383,6 @@ func (e *Engine) phase2(ctx context.Context, tk *Token) error {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Record the claim in the operation log so all peers converge on who
|
||||
// owns which forward (incremental log sync).
|
||||
if e.Log != nil {
|
||||
_, _ = e.Log.Append(e.ID, LogForwardAdd, map[string]any{
|
||||
"taskId": claimed.ID, "local": claimed.Local.Name, "remote": claimed.Remote.Name,
|
||||
@ -181,39 +390,24 @@ func (e *Engine) phase2(ctx context.Context, tk *Token) error {
|
||||
}
|
||||
e.state.AddTopology(claimed, e.ID)
|
||||
}
|
||||
// Publish our updated state back into the token so the next node carries
|
||||
// the fresh topology + pending set (do NOT revert local state to the
|
||||
// incoming snapshot — that would discard the claim we just made).
|
||||
tk.State = e.state
|
||||
tk.Passed = append(tk.Passed, e.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// OnToken receives the token: process by phase, return updated token.
|
||||
func (e *Engine) OnToken(ctx context.Context, tk *Token) (*Token, error) {
|
||||
e.curPhase = tk.Phase
|
||||
log.Printf("ring[%s] OnToken cycle=%d phase=%d passed=%v", e.ID, tk.Cycle, tk.Phase, tk.Passed)
|
||||
switch tk.Phase {
|
||||
case PhaseCollect:
|
||||
e.phase1(tk)
|
||||
case PhaseSync:
|
||||
if err := e.phase2(ctx, tk); err != nil {
|
||||
return tk, err
|
||||
}
|
||||
default:
|
||||
return tk, nil
|
||||
}
|
||||
return tk, nil
|
||||
}
|
||||
|
||||
// Forward hands the token to this node's successor over the injected send.
|
||||
// It is the transport hook used by the HTTP handler after OnToken.
|
||||
// It is the transport hook used by the HTTP handler after OnToken. If this
|
||||
// node just self-removed, its ID is no longer in state.Nodes so
|
||||
// AliveSuccessor would fail — use the original successor captured before
|
||||
// removal (plan §移除节点 step 3: "令牌传递给自身原本的下一家").
|
||||
func (e *Engine) Forward(ctx context.Context, tk *Token) error {
|
||||
nodes := make([]string, 0, len(e.state.Nodes))
|
||||
for _, n := range e.state.Nodes {
|
||||
nodes = append(nodes, n.ID)
|
||||
if e.removedNext != "" {
|
||||
next := e.removedNext
|
||||
e.removedNext = ""
|
||||
if e.send != nil {
|
||||
log.Printf("ring[%s] forward (self-removed) cycle=%d to %s", e.ID, tk.Cycle, next)
|
||||
return e.send(ctx, next, tk)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
log.Printf("ring[%s] fwd-debug id=%s nodes=%v", e.ID, e.ID, nodes)
|
||||
next, ok := e.state.AliveSuccessor(e.ID)
|
||||
if !ok {
|
||||
return nil // single-node ring
|
||||
@ -222,7 +416,7 @@ func (e *Engine) Forward(ctx context.Context, tk *Token) error {
|
||||
return nil // never forward to ourselves
|
||||
}
|
||||
if e.send != nil {
|
||||
log.Printf("ring[%s] forward cycle=%d phase=%d to %s", e.ID, tk.Cycle, tk.Phase, next)
|
||||
log.Printf("ring[%s] forward cycle=%d to %s", e.ID, tk.Cycle, next)
|
||||
return e.send(ctx, next, tk)
|
||||
}
|
||||
return nil
|
||||
@ -233,7 +427,7 @@ type RingSnapshot struct {
|
||||
SelfID string `json:"selfId"`
|
||||
LeaderID string `json:"leaderId"`
|
||||
Cycle int64 `json:"cycle"`
|
||||
Phase int `json:"phase"`
|
||||
LastSync int64 `json:"lastSync"`
|
||||
RoundDelay int64 `json:"roundDelayMs"`
|
||||
Nodes []Node `json:"nodes"`
|
||||
Pending []*Task `json:"pending"`
|
||||
@ -241,19 +435,12 @@ type RingSnapshot struct {
|
||||
Log []LogEntry `json:"log,omitempty"`
|
||||
}
|
||||
|
||||
func (e *Engine) currentPhase() int {
|
||||
if e.curPhase == 0 {
|
||||
return PhaseCollect
|
||||
}
|
||||
return e.curPhase
|
||||
}
|
||||
|
||||
func (e *Engine) Snapshot() *RingSnapshot {
|
||||
snap := &RingSnapshot{
|
||||
SelfID: e.ID,
|
||||
LeaderID: e.state.LeaderID,
|
||||
Cycle: e.state.Cycle,
|
||||
Phase: e.currentPhase(),
|
||||
LastSync: e.lastSyncAt,
|
||||
RoundDelay: e.state.RoundDelay.Milliseconds(),
|
||||
Nodes: e.state.Nodes,
|
||||
Pending: e.state.PendingList(),
|
||||
@ -265,40 +452,10 @@ func (e *Engine) Snapshot() *RingSnapshot {
|
||||
return snap
|
||||
}
|
||||
|
||||
// advanceToken decides the next recipient: normal successor; if we are leader
|
||||
// and everyone passed, flip phase for the second round.
|
||||
func (e *Engine) advanceToken(ctx context.Context, tk *Token) error {
|
||||
if tk.Phase == PhaseCollect {
|
||||
all := true
|
||||
for _, n := range e.state.Nodes {
|
||||
if !n.Alive {
|
||||
continue
|
||||
}
|
||||
if !contains(tk.Passed, n.ID) {
|
||||
all = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if all && e.state.LeaderID == e.ID {
|
||||
tk.Phase = PhaseSync
|
||||
tk.Passed = nil
|
||||
e.state.RoundDelay = tkDelaySince(tk)
|
||||
}
|
||||
}
|
||||
next, ok := e.state.AliveSuccessor(e.ID)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if e.send != nil {
|
||||
return e.send(ctx, next, tk)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tkDelaySince measures elapsed ms since token SentAt (leader round delay).
|
||||
// tkDelaySince measures elapsed since token SentAt (leader round delay).
|
||||
func tkDelaySince(tk *Token) time.Duration {
|
||||
if tk.SentAt == 0 {
|
||||
return 200 * time.Millisecond
|
||||
return 2 * time.Second
|
||||
}
|
||||
return time.Duration(time.Now().UnixMilli()-tk.SentAt) * time.Millisecond
|
||||
}
|
||||
@ -312,19 +469,19 @@ func contains(xs []string, v string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// StartRing kicks off the cycle from the leader by sending the first token
|
||||
// (phase 1) to the next node. Called once at leader boot.
|
||||
// StartRing kicks off a fresh token from the leader. The leader stamps a new
|
||||
// SentAt (so older in-flight tokens are dropped downstream) and marks inflight
|
||||
// so WatchTokenLoss can detect a lost first hop — previously StartRing sent
|
||||
// without marking, so a first-round loss was never noticed.
|
||||
func (e *Engine) StartRing(ctx context.Context) {
|
||||
if e.state.LeaderID != e.ID {
|
||||
return
|
||||
}
|
||||
// Single-node ring has no successor to hand the token to; do not POST to
|
||||
// ourselves. The cycle resumes once a newcomer joins (see JoinNode).
|
||||
if next, ok := e.state.AliveSuccessor(e.ID); !ok || next == e.ID {
|
||||
return
|
||||
next, ok := e.state.AliveSuccessor(e.ID)
|
||||
if !ok || next == e.ID {
|
||||
return // single-node ring; resumes once a newcomer joins
|
||||
}
|
||||
// Throttle: do not start a new cycle until roundDelay has elapsed since
|
||||
// the last one, so a healthy ring cycles at a deliberate pace.
|
||||
// Throttle: do not start a new cycle until RoundDelay has elapsed.
|
||||
if !e.lastRingStart.IsZero() && time.Since(e.lastRingStart) < e.state.RoundDelay {
|
||||
return
|
||||
}
|
||||
@ -332,14 +489,17 @@ func (e *Engine) StartRing(ctx context.Context) {
|
||||
e.state.Cycle++
|
||||
tk := &Token{
|
||||
Cycle: e.state.Cycle,
|
||||
Phase: PhaseCollect,
|
||||
State: e.state,
|
||||
Passed: []string{e.ID},
|
||||
SentAt: time.Now().UnixMilli(),
|
||||
}
|
||||
e.phase1(tk)
|
||||
if err := e.advanceToken(ctx, tk); err != nil {
|
||||
log.Printf("ring[%s] start cycle %d: %v", e.ID, tk.Cycle, err)
|
||||
if e.send != nil {
|
||||
if err := e.send(ctx, next, tk); err != nil {
|
||||
log.Printf("ring[%s] start cycle %d send: %v", e.ID, tk.Cycle, err)
|
||||
return
|
||||
}
|
||||
// Mark inflight ONLY on a successful first send so token-loss
|
||||
// detection covers the very first hop too.
|
||||
e.inflight.mark(e.state.RoundDelay)
|
||||
}
|
||||
}
|
||||
|
||||
@ -351,20 +511,42 @@ type JoinInfo struct {
|
||||
Cache []string `json:"cache,omitempty"`
|
||||
}
|
||||
|
||||
// JoinNode handles an incoming join request from a new node: it inserts the
|
||||
// newcomer right after this node (so the newcomer becomes our successor),
|
||||
// keeps this node the leader, and returns the updated ring state for the
|
||||
// newcomer to adopt.
|
||||
func (e *Engine) JoinNode(j JoinInfo) *State {
|
||||
n := Node{ID: j.ID, Addr: j.Addr, Alive: true,
|
||||
Load: Load{MemPct: 50, NetPct: 50}, Version: j.Version, Cache: j.Cache}
|
||||
e.state.InsertAfter(e.ID, n)
|
||||
if e.Log != nil {
|
||||
_, _ = e.Log.Append(e.ID, LogNodeJoin, map[string]string{"node": n.ID, "addr": n.Addr})
|
||||
// injectPendingJoin writes every queued newcomer into state right after
|
||||
// ourselves (so the newcomer becomes our successor) and logs node.join. Used
|
||||
// both by OnToken (plan: "令牌发到自己时把新节点加入令牌") and by JoinNode
|
||||
// when the leader is a single node with no token circulating.
|
||||
func (e *Engine) injectPendingJoin() {
|
||||
for _, ji := range e.pendingJoin {
|
||||
nn := Node{ID: ji.ID, Addr: ji.Addr, Alive: true,
|
||||
Load: Load{MemPct: 50, NetPct: 50}, Version: ji.Version, Cache: ji.Cache}
|
||||
e.state.InsertAfter(e.ID, nn)
|
||||
if e.Log != nil {
|
||||
_, _ = e.Log.Append(e.ID, LogNodeJoin, map[string]string{"node": nn.ID, "addr": nn.Addr})
|
||||
}
|
||||
log.Printf("ring[%s] injected newcomer %s after self", e.ID, nn.ID)
|
||||
}
|
||||
e.pendingJoin = nil
|
||||
}
|
||||
|
||||
// JoinNode accepts a newcomer's join request. Per plan §新节点加入 the
|
||||
// sponsor does NOT mutate its own state immediately (it would be washed out
|
||||
// by the next incoming token's authoritative merge). Instead it queues the
|
||||
// newcomer on pendingJoin; OnToken injects it into the token right after
|
||||
// the sponsor. EXCEPTION: a single-node leader has no token circulating
|
||||
// (StartRing refuses for lack of a successor), so the newcomer would never
|
||||
// be injected — in that case we inject immediately and kick off the ring.
|
||||
func (e *Engine) JoinNode(j JoinInfo) *State {
|
||||
e.pendingJoin = append(e.pendingJoin, j)
|
||||
if e.state.LeaderID == "" {
|
||||
e.state.LeaderID = e.ID
|
||||
}
|
||||
// Single-node leader: no successor, no circulating token → the pending
|
||||
// newcomer would never be injected. Inject now (safe: no foreign token
|
||||
// can overwrite a single-node leader) and start the ring.
|
||||
if e.state.LeaderID == e.ID && len(e.state.Nodes) == 1 {
|
||||
e.injectPendingJoin()
|
||||
go e.StartRing(context.Background())
|
||||
}
|
||||
return &e.state
|
||||
}
|
||||
|
||||
@ -392,12 +574,52 @@ func (e *Engine) AdoptState(s State) {
|
||||
e.state.UpsertNode(Node{ID: e.ID, Addr: e.myAddr, Alive: true,
|
||||
Load: e.loadSnapshot(), Version: e.Version, Cache: e.Cache})
|
||||
if e.Log != nil {
|
||||
_, _ = e.Log.Append(e.ID, LogNodeJoin, map[string]string{"node": e.ID, "addr": e.myAddr})
|
||||
e.Log.Append(e.ID, LogNodeJoin, map[string]string{"node": e.ID, "addr": e.myAddr})
|
||||
}
|
||||
}
|
||||
|
||||
// CreateCluster reseeds this node as a fresh standalone leader (single-node
|
||||
// ring). Used after a self-leave left the ring empty, or to (re)affirm seed
|
||||
// state on a standalone node. Refuses if this node is still a multi-node
|
||||
// member — reseeding mid-cluster would split the ring (split-brain). Safe to
|
||||
// call when standalone/empty: no token circulates to a non-member, so no
|
||||
// concurrent OnToken can overwrite the reset.
|
||||
func (e *Engine) CreateCluster() error {
|
||||
if e.IsMember() {
|
||||
return fmt.Errorf("node is a multi-node cluster member; leave first")
|
||||
}
|
||||
e.state = State{
|
||||
LeaderID: e.ID,
|
||||
Cycle: 0,
|
||||
PendingTasks: map[string]*Task{},
|
||||
Topology: map[string]*TopoEntry{},
|
||||
RoundDelay: 2 * time.Second,
|
||||
Seq: e.state.Seq,
|
||||
}
|
||||
e.state.UpsertNode(Node{
|
||||
ID: e.ID, Addr: e.myAddr, Alive: true, IsLeader: true,
|
||||
Load: e.loadSnapshot(), Version: e.Version, Cache: e.Cache,
|
||||
})
|
||||
e.selfRemoved = false
|
||||
e.removedNext = ""
|
||||
e.lastRingStart = time.Time{}
|
||||
e.lastTokenAt = 0
|
||||
e.inflight.clear()
|
||||
if e.Log != nil {
|
||||
_, _ = e.Log.Append(e.ID, LogLeaderChange, map[string]string{"leader": e.ID})
|
||||
}
|
||||
log.Printf("ring[%s] created/seeded fresh standalone cluster as leader", e.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SubmitTask adds a new forward request to pending; it rides the next token
|
||||
// round and is claimed by the lowest-load member.
|
||||
// RemoveNode publishes a node-removal command via the token; the target
|
||||
// node self-removes when the command reaches it.
|
||||
func (e *Engine) RemoveNode(nodeID string) *Task {
|
||||
return e.state.AddRemoveNode(nodeID)
|
||||
}
|
||||
|
||||
// RevokeTask publishes a revocation for an established forward through the
|
||||
// same token channel; the owning node stops the worker and drops topology.
|
||||
func (e *Engine) RevokeTask(local store.Local, remote store.Remote, link store.Link) *Task {
|
||||
@ -430,6 +652,25 @@ func (e *Engine) HasTask(local, remote string, port int) bool {
|
||||
// IsLeader reports whether this node is the current ring leader.
|
||||
func (e *Engine) IsLeader() bool { return e.state.LeaderID == e.ID }
|
||||
|
||||
// IsMember reports whether this node is currently an active multi-node member
|
||||
// (self is in the ring alongside others). Used by the create/join gates to
|
||||
// refuse actions that would split an active ring. A detached node (self not
|
||||
// in ring — e.g. after a self-leave) or a standalone node returns false and
|
||||
// may create/join freely. NOTE: after a self-leave the engine keeps the other
|
||||
// members in state.Nodes (it only dropped self), so a plain len>1 check would
|
||||
// wrongly block a detached node — the self-in-ring test is essential.
|
||||
func (e *Engine) IsMember() bool {
|
||||
if len(e.state.Nodes) <= 1 {
|
||||
return false
|
||||
}
|
||||
for _, n := range e.state.Nodes {
|
||||
if n.ID == e.ID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// LeaderAddr returns the current leader's address.
|
||||
func (e *Engine) LeaderAddr() string {
|
||||
i := e.state.Find(e.state.LeaderID)
|
||||
|
||||
@ -44,29 +44,28 @@ func TestSingleNodeCycle(t *testing.T) {
|
||||
|
||||
// TestTwoNodesPhaseCollect: leader n1 sends to n2; n1 collects both after n2
|
||||
// stamps; then phase flips and second round syncs.
|
||||
func TestTwoNodesCollectAndSync(t *testing.T) {
|
||||
var n2loaded bool
|
||||
func TestTwoNodesSingleRound(t *testing.T) {
|
||||
eng2 := NewEngine("n2", "n2:7500", "u", "p", "0.71.0", nil,
|
||||
&fakeHandler{load: Load{MemPct: 10, NetPct: 10}}, sendNull, "n2:7500", false)
|
||||
// n2's state must include both (leader + itself) so AliveSuccessor works
|
||||
eng2.state.UpsertNode(Node{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true, Load: Load{MemPct: 50, NetPct: 50}})
|
||||
|
||||
// n1 sends a collect-phase token with itself in Passed.
|
||||
eng2.state.UpsertNode(Node{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true, Load: eng2.state.Nodes[0].Load})
|
||||
tk := &Token{Cycle: 1, Phase: PhaseCollect, Passed: []string{"n1"},
|
||||
State: *func() *State {
|
||||
s := &State{}
|
||||
s.UpsertNode(Node{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true})
|
||||
s.UpsertNode(Node{ID: "n2", Addr: "n2:7500", Alive: true})
|
||||
return s
|
||||
}()}
|
||||
_ = n2loaded
|
||||
// n1 sends a token carrying the cluster picture; single-round processing:
|
||||
// the receiving node adopts it and appends its own info in one pass.
|
||||
tk := &Token{Cycle: 1, State: State{
|
||||
Nodes: []Node{
|
||||
{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true},
|
||||
{ID: "n2", Addr: "n2:7500", Alive: true},
|
||||
},
|
||||
}}
|
||||
|
||||
out, err := eng2.OnToken(context.Background(), tk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !contains(out.Passed, "n2") {
|
||||
t.Fatalf("n2 not stamped in round1, passed=%v", out.Passed)
|
||||
if eng2.State().Find("n2") < 0 {
|
||||
t.Fatal("n2 should be present in adopted state after single-round")
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatal("nil token after processing")
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,11 +1,8 @@
|
||||
// Leader-side cycle control + fault tolerance for the token ring:
|
||||
// - phase flip (round1 -> round2 -> next cycle) decided by the leader when
|
||||
// the token returns with all alive nodes passed;
|
||||
// - token-loss resend when it does not come back within roundDelay/2 + 20ms;
|
||||
// - predecessor monitors the leader by heartbeat and promotes itself when the
|
||||
// leader is gone;
|
||||
// - send failures (successor down) mark the neighbor offline, reattach its
|
||||
// tasks, and hop to the next alive successor.
|
||||
// Leader-side minimal roles for the token ring. Per the authoritative design:
|
||||
// the leader only (1) initiates the first token, and (2) judges token loss.
|
||||
// All other processing (adopt cluster picture, append own info, run commands,
|
||||
// forward) is IDENTICAL between leader and ordinary nodes — no phase flips,
|
||||
// no passed-accounting, no cycle bookkeeping in the leader.
|
||||
package cluster
|
||||
|
||||
import (
|
||||
@ -15,18 +12,23 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// LossTimeout returns the token-loss threshold per the authoritative design:
|
||||
// roundDelay/2 + 20ms, floored at 600ms so a healthy fast ring is never
|
||||
// misjudged as lost.
|
||||
// ringHopDelay paces each token hop (real-world cadence, avoids busy-loop).
|
||||
// 500ms keeps a 3-node ring well under the LossTimeout floor (2500ms) so a
|
||||
// healthy round is never misjudged lost, while not idly burning CPU/HTTP at
|
||||
// 20Hz like the old 50ms did.
|
||||
const ringHopDelay = 500 * time.Millisecond
|
||||
|
||||
// LossTimeout is the token-loss threshold per design: roundDelay/2 + 20ms,
|
||||
// floored so a healthy fast ring is never misjudged.
|
||||
func LossTimeout(roundDelay time.Duration) time.Duration {
|
||||
t := roundDelay/2 + 20*time.Millisecond
|
||||
if t < 600*time.Millisecond {
|
||||
return 600 * time.Millisecond
|
||||
if t < 2500*time.Millisecond {
|
||||
return 2500 * time.Millisecond
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// inFlight tracks token-in-flight state (leader only).
|
||||
// inFlight tracks token-in-flight state (leader only, for loss judging).
|
||||
type inFlight struct {
|
||||
mu sync.Mutex
|
||||
active bool
|
||||
@ -69,53 +71,22 @@ func (f *inFlight) lastDelay() time.Duration {
|
||||
return f.delay
|
||||
}
|
||||
|
||||
// flipIfComplete promotes the collect round to the sync round when every alive
|
||||
// node has been stamped; at the end of sync it completes the cycle. Only the
|
||||
// leader flips. Passed carries the set of nodes that saw this token this round;
|
||||
// it is NOT reset between phases (only between cycles).
|
||||
func (e *Engine) flipIfComplete(tk *Token) {
|
||||
if e.state.LeaderID != e.ID {
|
||||
return
|
||||
}
|
||||
switch tk.Phase {
|
||||
case PhaseCollect:
|
||||
if allAlivePassed(&e.state, tk.Passed) {
|
||||
// Round 1 done: go to round 2 (sync). Round-2 stamping restarts
|
||||
// from the leader so the token walks the ring once more.
|
||||
tk.Phase = PhaseSync
|
||||
tk.Passed = []string{e.ID}
|
||||
e.state.RoundDelay = tkDelaySince(tk)
|
||||
}
|
||||
case PhaseSync:
|
||||
if allAlivePassed(&e.state, tk.Passed) {
|
||||
// Cycle done: leader holds token; next StartRing bumps cycle.
|
||||
e.state.RoundDelay = tkDelaySince(tk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func allAlivePassed(s *State, passed []string) bool {
|
||||
for _, n := range s.Nodes {
|
||||
if !n.Alive {
|
||||
continue
|
||||
}
|
||||
if !contains(passed, n.ID) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Send moves the token onward (or completes the cycle at the leader). It is
|
||||
// called by the HTTP handler after OnToken. A full cycle stops at the leader.
|
||||
// Send is called after OnToken for the LEADER. The token has come full
|
||||
// circle (one round complete), so the leader bumps the cycle, stamps a fresh
|
||||
// SentAt (so any stale in-flight older token is dropped downstream), and
|
||||
// forwards to the successor to start the next round. This is what makes
|
||||
// `cycle` advance under the perpetual-flow model — without it cycle stuck
|
||||
// at the StartRing value forever.
|
||||
func (e *Engine) Send(ctx context.Context, tk *Token) error {
|
||||
if e.state.LeaderID == e.ID {
|
||||
e.flipIfComplete(tk)
|
||||
// If the sync round completed, the leader holds the token; watchdog
|
||||
// StartRing launches the next cycle. Clear in-flight marker.
|
||||
if tk.Phase == PhaseSync && allAlivePassed(&e.state, tk.Passed) {
|
||||
e.inflight.clear()
|
||||
return nil
|
||||
if !e.selfRemoved {
|
||||
e.state.Cycle = tk.Cycle + 1
|
||||
tk.Cycle = e.state.Cycle
|
||||
// Keep State.Cycle in lockstep with Token.Cycle so the snapshot
|
||||
// (which reads e.state.Cycle) reflects the real round number.
|
||||
tk.State = e.state
|
||||
tk.SentAt = time.Now().UnixMilli()
|
||||
if tk.SentAt > e.lastTokenAt {
|
||||
e.lastTokenAt = tk.SentAt
|
||||
}
|
||||
}
|
||||
return e.forwardToNext(ctx, tk)
|
||||
@ -135,14 +106,23 @@ func (e *Engine) forwardToNext(ctx context.Context, tk *Token) error {
|
||||
}
|
||||
err := e.send(ctx, next, tk)
|
||||
if err == nil {
|
||||
e.inflight.mark(e.state.RoundDelay)
|
||||
if e.state.LeaderID == e.ID {
|
||||
e.inflight.mark(e.state.RoundDelay)
|
||||
}
|
||||
// Pace the ring so tokens circulate at a realistic cadence.
|
||||
// Pacing handled by the parallel rhythm timer in OnToken.
|
||||
return nil
|
||||
}
|
||||
log.Printf("ring[%s] send to %s failed: %v", e.ID, next, err)
|
||||
e.state.MarkOffline(next)
|
||||
e.state.OfflineReassign(next)
|
||||
if e.state.LeaderID == next {
|
||||
e.becomeLeader()
|
||||
// Do not declare a neighbor offline on a single timeout: transient
|
||||
// send failures (network jitter, busy handler) must not break the
|
||||
// ring. Only after consecutive failures do we evict the node.
|
||||
e.failMu.Lock()
|
||||
e.failCount[next]++
|
||||
if e.failCount[next] >= 2 {
|
||||
e.state.MarkOffline(next)
|
||||
e.state.OfflineReassign(next)
|
||||
delete(e.failCount, next)
|
||||
}
|
||||
}
|
||||
e.inflight.clear()
|
||||
@ -158,8 +138,8 @@ func (e *Engine) becomeLeader() {
|
||||
log.Printf("ring[%s] promoted to leader", e.ID)
|
||||
}
|
||||
|
||||
// WatchLeader runs the heartbeat monitor: if our successor is the leader, ping
|
||||
// it; on failure mark it offline and promote ourselves. Stops on ctx cancel.
|
||||
// WatchLeader runs the leader liveness monitor: the predecessor of the leader
|
||||
// pings it; on failure it marks the leader offline and promotes itself.
|
||||
func (e *Engine) WatchLeader(ctx context.Context) {
|
||||
tick := time.NewTicker(2 * time.Second)
|
||||
defer tick.Stop()
|
||||
@ -168,44 +148,21 @@ func (e *Engine) WatchLeader(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
succ, ok := e.state.AliveSuccessor(e.ID)
|
||||
if !ok || succ == e.ID {
|
||||
continue // single-node ring: nothing to monitor
|
||||
if e.state.LeaderID == e.ID {
|
||||
continue // we are the leader; predecessor monitors us
|
||||
}
|
||||
// Only the leader's predecessor (the previous alive node) monitors
|
||||
// it; a node whose successor is the leader is the predecessor.
|
||||
if e.state.LeaderID != succ {
|
||||
continue
|
||||
}
|
||||
pred, ok := e.state.AlivePredecessor(e.ID)
|
||||
if !ok || pred == e.ID {
|
||||
continue
|
||||
}
|
||||
// We monitor the leader only if we ARE its predecessor.
|
||||
if succ != e.state.LeaderID {
|
||||
continue
|
||||
}
|
||||
// pred must be us: leader's predecessor sends the heartbeat.
|
||||
if e.state.Nodes[e.state.Find(succ)].ID == e.ID {
|
||||
continue
|
||||
}
|
||||
if e.state.Find(e.ID) == -1 {
|
||||
continue
|
||||
}
|
||||
// This node is the leader's predecessor iff leader's predecessor
|
||||
// (alive) equals our ID.
|
||||
leaderPred, ok := e.state.AlivePredecessor(e.state.LeaderID)
|
||||
if !ok || leaderPred != e.ID {
|
||||
continue
|
||||
continue // only the leader's predecessor pings it
|
||||
}
|
||||
if e.send != nil {
|
||||
hbCtx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond)
|
||||
err := e.send(hbCtx, succ, nil) // nil token = heartbeat ping
|
||||
err := e.send(hbCtx, e.state.LeaderID, nil) // nil = heartbeat
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("ring[%s] heartbeat to leader %s failed: %v", e.ID, succ, err)
|
||||
e.state.MarkOffline(succ)
|
||||
e.state.OfflineReassign(succ)
|
||||
log.Printf("ring[%s] heartbeat to leader %s failed: %v", e.ID, e.state.LeaderID, err)
|
||||
e.state.MarkOffline(e.state.LeaderID)
|
||||
e.state.OfflineReassign(e.state.LeaderID)
|
||||
e.becomeLeader()
|
||||
}
|
||||
}
|
||||
@ -213,8 +170,9 @@ func (e *Engine) WatchLeader(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// WatchTokenLoss runs on the leader: if a token was sent and does not return
|
||||
// within LossTimeout, resend it. Stops on ctx cancel.
|
||||
// WatchTokenLoss runs the leader's token-loss judge: if a token was sent and
|
||||
// does not return within LossTimeout, the leader issues a fresh token (all
|
||||
// nodes drop older stamps via the SentAt guard, so at most one circulates).
|
||||
func (e *Engine) WatchTokenLoss(ctx context.Context) {
|
||||
tick := time.NewTicker(200 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
|
||||
@ -65,9 +65,13 @@ func TestLogAfterWatermark(t *testing.T) {
|
||||
func TestClaimLogsToEngine(t *testing.T) {
|
||||
eng := newTestEngine("n1", true)
|
||||
eng.state.AddPending(store.Local{Name: "l1"}, store.Remote{Name: "r1"}, store.Link{})
|
||||
if err := eng.phase2(context.Background(), &Token{Cycle: 1, Phase: PhaseSync, State: eng.state}); err != nil {
|
||||
out, err := eng.OnToken(context.Background(), &Token{Cycle: 1, State: eng.state})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatal("nil token after OnToken")
|
||||
}
|
||||
snap := eng.Log.Snapshot()
|
||||
if len(snap) != 1 || snap[0].Kind != LogForwardAdd {
|
||||
t.Fatalf("engine log = %+v", snap)
|
||||
|
||||
121
internal/cluster/ring_remove_test.go
Normal file
121
internal/cluster/ring_remove_test.go
Normal file
@ -0,0 +1,121 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"webui4frpc/internal/store"
|
||||
)
|
||||
|
||||
// TestSelfRemoveViaToken: a node-removal command reaches the target node
|
||||
// through OnToken; it stops owned workers, re-queues its forwards, drops its
|
||||
// ring position, and the token flows to the original successor.
|
||||
func TestSelfRemoveViaToken(t *testing.T) {
|
||||
eng := newTestEngine("n1", true)
|
||||
eng.state.InsertAfter("n1", Node{ID: "n2", Addr: "n2:7500", Alive: true, Load: Load{MemPct: 10, NetPct: 10}})
|
||||
|
||||
// n1 owns a forward (via topology)
|
||||
p := eng.state.AddPending(store.Local{Name: "web"}, store.Remote{Name: "frps1"}, store.Link{RemotePort: 18081})
|
||||
eng.state.ClaimPending(p.ID)
|
||||
eng.state.AddTopology(p, "n1")
|
||||
|
||||
// n2 publishes remove-node for itself
|
||||
rm := eng.state.AddRemoveNode("n2")
|
||||
if rm.RemoveNode != "n2" {
|
||||
t.Fatalf("remove node cmd = %+v", rm)
|
||||
}
|
||||
|
||||
// run OnToken at n2 (lowest-load, so it claims the remove command).
|
||||
// State order is [n2, n1] so LowestAlive picks n2.
|
||||
eng2 := newTestEngine("n2", false)
|
||||
eng2.state.UpsertNode(Node{ID: "n2", Addr: "n2:7500", Alive: true, Load: Load{MemPct: 10, NetPct: 10}})
|
||||
eng2.state.UpsertNode(Node{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true, Load: Load{MemPct: 90, NetPct: 90}})
|
||||
eng2.state.Topology = map[string]*TopoEntry{
|
||||
p.ID: {TaskID: p.ID, OwnerID: "n2", Active: true, Local: p.Local, Remote: p.Remote, Link: p.Link},
|
||||
}
|
||||
eng2.state.PendingTasks = map[string]*Task{rm.ID: rm}
|
||||
out, err := eng2.OnToken(context.Background(), &Token{Cycle: 1, State: eng2.state})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatal("nil token after OnToken")
|
||||
}
|
||||
// n2 removed itself from ring (NOT re-added by the UpsertNode step)
|
||||
if eng2.state.Find("n2") >= 0 {
|
||||
t.Fatalf("n2 still in ring: %+v", eng2.state.Nodes)
|
||||
}
|
||||
// the forward n2 owned was re-queued as pending for another member
|
||||
if len(eng2.state.PendingList()) == 0 {
|
||||
t.Fatalf("no pending re-queue after self-remove: %+v", eng2.state.PendingList())
|
||||
}
|
||||
// removedNext captured so Forward can hand the token to the old successor
|
||||
if eng2.removedNext != "n1" {
|
||||
t.Fatalf("removedNext = %q want n1", eng2.removedNext)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRemoveCommandRidesPastLowest: a node-removal command directed at a
|
||||
// NON-lowest node must NOT be claimed by the lowest-load node (the old bug
|
||||
// corrupted it into a phantom forward via Handler.Claim). It stays pending
|
||||
// and rides the token until it reaches the target, which then self-removes.
|
||||
func TestRemoveCommandRidesPastLowest(t *testing.T) {
|
||||
// 3-node ring order [n2, n1, n3]: n2 is lowest, n3 is the remove target.
|
||||
eng2 := newTestEngine("n2", false)
|
||||
eng2.state.UpsertNode(Node{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true, Load: Load{MemPct: 50, NetPct: 50}})
|
||||
eng2.state.UpsertNode(Node{ID: "n3", Addr: "n3:7500", Alive: true, Load: Load{MemPct: 90, NetPct: 90}})
|
||||
rm := eng2.state.AddRemoveNode("n3")
|
||||
|
||||
// Token reaches n2 (lowest) carrying the remove command for n3.
|
||||
if _, err := eng2.OnToken(context.Background(), &Token{Cycle: 1, State: eng2.state}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// n2 must NOT have consumed the remove — it must still be pending, riding.
|
||||
if _, ok := eng2.state.PendingTasks[rm.ID]; !ok {
|
||||
t.Fatal("lowest node n2 consumed a remove command aimed at n3 (should ride to target)")
|
||||
}
|
||||
if i := eng2.state.Find("n3"); i < 0 {
|
||||
t.Fatal("n3 was removed by non-target n2 (should still be in ring)")
|
||||
}
|
||||
|
||||
// Token now reaches the TARGET (n3): it self-removes + captures successor.
|
||||
eng3 := newTestEngine("n3", false)
|
||||
eng3.state.UpsertNode(Node{ID: "n1", Addr: "n1:7500", Alive: true, IsLeader: true, Load: Load{MemPct: 50, NetPct: 50}})
|
||||
eng3.state.UpsertNode(Node{ID: "n2", Addr: "n2:7500", Alive: true, Load: Load{MemPct: 10, NetPct: 10}})
|
||||
eng3.state.PendingTasks = map[string]*Task{rm.ID: rm}
|
||||
if _, err := eng3.OnToken(context.Background(), &Token{Cycle: 2, State: eng3.state}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if eng3.state.Find("n3") >= 0 {
|
||||
t.Fatalf("n3 did not self-remove: %+v", eng3.state.Nodes)
|
||||
}
|
||||
// Ring order [n3, n1, n2] → n3's original successor is n1.
|
||||
if eng3.removedNext != "n1" {
|
||||
t.Fatalf("removedNext = %q want n1", eng3.removedNext)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRemoveCommandFulfilledNoPhantom: when a remove command's target has
|
||||
// already left the ring (or was never a member), the lowest-load node that
|
||||
// re-encounters it must DROP it — NOT fall through to Handler.Claim and spawn
|
||||
// a phantom empty forward (the remove task carries no local/remote/link).
|
||||
func TestRemoveCommandFulfilledNoPhantom(t *testing.T) {
|
||||
var claimed []*Task
|
||||
eng := NewEngine("n1", "n1:7500", "u", "p", "0.1.0", nil,
|
||||
&fakeHandler{load: Load{MemPct: 5, NetPct: 5},
|
||||
claim: func(ctx context.Context, tk *Task) error { claimed = append(claimed, tk); return nil }},
|
||||
sendNull, "n1:7500", true)
|
||||
rm := eng.state.AddRemoveNode("node-x:7500") // target not in the ring
|
||||
if _, err := eng.OnToken(context.Background(), &Token{Cycle: 1, State: eng.state}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := eng.state.PendingTasks[rm.ID]; ok {
|
||||
t.Fatal("fulfilled remove command was not dropped from pending")
|
||||
}
|
||||
if len(claimed) != 0 {
|
||||
t.Fatalf("Handler.Claim called for a remove command: %+v", claimed)
|
||||
}
|
||||
if len(eng.state.TopologyList()) != 0 {
|
||||
t.Fatalf("phantom topology entry created: %+v", eng.state.TopologyList())
|
||||
}
|
||||
}
|
||||
@ -20,7 +20,7 @@ func TestRevokeTaskRemovesTopology(t *testing.T) {
|
||||
// establish a forward
|
||||
eng.state.AddPending(store.Local{Name: "web"}, store.Remote{Name: "frps1"}, store.Link{RemotePort: 18081})
|
||||
// lowest-load is n1 (only node): it claims + builds topology
|
||||
if err := eng.phase2(context.Background(), &Token{Cycle: 1, Phase: PhaseSync, State: eng.state}); err != nil {
|
||||
if _, err := eng.OnToken(context.Background(), &Token{Cycle: 1, State: eng.state}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(eng.state.TopologyList()) != 1 {
|
||||
@ -29,7 +29,7 @@ func TestRevokeTaskRemovesTopology(t *testing.T) {
|
||||
|
||||
// publish a revoke task pointing at the same forward
|
||||
eng.state.AddRevoke(store.Local{Name: "web"}, store.Remote{Name: "frps1"}, store.Link{RemotePort: 18081})
|
||||
if err := eng.phase2(context.Background(), &Token{Cycle: 2, Phase: PhaseSync, State: eng.state}); err != nil {
|
||||
if _, err := eng.OnToken(context.Background(), &Token{Cycle: 2, State: eng.state}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(eng.state.TopologyList()) != 0 {
|
||||
@ -54,7 +54,7 @@ func TestRevokeTaskRemovesTopology(t *testing.T) {
|
||||
func TestRevokeIdempotent(t *testing.T) {
|
||||
eng := newTestEngine("n1", true)
|
||||
eng.state.AddRevoke(store.Local{Name: "ghost"}, store.Remote{Name: "frps1"}, store.Link{RemotePort: 1})
|
||||
if err := eng.phase2(context.Background(), &Token{Cycle: 1, Phase: PhaseSync, State: eng.state}); err != nil {
|
||||
if _, err := eng.OnToken(context.Background(), &Token{Cycle: 1, State: eng.state}); err != nil {
|
||||
t.Fatalf("revoke missing: %v", err)
|
||||
}
|
||||
// no topology entry, no panic
|
||||
|
||||
@ -45,3 +45,13 @@ func (e *Engine) JoinRing(ctx context.Context, targetAddr string, ji JoinInfo) e
|
||||
e.AdoptState(out.State)
|
||||
return nil
|
||||
}
|
||||
|
||||
// JoinRingAddr wraps JoinRing for HTTP handlers: it builds the newcomer's
|
||||
// JoinInfo from this node's own id/addr/version/cache (so the caller doesn't
|
||||
// touch the engine's unexported fields) and asks targetAddr to sponsor us
|
||||
// into its ring. This is the runtime "加入集群" path (vs. the startup
|
||||
// bootstrap call in cmd/webui4frpc/main.go).
|
||||
func (e *Engine) JoinRingAddr(ctx context.Context, targetAddr string) error {
|
||||
ji := JoinInfo{ID: e.ID, Addr: e.myAddr, Version: e.Version, Cache: e.Cache}
|
||||
return e.JoinRing(ctx, targetAddr, ji)
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
@ -48,11 +49,14 @@ func (t *TokenTransport) SendTo(getAddr func(nodeID string) string) func(ctx con
|
||||
req.SetBasicAuth(t.User, t.Pass)
|
||||
}
|
||||
cli := &http.Client{Timeout: 5 * time.Second}
|
||||
start := time.Now()
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("token-send %s: size=%dB elapsed=%v err=%v", addr, len(body), time.Since(start).Round(time.Millisecond), err)
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
log.Printf("token-send %s: size=%dB elapsed=%v -> %d", addr, len(body), time.Since(start).Round(time.Millisecond), resp.StatusCode)
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("token POST %s -> HTTP %d", url, resp.StatusCode)
|
||||
}
|
||||
|
||||
@ -28,11 +28,11 @@ func TestTokenTransportPOST(t *testing.T) {
|
||||
tr := &TokenTransport{User: "u", Pass: "p"}
|
||||
host := srv.Listener.Addr().String()
|
||||
send := tr.SendTo(func(nodeID string) string { return host })
|
||||
err := send(context.Background(), "peer1", &Token{Cycle: 3, Phase: PhaseCollect})
|
||||
err := send(context.Background(), "peer1", &Token{Cycle: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("send: %v", err)
|
||||
}
|
||||
if got.Cycle != 3 || got.Phase != PhaseCollect {
|
||||
if got.Cycle != 3 {
|
||||
t.Fatalf("got token = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
4
internal/httpapi/dist/index.html
vendored
4
internal/httpapi/dist/index.html
vendored
@ -4,8 +4,8 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>webui-frpc</title>
|
||||
<script type="module" crossorigin src="/assets/index-CNe69otc.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-Cq7Ykgzy.css">
|
||||
<script type="module" crossorigin src="/assets/index-CXBf2fLP.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-B_Hvtf6C.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@ -77,10 +77,19 @@ func (h *Handler) saveCanvas(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
} else if h.Ring != nil {
|
||||
if fwd, _ := s.LinksForLocal(old.Name); len(fwd) > 0 {
|
||||
if rem, ok := s.GetRemote(fwd[0].Remote); ok {
|
||||
h.Ring.RevokeTask(old, rem, store.Link{Local: old.Name, Remote: rem.Name, RemotePort: fwd[0].RemotePort})
|
||||
// Publish a REVOKE task for EVERY link of the removed local —
|
||||
// a local may fan out to several remotes, and revoking only the
|
||||
// first (fwd[0]) left the rest as orphan workers running on
|
||||
// their owning cluster nodes.
|
||||
fwd, _ := s.LinksForLocal(old.Name)
|
||||
for _, ln := range fwd {
|
||||
rem, ok := s.GetRemote(ln.Remote)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
h.Ring.RevokeTask(old, rem, store.Link{
|
||||
Local: old.Name, Remote: rem.Name, RemotePort: ln.RemotePort,
|
||||
})
|
||||
}
|
||||
}
|
||||
_ = s.DeleteLocal(old.Name)
|
||||
|
||||
@ -98,13 +98,23 @@ func NewServeMux(h *Handler) (http.Handler, error) {
|
||||
mux.HandleFunc(apiPrefix+"/cluster/ring", auth(h.handleClusterRing))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/join", auth(h.handleClusterJoin))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/task", auth(h.handleClusterTask))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/node-remove", auth(h.handleNodeRemove))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/create", auth(h.handleClusterCreate))
|
||||
mux.HandleFunc(apiPrefix+"/cluster/join-ring", auth(h.handleClusterJoinRing))
|
||||
|
||||
// M6: peer-to-peer binary exchange endpoint (Basic Auth, same creds).
|
||||
// Not under /api so peers hit it directly; auth still applied.
|
||||
mux.HandleFunc("/frpc/", auth(h.handleFrpcBinary))
|
||||
|
||||
// Static assets (also basic auth) under /.
|
||||
mux.HandleFunc("/", h.handleStatic)
|
||||
// The SPA shell is served behind the SAME Basic Auth as the API (plan:
|
||||
// "healthz 免认证,其余 Basic Auth"). Without this, the browser loads the
|
||||
// page without a 401 challenge, never caches credentials, and every
|
||||
// same-origin /api/* fetch (credentials:"same-origin") gets 401 — so the
|
||||
// SPA renders but all data pages read as empty ("集群未启动"). Wrapping /
|
||||
// in auth makes the browser prompt once, cache the Basic header for the
|
||||
// origin, and send it on every subsequent asset + API request.
|
||||
mux.HandleFunc("/", auth(h.handleStatic))
|
||||
|
||||
return mux, nil
|
||||
}
|
||||
@ -384,17 +394,26 @@ func (h *Handler) handleClusterToken(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "parse token: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
updated, err := h.Ring.OnToken(r.Context(), &tk)
|
||||
updated, err := h.Ring.OnToken(context.Background(), &tk)
|
||||
if err != nil {
|
||||
http.Error(w, "token process: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Phase flips and cycle completion happen at the leader; other nodes just
|
||||
// forward the token along the ring.
|
||||
// OnToken returns nil when it drops a stale/duplicate token: the token is
|
||||
// dead, do NOT hand a nil token onward (would nil-panic in Send/Forward).
|
||||
if updated == nil {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
// Onward forwarding is ASYNC: acknowledge receipt immediately (the
|
||||
// token is a relay baton, not a synchronous RPC chain). If we forwarded
|
||||
// synchronously, a slow next hop would make this handler hang for the
|
||||
// upstream client timeout, which would recursively stall the whole ring.
|
||||
nextTK := *updated
|
||||
if h.Ring.IsLeader() {
|
||||
_ = h.Ring.Send(r.Context(), updated)
|
||||
go func() { _ = h.Ring.Send(context.Background(), &nextTK) }()
|
||||
} else {
|
||||
_ = h.Ring.Forward(r.Context(), updated)
|
||||
go func() { _ = h.Ring.Forward(context.Background(), &nextTK) }()
|
||||
}
|
||||
writeJSON(w, http.StatusOK, updated)
|
||||
}
|
||||
@ -467,6 +486,90 @@ func (h *Handler) handleClusterTask(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"task": tk})
|
||||
}
|
||||
|
||||
// handleNodeRemove publishes a node-removal command via the token; the
|
||||
// target node self-removes when the command reaches it.
|
||||
func (h *Handler) handleNodeRemove(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w)
|
||||
return
|
||||
}
|
||||
if h.Ring == nil {
|
||||
http.Error(w, "ring engine not enabled", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "parse: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if req.ID == "" {
|
||||
http.Error(w, "node id required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
tk := h.Ring.RemoveNode(req.ID)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"task": tk})
|
||||
}
|
||||
|
||||
// handleClusterCreate reseeds this node as a fresh standalone leader (the
|
||||
// runtime "创建集群" path). Refuses with 409 if this node is still a
|
||||
// multi-node member — reseeding mid-cluster would split the ring.
|
||||
func (h *Handler) handleClusterCreate(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w)
|
||||
return
|
||||
}
|
||||
if h.Ring == nil {
|
||||
http.Error(w, "ring engine not enabled", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if err := h.Ring.CreateCluster(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, h.Ring.Snapshot())
|
||||
}
|
||||
|
||||
// handleClusterJoinRing is the runtime "加入集群" path: this node joins the
|
||||
// cluster at the given peer address (newcomer-side — it POSTs its own
|
||||
// JoinInfo to the peer's /cluster/join and adopts the returned state).
|
||||
// Synchronous so the UI gets a real success/failure; capped at 10s (the
|
||||
// peer dial itself times out at 8s). Refuses 409 if already a multi-node
|
||||
// member (would split the ring).
|
||||
func (h *Handler) handleClusterJoinRing(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w)
|
||||
return
|
||||
}
|
||||
if h.Ring == nil {
|
||||
http.Error(w, "ring engine not enabled", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Addr string `json:"addr"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "parse: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if req.Addr == "" {
|
||||
http.Error(w, "addr required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if h.Ring.IsMember() {
|
||||
http.Error(w, "already a multi-node cluster member; leave first", http.StatusConflict)
|
||||
return
|
||||
}
|
||||
jc, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := h.Ring.JoinRingAddr(jc, req.Addr); err != nil {
|
||||
http.Error(w, "join failed: "+err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, h.Ring.Snapshot())
|
||||
}
|
||||
|
||||
// handleClusterJoin accepts a newcomer join request: the target node inserts
|
||||
// the newcomer after itself in the ring and returns the updated ring state
|
||||
// for the newcomer to adopt.
|
||||
|
||||
Reference in New Issue
Block a user