mirror of
https://gitcode.com/JianFeeeee/webui4frpc.git
synced 2026-09-20 00:47:57 +00:00
- 主题: sakura×frost 玻璃拟态 (theme.css) + SCSS 变量重映射 - 侧栏: 玻璃侧栏 246px + 渐变品牌区 + 面包屑导航 - 状态页: 玻璃 KPI 卡 + 远程节点/本地服务卡片网格 - 集群页: 英雄玻璃卡 + 横向环拓扑链 + 待办命令/活跃拓扑/日志区 - 令牌环: 新增 lastSync 上次同步时间替代周期计数 - 模拟 frps: frps2/frps3 容器 + test-forward.sh 全链路验证脚本 - 修复: BinaryPath 空导致 worker 不启动, 撤销仅撤第一个 link, 任务复活风暴 (published 追踪)
682 lines
24 KiB
Go
682 lines
24 KiB
Go
// Package cluster implements the token-ring cooperative network.
|
|
// Authoritative design: plan.md §M6 (令牌环网拓扑).
|
|
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
|
|
"webui4frpc/internal/store"
|
|
)
|
|
|
|
// Handler is what the engine calls when the node must act on a task
|
|
// (create the frpc worker / forward). Injected to avoid import cycle.
|
|
type Handler interface {
|
|
Claim(ctx context.Context, tk *Task) error
|
|
Revoke(ctx context.Context, tk *Task) error
|
|
RuntimeLoad() Load
|
|
}
|
|
|
|
// Engine drives one member of the token ring.
|
|
type Engine struct {
|
|
ID string
|
|
Addr string
|
|
User string
|
|
Pass string
|
|
Version string
|
|
Cache []string
|
|
Handler Handler
|
|
|
|
state State
|
|
// myAddr maps our Node ID to the address peers dial.
|
|
myAddr string
|
|
|
|
// send moves the token to the next node (injected transport).
|
|
send func(ctx context.Context, next string, tk *Token) error
|
|
|
|
// inflight tracks token-in-flight state (leader only).
|
|
inflight inFlight
|
|
Log *ClusterLog
|
|
lastLogSent int64
|
|
// lastRingStart time of the previous cycle launch (leader throttle).
|
|
lastRingStart time.Time
|
|
// lastTokenAt: leader-stamped token timestamp of the newest valid token
|
|
// this node accepted. Older tokens (multi-token conflict leftovers) are
|
|
// dropped so only one token effectively circulates.
|
|
lastTokenAt int64
|
|
// lastSyncAt unix-seconds when this node last completed a sync round.
|
|
lastSyncAt int64
|
|
// failCount counts consecutive send failures per neighbor; a node is
|
|
// declared offline only after repeated failures (transient jitter must
|
|
// not break the ring).
|
|
failCount map[string]int
|
|
// failMu guards failCount (HTTP handlers run concurrently).
|
|
failMu sync.Mutex
|
|
// pendingJoin: newcomers accepted via JoinNode but NOT YET injected into
|
|
// the token. Per the authoritative design (plan §新节点加入): a sponsor
|
|
// injects the newcomer into the token only when the token reaches it
|
|
// ("令牌发到自己时,先转发给新节点,并把新节点加入令牌中的集群信息").
|
|
// Until then the newcomer is NOT in e.state.Nodes — so the OnToken
|
|
// state-merge (incoming authoritative) cannot wash it back out.
|
|
pendingJoin []JoinInfo
|
|
// selfRemoved is set when a node-remove command targeted THIS node and
|
|
// it has run SelfRemove; the OnToken "append own node info" step is then
|
|
// skipped (otherwise UpsertNode(self) would re-add the removed node).
|
|
selfRemoved bool
|
|
// removedNext: the original successor captured right before SelfRemove,
|
|
// so Forward can hand the token to it even though this node is no longer
|
|
// in state.Nodes (plan §移除节点 step 3: "令牌传递给自身原本的下一家").
|
|
removedNext string
|
|
// published records task IDs this node placed into the token it last
|
|
// forwarded. When the next token returns WITHOUT one of those IDs, the
|
|
// task was consumed downstream (claimed/revoked) — the localPending
|
|
// re-merge must NOT resurrect it, or the task rides forever (a re-claim /
|
|
// re-revoke storm centered on the submitter). Reset each round to the
|
|
// tasks actually leaving on this token.
|
|
published map[string]struct{}
|
|
}
|
|
|
|
// NewEngine builds the engine; state holds this node as initial leader unless
|
|
// a peer list says otherwise (creation node starts the ring).
|
|
func NewEngine(id, addr, user, pass, version string, cache []string, h Handler, send func(ctx context.Context, next string, tk *Token) error, selfAddr string, isLeader bool) *Engine {
|
|
e := &Engine{
|
|
ID: id, Addr: addr, User: user, Pass: pass,
|
|
Version: version, Cache: cache, Handler: h,
|
|
state: State{
|
|
LeaderID: "",
|
|
Cycle: 0,
|
|
PendingTasks: map[string]*Task{},
|
|
Topology: map[string]*TopoEntry{},
|
|
RoundDelay: 2 * time.Second,
|
|
},
|
|
myAddr: selfAddr,
|
|
send: send,
|
|
Log: NewClusterLog(),
|
|
failCount: map[string]int{},
|
|
published: map[string]struct{}{},
|
|
}
|
|
n := Node{ID: id, Addr: selfAddr, Alive: true, IsLeader: isLeader,
|
|
Load: Load{MemPct: 10, NetPct: 10}, Version: version, Cache: cache}
|
|
e.state.UpsertNode(n)
|
|
return e
|
|
}
|
|
|
|
// State returns the engine's current cluster picture.
|
|
func (e *Engine) State() *State { return &e.state }
|
|
|
|
// nextRecipient picks the successor to hand the token to, skipping dead nodes.
|
|
func (e *Engine) nextRecipient() (string, bool) {
|
|
return e.state.AliveSuccessor(e.ID)
|
|
}
|
|
|
|
// myNode returns this node's entry from state.
|
|
func (e *Engine) myNode() Node {
|
|
i := e.state.Find(e.ID)
|
|
if i < 0 {
|
|
return Node{ID: e.ID, Addr: e.myAddr, Alive: true}
|
|
}
|
|
return e.state.Nodes[i]
|
|
}
|
|
|
|
// loadSnapshot reads our runtime load (mem+net) from the handler.
|
|
func (e *Engine) loadSnapshot() Load {
|
|
if e.Handler != nil {
|
|
return e.Handler.RuntimeLoad()
|
|
}
|
|
return Load{MemPct: 20, NetPct: 20}
|
|
}
|
|
|
|
// OnToken is the SINGLE-ROUND token handler. Per the authoritative design
|
|
// (plan §令牌环协议): on receiving the token a node simultaneously
|
|
// (a) ADOPTS the carried cluster picture — incoming state is authoritative
|
|
// for membership + per-node fields. This is safe because structural
|
|
// changes (join/remove) are NOT kept in local state waiting to survive
|
|
// a merge: joins ride a separate pendingJoin channel injected INTO the
|
|
// token here, and a self-remove writes the node out of state so the
|
|
// downstream merge naturally drops it.
|
|
// (b) APPLIES the incremental log delta, then re-attaches own fresh entries.
|
|
// (c) APPENDS own node info (load/version) — skipped if we self-removed.
|
|
// (d) INJECTS pending newcomers right after ourselves + forwards to them.
|
|
// (e) Paces via a parallel rhythm timer (max(ops, timer)).
|
|
func (e *Engine) OnToken(ctx context.Context, tk *Token) (*Token, error) {
|
|
// Multi-token guard: only the newest leader-stamped token is kept.
|
|
if tk.SentAt > 0 && e.lastTokenAt > 0 && tk.SentAt < e.lastTokenAt {
|
|
log.Printf("ring[%s] drop stale token sentAt=%d (last=%d)", e.ID, tk.SentAt, e.lastTokenAt)
|
|
return nil, nil
|
|
}
|
|
if tk.SentAt > e.lastTokenAt {
|
|
e.lastTokenAt = tk.SentAt
|
|
}
|
|
log.Printf("ring[%s] OnToken cycle=%d", e.ID, tk.Cycle)
|
|
|
|
// Parallel rhythm timer: operations run while the pace clock ticks.
|
|
rhythm := time.NewTimer(ringHopDelay)
|
|
defer rhythm.Stop()
|
|
|
|
// (a) ADOPT the cluster picture. Incoming state is authoritative: joins
|
|
// ride pendingJoin (injected below), removes write the node out of state
|
|
// so a downstream merge drops it — so a plain assignment is correct and
|
|
// does NOT wash out local structural changes.
|
|
// EXCEPTION: pending COMMANDS (SubmitTask / RemoveNode / AddRevoke) are
|
|
// injected into e.state.PendingTasks by HTTP handlers OUTSIDE OnToken, so
|
|
// a blanket overwrite would drop them before the token carries them. Keep
|
|
// local-only pending commands and re-merge them after the adoption — BUT
|
|
// only those never yet published into a departing token. A task we already
|
|
// published that is now absent from the incoming token was consumed
|
|
// downstream (claimed/revoked); resurrecting it would make it ride forever
|
|
// (a re-claim / re-revoke storm centered on the submitter node).
|
|
localPending := map[string]*Task{}
|
|
for id, t := range e.state.PendingTasks {
|
|
if _, sent := e.published[id]; sent {
|
|
continue
|
|
}
|
|
if _, inToken := tk.State.PendingTasks[id]; inToken {
|
|
continue
|
|
}
|
|
localPending[id] = t
|
|
}
|
|
rd := e.state.RoundDelay // preserve if incoming carries none (zero-guard)
|
|
e.state = tk.State
|
|
if e.state.RoundDelay == 0 {
|
|
e.state.RoundDelay = rd
|
|
}
|
|
if e.state.PendingTasks == nil {
|
|
e.state.PendingTasks = map[string]*Task{}
|
|
}
|
|
for id, t := range localPending {
|
|
e.state.PendingTasks[id] = t
|
|
}
|
|
|
|
// (b) apply incremental log delta; trim consumed entries off the token.
|
|
if e.Log != nil && len(tk.Log) > 0 {
|
|
wm, err := e.Log.ApplyDelta(tk.Log)
|
|
if err != nil {
|
|
log.Printf("ring[%s] log delta gap: %v (request full sync later)", e.ID, err)
|
|
}
|
|
keep := tk.Log[:0]
|
|
for _, en := range tk.Log {
|
|
if en.Seq > wm {
|
|
keep = append(keep, en)
|
|
}
|
|
}
|
|
tk.Log = keep
|
|
}
|
|
|
|
// execute pending commands addressed to us or claimable by lowest load.
|
|
// runCommands may set e.selfRemoved + e.removedNext on a self-remove.
|
|
if err := e.runCommands(ctx, tk); err != nil {
|
|
return tk, err
|
|
}
|
|
|
|
// (c) append own node info (refresh load/lastSeen) — skipped when we
|
|
// just self-removed, otherwise UpsertNode(self) would resurrect us and
|
|
// undo the removal the command just performed.
|
|
if !e.selfRemoved {
|
|
e.state.UpsertNode(Node{
|
|
ID: e.ID, Addr: e.myAddr, Alive: true,
|
|
IsLeader: e.state.LeaderID == e.ID,
|
|
Load: e.loadSnapshot(),
|
|
Version: e.Version, Cache: e.Cache,
|
|
})
|
|
}
|
|
|
|
// (d) inject pending newcomers right after ourselves + log node.join.
|
|
// This is the plan's "令牌发到自己时把新节点加入令牌" step: the
|
|
// sponsor writes the newcomer into the token's state and the token then
|
|
// flows to the newcomer (its successor) so it can participate.
|
|
e.injectPendingJoin()
|
|
|
|
// re-attach own fresh log entries so peers converge.
|
|
if !e.selfRemoved && e.Log != nil {
|
|
mine := e.Log.EntriesAfter(e.lastLogSent)
|
|
if len(mine) > 0 {
|
|
tk.Log = append(tk.Log, mine...)
|
|
if last := mine[len(mine)-1]; last.Seq > e.lastLogSent {
|
|
e.lastLogSent = last.Seq
|
|
}
|
|
}
|
|
}
|
|
|
|
// Publish our consolidated state back into the token.
|
|
tk.State = e.state
|
|
e.lastSyncAt = time.Now().Unix()
|
|
// Record the tasks leaving on this token so their absence from the next
|
|
// incoming token is recognized as "consumed downstream" rather than
|
|
// "never sent" — otherwise the localPending re-merge above would
|
|
// resurrect them and they would ride the ring forever.
|
|
e.published = make(map[string]struct{}, len(e.state.PendingTasks))
|
|
for id := range e.state.PendingTasks {
|
|
e.published[id] = struct{}{}
|
|
}
|
|
// Forward only after BOTH the operations and the rhythm timer are done.
|
|
select {
|
|
case <-rhythm.C:
|
|
case <-ctx.Done():
|
|
return tk, ctx.Err()
|
|
}
|
|
return tk, nil
|
|
}
|
|
|
|
// runCommands executes pending commands carried by the token that this node
|
|
// must handle. Per the authoritative design (plan §M6), commands are DIRECTED
|
|
// to their executor — they are NOT blindly claimed by the lowest-load node:
|
|
// - node-removal: rides the token until it reaches the TARGET node, which
|
|
// self-removes (plan §移除节点: "令牌传递到被移除节点自身时,该节点执行
|
|
// 自移除"). A remove command for ANOTHER node is left in pending so it keeps
|
|
// riding; if its target is already gone/offline, the lowest node consumes
|
|
// it as a no-op so it cannot ride forever.
|
|
// - revocation: rides to the OWNING node (plan §任务撤销: "持有该转发的节点
|
|
// 收到撤销任务后取消"); a revoke whose forward is already absent from the
|
|
// topology is an idempotent no-op, consumed by the lowest-load node.
|
|
// - forward creation: claimed by the lowest-load node (plan §负载摘取).
|
|
func (e *Engine) runCommands(ctx context.Context, tk *Token) error {
|
|
for {
|
|
pending := e.state.PendingList()
|
|
if len(pending) == 0 {
|
|
break
|
|
}
|
|
selfIsLowest := false
|
|
if low := e.state.LowestAlive(); low != nil && low.ID == e.ID {
|
|
selfIsLowest = true
|
|
}
|
|
// Pick the first command this node may act on this pass.
|
|
var target *Task
|
|
for _, t := range pending {
|
|
if t.RemoveNode == e.ID {
|
|
target = t // directed at us — execute regardless of load
|
|
break
|
|
}
|
|
if t.RemoveNode != "" {
|
|
// directed at another node; let it ride unless the target is
|
|
// already gone (not in ring / offline) — then the lowest node
|
|
// consumes the stale command as a no-op so it cannot loop.
|
|
if selfIsLowest {
|
|
if idx := e.state.Find(t.RemoveNode); idx < 0 || !e.state.Nodes[idx].Alive {
|
|
target = t
|
|
break
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if t.Revoke {
|
|
if owner := e.state.TopologyOwner(t); owner == e.ID {
|
|
target = t // we own the forward — execute the revoke
|
|
break
|
|
} else if owner == "" && selfIsLowest {
|
|
target = t // forward already gone — idempotent no-op
|
|
break
|
|
}
|
|
continue // owned elsewhere — ride to the owner
|
|
}
|
|
if selfIsLowest {
|
|
target = t // generic forward create — lowest-load claim
|
|
break
|
|
}
|
|
}
|
|
if target == nil {
|
|
break
|
|
}
|
|
claimed := e.state.ClaimPending(target.ID)
|
|
if claimed == nil {
|
|
break
|
|
}
|
|
if claimed.Revoke {
|
|
if e.state.RemoveTopology(claimed.Local.Name, claimed.Remote.Name, claimed.Link.RemotePort) {
|
|
if e.Handler != nil {
|
|
if err := e.Handler.Revoke(ctx, claimed); err != nil {
|
|
log.Printf("ring[%s] revoke %s: %v", e.ID, claimed.ID, err)
|
|
}
|
|
}
|
|
if e.Log != nil {
|
|
_, _ = e.Log.Append(e.ID, LogForwardRemove, map[string]any{
|
|
"taskId": claimed.ID, "local": claimed.Local.Name, "remote": claimed.Remote.Name,
|
|
})
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if claimed.RemoveNode != "" && claimed.RemoveNode == e.ID {
|
|
// Plan §移除节点: self-remove re-queues own forwards, drops ring
|
|
// position, and the token continues to the original successor.
|
|
// Stop the local workers for every forward we own FIRST (plan:
|
|
// "完全取消一切集群远程转发,停其 worker"), then SelfRemove
|
|
// moves them back to pending for another member to claim.
|
|
if e.Handler != nil {
|
|
for _, te := range e.state.ForwardsOwnedBy(e.ID) {
|
|
t := &Task{ID: te.TaskID, Local: te.Local, Remote: te.Remote, Link: te.Link}
|
|
if err := e.Handler.Revoke(ctx, t); err != nil {
|
|
log.Printf("ring[%s] self-remove revoke %s: %v", e.ID, te.TaskID, err)
|
|
}
|
|
}
|
|
}
|
|
if succ, ok := e.state.AliveSuccessor(e.ID); ok && succ != e.ID {
|
|
e.removedNext = succ
|
|
}
|
|
e.state.SelfRemove(e.ID)
|
|
e.selfRemoved = true
|
|
if e.Log != nil {
|
|
_, _ = e.Log.Append(e.ID, LogNodeLeave, map[string]string{"node": e.ID})
|
|
}
|
|
log.Printf("ring[%s] self-removed from cluster (token command %s); next=%s",
|
|
e.ID, claimed.ID, e.removedNext)
|
|
continue
|
|
}
|
|
if claimed.RemoveNode != "" {
|
|
// A remove command whose target is no longer in the ring: the
|
|
// target already self-removed (or was never a member). Drop it
|
|
// here — it MUST NOT fall through to the forward-create path,
|
|
// which would spawn a phantom empty forward (the remove task
|
|
// carries no local/remote/link). This branch also absorbs the
|
|
// command after the injector's localPending merge resurrects a
|
|
// copy once the original has been consumed downstream.
|
|
log.Printf("ring[%s] drop fulfilled remove %s (target %s gone)",
|
|
e.ID, claimed.ID, claimed.RemoveNode)
|
|
continue
|
|
}
|
|
if e.Handler != nil {
|
|
if err := e.Handler.Claim(ctx, claimed); err != nil {
|
|
e.state.PendingTasks[claimed.ID] = claimed
|
|
log.Printf("ring[%s] claim %s failed: %v", e.ID, claimed.ID, err)
|
|
break
|
|
}
|
|
}
|
|
if e.Log != nil {
|
|
_, _ = e.Log.Append(e.ID, LogForwardAdd, map[string]any{
|
|
"taskId": claimed.ID, "local": claimed.Local.Name, "remote": claimed.Remote.Name,
|
|
})
|
|
}
|
|
e.state.AddTopology(claimed, e.ID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Forward hands the token to this node's successor over the injected send.
|
|
// It is the transport hook used by the HTTP handler after OnToken. If this
|
|
// node just self-removed, its ID is no longer in state.Nodes so
|
|
// AliveSuccessor would fail — use the original successor captured before
|
|
// removal (plan §移除节点 step 3: "令牌传递给自身原本的下一家").
|
|
func (e *Engine) Forward(ctx context.Context, tk *Token) error {
|
|
if e.removedNext != "" {
|
|
next := e.removedNext
|
|
e.removedNext = ""
|
|
if e.send != nil {
|
|
log.Printf("ring[%s] forward (self-removed) cycle=%d to %s", e.ID, tk.Cycle, next)
|
|
return e.send(ctx, next, tk)
|
|
}
|
|
return nil
|
|
}
|
|
next, ok := e.state.AliveSuccessor(e.ID)
|
|
if !ok {
|
|
return nil // single-node ring
|
|
}
|
|
if next == e.ID {
|
|
return nil // never forward to ourselves
|
|
}
|
|
if e.send != nil {
|
|
log.Printf("ring[%s] forward cycle=%d to %s", e.ID, tk.Cycle, next)
|
|
return e.send(ctx, next, tk)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Snapshot returns a serializable view of the ring for the frontend.
|
|
type RingSnapshot struct {
|
|
SelfID string `json:"selfId"`
|
|
LeaderID string `json:"leaderId"`
|
|
Cycle int64 `json:"cycle"`
|
|
LastSync int64 `json:"lastSync"`
|
|
RoundDelay int64 `json:"roundDelayMs"`
|
|
Nodes []Node `json:"nodes"`
|
|
Pending []*Task `json:"pending"`
|
|
Topology []*TopoEntry `json:"topology"`
|
|
Log []LogEntry `json:"log,omitempty"`
|
|
}
|
|
|
|
func (e *Engine) Snapshot() *RingSnapshot {
|
|
snap := &RingSnapshot{
|
|
SelfID: e.ID,
|
|
LeaderID: e.state.LeaderID,
|
|
Cycle: e.state.Cycle,
|
|
LastSync: e.lastSyncAt,
|
|
RoundDelay: e.state.RoundDelay.Milliseconds(),
|
|
Nodes: e.state.Nodes,
|
|
Pending: e.state.PendingList(),
|
|
Topology: e.state.TopologyList(),
|
|
}
|
|
if e.Log != nil {
|
|
snap.Log = e.Log.Snapshot()
|
|
}
|
|
return snap
|
|
}
|
|
|
|
// tkDelaySince measures elapsed since token SentAt (leader round delay).
|
|
func tkDelaySince(tk *Token) time.Duration {
|
|
if tk.SentAt == 0 {
|
|
return 2 * time.Second
|
|
}
|
|
return time.Duration(time.Now().UnixMilli()-tk.SentAt) * time.Millisecond
|
|
}
|
|
|
|
func contains(xs []string, v string) bool {
|
|
for _, x := range xs {
|
|
if x == v {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// StartRing kicks off a fresh token from the leader. The leader stamps a new
|
|
// SentAt (so older in-flight tokens are dropped downstream) and marks inflight
|
|
// so WatchTokenLoss can detect a lost first hop — previously StartRing sent
|
|
// without marking, so a first-round loss was never noticed.
|
|
func (e *Engine) StartRing(ctx context.Context) {
|
|
if e.state.LeaderID != e.ID {
|
|
return
|
|
}
|
|
next, ok := e.state.AliveSuccessor(e.ID)
|
|
if !ok || next == e.ID {
|
|
return // single-node ring; resumes once a newcomer joins
|
|
}
|
|
// Throttle: do not start a new cycle until RoundDelay has elapsed.
|
|
if !e.lastRingStart.IsZero() && time.Since(e.lastRingStart) < e.state.RoundDelay {
|
|
return
|
|
}
|
|
e.lastRingStart = time.Now()
|
|
e.state.Cycle++
|
|
tk := &Token{
|
|
Cycle: e.state.Cycle,
|
|
State: e.state,
|
|
SentAt: time.Now().UnixMilli(),
|
|
}
|
|
if e.send != nil {
|
|
if err := e.send(ctx, next, tk); err != nil {
|
|
log.Printf("ring[%s] start cycle %d send: %v", e.ID, tk.Cycle, err)
|
|
return
|
|
}
|
|
// Mark inflight ONLY on a successful first send so token-loss
|
|
// detection covers the very first hop too.
|
|
e.inflight.mark(e.state.RoundDelay)
|
|
}
|
|
}
|
|
|
|
// JoinInfo is a node's self-description sent when requesting to join a ring.
|
|
type JoinInfo struct {
|
|
ID string `json:"id"`
|
|
Addr string `json:"addr"`
|
|
Version string `json:"version,omitempty"`
|
|
Cache []string `json:"cache,omitempty"`
|
|
}
|
|
|
|
// injectPendingJoin writes every queued newcomer into state right after
|
|
// ourselves (so the newcomer becomes our successor) and logs node.join. Used
|
|
// both by OnToken (plan: "令牌发到自己时把新节点加入令牌") and by JoinNode
|
|
// when the leader is a single node with no token circulating.
|
|
func (e *Engine) injectPendingJoin() {
|
|
for _, ji := range e.pendingJoin {
|
|
nn := Node{ID: ji.ID, Addr: ji.Addr, Alive: true,
|
|
Load: Load{MemPct: 50, NetPct: 50}, Version: ji.Version, Cache: ji.Cache}
|
|
e.state.InsertAfter(e.ID, nn)
|
|
if e.Log != nil {
|
|
_, _ = e.Log.Append(e.ID, LogNodeJoin, map[string]string{"node": nn.ID, "addr": nn.Addr})
|
|
}
|
|
log.Printf("ring[%s] injected newcomer %s after self", e.ID, nn.ID)
|
|
}
|
|
e.pendingJoin = nil
|
|
}
|
|
|
|
// JoinNode accepts a newcomer's join request. Per plan §新节点加入 the
|
|
// sponsor does NOT mutate its own state immediately (it would be washed out
|
|
// by the next incoming token's authoritative merge). Instead it queues the
|
|
// newcomer on pendingJoin; OnToken injects it into the token right after
|
|
// the sponsor. EXCEPTION: a single-node leader has no token circulating
|
|
// (StartRing refuses for lack of a successor), so the newcomer would never
|
|
// be injected — in that case we inject immediately and kick off the ring.
|
|
func (e *Engine) JoinNode(j JoinInfo) *State {
|
|
e.pendingJoin = append(e.pendingJoin, j)
|
|
if e.state.LeaderID == "" {
|
|
e.state.LeaderID = e.ID
|
|
}
|
|
// Single-node leader: no successor, no circulating token → the pending
|
|
// newcomer would never be injected. Inject now (safe: no foreign token
|
|
// can overwrite a single-node leader) and start the ring.
|
|
if e.state.LeaderID == e.ID && len(e.state.Nodes) == 1 {
|
|
e.injectPendingJoin()
|
|
go e.StartRing(context.Background())
|
|
}
|
|
return &e.state
|
|
}
|
|
|
|
// AdoptState replaces this node's cluster picture with the state provided by
|
|
// the join target, then re-inserts us (in case we were absent).
|
|
func (e *Engine) AdoptState(s State) {
|
|
ns := make([]string, 0, len(s.Nodes))
|
|
for _, n := range s.Nodes {
|
|
ns = append(ns, n.ID)
|
|
}
|
|
log.Printf("ring[%s] adopt-state nodes=%v", e.ID, ns)
|
|
kept := map[string]*Task{}
|
|
for id, t := range e.state.PendingTasks {
|
|
if _, ok := s.PendingTasks[id]; !ok {
|
|
kept[id] = t
|
|
}
|
|
}
|
|
e.state = s
|
|
if e.state.PendingTasks == nil {
|
|
e.state.PendingTasks = map[string]*Task{}
|
|
}
|
|
for id, t := range kept {
|
|
e.state.PendingTasks[id] = t
|
|
}
|
|
e.state.UpsertNode(Node{ID: e.ID, Addr: e.myAddr, Alive: true,
|
|
Load: e.loadSnapshot(), Version: e.Version, Cache: e.Cache})
|
|
if e.Log != nil {
|
|
e.Log.Append(e.ID, LogNodeJoin, map[string]string{"node": e.ID, "addr": e.myAddr})
|
|
}
|
|
}
|
|
|
|
// CreateCluster reseeds this node as a fresh standalone leader (single-node
|
|
// ring). Used after a self-leave left the ring empty, or to (re)affirm seed
|
|
// state on a standalone node. Refuses if this node is still a multi-node
|
|
// member — reseeding mid-cluster would split the ring (split-brain). Safe to
|
|
// call when standalone/empty: no token circulates to a non-member, so no
|
|
// concurrent OnToken can overwrite the reset.
|
|
func (e *Engine) CreateCluster() error {
|
|
if e.IsMember() {
|
|
return fmt.Errorf("node is a multi-node cluster member; leave first")
|
|
}
|
|
e.state = State{
|
|
LeaderID: e.ID,
|
|
Cycle: 0,
|
|
PendingTasks: map[string]*Task{},
|
|
Topology: map[string]*TopoEntry{},
|
|
RoundDelay: 2 * time.Second,
|
|
Seq: e.state.Seq,
|
|
}
|
|
e.state.UpsertNode(Node{
|
|
ID: e.ID, Addr: e.myAddr, Alive: true, IsLeader: true,
|
|
Load: e.loadSnapshot(), Version: e.Version, Cache: e.Cache,
|
|
})
|
|
e.selfRemoved = false
|
|
e.removedNext = ""
|
|
e.lastRingStart = time.Time{}
|
|
e.lastTokenAt = 0
|
|
e.inflight.clear()
|
|
if e.Log != nil {
|
|
_, _ = e.Log.Append(e.ID, LogLeaderChange, map[string]string{"leader": e.ID})
|
|
}
|
|
log.Printf("ring[%s] created/seeded fresh standalone cluster as leader", e.ID)
|
|
return nil
|
|
}
|
|
|
|
// SubmitTask adds a new forward request to pending; it rides the next token
|
|
// round and is claimed by the lowest-load member.
|
|
// RemoveNode publishes a node-removal command via the token; the target
|
|
// node self-removes when the command reaches it.
|
|
func (e *Engine) RemoveNode(nodeID string) *Task {
|
|
return e.state.AddRemoveNode(nodeID)
|
|
}
|
|
|
|
// RevokeTask publishes a revocation for an established forward through the
|
|
// same token channel; the owning node stops the worker and drops topology.
|
|
func (e *Engine) RevokeTask(local store.Local, remote store.Remote, link store.Link) *Task {
|
|
return e.state.AddRevoke(local, remote, link)
|
|
}
|
|
|
|
func (e *Engine) SubmitTask(local store.Local, remote store.Remote, link store.Link) *Task {
|
|
if e.HasTask(local.Name, remote.Name, link.RemotePort) {
|
|
return nil
|
|
}
|
|
return e.state.AddPending(local, remote, link)
|
|
}
|
|
|
|
// HasTask reports whether a forward with the same local/remote/remotePort is
|
|
// already pending or active in the topology (idempotency guard for resaves).
|
|
func (e *Engine) HasTask(local, remote string, port int) bool {
|
|
for _, t := range e.state.PendingList() {
|
|
if t.Local.Name == local && t.Remote.Name == remote && t.Link.RemotePort == port {
|
|
return true
|
|
}
|
|
}
|
|
for _, t := range e.state.TopologyList() {
|
|
if t.Local.Name == local && t.Remote.Name == remote && t.Link.RemotePort == port {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// IsLeader reports whether this node is the current ring leader.
|
|
func (e *Engine) IsLeader() bool { return e.state.LeaderID == e.ID }
|
|
|
|
// IsMember reports whether this node is currently an active multi-node member
|
|
// (self is in the ring alongside others). Used by the create/join gates to
|
|
// refuse actions that would split an active ring. A detached node (self not
|
|
// in ring — e.g. after a self-leave) or a standalone node returns false and
|
|
// may create/join freely. NOTE: after a self-leave the engine keeps the other
|
|
// members in state.Nodes (it only dropped self), so a plain len>1 check would
|
|
// wrongly block a detached node — the self-in-ring test is essential.
|
|
func (e *Engine) IsMember() bool {
|
|
if len(e.state.Nodes) <= 1 {
|
|
return false
|
|
}
|
|
for _, n := range e.state.Nodes {
|
|
if n.ID == e.ID {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// LeaderAddr returns the current leader's address.
|
|
func (e *Engine) LeaderAddr() string {
|
|
i := e.state.Find(e.state.LeaderID)
|
|
if i < 0 {
|
|
return ""
|
|
}
|
|
return e.state.Nodes[i].Addr
|
|
}
|