// Leader-side minimal roles for the token ring. Per the authoritative design: // the leader only (1) initiates the first token, and (2) judges token loss. // All other processing (adopt cluster picture, append own info, run commands, // forward) is IDENTICAL between leader and ordinary nodes — no phase flips, // no passed-accounting, no cycle bookkeeping in the leader. package cluster import ( "context" "log" "sync" "time" ) // Per-hop pacing bounds. The hop delay scales DOWN as the ring grows so the // round time stays ~ringHopDelayMax regardless of node count — a static // 500ms made large clusters slow (3 nodes=1.5s, 5 nodes=2.5s, 10 nodes=5s // per round); now 3/5/10 nodes all round at ~500ms (until the floor bites), // keeping sync real-time without a token storm (round freq ≈2Hz). const ( ringHopDelayMax = 500 * time.Millisecond ringHopDelayMin = 50 * time.Millisecond ) // hopDelayFor returns the per-hop pace for a ring of aliveNodes members. // Nodes越多延迟越低: delay = ringHopDelayMax / aliveNodes, floored at min. // n=2→250ms, n=3→167ms, n=5→100ms, n=10→50ms(floor) — round time ≈500ms. func hopDelayFor(aliveNodes int) time.Duration { if aliveNodes < 2 { aliveNodes = 2 } d := ringHopDelayMax / time.Duration(aliveNodes) if d < ringHopDelayMin { d = ringHopDelayMin } return d } // LossTimeout is the token-loss threshold per design: roundDelay/2 + 20ms, // floored so a healthy fast ring is never misjudged. func LossTimeout(roundDelay time.Duration) time.Duration { t := roundDelay/2 + 20*time.Millisecond if t < 2500*time.Millisecond { return 2500 * time.Millisecond } return t } // inFlight tracks token-in-flight state (leader only, for loss judging). type inFlight struct { mu sync.Mutex active bool sentAt time.Time delay time.Duration } func (f *inFlight) mark(delay time.Duration) { f.mu.Lock() f.active = true f.sentAt = time.Now() f.delay = delay f.mu.Unlock() } func (f *inFlight) clear() { f.mu.Lock() f.active = false f.mu.Unlock() } func (f *inFlight) inflight() bool { f.mu.Lock() defer f.mu.Unlock() return f.active } func (f *inFlight) age() time.Duration { f.mu.Lock() defer f.mu.Unlock() if !f.active { return 0 } return time.Since(f.sentAt) } func (f *inFlight) lastDelay() time.Duration { f.mu.Lock() defer f.mu.Unlock() return f.delay } // Send is called after OnToken for the LEADER. The token has come full // circle (one round complete), so the leader bumps the cycle, stamps a fresh // SentAt (so any stale in-flight older token is dropped downstream), and // forwards to the successor to start the next round. This is what makes // `cycle` advance under the perpetual-flow model — without it cycle stuck // at the StartRing value forever. func (e *Engine) Send(ctx context.Context, tk *Token) error { if !e.selfRemoved { e.state.Cycle = tk.Cycle + 1 tk.Cycle = e.state.Cycle // Keep State.Cycle in lockstep with Token.Cycle so the snapshot // (which reads e.state.Cycle) reflects the real round number. tk.State = e.state tk.SentAt = time.Now().UnixMilli() if tk.SentAt > e.lastTokenAt { e.lastTokenAt = tk.SentAt } } return e.forwardToNext(ctx, tk) } // forwardToNext sends the token to the next alive successor. On send // failure (no receipt within the HTTP timeout = neighbor unreachable), // the normal node-death procedure fires: mark offline, reassign the dead // node's tasks, and try the next hop. If the dead node was the leader, // the predecessor reuses the same procedure and additionally promotes // itself to leader + starts a fresh cycle (the token destined for the // dead leader is lost; a new cycle must begin). This is the PRIMARY // leader-death detection path per plan §故障自幽 + §leader 补充/监控. func (e *Engine) forwardToNext(ctx context.Context, tk *Token) error { for hops := 0; hops < len(e.state.Nodes); hops++ { next, ok := e.nextRecipient() if !ok { e.inflight.clear() return nil } if e.send == nil { return nil } log.Printf("ring[%s] forward cycle=%d to %s", e.ID, tk.Cycle, next) err := e.send(ctx, next, tk) if err == nil { if e.state.LeaderID == e.ID { e.inflight.mark(e.state.RoundDelay) } return nil } // Send failed = no receipt within timeout = neighbor offline. // Normal node-death: mark offline, reassign tasks to pending. log.Printf("ring[%s] send to %s failed (no receipt): %v", e.ID, next, err) e.state.MarkOffline(next) e.state.OfflineReassign(next) // If the dead node was the leader, promote self and start a new // cycle. The token was going to the leader; with the leader dead // the token is lost — start fresh as the new leader (plan §leader // 补充/监控: "上家邻居探测到 leader 崩溃 → 自身成为新 leader"). if next == e.state.LeaderID { e.becomeLeader() if e.Log != nil { _, _ = e.Log.Append(e.ID, LogLeaderChange, map[string]string{"leader": e.ID}) } e.StartRing(ctx) return nil } // Non-leader neighbor death: continue to the next recipient. } e.inflight.clear() return nil } // becomeLeader promotes this node (used when the monitored leader dies). func (e *Engine) becomeLeader() { e.state.LeaderID = e.ID for i := range e.state.Nodes { e.state.Nodes[i].IsLeader = e.state.Nodes[i].ID == e.ID } log.Printf("ring[%s] promoted to leader", e.ID) } // WatchLeader runs the FALLBACK leader liveness monitor. The PRIMARY path is // forwardToNext: when the predecessor sends a token to the leader and the send // fails (leader's HTTP server down), forwardToNext marks the leader offline and // promotes self. WatchLeader covers the case forwardToNext CANNOT detect: // the leader received the token (POST returned 200) but then crashed/restarted/ // detached before forwarding it — the send succeeded, so forwardToNext sees no // error. In this case the predecessor pings the leader; if the leader is down // (connection refused) or restarted/detached (standalone → 409), the heartbeat // fails and the predecessor takes over. // // The predecessor role is NOT permanent — it shifts as the ring topology // changes (nodes join/leave). Each tick re-evaluates AlivePredecessor(LeaderID) // so the correct node monitors the leader at all times. Per design: // "上邻居也不是永久的,也要有普通节点按照令牌传递的拓扑变换转换为上邻居的逻辑". // // Interval = 1s so worst-case detection (tick + 1.5s ping timeout ≈ 2.5s) // aligns with LossTimeout (roundDelay/2 + 20ms, floored at 2500ms), per design: // "与leader超时重发时间一致". func (e *Engine) WatchLeader(ctx context.Context) { tick := time.NewTicker(1 * time.Second) defer tick.Stop() for { select { case <-ctx.Done(): return case <-tick.C: if e.state.LeaderID == e.ID { continue // we are the leader; predecessor monitors us } leaderPred, ok := e.state.AlivePredecessor(e.state.LeaderID) if !ok || leaderPred != e.ID { continue // only the leader's predecessor pings it } if e.send != nil { hbCtx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond) err := e.send(hbCtx, e.state.LeaderID, nil) // nil = heartbeat cancel() if err != nil { log.Printf("ring[%s] heartbeat to leader %s failed: %v", e.ID, e.state.LeaderID, err) e.state.MarkOffline(e.state.LeaderID) e.state.OfflineReassign(e.state.LeaderID) e.becomeLeader() // Kick off a fresh cycle: the ring died with the old // leader (no token inflight → WatchTokenLoss won't // fire). Without this the newly promoted leader would // sit idle and the ring would stay dead. e.StartRing(ctx) } } } } } // WatchTokenLoss runs the leader's token-loss judge: if a token was sent and // does not return within LossTimeout, the leader issues a fresh token (all // nodes drop older stamps via the SentAt guard, so at most one circulates). func (e *Engine) WatchTokenLoss(ctx context.Context) { tick := time.NewTicker(200 * time.Millisecond) defer tick.Stop() for { select { case <-ctx.Done(): return case <-tick.C: if e.ID != e.state.LeaderID { continue } if !e.inflight.inflight() { continue } delay := e.inflight.lastDelay() if delay <= 0 { delay = 200 * time.Millisecond } if e.inflight.age() > LossTimeout(delay) { log.Printf("ring[%s] token lost, resending cycle %d", e.ID, e.state.Cycle) e.inflight.clear() e.StartRing(ctx) } } } }