webui4frpc/internal/cluster/ring_leader.go

// Leader-side minimal roles for the token ring. Per the authoritative design:
// the leader only (1) initiates the first token, and (2) judges token loss.
// All other processing (adopt cluster picture, append own info, run commands,
// forward) is IDENTICAL between leader and ordinary nodes — no phase flips,
// no passed-accounting, no cycle bookkeeping in the leader.
package cluster

import (
	"context"
	"log"
	"sync"
	"time"
)

// Per-hop pacing bounds. The hop delay scales DOWN as the ring grows so the
// round time stays ~ringHopDelayMax regardless of node count — a static
// 500ms made large clusters slow (3 nodes=1.5s, 5 nodes=2.5s, 10 nodes=5s
// per round); now 3/5/10 nodes all round at ~500ms (until the floor bites),
// keeping sync real-time without a token storm (round freq ≈2Hz).
const (
	ringHopDelayMax = 500 * time.Millisecond
	ringHopDelayMin = 50 * time.Millisecond
)

// hopDelayFor returns the per-hop pace for a ring of aliveNodes members.
// Nodes越多延迟越低: delay = ringHopDelayMax / aliveNodes, floored at min.
// n=2→250ms, n=3→167ms, n=5→100ms, n=10→50ms(floor) — round time ≈500ms.
func hopDelayFor(aliveNodes int) time.Duration {
	if aliveNodes < 2 {
		aliveNodes = 2
	}
	d := ringHopDelayMax / time.Duration(aliveNodes)
	if d < ringHopDelayMin {
		d = ringHopDelayMin
	}
	return d
}

// LossTimeout is the token-loss threshold per design: roundDelay/2 + 20ms,
// floored so a healthy fast ring is never misjudged.
func LossTimeout(roundDelay time.Duration) time.Duration {
	t := roundDelay/2 + 20*time.Millisecond
	if t < 2500*time.Millisecond {
		return 2500 * time.Millisecond
	}
	return t
}

// inFlight tracks token-in-flight state (leader only, for loss judging).
type inFlight struct {
	mu     sync.Mutex
	active bool
	sentAt time.Time
	delay  time.Duration
}

func (f *inFlight) mark(delay time.Duration) {
	f.mu.Lock()
	f.active = true
	f.sentAt = time.Now()
	f.delay = delay
	f.mu.Unlock()
}

func (f *inFlight) clear() {
	f.mu.Lock()
	f.active = false
	f.mu.Unlock()
}

func (f *inFlight) inflight() bool {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.active
}

func (f *inFlight) age() time.Duration {
	f.mu.Lock()
	defer f.mu.Unlock()
	if !f.active {
		return 0
	}
	return time.Since(f.sentAt)
}

func (f *inFlight) lastDelay() time.Duration {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.delay
}

// Send is called after OnToken for the LEADER. The token has come full
// circle (one round complete), so the leader bumps the cycle, stamps a fresh
// SentAt (so any stale in-flight older token is dropped downstream), and
// forwards to the successor to start the next round. This is what makes
// `cycle` advance under the perpetual-flow model — without it cycle stuck
// at the StartRing value forever.
func (e *Engine) Send(ctx context.Context, tk *Token) error {
	if !e.selfRemoved {
		e.state.Cycle = tk.Cycle + 1
		tk.Cycle = e.state.Cycle
		// Keep State.Cycle in lockstep with Token.Cycle so the snapshot
		// (which reads e.state.Cycle) reflects the real round number.
		tk.State = e.state
		tk.SentAt = time.Now().UnixMilli()
		if tk.SentAt > e.lastTokenAt {
			e.lastTokenAt = tk.SentAt
		}
	}
	return e.forwardToNext(ctx, tk)
}

// forwardToNext sends the token to the next alive successor. On send
// failure (no receipt within the HTTP timeout = neighbor unreachable),
// the normal node-death procedure fires: mark offline, reassign the dead
// node's tasks, and try the next hop. If the dead node was the leader,
// the predecessor reuses the same procedure and additionally promotes
// itself to leader + starts a fresh cycle (the token destined for the
// dead leader is lost; a new cycle must begin). This is the PRIMARY
// leader-death detection path per plan §故障自幽 + §leader 补充/监控.
func (e *Engine) forwardToNext(ctx context.Context, tk *Token) error {
	for hops := 0; hops < len(e.state.Nodes); hops++ {
		next, ok := e.nextRecipient()
		if !ok {
			e.inflight.clear()
			return nil
		}
		if e.send == nil {
			return nil
		}
		debugToken("ring[%s] forward cycle=%d to %s", e.ID, tk.Cycle, next)
		err := e.send(ctx, next, tk)
		if err == nil {
			if e.state.LeaderID == e.ID {
				e.inflight.mark(e.state.RoundDelay)
			}
			return nil
		}
		// Send failed = no receipt within timeout = neighbor offline.
		// Normal node-death: mark offline, reassign tasks to pending.
		log.Printf("ring[%s] send to %s failed (no receipt): %v", e.ID, next, err)
		e.state.MarkOffline(next)
		e.state.OfflineReassign(next)
		// If the dead node was the leader, promote self and start a new
		// cycle. The token was going to the leader; with the leader dead
		// the token is lost — start fresh as the new leader (plan §leader
		// 补充/监控: "上家邻居探测到 leader 崩溃 → 自身成为新 leader").
		if next == e.state.LeaderID {
			if !e.becomeLeader() {
				// Another live leader exists (we are the partitioned side):
				// keep its ring intact and let it re-drive the token. Re-raise
				// the node we just marked dead only if the send really was the
				// problem — otherwise we would paper over a real outage.
				log.Printf("ring[%s] not promoting: deferring to live leader", e.ID)
				return nil
			}
			if e.Log != nil {
				_, _ = e.Log.Append(e.ID, LogLeaderChange, map[string]string{"leader": e.ID})
			}
			e.StartRing(ctx)
			return nil
		}
		// Non-leader neighbor death: continue to the next recipient.
	}
	// 全环遍历完毕，所有后继都不可达（或已全部尝试过）。
	// 保持 inflight（不清空），让 WatchTokenLoss 超时后重发。
	// 重发时 AliveSuccessor 重新计算，可能已被心跳复活。
	return nil
}

// becomeLeader promotes this node, but ONLY when the ring agrees there is no
// live leader to defer to.
//
// The guard matters because leadership was previously self-declared: any node
// that failed to reach the leader it believed in would immediately promote
// itself. Two nodes partitioned from each other both hit that path and both
// ended up believing they were the leader — a split brain with two token
// rings, duplicate claims, and each side reassigning the other's forwards. That
// was observed live: after a rolling restart, two nodes reported the same
// leader while marking each other dead, and the topology was emptied into
// pending on both sides.
//
// Before promoting, ask the ring: is any OTHER node currently alive and claiming
// leadership? If so, we are the partitioned one — demote ourselves and let the
// live leader keep the ring. A genuine single-leader loss (the real leader is
// dead or unreachable) leaves no such node, so promotion still happens.
//
// CreateCluster does not come through here: it assigns LeaderID directly, so
// the "no other leader exists" case never needs an override.
func (e *Engine) becomeLeader() bool {
	for i := range e.state.Nodes {
		n := &e.state.Nodes[i]
		if n.ID == e.ID {
			continue
		}
		if n.Alive && n.IsLeader {
			log.Printf("ring[%s] decline promotion: %s is alive and is the leader", e.ID, n.ID)
			return false
		}
	}
	e.state.LeaderID = e.ID
	for i := range e.state.Nodes {
		e.state.Nodes[i].IsLeader = e.state.Nodes[i].ID == e.ID
	}
	log.Printf("ring[%s] promoted to leader", e.ID)
	return true
}

// WatchLeader runs the FALLBACK leader liveness monitor. The PRIMARY path is
// forwardToNext: when the predecessor sends a token to the leader and the send
// fails (leader's HTTP server down), forwardToNext marks the leader offline and
// promotes self. WatchLeader covers the case forwardToNext CANNOT detect:
// the leader received the token (POST returned 200) but then crashed/restarted/
// detached before forwarding it — the send succeeded, so forwardToNext sees no
// error. In this case the predecessor pings the leader; if the leader is down
// (connection refused) or restarted/detached (standalone → 409), the heartbeat
// fails and the predecessor takes over.
//
// The predecessor role is NOT permanent — it shifts as the ring topology
// changes (nodes join/leave). Each tick re-evaluates AlivePredecessor(LeaderID)
// so the correct node monitors the leader at all times. Per design:
// "上邻居也不是永久的，也要有普通节点按照令牌传递的拓扑变换转换为上邻居的逻辑".
//
// Interval = 1s so worst-case detection (tick + 1.5s ping timeout ≈ 2.5s)
// aligns with LossTimeout (roundDelay/2 + 20ms, floored at 2500ms), per design:
// "与leader超时重发时间一致".
func (e *Engine) WatchLeader(ctx context.Context) {
	tick := time.NewTicker(1 * time.Second)
	defer tick.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
			if e.state.LeaderID == e.ID {
				continue // we are the leader; predecessor monitors us
			}
			leaderPred, ok := e.state.AlivePredecessor(e.state.LeaderID)
			if !ok || leaderPred != e.ID {
				continue // only the leader's predecessor pings it
			}
			if e.send != nil {
				hbCtx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond)
				err := e.send(hbCtx, e.state.LeaderID, nil) // nil = heartbeat
				cancel()
				if err != nil {
					log.Printf("ring[%s] heartbeat to leader %s failed: %v", e.ID, e.state.LeaderID, err)
					e.takeOverFromDeadLeader(ctx)
				}
			}
		}
	}
}

// takeOverFromDeadLeader handles "the leader I could not reach is gone": mark it
// dead and, if the ring agrees there is no live leader left, promote this node
// and re-seed the cycle.
//
// The promotion is deliberately conditional, and so is the reassignment. Doing
// the reassignment unconditionally is what destroyed the live ring: a node on
// the partitioned side would mark the (still very much alive) leader dead and
// move ALL of its forwards into pending, so both sides ended up with an empty
// topology and a queue nobody was serving. Sequencing matters — the forward
// re-hang is only correct once we have actually taken leadership.
//
// Extracted from WatchLeader's loop so the decision is testable (the loop itself
// is an endless goroutine).
func (e *Engine) takeOverFromDeadLeader(ctx context.Context) {
	old := e.state.LeaderID
	e.state.MarkOffline(old)
	if !e.becomeLeader() {
		log.Printf("ring[%s] leader %s unreachable but a live leader remains; "+
			"declining takeover and leaving its forwards untouched", e.ID, old)
		return
	}
	e.state.OfflineReassign(old)
	if e.Log != nil {
		_, _ = e.Log.Append(e.ID, LogLeaderChange, map[string]string{"leader": e.ID})
	}
	// The ring died with the old leader (no token inflight → WatchTokenLoss
	// won't fire). Without this the newly promoted leader would sit idle.
	e.StartRing(ctx)
}

// WatchTokenLoss runs the leader's token-loss judge: if a token was sent and
// does not return within LossTimeout, the leader issues a fresh token (all
// nodes drop older stamps via the SentAt guard, so at most one circulates).
func (e *Engine) WatchTokenLoss(ctx context.Context) {
	tick := time.NewTicker(200 * time.Millisecond)
	defer tick.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
			if e.ID != e.state.LeaderID {
				continue
			}
			if !e.inflight.inflight() {
				continue
			}
			delay := e.inflight.lastDelay()
			if delay <= 0 {
				delay = 200 * time.Millisecond
			}
			if e.inflight.age() > LossTimeout(delay) {
				log.Printf("ring[%s] token lost, resending cycle %d", e.ID, e.state.Cycle)
				e.inflight.clear()
				e.StartRing(ctx)
			}
		}
	}
}