feat: cluster reliability (leader failover, crash rejoin, key exchange) + auth/users + canvas/forwards enhancements + comprehensive README + API docs

- Cluster: forwardToNext offline detection (leader+non-leader), WatchLeader 1s heartbeat fallback, 409 for standalone nodes, Node.NodeKey key exchange via token ring, ClusterPeers persistence + auto-rejoin, Forward delegates to forwardToNext (bugfix)
- Auth: Basic Auth (flag-creds fast path) + bcrypt users (admin/viewer) + Bearer API keys (read/write/admin scope)
- Frontend: UsersView (accounts+API keys), ClusterView (ring/nodeKey/tasks/topology/log), StatusView (group management, per-proxy status), CanvasView (edge toggle/group), PortEdge (disabled/group labels)
- API: handlers split (canvas/forwards/users/logs), canvas export/import, forwards group start/stop/assign/delete, cluster endpoints
- Docs: comprehensive README rewrite (all flags/APIs/auth/cluster), docs/cluster-api.md (cluster management API reference)
- Deploy: run-cluster.sh now 4-node ring + 1 isolated standalone, test-forward.sh updated for 4 nodes
- Removed plan.md (design notes consolidated into README + API docs)

internal/cluster/ring_leader.go

@@ -12,11 +12,29 @@ import (
 	"time"
 )
 
-// ringHopDelay paces each token hop (real-world cadence, avoids busy-loop).
-// 500ms keeps a 3-node ring well under the LossTimeout floor (2500ms) so a
-// healthy round is never misjudged lost, while not idly burning CPU/HTTP at
-// 20Hz like the old 50ms did.
-const ringHopDelay = 500 * time.Millisecond
+// Per-hop pacing bounds. The hop delay scales DOWN as the ring grows so the
+// round time stays ~ringHopDelayMax regardless of node count — a static
+// 500ms made large clusters slow (3 nodes=1.5s, 5 nodes=2.5s, 10 nodes=5s
+// per round); now 3/5/10 nodes all round at ~500ms (until the floor bites),
+// keeping sync real-time without a token storm (round freq ≈2Hz).
+const (
+	ringHopDelayMax = 500 * time.Millisecond
+	ringHopDelayMin = 50 * time.Millisecond
+)
+
+// hopDelayFor returns the per-hop pace for a ring of aliveNodes members.
+// Nodes越多延迟越低: delay = ringHopDelayMax / aliveNodes, floored at min.
+// n=2→250ms, n=3→167ms, n=5→100ms, n=10→50ms(floor) — round time ≈500ms.
+func hopDelayFor(aliveNodes int) time.Duration {
+	if aliveNodes < 2 {
+		aliveNodes = 2
+	}
+	d := ringHopDelayMax / time.Duration(aliveNodes)
+	if d < ringHopDelayMin {
+		d = ringHopDelayMin
+	}
+	return d
+}
 
 // LossTimeout is the token-loss threshold per design: roundDelay/2 + 20ms,
 // floored so a healthy fast ring is never misjudged.
@@ -92,8 +110,14 @@ func (e *Engine) Send(ctx context.Context, tk *Token) error {
 	return e.forwardToNext(ctx, tk)
 }
 
-// forwardToNext sends the token to the next alive successor; on failure it
-// marks that node offline, reattaches its tasks, and tries the next hop.
+// forwardToNext sends the token to the next alive successor. On send
+// failure (no receipt within the HTTP timeout = neighbor unreachable),
+// the normal node-death procedure fires: mark offline, reassign the dead
+// node's tasks, and try the next hop. If the dead node was the leader,
+// the predecessor reuses the same procedure and additionally promotes
+// itself to leader + starts a fresh cycle (the token destined for the
+// dead leader is lost; a new cycle must begin). This is the PRIMARY
+// leader-death detection path per plan §故障自幽 + §leader 补充/监控.
 func (e *Engine) forwardToNext(ctx context.Context, tk *Token) error {
 	for hops := 0; hops < len(e.state.Nodes); hops++ {
 		next, ok := e.nextRecipient()
@@ -104,26 +128,32 @@ func (e *Engine) forwardToNext(ctx context.Context, tk *Token) error {
 		if e.send == nil {
 			return nil
 		}
+		log.Printf("ring[%s] forward cycle=%d to %s", e.ID, tk.Cycle, next)
 		err := e.send(ctx, next, tk)
 		if err == nil {
 			if e.state.LeaderID == e.ID {
 				e.inflight.mark(e.state.RoundDelay)
 			}
-			// Pace the ring so tokens circulate at a realistic cadence.
-			// Pacing handled by the parallel rhythm timer in OnToken.
 			return nil
 		}
-		log.Printf("ring[%s] send to %s failed: %v", e.ID, next, err)
-		// Do not declare a neighbor offline on a single timeout: transient
-		// send failures (network jitter, busy handler) must not break the
-		// ring. Only after consecutive failures do we evict the node.
-		e.failMu.Lock()
-		e.failCount[next]++
-		if e.failCount[next] >= 2 {
-			e.state.MarkOffline(next)
-			e.state.OfflineReassign(next)
-			delete(e.failCount, next)
+		// Send failed = no receipt within timeout = neighbor offline.
+		// Normal node-death: mark offline, reassign tasks to pending.
+		log.Printf("ring[%s] send to %s failed (no receipt): %v", e.ID, next, err)
+		e.state.MarkOffline(next)
+		e.state.OfflineReassign(next)
+		// If the dead node was the leader, promote self and start a new
+		// cycle. The token was going to the leader; with the leader dead
+		// the token is lost — start fresh as the new leader (plan §leader
+		// 补充/监控: "上家邻居探测到 leader 崩溃 → 自身成为新 leader").
+		if next == e.state.LeaderID {
+			e.becomeLeader()
+			if e.Log != nil {
+				_, _ = e.Log.Append(e.ID, LogLeaderChange, map[string]string{"leader": e.ID})
+			}
+			e.StartRing(ctx)
+			return nil
 		}
+		// Non-leader neighbor death: continue to the next recipient.
 	}
 	e.inflight.clear()
 	return nil
@@ -138,10 +168,26 @@ func (e *Engine) becomeLeader() {
 	log.Printf("ring[%s] promoted to leader", e.ID)
 }
 
-// WatchLeader runs the leader liveness monitor: the predecessor of the leader
-// pings it; on failure it marks the leader offline and promotes itself.
+// WatchLeader runs the FALLBACK leader liveness monitor. The PRIMARY path is
+// forwardToNext: when the predecessor sends a token to the leader and the send
+// fails (leader's HTTP server down), forwardToNext marks the leader offline and
+// promotes self. WatchLeader covers the case forwardToNext CANNOT detect:
+// the leader received the token (POST returned 200) but then crashed/restarted/
+// detached before forwarding it — the send succeeded, so forwardToNext sees no
+// error. In this case the predecessor pings the leader; if the leader is down
+// (connection refused) or restarted/detached (standalone → 409), the heartbeat
+// fails and the predecessor takes over.
+//
+// The predecessor role is NOT permanent — it shifts as the ring topology
+// changes (nodes join/leave). Each tick re-evaluates AlivePredecessor(LeaderID)
+// so the correct node monitors the leader at all times. Per design:
+// "上邻居也不是永久的，也要有普通节点按照令牌传递的拓扑变换转换为上邻居的逻辑".
+//
+// Interval = 1s so worst-case detection (tick + 1.5s ping timeout ≈ 2.5s)
+// aligns with LossTimeout (roundDelay/2 + 20ms, floored at 2500ms), per design:
+// "与leader超时重发时间一致".
 func (e *Engine) WatchLeader(ctx context.Context) {
-	tick := time.NewTicker(2 * time.Second)
+	tick := time.NewTicker(1 * time.Second)
 	defer tick.Stop()
 	for {
 		select {
@@ -164,6 +210,11 @@ func (e *Engine) WatchLeader(ctx context.Context) {
 					e.state.MarkOffline(e.state.LeaderID)
 					e.state.OfflineReassign(e.state.LeaderID)
 					e.becomeLeader()
+					// Kick off a fresh cycle: the ring died with the old
+					// leader (no token inflight → WatchTokenLoss won't
+					// fire). Without this the newly promoted leader would
+					// sit idle and the ring would stay dead.
+					e.StartRing(ctx)
 				}
 			}
 		}