webui4frpc/cmd/webui4frpc/main.go

// webui-frpc is a standalone web controller for frpc: configure local forward
// items and remote servers on a visual canvas, and it spawns one frpc worker
// per remote server. No frp source code is bundled; frpc binaries are
// downloaded on demand or pointed at manually.
package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"webui4frpc/internal/cluster"
	"webui4frpc/internal/httpapi"
	"webui4frpc/internal/install"
	"webui4frpc/internal/process"
	"webui4frpc/internal/render"
	"webui4frpc/internal/store"
	"webui4frpc/internal/version"
)

func main() {
	addr := flag.String("addr", "127.0.0.1:7500", "listen address")
	user := flag.String("user", "admin", "basic auth user")
	pass := flag.String("password", "admin", "basic auth password")
	workdir := flag.String("workdir", "./webui-frpc", "data directory (db, configs, logs, bin)")
	bin := flag.String("bin", "", "initial frpc binary path (optional; defaults to manual/auto-install)")
	frpc := flag.String("frpc", "", "default worker frpc binary path (fallback when settings has none)")
	var peers multiFlag
	flag.Var(&peers, "peer", "cluster peer host:port (repeatable; same creds)")
	joinKey := flag.String("join-key", os.Getenv("W4F_JOIN_KEY"), "cluster join key = the sponsor node's nodeKey (required with -peer to join an existing cluster)")
	flag.Parse()

	wd, err := filepath.Abs(*workdir)
	if err != nil {
		log.Fatalf("resolve workdir: %v", err)
	}
	for _, d := range []string{wd, filepath.Join(wd, "configs"), filepath.Join(wd, "logs"), filepath.Join(wd, "bin")} {
		if err := os.MkdirAll(d, 0o755); err != nil {
			log.Fatalf("create dir %s: %v", d, err)
		}
	}

	st, err := store.New(filepath.Join(wd, "manager.db"))
	if err != nil {
		log.Fatalf("open store: %v", err)
	}
	defer st.Close()

	// Sync the flag credentials into the users table as a system=1 admin account
	// on every startup, so -user/-password changes propagate without manual
	// account setup. The auth middleware's Basic branch still falls back to the
	// raw flag creds, so inter-node cluster traffic (which uses SetBasicAuth
	// with these flags) never breaks even before this row exists.
	if err := st.SyncSystemUser(*user, *pass); err != nil {
		log.Fatalf("sync system user: %v", err)
	}

	// Seed settings with the initial binary path if provided.
	if *bin != "" {
		cfg, _ := st.Settings()
		cfg.BinaryPath = *bin
		_ = st.UpdateSettings(cfg)
	}

	// Determine the fallback binary: explicit -bin, else this executable (if it
	// looks like a usable frpc) — but we never want to accidentally treat our
	// own webui binary as frpc. So fallback is empty unless -bin is set; the UI
	// then asks the user to install or provide frpc.
	fallbackBin := ""
	if *frpc != "" {
		fallbackBin = *frpc
	} else if *bin != "" {
		fallbackBin = *bin
	}

	pm := process.NewManager(process.Options{
		ConfigsDir: filepath.Join(wd, "configs"),
		LogsDir:    filepath.Join(wd, "logs"),
		BinaryPath: func() string {
			if s, _ := st.Settings(); s.BinaryPath != "" {
				return s.BinaryPath
			}
			return fallbackBin
		},
		Render: renderForward(st),
		AutoRestart: func(name string) bool {
			s, _ := st.Settings()
			if !s.RestartOnExit {
				return false
			}
			rem, ok := st.GetRemote(name)
			return ok && rem.Enabled
		},
		RestartInterval: func() int {
			s, _ := st.Settings()
			if s.RestartIntervalSeconds > 0 {
				return s.RestartIntervalSeconds
			}
			return 5
		},
	})

	reg := cluster.NewRegistry(filepath.Join(wd, "bin"))
	// M6: register cluster peers from -peer flags (same creds as this node).
	for _, p := range peers {
		reg.RegisterPeer(p, p, *user, *pass)
	}

	// M6 token ring: this node is the initial leader of the cluster ring
	// (creation node). Engine drives two-phase cycles; transport POSTs the
	// token to the successor over HTTP.
	selfID := reachableAddr(*addr)
	ringTransport := &cluster.TokenTransport{User: *user, Pass: *pass}
	// nodeKey is this node's cluster admission key. It is NOT generated at
	// startup — a fresh node has no key until it creates or joins a cluster.
	// A creation node (no -peer) generates its key on CreateCluster; a joiner
	// (has -peer) generates its key on AdoptState after a successful join.
	// If the store has a persisted key (from a previous cluster membership),
	// restore it so a restart-in-cluster node keeps its identity.
	nodeKey := ""
	if s, _ := st.Settings(); s.NodeKey != "" {
		nodeKey = s.NodeKey
	}
	var ring *cluster.Engine
	ring = cluster.NewEngine(
		selfID, selfID, *user, *pass, version.Version, nil,
		&cluster.AppHandler{
			LoadFn: func() (float64, float64) {
				// NetPct: real NIC saturation from /proc/net/dev deltas vs
				// interface link speed (Linux); 0 fallback elsewhere. A NIC
				// already near capacity should stop attracting new forwards —
				// the forward itself is a network hop.
				return cluster.SampleMemLoad(), cluster.SampleNetLoad()
			},
			// Claim persists the forward locally (intermediate config -> store),
			// then syncs workers so the frpc worker for that remote starts.
			ClaimFn: func(ctx context.Context, tk *cluster.Task) error {
				if err := st.UpsertLocal(tk.Local); err != nil {
					return err
				}
				if err := st.UpsertRemote(tk.Remote); err != nil {
					return err
				}
				links, err := st.ListLinks()
				if err != nil {
					return err
				}
				// Upsert the task's link (dedup by local+remote+remotePort).
				// The submitter persists the canvas — including this link — via
				// saveCanvas, and when this same node is also the claimer (lowest
				// load) blindly re-appending would duplicate the proxy in the
				// rendered frpc config (two proxies, same remotePort) and the
				// worker would fail to register. Skip only if already present.
				dup := false
				for _, l := range links {
					if l.Local == tk.Link.Local && l.Remote == tk.Link.Remote && l.RemotePort == tk.Link.RemotePort {
						dup = true
						break
					}
				}
				if !dup {
					links = append(links, tk.Link)
				}
				if err := st.ReplaceLinks(links); err != nil {
					return err
				}
				// A re-claim after a forward-centric stop leaves the link flagged
				// disabled (by RevokeFn). Do NOT clear that flag here: the ring
				// re-claims automatically (startup reconcile re-spawns any owned
				// forward whose worker is missing), so clearing it on claim made
				// "stopped" un-durable — every restart resurrected a forward the user
				// had explicitly stopped, and it then failed forever against a local
				// service that was intentionally not running.
				//
				// The flag is now owned by the two user-facing actions:
				//   startForward → SetLinkDisabled(false) before submitting the task
				//   stopForward  → SetLinkDisabled(true)  before revoking it
				// Claim only READS it to decide whether to bring the worker up. A fresh
				// claim of a link with no row still starts, because the lookup miss
				// below is treated as "not disabled".
				disabled := false
				if existing, found, err := st.LinkByTriple(tk.Link.Local, tk.Link.Remote, tk.Link.RemotePort); err != nil {
					return err
				} else if found {
					disabled = existing.Disabled
				}
				// Start (or restart) the per-forward worker for exactly this link.
				// Each forward has its own frpc process (keyed by the forward
				// triple); restarting only this key leaves sibling forwards'
				// processes untouched.
				if disabled {
					// Mark the entry stopped rather than deleting it. The entry is the
					// carrier that keeps the flag travelling around the ring, and
					// UpdateTopologyDisabled also clears Active — which is what makes
					// this entry inert for OfflineReassign() and the startup
					// reconcile, so it is not resurrected later.
					if ring != nil {
						ring.UpdateTopologyDisabled(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort, true)
					}
					log.Printf("ring[%s] claim %s skipped: %s→%s:%d is disabled", selfID, tk.ID, tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
					return nil
				}
				if tk.Remote.Enabled {
					key := process.WorkerKey(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
					if _, has := pm.Status(key); has {
						_ = pm.Restart(key)
					} else {
						_ = pm.Start(key)
					}
				}
				log.Printf("ring[%s] claimed task %s: %s→%s:%d", selfID, tk.ID, tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
				return nil
			},
			// Revoke cancels the forward NON-destructively: stop THIS forward's
			// own worker process (per-forward model — no siblings share it) and
			// flag the exact link disabled so a later forward-centric start can
			// re-enable it. The old delete-local/remote/drop-all-matching-links
			// form served the topology-derived-canvas model but broke per-forward
			// stop/start: it wiped sibling forwards sharing the local or remote.
			RevokeFn: func(ctx context.Context, tk *cluster.Task) error {
				_ = st.SetLinkDisabled(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort, true)
				// The links table is node-local: only the node that received the
				// stop request has the flag persisted, while the node that OWNS the
				// forward is the one holding the worker (and is often a different
				// machine). Trusting only the local row meant the owner's copy could
				// still read disabled=false. The task now carries the flag, so use it
				// as the authority and write it here.
				ln, found, err := st.LinkByTriple(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
				switch {
				case err != nil:
					return err
				case found:
					ln.Disabled = true
					// Persist via the same rewrite path the canvas uses, so the flag
					// survives a restart on THIS node too.
					links, err := st.ListLinks()
					if err != nil {
						return err
					}
					for i := range links {
						if links[i].Local == ln.Local && links[i].Remote == ln.Remote && links[i].RemotePort == ln.RemotePort {
							links[i].Disabled = true
						}
					}
					if err := st.ReplaceLinks(links); err != nil {
						return err
					}
				case tk.Link.Disabled:
					// No local row yet (this node never claimed the forward) but the
					// task asserts the stop — record it so a later claim on this node
					// cannot resurrect the forward.
					loc, rem := tk.Local, tk.Remote
					links, err := st.ListLinks()
					if err != nil {
						return err
					}
					links = append(links, store.Link{
						Local: loc.Name, Remote: rem.Name, RemotePort: tk.Link.RemotePort,
						OffsetX: tk.Link.OffsetX, OffsetY: tk.Link.OffsetY,
						Group: tk.Link.Group, Disabled: true,
					})
					if err := st.ReplaceLinks(links); err != nil {
						return err
					}
				}
				key := process.WorkerKey(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
				if _, running := pm.Status(key); running {
					_ = pm.Stop(key)
				}
				// Mark the entry stopped instead of deleting it: the entry carries
				// the flag around the ring, and clearing Active keeps it inert for
				// the rebalancing paths (OfflineReassign skips inactive entries, and
				// the startup reconcile skips disabled forwards), so the worker is
				// not re-spawned on the next restart. Deleting it here is what used
				// to leave the stop with no carrier at all.
				if ring != nil {
					ring.UpdateTopologyDisabled(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort, true)
				}
				log.Printf("ring[%s] revoked task %s: %s→%s:%d", selfID, tk.ID, tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
				return nil
			},
			// ReleaseFn stops this forward's worker for a REBALANCE, leaving the
			// link ENABLED. This must not reuse RevokeFn: that marks the link
			// stopped, and the released forward is about to run on another node —
			// a "stopped" flag would make the claim path treat the re-published
			// task as a disabled forward and refuse to start a worker for it,
			// leaving the forward dead everywhere.
			ReleaseFn: func(ctx context.Context, tk *cluster.Task) error {
				key := process.WorkerKey(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
				if _, running := pm.Status(key); running {
					_ = pm.Stop(key)
				}
				log.Printf("ring[%s] released worker %s (rebalancing): %s→%s:%d", selfID, key,
					tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
				return nil
			},
			// RestartFn brings an already-claimed forward back after a stop. It
			// must NOT run the claim bookkeeping (ClaimFn), which appends links and
			// writes topology state: the forward is already established, and a stop
			// deliberately keeps its topology entry, so all that is needed is to
			// clear the flag and respawn the worker. The engine has already
			// re-marked the topology entry enabled before calling this.
			RestartFn: func(ctx context.Context, tk *cluster.Task) error {
				// Clear the stop on this node's own store, using the same path the
				// start handler uses, so the local copy cannot disagree with the
				// ring and block a later restart.
				if ln, found, err := st.LinkByTriple(tk.Link.Local, tk.Link.Remote, tk.Link.RemotePort); err != nil {
					return err
				} else if found && ln.Disabled {
					links, err := st.ListLinks()
					if err != nil {
						return err
					}
					for i := range links {
						if links[i].Local == ln.Local && links[i].Remote == ln.Remote && links[i].RemotePort == ln.RemotePort {
							links[i].Disabled = false
						}
					}
					if err := st.ReplaceLinks(links); err != nil {
						return err
					}
				}
				if tk.Remote.Enabled {
					key := process.WorkerKey(tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
					if _, has := pm.Status(key); has {
						_ = pm.Restart(key)
					} else {
						_ = pm.Start(key)
					}
				}
				log.Printf("ring[%s] restarted %s: %s→%s:%d", selfID, tk.ID, tk.Local.Name, tk.Remote.Name, tk.Link.RemotePort)
				return nil
			},
		},
		ringTransport.SendTo(func(nodeID string) string {
			if ring == nil {
				return nodeID
			}
			for _, n := range ring.State().Nodes {
				if n.ID == nodeID {
					return n.Addr
				}
			}
			return nodeID
		}),
		selfID, len(peers) == 0, nodeKey, // creation node (no -peer) is the initial leader
	)
	// Install the persistence callbacks so ensureNodeKey (called from
	// CreateCluster / AdoptState) and detachAsStandalone can persist/clear
	// the key in the settings store without the engine depending on store.
	ring.SetKeyPersist(func(key string) error { return st.SetNodeKey(key) })
	// peerPersist saves the cached peer list (addr+key) every token cycle so
	// a crashed node can auto-rejoin via any cached peer on restart. Cleared
	// on explicit detach (detachAsStandalone) so a node that intentionally
	// left does NOT auto-rejoin.
	ring.SetPeerPersist(func(peersJSON string) error { return st.SetClusterPeers(peersJSON) })

	// Re-apply local store state onto the ring topology after each state
	// adoption, and learn the ring's disabled decisions back into the store.
	//
	// Without the group half, group changes made via HTTP handlers
	// (POST /forwards/assign) are overwritten by the next e.state = tk.State
	// and never propagate.
	//
	// The disabled half makes the RING authoritative for stop/start. That is the
	// only source all members can agree on: the links table is a per-node copy, so
	// treating it as authoritative let two nodes disagree indefinitely (observed
	// live: one node reading disabled=1 while the owner still read 0 and kept its
	// worker running). This is a write-through, not a listener — it runs on every
	// adoption, so any peer's decision lands here within a token cycle.
	//
	// Ordering matters in one direction only: the engine has already re-applied
	// this node's not-yet-confirmed local decisions onto the adopted state (see
	// reconcileLocalDisabled), so reading the ring here can never clobber an
	// action the user just took on this node.
	ring.SetTopologySync(func() {
		links, err := st.ListLinks()
		if err != nil {
			return
		}
		for _, ln := range links {
			ring.UpdateTopologyGroup(ln.Local, ln.Remote, ln.RemotePort, ln.Group)
		}
		for _, te := range ring.State().TopologyList() {
			disabled, known := ring.TopologyDisabled(te.Local.Name, te.Remote.Name, te.Link.RemotePort)
			if !known {
				continue
			}
			if ln, found, err := st.LinkByTriple(te.Local.Name, te.Remote.Name, te.Link.RemotePort); err == nil && found && ln.Disabled == disabled {
				continue // store already agrees with the ring
			}
			_ = st.ReconcileLinkDisabled(te.Local.Name, te.Remote.Name, te.Link.RemotePort, disabled)
		}
	})

	// Load balancer is created here (before the Handler) so the HTTP layer can
	// expose its read-only report. Mode/parameters are applied further down, once
	// the W4F_BALANCE_* environment has been parsed.
	balancer := cluster.NewBalancer(ring, 600)

	h := &httpapi.Handler{
		Store:    st,
		Process:  pm,
		WorkDir:  wd,
		BinDir:   filepath.Join(wd, "bin"),
		Sessions: httpapi.NewSessionStore(),
		Cluster:  reg,
		Ring:     ring,
		// Balancer is created above, before the environment that configures it is
		// parsed, because the HTTP layer needs the pointer.
		Balancer: balancer,
		// SelfAddr must be reachable by peers. 0.0.0.0/empty on a real host
		// would break peer reconnect, so fall back to the hostname (container
		// name inside docker is a routable addr on the compose network).
		SelfAddr: reachableAddr(*addr),
		User:     *user,
		Password: *pass,
		BinaryPath: func() string {
			s, _ := st.Settings()
			if s.BinaryPath != "" {
				return s.BinaryPath
			}
			return fallbackBin
		},
		InstallBinary: func(version string) (string, string, error) {
			// M6: prefer cluster peer exchange (local cache -> peers), fall
			// back to the official GitHub URL only when no peer has it.
			ictx, icancel := context.WithTimeout(context.Background(), 90*time.Second)
			defer icancel()
			if p, err := reg.EnsureVersion(ictx, version); err == nil {
				s, _ := st.Settings()
				s.BinaryPath = p
				if err := st.UpdateSettings(s); err != nil {
					return p, version, err
				}
				return p, version, nil
			}
			path, ver, err := install.Install(context.Background(), filepath.Join(wd, "bin"), version)
			if err != nil {
				return "", "", err
			}
			s, _ := st.Settings()
			s.BinaryPath = path
			if err := st.UpdateSettings(s); err != nil {
				return path, ver, err
			}
			return path, ver, nil
		},
		SyncWorkers: func() {
			// PER-FORWARD worker model: every enabled, non-disabled link on THIS
			// node gets its own frpc process keyed by the forward triple.
			// - localOnly forwards: always owned here (loopback stays local).
			// - cluster forwards: only started by the ring claim (ClaimFn) —
			//   SyncWorkers deliberately skips them so a node that merely holds
			//   a canvas copy does not spawn workers for forwards it doesn't own
			//   (the old "every node registers every proxy" bug).
			links, err := st.ListLinks()
			if err != nil {
				return
			}
			for _, ln := range links {
				loc, ok := st.GetLocal(ln.Local)
				if !ok || !loc.LocalOnly {
					continue // cluster-owned or unknown local
				}
				rem, ok := st.GetRemote(ln.Remote)
				if !ok || !rem.Enabled {
					continue
				}
				key := process.WorkerKey(ln.Local, ln.Remote, ln.RemotePort)
				if ln.Disabled {
					_ = pm.Stop(key) // per-forward stop kills exactly its process
					continue
				}
				if _, has := pm.Status(key); has {
					_ = pm.Restart(key)
				} else {
					_ = pm.Start(key)
				}
			}
		},
	}

	// Auto-start localOnly forwards on boot. Cluster forwards are NOT started
	// here — they are re-submitted by the leader's startup auto-submit below
	// and claimed (spawning their own worker) wherever the ring places them.
	s, _ := st.Settings()
	if s.AutoStartProfiles {
		links, _ := st.ListLinks()
		remotesByName := map[string]store.Remote{}
		if remotes, err := st.ListRemotes(); err == nil {
			for _, r := range remotes {
				remotesByName[r.Name] = r
			}
		}
		localsByName := map[string]store.Local{}
		if locals, err := st.ListLocals(); err == nil {
			for _, l := range locals {
				localsByName[l.Name] = l
			}
		}
		for _, ln := range links {
			if ln.Disabled {
				continue
			}
			loc, ok := localsByName[ln.Local]
			if !ok || !loc.LocalOnly {
				continue
			}
			rem, ok := remotesByName[ln.Remote]
			if !ok || !rem.Enabled {
				continue
			}
			_ = pm.Start(process.WorkerKey(ln.Local, ln.Remote, ln.RemotePort))
		}
	}

	handler, err := httpapi.NewServeMux(h)
	if err != nil {
		log.Fatalf("build handler: %v", err)
	}
	srv := &http.Server{Addr: *addr, Handler: handler}

	go func() {
		log.Printf("webui-frpc listening on %s (workdir %s)", *addr, wd)
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Fatalf("server error: %v", err)
		}
	}()

	// M6: cluster heartbeat + new-node bootstrap auto-propagation. The
	// bootstrap loop retries every 10s for the first minutes so binaries are
	// pulled as soon as neighbor info appears (a fresh node has none at boot).
	clusterCtx, clusterCancel := context.WithCancel(context.Background())
	defer clusterCancel()
	go reg.Discover(clusterCtx, 30*time.Second)
	// M6 token ring startup. Cached peers (persisted on every token cycle)
	// take priority over the -peer flag — -peer is only for FIRST-TIME
	// bootstrap; on crash/restart the node always rejoin via cached peers.
	//
	//  1. Persisted ClusterPeers → crash/restart recovery: try to rejoin
	//     via any cached peer (addr+key from the last token cycle before
	//     the crash). First node up finds all peers down → falls back to
	//     CreateCluster; others rejoin it via cached peers.
	//  2. No cached peers + -peer flag → first-time bootstrap: join the
	//     specified peer with the given join key. Retry every 10s for 5 min.
	//  3. No cached peers, no -peer → first-time creation node: seed a
	//     fresh standalone cluster (CreateCluster).
	//
	// Explicit detach (detachAsStandalone) clears ClusterPeers, so a node
	// that intentionally left falls into case 2 or 3 and does NOT auto-rejoin.
	persistedPeers := ""
	if s, _ := st.Settings(); s.ClusterPeers != "" {
		persistedPeers = s.ClusterPeers
	}
	if persistedPeers != "" {
		go retryRejoinCached(clusterCtx, ring, persistedPeers)
	} else if len(peers) > 0 {
		go retryJoinCluster(clusterCtx, ring, peers[0], *joinKey)
	} else {
		if err := ring.CreateCluster(); err != nil {
			log.Printf("ring[%s] create cluster at startup: %v", selfID, err)
		}
		go ring.StartRing(clusterCtx)
	}
	go ring.WatchTokenLoss(clusterCtx)
	go ring.WatchLeader(clusterCtx)

	// Load balancing — SELF-DIRECTED, and dry-run by default.
	//
	// Every node runs this check against its OWN forwards. There is no
	// coordinator: a node's load only actually falls when its own topology entry
	// goes away, so only the owner can produce the state change that makes the
	// ring hand a forward to somebody else. The node gives a forward up and
	// re-publishes it as an ordinary unclaimed task; the ring's normal
	// lowest-load claim then places it. (The earlier design had the leader pick a
	// victim and a destination — but it can only see the imbalance, not resolve
	// it, so it had to reach into another node's bookkeeping to make its decision
	// true.)
	//
	// A release stops a worker and drops a live connection, so acting on the ring
	// is opt-in:
	//
	//   W4F_BALANCE_MODE=dry-run   report the decision, release nothing (default)
	//   W4F_BALANCE_MODE=on        actually release
	//   W4F_BALANCE_MODE=off       do nothing at all
	//   W4F_BALANCE_COOLDOWN=600   min seconds between two releases by one node
	//   W4F_BALANCE_INTERVAL=10    check every N seconds
	balMode := strings.ToLower(strings.TrimSpace(os.Getenv("W4F_BALANCE_MODE")))
	if balMode == "" {
		balMode = cluster.BalanceDryRun
	}
	balInterval := time.Duration(envFloat("W4F_BALANCE_INTERVAL", 10)) * time.Second
	if balInterval < time.Second {
		balInterval = 10 * time.Second
	}
	balCooldown := int64(envFloat("W4F_BALANCE_COOLDOWN", 600))
	balancer.SetCooldown(balCooldown)
	balancer.SetMode(balMode)
	// A mode stored from the cluster page wins over the environment, so flipping
	// the switch is not silently reverted by the next restart. SetMode is applied
	// first so a stored value only has to be a no-op when the two agree.
	balancer.SetModePersistence(
		func() (string, bool) {
			m, ok, err := st.BalanceMode()
			if err != nil {
				log.Printf("ring[%s] read stored balance mode: %v", selfID, err)
				return "", false
			}
			return m, ok
		},
		func(m string) error {
			s, err := st.Settings()
			if err != nil {
				return err
			}
			s.BalanceMode = m
			return st.UpdateSettings(s)
		},
	)
	// Report the EFFECTIVE mode, not balMode: a stored value may have overridden
	// the environment, and a startup line naming the wrong mode is exactly how an
	// operator ends up believing releasing is off while it is on.
	log.Printf("ring[%s] load balancing %s (source=%s cooldown=%ds interval=%s, self-directed)",
		selfID, strings.ToUpper(balancer.Mode()), balancer.ModeSource(), balCooldown, balInterval)
	go balancer.Watch(clusterCtx, balInterval)

	// Startup reconcile: every node (leader or not) checks its own topology
	// entries and re-claims any whose per-forward frpc worker process is
	// absent — the topology was recovered via AdoptState but the worker
	// processes died with the restart. This is safer than the leader clearing
	// and re-submitting all topology, which races with leader changes.
	// LocalOnly forwards are never in the topology and are handled by the
	// auto-start loop above.
	go func() {
		// No leader gate here. The reconcile body is per-NODE work: it re-claims
		// the forwards THIS node owns, whose worker processes died with the
		// restart. Every member owns forwards (a 3-node ring splits them), so
		// gating on leadership meant non-leaders waited forever and never
		// rebuilt their own workers — a node could come back up "in the cluster"
		// while owning a forward with no process behind it. Observed live: after a
		// rolling restart a non-leader rejoined, adopted state listing itself as
		// the owner of a forward, and never started it, because it sat waiting to
		// become leader.
		//
		// What actually needs settling is the ring, not the leadership: wait for
		// the topology to show us (with a bounded fallback so a lone node that
		// rejoined a ring whose leader is unreachable still makes progress).
		deadline := time.After(30 * time.Second)
		for !ring.HasSelfInTopology() {
			select {
			case <-deadline:
				log.Printf("ring[%s] startup reconcile: self not in topology after 30s, proceeding anyway", ring.ID)
			case <-time.After(500 * time.Millisecond):
				continue
			case <-clusterCtx.Done():
				return
			}
			break
		}
		// Let the ring stabilize after election / rejoin before acting on it.
		time.Sleep(3 * time.Second)
		snap := ring.Snapshot()
		reclaimed := 0
		for _, t := range snap.Topology {
			if t.OwnerID != selfID {
				continue
			}
			// A forward the user stopped must stay stopped: skip it here so a
			// restart does not re-spawn its worker. The ClaimFn enforces the
			// same rule (belt and braces — this also avoids a pointless
			// claim→skip round trip per disabled forward on every boot).
			if ln, found, err := st.LinkByTriple(t.Local.Name, t.Remote.Name, t.Link.RemotePort); err != nil {
				log.Printf("ring[%s] reconcile: lookup %s→%s:%d: %v", ring.ID, t.Local.Name, t.Remote.Name, t.Link.RemotePort, err)
				continue
			} else if found && ln.Disabled {
				continue
			}
			key := process.WorkerKey(t.Local.Name, t.Remote.Name, t.Link.RemotePort)
			if _, has := pm.Status(key); has {
				continue // worker already running
			}
			ctx, cancel := context.WithTimeout(clusterCtx, 15*time.Second)
			if err := ring.Handler.Claim(ctx, &cluster.Task{
				ID:    fmt.Sprintf("reclaim-%s-%s-%d", t.Local.Name, t.Remote.Name, t.Link.RemotePort),
				Local: t.Local, Remote: t.Remote, Link: t.Link,
			}); err != nil {
				log.Printf("ring[%s] reclaim %s→%s:%d: %v", ring.ID, t.Local.Name, t.Remote.Name, t.Link.RemotePort, err)
			} else {
				reclaimed++
			}
			cancel()
		}
		// Leader also re-submits any non-disabled, non-localOnly forwards
		// that are NOT yet in the topology (new entries the canvas has but
		// the ring hasn't processed yet).
		links, err := st.ListLinks()
		if err != nil {
			log.Printf("ring[%s] reconcile: %v", ring.ID, err)
			return
		}
		type ownerKey struct {
			local, remote string
			port          int
		}
		ownedBy := map[ownerKey]string{}
		for _, t := range snap.Topology {
			k := ownerKey{t.Local.Name, t.Remote.Name, t.Link.RemotePort}
			if _, ok := ownedBy[k]; !ok {
				ownedBy[k] = t.OwnerID
			}
		}
		submitted := 0
		for _, ln := range links {
			if ln.Disabled {
				continue
			}
			loc, ok := st.GetLocal(ln.Local)
			if !ok || loc.LocalOnly {
				continue
			}
			rem, ok := st.GetRemote(ln.Remote)
			if !ok {
				continue
			}
			k := ownerKey{ln.Local, ln.Remote, ln.RemotePort}
			if _, inTopo := ownedBy[k]; inTopo {
				continue // already reclaimed above
			}
			ring.SubmitTask(loc, rem, ln)
			submitted++
		}
		log.Printf("ring[%s] startup reconcile: reclaimed=%d submitted=%d", ring.ID, reclaimed, submitted)
	}()

	go func() {
		ticker := time.NewTicker(10 * time.Second)
		defer ticker.Stop()
		bootTimeout := time.NewTimer(90 * time.Second)
		defer bootTimeout.Stop()
		for {
			bootCtx, bootCancel := context.WithTimeout(clusterCtx, 20*time.Second)
			err := reg.SyncFromPeers(bootCtx)
			bootCancel()
			log.Printf("cluster bootstrap sync iteration (peers=%d caches=%d err=%v)", len(reg.PeerList()), len(reg.CachedVersions()), err)
			if err != nil {
				log.Printf("cluster bootstrap sync: %v", err)
			}
			select {
			case <-ticker.C:
			case <-bootTimeout.C:
				return
			case <-clusterCtx.Done():
				return
			}
		}
	}()

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
	<-sig
	log.Println("shutting down...")
	clusterCancel()
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()
	pm.StopAll()
	_ = srv.Shutdown(ctx)
}

// retryJoinCluster joins the ring at targetAddr using joinKey, retrying every
// 10s for up to 5 min in case the peer isn't up yet. On success AdoptState is
// called by JoinRing (which persists peers). On exhaustion, falls back to
// CreateCluster so the node is at least functional as standalone.
func retryJoinCluster(ctx context.Context, ring *cluster.Engine, targetAddr, joinKey string) {
	selfNode := ring.State().Nodes[0]
	ji := cluster.JoinInfo{ID: selfNode.ID, Addr: selfNode.Addr, Version: version.Version, JoinKey: joinKey}
	deadline := time.NewTimer(5 * time.Minute)
	defer deadline.Stop()
	ticker := time.NewTicker(10 * time.Second)
	defer ticker.Stop()
	// First attempt immediately, then on each tick.
	for {
		// If another node already joined us (we're now multi-node), stop
		// trying to join — we're already in a ring.
		if ring.IsMember() {
			log.Printf("ring[%s] already a cluster member, skipping join", selfNode.ID)
			return
		}
		jc, cancel := context.WithTimeout(ctx, 8*time.Second)
		err := ring.JoinRing(jc, targetAddr, ji)
		cancel()
		if err == nil {
			log.Printf("ring[%s] joined cluster via %s", selfNode.ID, targetAddr)
			return
		}
		log.Printf("ring[%s] join %s failed: %v; retrying", selfNode.ID, targetAddr, err)
		select {
		case <-ticker.C:
		case <-deadline.C:
			log.Printf("ring[%s] join retry exhausted, falling back to standalone", selfNode.ID)
			_ = ring.CreateCluster()
			go ring.StartRing(ctx)
			return
		case <-ctx.Done():
			return
		}
	}
}

// envFloat reads a numeric environment variable, returning def when unset or
// unparseable. A malformed value must not silently change behaviour, so it falls
// back to the documented default rather than 0.
func envFloat(key string, def float64) float64 {
	v := strings.TrimSpace(os.Getenv(key))
	if v == "" {
		return def
	}
	f, err := strconv.ParseFloat(v, 64)
	if err != nil {
		log.Printf("env %s=%q is not a number; using %v", key, v, def)
		return def
	}
	return f
}

// retryRejoinCached is the crash/restart recovery path: parse the persisted
// peer list (JSON of [{addr,key},...]) and try to rejoin via each peer in
// order. The first peer that accepts wins. Retry the whole list every 10s
// for up to 5 min. On exhaustion, fall back to CreateCluster (standalone).
func retryRejoinCached(ctx context.Context, ring *cluster.Engine, peersJSON string) {
	type peerEntry struct {
		Addr string `json:"addr"`
		Key  string `json:"key"`
	}
	var peers []peerEntry
	if err := json.Unmarshal([]byte(peersJSON), &peers); err != nil || len(peers) == 0 {
		log.Printf("ring[%s] cached peers parse error, falling back to standalone", ring.ID)
		_ = ring.CreateCluster()
		go ring.StartRing(ctx)
		return
	}
	selfNode := ring.State().Nodes[0]
	deadline := time.NewTimer(5 * time.Minute)
	defer deadline.Stop()
	for {
		// If another node already joined us, stop trying to rejoin.
		if ring.IsMember() {
			log.Printf("ring[%s] already a cluster member, skipping rejoin", ring.ID)
			return
		}
		for _, p := range peers {
			ji := cluster.JoinInfo{ID: selfNode.ID, Addr: selfNode.Addr, Version: version.Version, JoinKey: p.Key}
			jc, cancel := context.WithTimeout(ctx, 8*time.Second)
			err := ring.JoinRing(jc, p.Addr, ji)
			cancel()
			if err == nil {
				log.Printf("ring[%s] rejoined cluster via cached peer %s", ring.ID, p.Addr)
				// A rejoined node must be able to drive the ring itself. The
				// normal case is that the existing leader keeps circulating the
				// token, but if WE are the leader (typical after a rolling
				// restart where the leader came back last, or the leader died
				// while we were down) nobody else will: StartRing is a no-op on
				// non-leaders, and the ring sits still with every task stranded
				// in pending. Observed live: the leader restarted, rejoined fine,
				// and then never forwarded a token — cycle frozen, topology
				// empty, pending=3.
				//
				// StartRing is safe to call unconditionally: it returns
				// immediately unless this node leads, and it throttles itself
				// against RoundDelay so a redundant call cannot double-send.
				go ring.StartRing(ctx)
				return
			}
			log.Printf("ring[%s] rejoin %s failed: %v", ring.ID, p.Addr, err)
		}
		select {
		case <-time.After(10 * time.Second):
		case <-deadline.C:
			log.Printf("ring[%s] rejoin retry exhausted, falling back to standalone", ring.ID)
			_ = ring.CreateCluster()
			go ring.StartRing(ctx)
			return
		case <-ctx.Done():
			return
		}
	}
}

// reachableAddr returns an address peers can dial back: an explicit
// W4F_HOST (compose DNS name) wins; otherwise a non-wildcard listen addr;
// a wildcard falls back to the primary non-loopback IPv4 detected from the
// interfaces (routable on plain LAN hosts); hostname is the last resort.
func reachableAddr(addr string) string {
	if h := os.Getenv("W4F_HOST"); h != "" {
		if _, port, err := net.SplitHostPort(addr); err == nil {
			return net.JoinHostPort(h, port)
		}
		return h
	}
	host, port, err := net.SplitHostPort(addr)
	if err != nil {
		return addr
	}
	if host == "" || host == "0.0.0.0" || host == "::" || host == "[::]" {
		if ip := primaryLANIPv4(); ip != "" {
			return net.JoinHostPort(ip, port)
		}
		if h, herr := os.Hostname(); herr == nil && h != "" {
			return net.JoinHostPort(h, port)
		}
	}
	return addr
}

// primaryLANIPv4 returns the first global-scope non-loopback IPv4 address of
// any up interface, preferring RFC1918 ranges. Empty string when none found.
func primaryLANIPv4() string {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return ""
	}
	var fallback string
	for _, a := range addrs {
		ipnet, ok := a.(*net.IPNet)
		if !ok || ipnet.IP.To4() == nil || ipnet.IP.IsLoopback() || !ipnet.IP.IsGlobalUnicast() {
			continue
		}
		ip := ipnet.IP.To4()
		if fallback == "" {
			fallback = ip.String()
		}
		// prefer private ranges over link-local / unusual globals
		if ip[0] == 10 || (ip[0] == 172 && ip[1]&0xf0 == 16) || (ip[0] == 192 && ip[1] == 168) {
			return ip.String()
		}
	}
	return fallback
}

// splitCSV splits a comma-separated list into a slice (empty -> nil).
func splitCSV(s string) []string {
	if strings.TrimSpace(s) == "" {
		return nil
	}
	parts := strings.Split(s, ",")
	out := make([]string, 0, len(parts))
	for _, p := range parts {
		if p = strings.TrimSpace(p); p != "" {
			out = append(out, p)
		}
	}
	return out
}

// renderForward builds the process.Render closure for the PER-FORWARD worker
// model: each (local, remote, remotePort) link gets its OWN frpc config
// containing exactly ONE proxy. The worker key is the forward triple
// (process.WorkerKey), so every node only ever registers its own proxies on
// frps — this eliminates the multi-node "proxy already exists" fight of the
// old per-remote model, where every node rendered ALL forwards of a remote.
// A disabled link renders an empty proxy list: frpc exits cleanly with no
// proxies (the supervisor sees a clean exit and stops restarting it), which
// is exactly the per-forward stop semantics.
func renderForward(st *store.Store) func(string) ([]byte, error) {
	return func(key string) ([]byte, error) {
		localName, remoteName, port, ok := process.ParseWorkerKey(key)
		if !ok {
			return nil, fmt.Errorf("invalid worker key %q", key)
		}
		rem, ok := st.GetRemote(remoteName)
		if !ok {
			return nil, store.ErrNotFound
		}
		loc, ok := st.GetLocal(localName)
		if !ok {
			return nil, store.ErrNotFound
		}
		disabled := false
		for _, ln := range mustListLinks(st) {
			if ln.Local == localName && ln.Remote == remoteName && ln.RemotePort == port {
				disabled = ln.Disabled
				break
			}
		}
		proxies := []render.Proxy(nil)
		if !disabled {
			proxies = append(proxies, render.Proxy{
				Name:              loc.Name,
				Type:              loc.Protocol,
				LocalIP:           loc.IP,
				LocalPort:         loc.Port,
				RemotePort:        port,
				UseEncryption:     loc.UseEncryption,
				UseCompression:    loc.UseCompression,
				BandwidthLimit:    loc.BandwidthLimit,
				PoolCount:         loc.PoolCount,
				Metadatas:         loc.Metadatas,
				Annotations:       loc.Annotations,
				CustomDomains:     splitCSV(loc.CustomDomains),
				SubDomain:         loc.SubDomain,
				Locations:         loc.Locations,
				HostHeaderRewrite: loc.HostHeaderRewrite,
				HTTPHeaders:       loc.HTTPHeaders,
				BasicAuthUser:     loc.BasicAuthUser,
				BasicAuthPassword: loc.BasicAuthPassword,

				// M3: load balancing group + health check.
				LBGroup:              loc.LBGroup,
				LBGroupKey:           loc.LBGroupKey,
				HealthCheckType:      loc.HealthCheckType,
				HealthCheckPath:      loc.HealthCheckPath,
				HealthCheckTimeout:   loc.HealthCheckTimeout,
				HealthCheckMaxFailed: loc.HealthCheckMaxFailed,
				HealthCheckInterval:  loc.HealthCheckInterval,
			})
		}
		return render.Render(rem, proxies)
	}
}

// mustListLinks is ListLinks with errors swallowed (best-effort reads).
func mustListLinks(st *store.Store) []store.Link {
	links, _ := st.ListLinks()
	return links
}

// multiFlag collects repeated string flags (-peer a -peer b ...).
type multiFlag []string

func (m *multiFlag) String() string { return strings.Join(*m, ",") }
func (m *multiFlag) Set(v string) error {
	*m = append(*m, v)
	return nil
}