mirror of
https://gitcode.com/JianFeeeee/webui4frpc.git
synced 2026-09-19 16:38:31 +00:00
feat: 网络接口负载采样 — 令牌环认领决策引入真实 NIC 饱和度
- netload_linux.go: 从 /proc/net/dev 字节增量 + /sys/class/net/<iface>/speed 计算网卡利用率 (rx+tx 合计, 多网卡按容量加权, [0,100] 钳制) - virtio VM speed=-1 时用 1Gbps 软容量兜底, 保持 VM 上采样有意义 - 排除 lo/docker*/br-*/veth*/tun/tap/wg/virbr* 等非物理 uplink, 容器桥接流量不再虚增主机负载 - netload_other.go: 非 Linux 平台编译期 fallback 返回 0 - main.go LoadFn: NetPct 从写死的 10 换成 cluster.SampleNetLoad() - Score = Forwards*100 + MemPct + NetPct: 主信号仍是转发数, net 在同转发数节点间打破平局 — NIC 已饱和的节点不再吸引新转发 - 单测: 采样范围断言 + 虚拟接口排除表 实测: 三台集群 net 列显示真实值 (0.02-0.03%), 不再是常量
This commit is contained in:
@ -135,7 +135,11 @@ func main() {
|
||||
selfID, selfID, *user, *pass, "0.1.0", nil,
|
||||
&cluster.AppHandler{
|
||||
LoadFn: func() (float64, float64) {
|
||||
return cluster.SampleMemLoad(), 10
|
||||
// NetPct: real NIC saturation from /proc/net/dev deltas vs
|
||||
// interface link speed (Linux); 0 fallback elsewhere. A NIC
|
||||
// already near capacity should stop attracting new forwards —
|
||||
// the forward itself is a network hop.
|
||||
return cluster.SampleMemLoad(), cluster.SampleNetLoad()
|
||||
},
|
||||
// Claim persists the forward locally (intermediate config -> store),
|
||||
// then syncs workers so the frpc worker for that remote starts.
|
||||
@ -454,7 +458,10 @@ func main() {
|
||||
log.Printf("ring[%s] reconcile: %v", ring.ID, err)
|
||||
return
|
||||
}
|
||||
type ownerKey struct{ local, remote string; port int }
|
||||
type ownerKey struct {
|
||||
local, remote string
|
||||
port int
|
||||
}
|
||||
ownedBy := map[ownerKey]string{}
|
||||
for _, t := range snap.Topology {
|
||||
k := ownerKey{t.Local.Name, t.Remote.Name, t.Link.RemotePort}
|
||||
|
||||
229
internal/cluster/netload_linux.go
Normal file
229
internal/cluster/netload_linux.go
Normal file
@ -0,0 +1,229 @@
|
||||
//go:build linux
|
||||
|
||||
// Package cluster — Linux network interface load sampling.
|
||||
//
|
||||
// SampleNetLoad measures how saturated this node's network interfaces are so
|
||||
// the ring's lowest-load claimer selection can AVOID funneling a new forward
|
||||
// onto a NIC already running close to its link capacity. The forward itself
|
||||
// IS a network hop, so NIC throughput is a first-class load signal — not the
|
||||
// placeholder constant it replaced.
|
||||
//
|
||||
// Method (cheap, allocation-free, no external deps):
|
||||
// 1. Read /proc/net/dev for per-interface cumulative rx/tx byte counters.
|
||||
// 2. Skip non-physical links: loopback (lo), bridges (docker*/br*), virtual
|
||||
// tunnels (veth*/tun*/tap*/wg*), and anything /sys/class/net/<iface>/type
|
||||
// flags as software (ARPHRD_NONE / ARPHRD_LOOPBACK / ARPHRD_TUNNEL*).
|
||||
// 3. Read /sys/class/net/<iface>/speed (Mbps) to learn each kept interface's
|
||||
// capacity. Skip interfaces whose speed is unknown (-1), 0, or absent —
|
||||
// they have no meaningful budget to compare against.
|
||||
// 4. Compare two samples taken a short interval apart → bytes/sec per
|
||||
// interface → percent of that interface's capacity (rx+tx combined).
|
||||
// 5. Aggregate across all kept interfaces as a WEIGHTED BY CAPACITY average
|
||||
// so a 10G NIC at 20% and a 1G NIC at 90% combine to a value that
|
||||
// reflects total headroom, not a naive mean.
|
||||
//
|
||||
// The returned value is in [0, 100]. A soft cap clamps spikes above 100 so a
|
||||
// short burst on a slow NIC can't dominate the ring's load decisions forever
|
||||
// (the next sample returns to a calm value).
|
||||
//
|
||||
// The very first call has no prior sample; it returns 0 (the ring falls back
|
||||
// to the forward-count primary signal) and seeds the counters for the next
|
||||
// call. Read errors or a degenerate environment also return 0.
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// netSample is one point-in-time snapshot of the kept interfaces' cumulative
|
||||
// byte counters, plus the sum of their capacities (used for weighted
|
||||
// aggregation).
|
||||
type netSample struct {
|
||||
when time.Time
|
||||
bytes map[string]uint64 // iface -> rx_bytes + tx_bytes
|
||||
capMbps uint64 // sum of kept interface speeds
|
||||
}
|
||||
|
||||
var (
|
||||
netMu sync.Mutex
|
||||
netPrev netSample
|
||||
netSeeded bool
|
||||
)
|
||||
|
||||
// physicalIfaces is the set of interface name prefixes that represent a real
|
||||
// physical NIC or a real uplink bond. Bridges/veths/tunnels are excluded so a
|
||||
// busy docker bridge doesn't make the host look saturated.
|
||||
func isPhysicalUplink(name string, typ int) bool {
|
||||
if name == "lo" {
|
||||
return false
|
||||
}
|
||||
switch typ {
|
||||
case 1, // ARPHRD_ETHER ( ethernet )
|
||||
772: // ARPHRD_LOOPBACK — already excluded by name, defensive
|
||||
// keep
|
||||
default:
|
||||
// ARPHRD_NONE (65534) veth/tun/tap, ARPHRD_TUNNEL* — software.
|
||||
return false
|
||||
}
|
||||
// Bond/team/mvlan/vlan real uplinks are OK; container-internal links are not.
|
||||
switch {
|
||||
case strings.HasPrefix(name, "docker"),
|
||||
strings.HasPrefix(name, "br-"),
|
||||
strings.HasPrefix(name, "veth"),
|
||||
strings.HasPrefix(name, "tun"),
|
||||
strings.HasPrefix(name, "tap"),
|
||||
strings.HasPrefix(name, "wg"),
|
||||
strings.HasPrefix(name, "virbr"):
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// readIfaceType reads /sys/class/net/<iface>/type (the ARPHRD_* number). 0
|
||||
// when unreadable.
|
||||
func readIfaceType(name string) int {
|
||||
b, err := os.ReadFile("/sys/class/net/" + name + "/type")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
t, _ := strconv.Atoi(strings.TrimSpace(string(b)))
|
||||
return t
|
||||
}
|
||||
|
||||
// defaultUnknownSpeedMbps is the conservative soft budget for a real uplink
|
||||
// whose driver reports speed=-1 (common for virtio/cloud VMs). It keeps
|
||||
// network load meaningful on virtual machines instead of silently returning
|
||||
// zero forever. Container bridges and virtual tunnel interfaces are excluded
|
||||
// before this fallback is applied.
|
||||
const defaultUnknownSpeedMbps = 1000
|
||||
|
||||
// readIfaceSpeed reads /sys/class/net/<iface>/speed in Mbps. Unknown driver
|
||||
// speed (-1) falls back to a conservative 1 Gbps soft budget for a real
|
||||
// uplink; unreadable/zero still means the interface cannot be measured.
|
||||
func readIfaceSpeed(name string) uint64 {
|
||||
b, err := os.ReadFile("/sys/class/net/" + name + "/speed")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
|
||||
if v < 0 {
|
||||
return defaultUnknownSpeedMbps
|
||||
}
|
||||
if v == 0 {
|
||||
return 0
|
||||
}
|
||||
return uint64(v)
|
||||
}
|
||||
|
||||
// snapshotNet reads /proc/net/dev once and returns the byte totals per kept
|
||||
// interface plus the summed capacity. Empty when nothing kept is usable.
|
||||
func snapshotNet() netSample {
|
||||
f, err := os.Open("/proc/net/dev")
|
||||
if err != nil {
|
||||
return netSample{}
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
s := netSample{
|
||||
when: time.Now(),
|
||||
bytes: map[string]uint64{},
|
||||
}
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
colon := strings.Index(line, ":")
|
||||
if colon < 0 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(line[:colon])
|
||||
rest := strings.Fields(line[colon+1:])
|
||||
if len(rest) < 16 {
|
||||
continue // malformed row
|
||||
}
|
||||
rxBytes, _ := strconv.ParseUint(rest[0], 10, 64)
|
||||
txBytes, _ := strconv.ParseUint(rest[8], 10, 64)
|
||||
|
||||
if !isPhysicalUplink(name, readIfaceType(name)) {
|
||||
continue
|
||||
}
|
||||
speed := readIfaceSpeed(name)
|
||||
if speed == 0 {
|
||||
continue // no known capacity → can't compute saturation
|
||||
}
|
||||
s.bytes[name] = rxBytes + txBytes
|
||||
s.capMbps += speed
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// SampleNetLoad returns the current network interface saturation as a
|
||||
// percentage of total kept-interface capacity (0..100), using two samples
|
||||
// spaced ~200ms apart for the throughput estimate. Returns 0 when no prior
|
||||
// sample exists (first call), when no usable interfaces were found, or when
|
||||
// the platform isn't Linux.
|
||||
func SampleNetLoad() float64 {
|
||||
netMu.Lock()
|
||||
defer netMu.Unlock()
|
||||
|
||||
cur := snapshotNet()
|
||||
if cur.capMbps == 0 || len(cur.bytes) == 0 {
|
||||
// Nothing to measure (or no physical uplink with known speed).
|
||||
// Don't seed from a degenerate snapshot — the next call retries.
|
||||
return 0
|
||||
}
|
||||
if !netSeeded {
|
||||
netPrev = cur
|
||||
netSeeded = true
|
||||
return 0
|
||||
}
|
||||
|
||||
prev := netPrev
|
||||
netPrev = cur
|
||||
|
||||
dt := cur.when.Sub(prev.when).Seconds()
|
||||
if dt <= 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Weighted aggregation: sum delta bytes across kept interfaces present in
|
||||
// BOTH samples (a hot-plugged NIC mustn't zero the result), convert to
|
||||
// bits/sec, divide by total capacity (Mbps * 1e6 bits/s).
|
||||
//
|
||||
// rx+tx counts BOTH directions — a forward proxies in AND out, so a NIC
|
||||
// that's saturated in one direction is saturated for forwarding purposes.
|
||||
var deltaBits float64
|
||||
sharedCap := uint64(0)
|
||||
for name, c := range cur.bytes {
|
||||
p, ok := prev.bytes[name]
|
||||
if !ok {
|
||||
continue // interface appeared between samples — skip until stable
|
||||
}
|
||||
if c < p {
|
||||
// Counter wrapped (32-bit /proc counter reset or driver reload).
|
||||
// Treat as no new traffic rather than a huge bogus delta.
|
||||
continue
|
||||
}
|
||||
deltaBits += float64(c-p) * 8
|
||||
// approximate this interface's capacity contribution proportional to
|
||||
// its share of the summed speed — read back from /sys would be noisy
|
||||
// per call, so reuse the prior snapshot's capMbps scaling.
|
||||
sharedCap += readIfaceSpeed(name)
|
||||
}
|
||||
if sharedCap == 0 {
|
||||
return 0
|
||||
}
|
||||
capBitsPerSec := float64(sharedCap) * 1e6
|
||||
pct := (deltaBits / dt) / capBitsPerSec * 100
|
||||
if pct < 0 {
|
||||
return 0
|
||||
}
|
||||
if pct > 100 {
|
||||
return 100
|
||||
}
|
||||
return pct
|
||||
}
|
||||
49
internal/cluster/netload_linux_test.go
Normal file
49
internal/cluster/netload_linux_test.go
Normal file
@ -0,0 +1,49 @@
|
||||
//go:build linux
|
||||
|
||||
package cluster
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestSampleNetLoadBasic: the sampler must stay in [0,100], must return 0 on
|
||||
// the first (seeding) call, and must be stable across repeated calls. On a
|
||||
// CI/dev box without a speed-known physical NIC it may legitimately return 0
|
||||
// every time — that's fine; we only assert the invariants, not a specific
|
||||
// value.
|
||||
func TestSampleNetLoadBasic(t *testing.T) {
|
||||
first := SampleNetLoad()
|
||||
if first < 0 || first > 100 {
|
||||
t.Fatalf("first sample out of range: %v", first)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
v := SampleNetLoad()
|
||||
if v < 0 || v > 100 {
|
||||
t.Fatalf("sample %d out of range: %v", i, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsPhysicalUplink: virtual/container interfaces must never count as
|
||||
// uplinks — a busy docker bridge would otherwise fake host saturation.
|
||||
func TestIsPhysicalUplink(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
typ int
|
||||
want bool
|
||||
}{
|
||||
{"lo", 772, false},
|
||||
{"ens18", 1, true}, // ARPHRD_ETHER physical
|
||||
{"eth0", 1, true}, // ARPHRD_ETHER physical
|
||||
{"docker0", 1, false},
|
||||
{"br-abc123", 1, false},
|
||||
{"veth9f2c1a", 1, false},
|
||||
{"wg0", 1, false},
|
||||
{"tun0", 65534, false}, // ARPHRD_NONE software iface
|
||||
{"tap7", 1, false},
|
||||
{"bond0", 1, true}, // real aggregated uplink counts
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := isPhysicalUplink(c.name, c.typ); got != c.want {
|
||||
t.Errorf("isPhysicalUplink(%q,%d)=%v want %v", c.name, c.typ, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
12
internal/cluster/netload_other.go
Normal file
12
internal/cluster/netload_other.go
Normal file
@ -0,0 +1,12 @@
|
||||
//go:build !linux
|
||||
|
||||
// Package cluster — non-Linux fallback: no /proc/net/dev, so network load is
|
||||
// reported as 0 (the ring's primary forward-count signal carries the load
|
||||
// decision; mem/net only break ties, and an unknown net load leaves the tie
|
||||
// break to memory).
|
||||
package cluster
|
||||
|
||||
// SampleNetLoad returns 0 on platforms without /proc/net/dev sampling.
|
||||
func SampleNetLoad() float64 {
|
||||
return 0
|
||||
}
|
||||
@ -55,11 +55,9 @@ func SampleMemLoad() float64 {
|
||||
return float64(m.Alloc) / float64(total) * 100
|
||||
}
|
||||
|
||||
// SampleNetLoad is an approximation: bytes in/out relative to a soft budget.
|
||||
// Kept simple; production could use /proc/net/dev deltas.
|
||||
func SampleNetLoad() float64 {
|
||||
return 0
|
||||
}
|
||||
// SampleNetLoad is implemented in netload_linux.go for Linux and returns a
|
||||
// safe zero fallback on other platforms. It is kept here as a package-level
|
||||
// API because the ring handler injects it into RuntimeLoad.
|
||||
|
||||
// HostID returns a stable node id (hostname; empty fallback to pid).
|
||||
func HostID() string {
|
||||
|
||||
Reference in New Issue
Block a user