jianf/ModelRouter
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/ModelRouter.git synced 2026-09-20 17:07:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(lua): elastic adapter worker pools instead of monotonic growth

Browse Source 
ConfigureConcurrency() set each adapter pool's target to the sum of
max_concurrent over its sources (108 on this deployment) and `created` only
ever went UP: once a Lua state was booted it was parked forever, so a
long-running gateway's resident state count was a high-water mark of all
traffic it had ever seen, never of what it currently needs.

Pools are now sized from three live inputs:

  * the adapter's MAX CONCURRENCY (sum of max_concurrent) is a ceiling, not a
    preallocation, and it sets the growth step:
    growStep = clamp(ceil(maxW/8), 1, 8). A 64-wide adapter warms 8 states at
    once on a spike; an 8-wide one creeps up one at a time.
  * the LIVE CONNECTION COUNT (inUse, i.e. checked-out states) sets the shrink
    step: shrinkStep = clamp(ceil(excess/(1+inUse)), 1, excess). With no
    connections the slack collapses in a single round; a busy adapter gives up
    one state per round so the hot path keeps its warm states.
  * how many states already exist (created / len(idle)) decides how much room
    is left to grow and how much can be reclaimed.

Batch prewarm only fires on genuine contention (a miss while every existing
state is checked out), so a single sequential caller keeps reusing one state
rather than burning a whole grow step on a cold start. A single VM-level
janitor goroutine (not one per adapter) reclaims idle states every 30s, and
shrinkGraceRounds=2 plus idleHeadroom=1 keep a gap between requests from being
mistaken for the end of a load period; any checkout resets the grace counter.

`created` now decrements on reclaim and on shutdown, and release() closes a
state outright when the ceiling was lowered underneath it, so shrinking
max_concurrent in the config gives memory back immediately instead of parking
orphans until restart.

PoolStats() exposes created/idle/in_use/waiting/max/resident/grow_step/
shrink_step/peak_in_use for the status API.

Measured on the test instance (single mock source, max_concurrent=64):
idle created=1; 50 concurrent requests -> created=10 (peak_in_use=5, ceiling
respected); after 95s of silence -> created=1. On production after deploy: 13
adapters, 1 resident Lua state total with ceilings up to 76.
This commit is contained in:
JianFeeeee
2026-08-30 08:05:13 +08:00
parent 21cd0429a2
commit df9aeed5fb
2 changed files with 909 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

440
internal/lua/vm.go
View File

@ -4,9 +4,15 @@
//
// The runtime uses LuaJIT (github.com/aarzilli/golua) so every adapter runs on
// an independent interpreter state. Because a lua.State is NOT goroutine-safe,
// each adapter owns a pool of workers (states); the pool size equals the sum
// of max_concurrent over all sources that use the adapter, and a worker is
// checked out for the duration of a single hook call.
// each adapter owns a pool of workers (states) and a worker is checked out for
// the duration of a single hook call.
//
// Pools are ELASTIC: the sum of max_concurrent over the sources using an adapter
// is a ceiling, not a preallocation. States are booted on demand in batches
// sized by that ceiling, and reclaimed by a single background janitor at a rate
// scaled inversely to the adapter's live connection count. An idle gateway
// therefore holds close to zero Lua states regardless of how much aggregate
// concurrency is configured. See adapterPool for the sizing rules.
package lua
import (
@ -23,6 +29,7 @@ import (
"strconv"
"strings"
"sync"
"time"
// golua exposes the LuaJIT C API. Build with -tags luajit so the cgo
// LDFLAGS resolve to -lluajit-5.1 (see golua's lua.go).
@ -64,7 +71,20 @@ type staticInfo struct {
}
// adapterPool owns the worker states for one adapter. Idle workers are held in
// a slice guarded by a cond; up to target workers are created lazily.
// a slice guarded by a cond; workers are created lazily on demand and reclaimed
// by the VM janitor once demand drops.
//
// Sizing is elastic rather than fixed (plan 阶段 2). Three inputs drive it:
//
// - maxW — the adapter's maximum concurrency (Σ max_concurrent of every source
// using it). It is the hard ceiling AND sets the growth step, so a hot,
// high-concurrency adapter ramps in batches while a low-concurrency one
// creeps up one state at a time.
// - created / len(idle) — how many states already exist, i.e. how much room is
// left to grow and how much is available to reclaim.
// - inUse — how many states are checked out right now, i.e. the live
// connection count for this adapter. It sets the shrink step: the fewer
// connections, the more aggressively idle states are released.
type adapterPool struct {
name string
script string
@ -73,24 +93,160 @@ type adapterPool struct {
mu sync.Mutex
cond *sync.Cond
idle []*worker
created int
target int
created int // live states: len(idle) + inUse (+ states being booted)
inUse int // states checked out right now == live connections
waiting int // goroutines blocked waiting for a free state
maxW int // hard ceiling = Σ max_concurrent of the sources using this adapter
used bool
closed bool
lastGrow time.Time
idleRounds int // consecutive janitor rounds that saw reclaimable slack
peakInUse int // high-water mark of inUse, for observability
}
const (
// growDivisor turns the adapter's max concurrency into a growth step:
// step = ceil(maxW / growDivisor), clamped to [1, growStepCap]. A source set
// at max_concurrent=8 grows one state at a time; a 64-wide adapter warms 8
// at once instead of paying 8 sequential boots on a traffic spike.
growDivisor = 8
growStepCap = 8
// growCooldown keeps a burst of misses from batching repeatedly while the
// previous batch is still booting.
growCooldown = time.Second
// residentWorkers is how many states an adapter keeps warm once it has
// served at least one request. Booting is milliseconds, but keeping one warm
// removes that from the critical path of the next request. Adapters that
// were never used keep nothing.
residentWorkers = 1
// idleHeadroom is the spare idle state kept above the live connection count,
// so the next concurrent request does not have to boot.
idleHeadroom = 1
// shrinkInterval is how often the VM janitor reclaims idle states.
shrinkInterval = 30 * time.Second
// shrinkGraceRounds is how many consecutive janitor rounds must see slack
// before anything is released, so a gap between requests is not mistaken
// for the end of a load period.
shrinkGraceRounds = 2
)
// PoolStats is a snapshot of one adapter pool's elastic sizing, surfaced by the
// status API so the algorithm is observable instead of a black box.
type PoolStats struct {
Name string `json:"name"`
Created int `json:"created"`
Idle int `json:"idle"`
InUse int `json:"in_use"`
Waiting int `json:"waiting"`
Max int `json:"max"`
Resident int `json:"resident"`
GrowStep int `json:"grow_step"`
ShrinkStep int `json:"shrink_step"`
PeakInUse int `json:"peak_in_use"`
}
func newAdapterPool(name, script string, static staticInfo) *adapterPool {
p := &adapterPool{name: name, script: script, static: static, target: 1}
p := &adapterPool{name: name, script: script, static: static, maxW: 1}
p.cond = sync.NewCond(&p.mu)
return p
}
func (p *adapterPool) setTarget(n int) {
// setMax updates the pool's hard ceiling (Σ max_concurrent of its sources).
// Lowering it does not kill live states: the janitor reclaims the excess.
func (p *adapterPool) setMax(n int) {
if n < 1 {
n = 1
}
p.mu.Lock()
p.target = n
p.maxW = n
p.cond.Broadcast()
p.mu.Unlock()
}
func ceilDiv(a, b int) int {
if b <= 0 {
return a
}
return (a + b - 1) / b
}
// growStepLocked is how many states to warm on a miss, derived from the
// adapter's maximum concurrency. Caller holds p.mu.
func (p *adapterPool) growStepLocked() int {
step := ceilDiv(p.maxW, growDivisor)
if step < 1 {
return 1
}
if step > growStepCap {
return growStepCap
}
return step
}
// residentLocked is the floor the janitor will not shrink below. Caller holds p.mu.
func (p *adapterPool) residentLocked() int {
if !p.used {
return 0
}
if residentWorkers > p.maxW {
return p.maxW
}
return residentWorkers
}
// shrinkStepLocked is how many idle states to release this round. The step is
// inversely proportional to the live connection count: with no connections the
// slack collapses in one round, while a busy adapter gives up one state at a
// time so the hot path keeps its warm states. Caller holds p.mu.
func (p *adapterPool) shrinkStepLocked() int {
excess := p.created - p.keepLocked()
if excess <= 0 {
return 0
}
step := ceilDiv(excess, 1+p.inUse)
if step > excess {
step = excess
}
if step > len(p.idle) {
step = len(p.idle)
}
if step < 0 {
return 0
}
return step
}
// keepLocked is the number of live states this pool should retain right now.
// Caller holds p.mu.
func (p *adapterPool) keepLocked() int {
keep := p.residentLocked()
if h := p.inUse + idleHeadroom; h > keep {
keep = h
}
if keep > p.maxW {
keep = p.maxW
}
return keep
}
func (p *adapterPool) stats() PoolStats {
p.mu.Lock()
defer p.mu.Unlock()
return PoolStats{
Name: p.name,
Created: p.created,
Idle: len(p.idle),
InUse: p.inUse,
Waiting: p.waiting,
Max: p.maxW,
Resident: p.residentLocked(),
GrowStep: p.growStepLocked(),
ShrinkStep: p.shrinkStepLocked(),
PeakInUse: p.peakInUse,
}
}
// boot creates a fresh LuaJIT state, registers the shared globals and executes
// the adapter script, storing the returned adapter table in the state.
func (p *adapterPool) boot() (*worker, error) {
@ -110,6 +266,81 @@ func (p *adapterPool) boot() (*worker, error) {
return &worker{L: L}, nil
}
// growPlanLocked reserves capacity for a batch prewarm and returns how many
// EXTRA states to boot in the background (the caller already reserved one for
// itself). contended says the miss happened while every existing state was
// already checked out, which is the real ramp signal: sequential traffic keeps
// reusing one warm state and must never trigger a batch, while genuinely
// concurrent traffic warms a whole step at once instead of paying one boot per
// request all the way up the ramp. Caller holds p.mu.
func (p *adapterPool) growPlanLocked(contended bool) int {
if !contended {
return 0
}
if time.Since(p.lastGrow) < growCooldown {
return 0
}
extra := p.growStepLocked() - 1
if extra <= 0 {
return 0
}
if room := p.maxW - p.created; extra > room {
extra = room
}
if extra <= 0 {
return 0
}
// Reserve immediately so concurrent acquires respect the ceiling while the
// batch is still booting.
p.created += extra
p.lastGrow = time.Now()
return extra
}
// prewarm boots n reserved states and publishes them as idle. A boot failure
// releases its reservation; it is not fatal, since the synchronous path
// surfaces adapter errors already.
func (p *adapterPool) prewarm(n int) {
for i := 0; i < n; i++ {
p.mu.Lock()
closed := p.closed
p.mu.Unlock()
if closed {
p.releaseReservation(n - i)
return
}
w, err := p.boot()
if err != nil {
p.releaseReservation(n - i)
return
}
p.mu.Lock()
if p.closed {
p.created--
p.mu.Unlock()
w.close()
p.releaseReservation(n - i - 1)
return
}
p.idle = append(p.idle, w)
p.cond.Signal()
p.mu.Unlock()
}
}
func (p *adapterPool) releaseReservation(n int) {
if n <= 0 {
return
}
p.mu.Lock()
p.created -= n
if p.created < 0 {
p.created = 0
}
p.cond.Broadcast()
p.mu.Unlock()
}
func (p *adapterPool) acquire() (*worker, error) {
p.mu.Lock()
for {
@ -120,32 +351,67 @@ func (p *adapterPool) acquire() (*worker, error) {
if n := len(p.idle); n > 0 {
w := p.idle[n-1]
p.idle = p.idle[:n-1]
p.checkoutLocked()
p.mu.Unlock()
return w, nil
}
if p.created < p.target {
if p.created < p.maxW {
// Every existing state busy on a miss = real concurrency, not a
// sequential caller reusing one warm state.
contended := p.created > 0 && p.inUse >= p.created
p.created++
break
extra := p.growPlanLocked(contended)
p.mu.Unlock()
if extra > 0 {
go p.prewarm(extra)
}
w, err := p.boot()
if err != nil {
p.mu.Lock()
p.created--
p.cond.Signal()
p.mu.Unlock()
return nil, err
}
p.mu.Lock()
p.checkoutLocked()
p.mu.Unlock()
return w, nil
}
p.waiting++
p.cond.Wait()
p.waiting--
}
p.mu.Unlock()
}
w, err := p.boot()
if err != nil {
p.mu.Lock()
p.created--
p.cond.Signal()
p.mu.Unlock()
return nil, err
// checkoutLocked accounts one state as handed out. Caller holds p.mu.
func (p *adapterPool) checkoutLocked() {
p.inUse++
p.used = true
if p.inUse > p.peakInUse {
p.peakInUse = p.inUse
}
return w, nil
// Live demand invalidates any pending shrink decision.
p.idleRounds = 0
}
func (p *adapterPool) release(w *worker) {
w.L.SetTop(0)
p.mu.Lock()
if p.inUse > 0 {
p.inUse--
}
if p.closed {
p.created--
p.mu.Unlock()
w.close()
return
}
// A lowered ceiling is honoured on the spot rather than waiting for the
// janitor: returning a state the pool may no longer keep closes it.
if p.created > p.maxW {
p.created--
p.cond.Signal()
p.mu.Unlock()
w.close()
return
@ -155,11 +421,42 @@ func (p *adapterPool) release(w *worker) {
p.mu.Unlock()
}
// reclaim releases idle states down to the current keep target, using a step
// scaled by the live connection count. It returns how many states were closed.
func (p *adapterPool) reclaim() int {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return 0
}
step := p.shrinkStepLocked()
if step <= 0 {
p.idleRounds = 0
p.mu.Unlock()
return 0
}
p.idleRounds++
if p.idleRounds < shrinkGraceRounds {
p.mu.Unlock()
return 0
}
victims := append([]*worker(nil), p.idle[len(p.idle)-step:]...)
p.idle = p.idle[:len(p.idle)-step]
p.created -= step
p.idleRounds = 0
p.mu.Unlock()
for _, w := range victims {
w.close()
}
return step
}
func (p *adapterPool) shutdown() {
p.mu.Lock()
p.closed = true
idle := p.idle
p.idle = nil
p.created -= len(idle)
p.cond.Broadcast()
p.mu.Unlock()
for _, w := range idle {
@ -173,13 +470,84 @@ type VM struct {
mu sync.RWMutex
dir string
pools map[string]*adapterPool
// janitor drives elastic shrinking. One goroutine serves every pool (not
// one per adapter), so an idle gateway costs a single sleeping goroutine.
janitorOnce sync.Once
janitorStop chan struct{}
janitorDone chan struct{}
}
func NewVM(dir string) *VM {
return &VM{dir: dir, pools: map[string]*adapterPool{}}
return &VM{
dir: dir,
pools: map[string]*adapterPool{},
janitorStop: make(chan struct{}),
janitorDone: make(chan struct{}),
}
}
// startJanitor launches the single shrink loop. Idempotent.
func (v *VM) startJanitor() {
v.janitorOnce.Do(func() {
go v.janitorLoop(shrinkInterval)
})
}
func (v *VM) janitorLoop(every time.Duration) {
defer close(v.janitorDone)
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-v.janitorStop:
return
case <-t.C:
v.reclaimIdle()
}
}
}
// reclaimIdle runs one shrink round over every pool. Exported behaviour is
// tested through ReclaimIdleNow.
func (v *VM) reclaimIdle() int {
v.mu.RLock()
pools := make([]*adapterPool, 0, len(v.pools))
for _, p := range v.pools {
pools = append(pools, p)
}
v.mu.RUnlock()
n := 0
for _, p := range pools {
n += p.reclaim()
}
return n
}
// ReclaimIdleNow forces one shrink round and reports how many worker states
// were closed. Intended for tests and for an explicit "release idle memory"
// action; normal operation relies on the janitor.
func (v *VM) ReclaimIdleNow() int { return v.reclaimIdle() }
// PoolStats returns the elastic sizing snapshot of every loaded adapter,
// sorted by name.
func (v *VM) PoolStats() []PoolStats {
v.mu.RLock()
pools := make([]*adapterPool, 0, len(v.pools))
for _, p := range v.pools {
pools = append(pools, p)
}
v.mu.RUnlock()
out := make([]PoolStats, 0, len(pools))
for _, p := range pools {
out = append(out, p.stats())
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out
}
func (v *VM) Start() error {
v.startJanitor()
if v.dir == "" {
return nil
}
@ -214,6 +582,12 @@ func (v *VM) Start() error {
}
func (v *VM) Stop() {
select {
case <-v.janitorStop:
// already stopped
default:
close(v.janitorStop)
}
v.mu.Lock()
pools := v.pools
v.pools = map[string]*adapterPool{}
@ -223,17 +597,17 @@ func (v *VM) Stop() {
}
}
// ConfigureConcurrency sets each adapter's worker target to the sum of
// max_concurrent of every source using it. Values below one are clamped to 1.
// ConfigureConcurrency sets each adapter's worker CEILING to the sum of
// max_concurrent of every source using it. It is a ceiling, not a preallocation:
// states are booted on demand and reclaimed when demand drops, so a config with
// a large aggregate concurrency no longer implies a large resident pool.
// Values below one are clamped to 1.
func (v *VM) ConfigureConcurrency(adapterConcurrency map[string]int) {
v.mu.RLock()
defer v.mu.RUnlock()
for name, n := range adapterConcurrency {
if p, ok := v.pools[name]; ok {
if n < 1 {
n = 1
}
p.setTarget(n)
p.setMax(n)
}
}
}
@ -260,16 +634,19 @@ func (v *VM) LoadAdapter(path string) error {
}
// LoadAdapterSource registers an adapter from source code. Replacing an
// existing adapter shuts the old pool down and re-boots lazily.
// existing adapter shuts the old pool down and re-boots lazily, carrying the
// configured ceiling over so a hot-reload does not reset elastic sizing.
func (v *VM) LoadAdapterSource(name, code string) error {
static, err := inspectScript(name, code)
if err != nil {
return err
}
target := 1
maxW := 1
v.mu.RLock()
if p, ok := v.pools[static.name]; ok {
target = p.target
p.mu.Lock()
maxW = p.maxW
p.mu.Unlock()
}
v.mu.RUnlock()
@ -278,9 +655,10 @@ func (v *VM) LoadAdapterSource(name, code string) error {
p.shutdown()
}
p := newAdapterPool(static.name, code, *static)
p.setTarget(target)
p.setMax(maxW)
v.pools[static.name] = p
v.mu.Unlock()
v.startJanitor()
return nil
}