-- Provider quota — plan balance across upstreams, one card per source. -- -- This started as a Command Code plugin and grew, because only one vendor was -- being asked. The shape that works across providers is a two-part contract: -- -- 1. A small set of PROBE ADAPTERS, each naming an endpoint, how to call it -- and which JSON paths hold the numbers. Adapters are tried in order and -- the first one that returns a usable answer wins. -- 2. A DECLARATIVE override, so a provider nobody anticipated can be added -- from the plugin's own state (endpoints + path mapping) without touching -- code. -- -- What the adapters share, and why it is worth generalizing: -- -- - The DeepSeek family (deepseek, and the many re-sells that copied its -- dashboard) answers GET /user/balance with -- {is_available, balance_infos:[{currency,total_balance,granted,topped_up}]}. -- Granted vs topped-up matters: gifted credit expiring on a schedule is a -- different operational fact from prepaid money. -- - The OpenAI-compatible dashboard shape answers -- /v1/dashboard/billing/subscription plus /v1/dashboard/billing/usage, -- which is what most re-sells of an OpenAI console expose. Note these -- report LIMITS and SPEND, not a balance: the provider here has a hard limit -- of 1e8 and 660 spent, so "remaining" must not be computed as -- limit - spend and presented as money left — it is a ceiling nobody is -- near. Reporting that honestly matters more than a green bar. -- - Command Code exposes rolling 5-hour / weekly / monthly windows plus a -- billing period, which no other provider here does. -- -- A provider with no working adapter is SKIPPED, silently. Half these upstreams -- are small re-sells whose dashboard endpoints return an HTML page, not JSON; a -- list full of "unknown" rows trains the operator to ignore the page entirely, -- which is worse than showing only what is real. Failures that are NOT "no such -- endpoint" — a 401 on a key we hold, say — are surfaced, because those are the -- ones worth fixing. -- -- Only hosts of CONFIGURED SOURCES are reachable (__llmsproxy_http enforces -- that), so adding a probe never widens what the gateway can talk to. local PLUGIN = {} PLUGIN.name = "quota" PLUGIN.version = "1.0.0" PLUGIN.description = "多源套餐余量(余额 / 用量 / 滚动窗口)" PLUGIN.exports = { "collect_source", "collect_all", "due_sources", "memo_record" } -- plugin.state must exist before anything indexes it. The kernel restores -- persisted state OVER this table, so declaring it here is safe: a restart with -- no saved state still has an empty table to write into, rather than nil. -- -- It is initialised at the TOP on purpose. Every collect path below touches it -- (to restore the probe memo on first use, to persist it afterwards), and a -- declaration further down would be nil for the reads that run first. PLUGIN.state = {} -- ---------------------------------------------------------------- helpers local function tonum(v) return tonumber(v) or 0 end -- hostOf extracts the host from a base_url, for showing WHICH upstream a row is -- about. Two sources on the same vendor (two Command Code keys) are -- indistinguishable without it. -- hostOf returns the host of a URL — the part memoisation is keyed on, so two -- sources pointing at the same host with different paths share one answer. -- -- It used to be written with string.gmatch and a capture group. gmatch yields -- WHOLE matches, not captures, and with `^` it never advances, so the iterator -- returned nil on the first step and hostOf returned its `or u` fallback — -- the FULL URL. -- -- The consequence was invisible and total. Every lookup was HOST_MEMO[nil] and -- every write was HOST_MEMO[nil], so: -- - the memo table never returned anything, for any host, ever -- - the negative memo, the TTL and (later) the backoff were all dead code -- - the 19 entries in this gateway's production state are unreachable: their -- keys are full URLs, while a lookup asks for nil -- Measured before the fix: hostOf(key) matched 0 of 19 production entries. local function hostOf(u) return (string.match(u, "^https?://([^/]+)")) or u end -- get walks a dotted path through nested tables, returning nil if any hop is -- missing. A missing field must stay nil rather than becoming 0: "this provider -- did not report it" and "this provider reported zero" are different, and -- collapsing them shows a healthy $0.00 for a provider that never answered. local function get(root, path) if root == nil then return nil end local cur = root for seg in string.gmatch(path, "[^%.]+") do if type(cur) ~= "table" then return nil end cur = cur[seg] if cur == nil then return nil end end return cur end local function getnum(root, path) local v = get(root, path) if v == nil then return nil end if type(v) == "boolean" then return v and 1 or 0 end local n = tonumber(v) return n end -- strang reads a string field, returning nil rather than "". An empty string -- and an absent field must stay distinguishable: the UI prints the first as -- blank and the second as "未探测", and collapsing them loses that. local function strang(root, path) local v = get(root, path) if v == nil then return nil end if type(v) ~= "string" then return nil end if v == "" then return nil end return v end -- request performs one GET with the standard retry. Retrying is not -- politeness here: several of these hosts sit behind DNS that intermittently -- refuses, and a single 000 turned into "provider unknown" on every poll. local function request(path, key, headers, tries) tries = tries or 2 local hdrs = {} for k, v in pairs(headers or {}) do hdrs[k] = v end hdrs["Authorization"] = "Bearer " .. tostring(key or "") local lastErr = "no attempt" for _ = 1, tries do local res, err = __llmsproxy_http({ -- 4s, not 10s: probing is network-bound and the whole page shares ONE -- Lua state under a write lock, so a slow upstream is not just a slow card -- — it stalls every request_end hook (i.e. billing) behind it. url = path, method = "GET", headers = hdrs, timeout_ms = 4000, }) if res ~= nil then if res.status >= 200 and res.status < 300 then local ok, decoded = pcall(json.decode, res.body or "") if ok and type(decoded) == "table" then return decoded, nil end -- A 200 that is not JSON means we hit a marketing page, not an API. return nil, "not json (HTTP " .. tostring(res.status) .. ")" end if res.status == 401 or res.status == 403 then -- Retrying a rejected credential only wastes the operator's time. return nil, "HTTP " .. tostring(res.status) .. " (key rejected)" end lastErr = "HTTP " .. tostring(res.status) else lastErr = tostring(err or "request failed") end end return nil, lastErr end -- Monthly credit caps per planId. The credits endpoint never reports a cap, so -- this is a lookup against the vendor's published pricing ($10 Go, $70 -- GOAT/Pro/Max), cross-checked against monthly_used + balance. Declared before -- the adapters that read it — a Lua local is nil until its assignment runs, and -- the Command Code probe hit that as a runtime nil-index. local PLAN_CAPS = { ["individual-go"] = 10, ["individual-goat"] = 70, ["individual-pro"] = 70, ["individual-max"] = 70, ["team-goat"] = 70, } -- ---------------------------------------------------------------- adapters -- Each adapter returns (report, nil) when it produced something usable, or -- (nil, reason) to let the next one try. `skip` is the quiet reason — a 404 or -- an HTML page means "this provider does not expose balance here", which is not -- worth showing. Everything else becomes a visible failure. local function deepseekBalance(base, key) local r, err = request(base .. "/user/balance", key) if r == nil then if err and (err:match("not json") or err:match("HTTP 404")) then return nil, "skip", err end return nil, "error", err end local infos = get(r, "balance_infos") if type(infos) ~= "table" or #infos == 0 then return nil, "skip", "no balance_infos" end local first = infos[1] local rep = { vendor = "deepseek-style", currency = tostring(first.currency or "USD"), balance = tonum(first.total_balance), granted = getnum(first, "granted_balance"), topped_up = getnum(first, "topped_up_balance"), available = get(r, "is_available") == true, } return rep, nil end -- openaiDashboard reports LIMITS and SPEND. There is deliberately no -- `remaining` here: hard_limit_usd is a ceiling (1e8 on the live provider), and -- ceiling - spend would read as "$99,999,339 left", which is a number nobody -- should ever believe. local function openaiDashboard(base, key) local sub, subErr = request(base .. "/v1/dashboard/billing/subscription", key, nil, 1) if sub == nil then if subErr and (subErr:match("not json") or subErr:match("HTTP 404")) then return nil, "skip", subErr end end local use, useErr = request(base .. "/v1/dashboard/billing/usage", key, nil, 1) if sub == nil and use == nil then if subErr and subErr:match("key rejected") then return nil, "error", subErr end return nil, "skip", (subErr or useErr or "no dashboard") end local rep = { vendor = "openai-dashboard", hard_limit = getnum(sub or {}, "hard_limit_usd"), soft_limit = getnum(sub or {}, "soft_limit_usd"), total_usage = getnum(use or {}, "total_usage"), has_payment_method = get(sub or {}, "has_payment_method") == true, } -- Only call it a balance when the vendor says so. A hard limit with no usage -- endpoint is a ceiling with nothing spent against it; that still renders, but -- as a limit rather than as money left. rep.balance = nil if rep.hard_limit and rep.hard_limit < 1000000 and rep.total_usage then rep.balance = rep.hard_limit - rep.total_usage rep.cap = rep.hard_limit rep.used = rep.total_usage end return rep, nil end -- openDesign probes OpenDesign / Vela (amr-link.open-design.ai). -- -- This vendor needs a credential the source does not carry. Inference runs on -- `runtimeKey`, but the billing endpoints on amr-api.open-design.ai only accept -- `controlKey` — a second, separate secret handed out when the desktop app signs -- in and stored in ~/.amr/config.json under profiles.prod. -- -- Verified against the live service (200 on both, ~1.3s each): -- GET https://amr-api.open-design.ai/api/v1/wallet/balance (controlKey) -- {balanceUsd, expiresAt, workspaceId, ...} -- GET https://amr-api.open-design.ai/api/v1/billing/summary (controlKey) -- {creditsPerUsd, membershipTier, subscriptionStatus, -- subscriptionCreditsUsd, rechargeCreditsUsd, -- todayConsumedUsd, totalConsumedUsd, subscriptionCurrentPeriodEnd, ...} -- -- The two keys are NOT interchangeable: sending runtimeKey to either endpoint -- answers 401 {"error":"unauthenticated"}. That is why this adapter reads -- PLUGIN.prices instead of reusing the source's api_key. -- -- config location: `prices.control_key` in the plugin's persisted config (the -- same file billing uses for its price table — the plugin system persists one -- `prices` table per plugin, separate from `state`). Set it through the admin -- API: -- PUT /api/plugins/quota/state {"prices": {"control_key": "vela_ctrl_..."}} -- `prices.api_base` is optional and overrides the amr-link -> amr-api host -- derivation. -- -- Three things this adapter deliberately does NOT do: -- -- * It does not invent an exchange rate. creditsPerUsd is the only thing that -- turns credits into dollars, and a missing or non-positive value is an -- error, not a default. The vendor serves pre-computed *Usd fields too, so -- the rate is only needed for the credit breakdown. -- * It does not report a period window when the account has no Coding Plan. -- `vela billing workspace-snapshot` returns billingState:"free" and NO -- codingPlan.windows for such workspaces, and rendering "100% remaining" -- from an absent field is the exact lie this plugin was written to avoid. -- * It does not treat a zero balance as an exhausted plan. balanceUsd and the -- period windows are different things; conflating them was a previous bug. local function openDesign(base, key) -- The source's own api_key is the RUNTIME key, which these endpoints reject. -- The control key comes from the plugin's persisted config. local cfg = PLUGIN.prices local ck = nil if type(cfg) == "table" then ck = cfg.control_key end if type(ck) ~= "string" or ck == "" then return nil, "skip", "control_key not configured (set prices.control_key)" end -- amr-link serves inference; amr-api serves billing. Rewriting the host here -- rather than asking the operator to configure a second source keeps one -- logical provider in the UI. -- -- `api_base` overrides the derivation: the vendor has already moved these -- hosts once, and an operator who can see the new one in a browser should not -- be blocked on a code change to use it. local api = nil if type(cfg) == "table" then api = cfg.api_base end if type(api) ~= "string" or api == "" then -- Host only: base_url carries a path (`.../v1`), and matching the hostname -- rule against it would silently skip the rewrite and send the probe to the -- inference host — which serves no billing route. local origin = base:gsub("^https?://", ""):gsub("/.*$", "") origin = origin:gsub("^amr%-link%.open%-design%.ai$", "amr-api.open-design.ai") api = "https://" .. origin .. "/api/v1" end api = api:gsub("/+$", "") -- The gateway's outbound allowlist is the hosts of configured sources. It -- therefore contains amr-link (the inference host) but NOT amr-api (the -- billing host), and every request below would be refused with "host ... is -- not an allowed outbound destination" — a message that reads like a plugin -- bug rather than a missing allowlist entry. Add the derived host alongside -- the source hosts in core.sourceHosts when this adapter is enabled. local wallet, wErr = request(api .. "/wallet/balance", ck, nil, 1) local summary, sErr = request(api .. "/billing/summary", ck, nil, 1) if wallet == nil and summary == nil then if (wErr or ""):match("key rejected") or (sErr or ""):match("key rejected") then return nil, "error", "control_key rejected (HTTP 401)" end -- A 404 here means the host is not amr-link/amr-api at all; skip rather -- than error so the memo does not blacklist an unrelated provider. if (wErr or ""):match("HTTP 404") and (sErr or ""):match("HTTP 404") then return nil, "skip", "no OpenDesign billing endpoints" end return nil, "skip", (wErr or sErr or "no billing response") end local w = wallet or {} local b = summary or {} local rep = { vendor = "opendesign", -- tonum would turn a missing balance into 0, i.e. "you are broke", which is -- the failure mode this whole plugin exists to avoid. A balance that is -- absent stays absent and the guard below turns it into a skip. balance = getnum(w, "balanceUsd") or getnum(b, "balanceUsd"), balance_usd = getnum(w, "balanceUsd") or getnum(b, "balanceUsd"), tier = strang(b, "membershipTier"), status = strang(b, "subscriptionStatus"), interval = strang(b, "billingInterval"), } -- Credits breakdown. Converted with the vendor's OWN rate rather than -- assuming 10000 — that is this vendor's current value and hardcoding it -- would silently produce wrong numbers the day it changes. local rate = getnum(b, "creditsPerUsd") if rate and rate > 0 then rep.credits_per_usd = rate local sub = getnum(b, "subscriptionCredits") local rec = getnum(b, "rechargeCredits") if sub or rec then rep.credits = (sub or 0) + (rec or 0) rep.subscription_usd = sub and (sub / rate) or nil rep.recharge_usd = rec and (rec / rate) or nil end rep.total_available_credits = getnum(b, "totalAvailableCredits") else -- No usable rate. Flag it instead of guessing; the *Usd fields below still -- give a correct balance, so the card renders with the split missing. rep.credits_rate_missing = true end -- The vendor's own pre-computed dollar amounts win over our conversion. local sUsd = getnum(b, "subscriptionCreditsUsd") local rUsd = getnum(b, "rechargeCreditsUsd") if sUsd then rep.subscription_usd = sUsd end if rUsd then rep.recharge_usd = rUsd end -- Spend, shown next to the balance. rep.today_used = getnum(b, "todayConsumedUsd") rep.period_used = getnum(b, "totalConsumedUsd") rep.total_recharged = getnum(b, "totalRechargedUsd") rep.usage_count = getnum(b, "usageCount") -- Wallet expiry: this balance does not last forever. Surfaced because a -- balance that silently stops being spendable is worse than a warning. rep.balance_expires_at = strang(w, "expiresAt") rep.period_start_at = strang(b, "subscriptionCurrentPeriodStart") rep.period_end_at = strang(b, "subscriptionCurrentPeriodEnd") rep.workspace_id = strang(w, "workspaceId") -- Period windows: deliberately NOT set. codingPlan.windows only exists for a -- workspace on a Coding Plan, and this one reports billingState:"free". -- Leaving rep.windows nil makes the page say "no period data" rather than -- fabricating a 100% that an operator would read as "plenty left". -- -- When a workspace does have a plan the windows arrive as -- {durationSeconds, usedCredits, limitCredits, remainingCredits, resetsAt} -- and need the same credits->USD conversion; that branch belongs here, gated -- on the response actually carrying codingPlan. if rep.balance == nil then return nil, "skip", "no balanceUsd in response" end return rep, nil end -- commandCode keeps its own shape: rolling windows plus a billing period. local function commandCode(base, key) local H = { ["x-command-code-version"] = "1.54.2", ["x-cli-environment"] = "production", ["User-Agent"] = "quota-plugin/1.0.0", } local origin = base:gsub("/provider/v1/?$", "") local credits, err = request(origin .. "/alpha/billing/credits", key, H) if credits == nil then if err and err:match("key rejected") then return nil, "error", err end return nil, "skip", err end local rep = { vendor = "commandcode", currency = "USD", balance = tonum(get(credits, "credits.monthlyCredits")) + tonum(get(credits, "credits.purchasedCredits")) + tonum(get(credits, "credits.freeCredits")), below_threshold = get(credits, "credits.belowThreshold") == true, limited = get(credits, "windowLimits.limited") == true, exceeded = get(credits, "windowLimits.exceeded"), } local fh = get(credits, "windowLimits.fiveHour") if type(fh) == "table" then rep.windows = rep.windows or {} rep.windows[#rep.windows + 1] = { label = "5 小时", used = tonum(fh.used), cap = tonum(fh.cap), exceeded = fh.exceeded == true, reset_at = tonum(fh.resetAt), } end local wk = get(credits, "windowLimits.weekly") if type(wk) == "table" then rep.windows = rep.windows or {} rep.windows[#rep.windows + 1] = { label = "每周", used = tonum(wk.used), cap = tonum(wk.cap), exceeded = wk.exceeded == true, reset_at = tonum(wk.resetAt), } end -- Subscriptions and usage are enrichment: their absence must not blank the -- balance that just came back. local subs = request(origin .. "/alpha/billing/subscriptions", key, H, 1) if subs ~= nil then rep.plan_id = get(subs, "data.planId") rep.plan_status = get(subs, "data.status") rep.period_end = get(subs, "data.currentPeriodEnd") rep.plan_cap = PLAN_CAPS[tostring(rep.plan_id or "")] rep.cancel_at_period_end = get(subs, "data.cancelAtPeriodEnd") == true end local usage = request(origin .. "/alpha/usage/summary", key, H, 1) if usage ~= nil then rep.monthly_used = getnum(usage, "totalMonthlyCredits") rep.requests = getnum(usage, "totalCount") rep.tokens = getnum(usage, "totalTokens") rep.success_rate = getnum(usage, "successRate") end if rep.plan_cap and rep.monthly_used and rep.monthly_used > 0 then local implied = rep.monthly_used + rep.balance if math.abs(implied - rep.plan_cap) > (rep.plan_cap * 0.02 + 0.5) then rep.cap_mismatch = true rep.implied_cap = implied end end return rep, nil end -- Ordered probes. DeepSeek-family first because it is a single cheap endpoint, -- then the OpenAI console shape, then Command Code's four. local ADAPTERS = { { vendor = "commandcode", probe = commandCode }, { vendor = "deepseek", probe = deepseekBalance }, { vendor = "openai-dash", probe = openaiDashboard }, { vendor = "opendesign", probe = openDesign }, } -- The probe memo remembers, per host, which adapter answered. Persisted in the -- plugin's state so the expensive part — discovering that 10 of 21 upstreams -- expose no balance endpoint at all — is paid once, not on every page open. -- -- This matters more than it looks. The answers depend only on the host, never -- on the key, and re-discovering them cost 21 sources x 3 adapters x 2 attempts -- of timeouts on the first load. -- HOST_MEMO maps host -> adapter vendor (or false when nothing answered). It -- is restored from the plugin's own state on the first collect of a process, so -- a restart does not re-pay the cost of discovering which upstreams are silent. local HOST_MEMO = {} -- HOST_MEMO_AT[host] = when that host last produced data. A negative memo older -- than PROBE_MEMO_TTL is ignored, so a vendor that adds an endpoint is picked up -- within the TTL without a restart. local HOST_MEMO_AT = {} local PROBE_MEMO_TTL = 1800 -- 30 minutes -- Exponential backoff for hosts that answer nothing. -- -- A flat TTL is the wrong shape for a dead upstream. Measured on this gateway: -- two hosts had been unreachable for 10 HOURS, and because PROBE_MEMO_TTL is 30 -- minutes their negative memo was stale on every single page open — so each -- refresh re-paid the full cost of three adapters (measured: 24s per source -- before the probe budget was fixed, 16s after). A host that is down should not -- be re-examined every 30 minutes forever; each consecutive failure doubles the -- wait, so a long-dead host costs almost nothing while a host that was briefly -- down is retried within the TTL as before. -- -- The cap matters as much as the doubling: without it a host that is down for a -- week stops being probed at all, and the page would quietly and permanently -- misreport it as having no balance endpoint. -- -- SUCCESS resets the streak. This is only a probe-scheduling decision — a source -- that stops answering is still shown, still counted, and still retried. local BACKOFF_BASE = 1800 -- first retry after the usual TTL local BACKOFF_FACTOR = 2 -- doubles per consecutive miss local BACKOFF_MAX = 21600 -- 6 hours; beyond this we always re-probe -- backoffSeconds returns how long to wait before re-probing a host that has -- missed `streak` times in a row, and 0 when it is due now. -- -- `at` is when the host last failed. streak is stored inside the memo table -- itself so it survives a restart: a process restart must not reset the backoff -- and re-pay the cost it was avoiding. -- Returns a BOOLEAN and nothing else. -- -- In Lua only nil and false are falsy, so every path here must return one of -- those. Two versions of this got it wrong and both were caught by a test that -- asserts the type: `return 0` for "no streak" (0 is truthy, so a host with no -- streak — which is every memo written before this change — would be skipped -- forever, silently), and `return BACKOFF_MAX` on the cap path (21600 is -- truthy too, so a long-dead host would never be re-probed, which is the exact -- failure the cap exists to prevent). local function backoffSeconds(at, streak) if streak == nil or streak <= 0 then return false end local wait = BACKOFF_BASE for _ = 2, streak do wait = wait * BACKOFF_FACTOR if wait >= BACKOFF_MAX then wait = BACKOFF_MAX break end end return (os.time() - (at or 0)) < wait end -- PROBE_BUDGET_MS caps one collect pass. Without it a page open over a set of -- dead upstreams runs to the sum of every timeout — measured at 149s — and the -- plugin's write lock is held the whole time. local PROBE_BUDGET_MS = 12000 -- ---------------------------------------------------------------- source scan -- collectSource probes one source and returns a report, or nil when the source -- simply has no balance endpoint (silent) — the caller distinguishes the two -- through `skipped`. local function collectSource(src, budget) local base = tostring(src.base_url or "") if base == "" then return nil end base = base:gsub("/+$", "") local key = tostring(src.api_key or "") local rep, kind, why -- Try the adapter this host answered with last time first. Same host with a -- different key (two Command Code seats) gives the same answers, so this turns -- 21 sources into a handful of distinct round trips after the first pass. local host = hostOf(base) local known = HOST_MEMO[host] -- A negative memo is a TABLE ({at=timestamp}), distinguishable from a vendor -- string. JSON round-trips it fine; a bare false would not survive the trip -- through the Go-side state as cleanly, and the two being different types is -- exactly the confusion that made an earlier revision drop five sources. if type(known) == "table" then local seen = known.at or 0 -- A streak is a host that has failed repeatedly. Skip it outright until its -- backoff expires, instead of re-paying three adapter timeouts every TTL. if backoffSeconds(seen, known.streak) then return { name = src.name, host = host, skipped = true, backing_off = true, reason = "连续探测失败 " .. tostring(known.streak) .. " 次,退避中(等待下次重试)" } end if PROBE_MEMO_TTL > 0 and (os.time() - seen) > PROBE_MEMO_TTL then known = nil else known = false -- still "nothing answered, try cheapest first" end end -- known == false means every adapter already came back "no endpoint here" for -- this host during this process. Skip the probing entirely; one cheap request -- confirms it is still the case, so a vendor that adds a dashboard later is -- picked up without a restart. -- A cheap liveness probe, NOT a skip: the hosts behind these adapters are -- slow (12s+ to answer a root request), so treating a slow answer as "down" -- lost five sources that do have real data. The probe only short-circuits the -- genuinely unreachable case; a reachable host still goes through the -- adapters. if known == false then local probe = __llmsproxy_http({ url = base .. "/", method = "GET", timeout_ms = 2500, headers = { ["Authorization"] = "Bearer " .. tostring(key or "") }, }) if probe == nil then return { name = src.name, host = host, skipped = true, reason = "host unreachable" } end -- Reachable: fall through and try the adapters again, since a slow host -- that was merely timing out may answer fine. end local order = ADAPTERS if known then local reordered, seen = {}, {} for _, a in ipairs(ADAPTERS) do if a.vendor == known then reordered[#reordered + 1] = a seen[a.vendor] = true end end for _, a in ipairs(ADAPTERS) do if not seen[a.vendor] then reordered[#reordered + 1] = a end end order = reordered end for _, a in ipairs(order) do -- Budget check BETWEEN adapters, so an exhausted budget still returns the -- sources already answered instead of nothing at all. if budget and budget.exhausted() then return { name = src.name, host = host, skipped = true, reason = "probe budget exhausted" } end local r, k, e = a.probe(base, key) if r ~= nil then rep = r HOST_MEMO[host] = a.vendor break end kind, why = k, e end -- Remember the NEGATIVE result too. Only recording successes left every -- source that has no balance endpoint re-probing all three adapters (2 -- attempts, 4s each) on every single page open — measured at 60s per source, -- unchanged between runs, because "nothing answered" was never recorded. -- -- The value is the adapter that came closest (an error rather than a skip), so -- a host that later starts working is still retried first. -- Remember that this host answered nothing, but only as an ORDERING hint, and -- with a stamp so it expires. "This upstream has no balance endpoint" is a -- fact that can change — vendors ship dashboards — so a permanent negative -- would make the page quietly stop reporting a source that started working. -- The stamp is what turns a 60s first load into a fast steady state without -- freezing what the page can see. -- Record the negative result, and GROW THE STREAK. -- -- The guard used to be `not HOST_MEMO[host]`, which is true only the FIRST time -- a host fails: on every later failure the memo already held a table, the -- guard was false, and nothing was written. The streak therefore stayed pinned -- at 1 forever and the backoff never grew. Measured while writing this: the -- memo read back `{at:... streak:1}` after four consecutive failures. -- -- A success overwrites the memo with a vendor string, so a host that recovers -- resets its streak automatically. if rep == nil then local prev = HOST_MEMO[host] local streak = 1 if type(prev) == "table" then streak = (prev.streak or 1) + 1 end HOST_MEMO[host] = { at = os.time(), streak = streak } end if rep ~= nil then HOST_MEMO_AT[host] = os.time() end if rep == nil then if kind == "error" then return { name = src.name, vendor = "", error = tostring(why), skipped = false, host = hostOf(base) } end -- The skip REASON travels. Discarding it here is why -- TestQuotaOpenDesignSkipsWithoutControlKey could not tell a configured -- adapter that refused from a source that never reached it: both arrived as -- {skipped=true}, so a mutation that ignored the missing credential was -- invisible at this boundary. "Not configured" is actionable (set the key); -- "no balance endpoint" is not. local out = { name = src.name, skipped = true, host = hostOf(base) } if why and why ~= "" then out.reason = tostring(why) end return out end rep.name = src.name rep.host = hostOf(base) rep.skipped = false rep.checked_at = os.time() return rep end -- collect_all walks every source. Prefer it for a SMALL number of sources. -- -- The page calls collect_source concurrently instead: probing is almost entirely -- network wait (three adapters, up to two attempts, ten second timeouts each), -- and doing 21 sources in sequence overran the HTTP call's own budget — the -- whole page returned 502 while individual probes were still in flight. The -- sequential entry point stays for tests and for anyone scripting the plugin. -- due_sources answers, on the SHARED VM, which sources are worth probing right -- now — and it makes NO network requests. -- -- This exists because of a conflict between two facts: -- -- - Probing is network-bound, and Plugins.Call holds the plugin mutex for the -- whole Lua function, so twenty probes through /call/ serialise. Measured: -- 29s on a 20-source gateway. The fix is ShardCall, which runs each call on -- its own VM — and a shard VM has its own PLUGIN.state. -- - The backoff lives in PLUGIN.state (HOST_MEMO). A shard therefore cannot -- see it, cannot record it, and a host that fails keeps being probed on every -- page open. Measured: fanout left the memo at 0 entries, collect_all reached -- 17. -- -- So the split is: the DECISION runs where the state is (one shared-VM call, -- microseconds), and only the hosts that pass it get sharded. -- -- Cheap on purpose. It reads HOST_MEMO and nothing else; if the state has not -- been restored yet, every host is due, which is the correct answer for a first -- call. -- memo_record writes the outcome of a sharded pass into the SHARED VM's memo. -- -- A shard runs on its own VM and therefore cannot record anything: measured, a -- page of fanouts left the memo at 0 entries, so the backoff never accumulated -- and every refresh re-probed the dead hosts. This is the other half of -- due_sources — the decision is taken here, the outcome is written back here, and -- only the network wait happens on the shards. -- -- It records FAILURES as well as successes. Recording only successes is what -- left every source with no balance endpoint re-probing three adapters forever. PLUGIN.memo_record = function(payload) if type(payload) ~= "table" then return nil, "expected table" end if not next(HOST_MEMO) then local st = PLUGIN.state if type(st) == "table" and type(st.probe_memo) == "table" then for k, v in pairs(st.probe_memo) do HOST_MEMO[k] = v end end end local rows = payload.results if type(rows) ~= "table" then return nil, "missing results" end local now = os.time() for _, r in ipairs(rows) do if type(r) == "table" and type(r.host) == "string" and r.host ~= "" then local host = hostOf(r.host) if r.vendor then -- A source that answered resets its streak: the memo becomes a vendor -- string, which every reader already treats as a positive memo. HOST_MEMO[host] = r.vendor else local prev = HOST_MEMO[host] local streak = 1 if type(prev) == "table" then streak = (prev.streak or 1) + 1 end HOST_MEMO[host] = { at = now, streak = streak } end end end PLUGIN.state.probe_memo = HOST_MEMO return true end PLUGIN.due_sources = function(payload) if type(payload) ~= "table" then return nil, "expected table" end if not next(HOST_MEMO) then local st = PLUGIN.state if type(st) == "table" and type(st.probe_memo) == "table" then for k, v in pairs(st.probe_memo) do HOST_MEMO[k] = v end end end local srcs = payload.sources if type(srcs) ~= "table" then return nil, "missing sources" end local due, backing = {}, {} for _, s in ipairs(srcs) do if type(s) == "table" and type(s.base_url) == "string" and s.base_url ~= "" then local base = (s.base_url:gsub("/+$", "")) local known = HOST_MEMO[hostOf(base)] if type(known) == "table" and backoffSeconds(known.at or 0, known.streak) then -- skipped matters: the page filters on it. Without the flag a backed-off -- host reaches the card renderer, which has no field to match and falls -- through to its "has data" default — so thirteen dead upstreams were -- displayed as if they had answered. backing[#backing + 1] = { name = s.name, host = hostOf(base), skipped = true, backing_off = true, reason = "连续探测失败 " .. tostring(known.streak) .. " 次,退避中(等待下次重试)" } else due[#due + 1] = s end end end return { due = due, backing_off = backing } end PLUGIN.collect_all = function(payload) if type(payload) ~= "table" then return nil, "expected table" end if not next(HOST_MEMO) then local st = PLUGIN.state if type(st) == "table" and type(st.probe_memo) == "table" then for k, v in pairs(st.probe_memo) do HOST_MEMO[k] = v end end end local srcs = payload.sources if type(srcs) ~= "table" then return nil, "missing sources" end -- os.time, NOT os.clock. os.clock measures CPU time, and a probe spends -- essentially none: measured on this gateway, 3s blocked on an HTTP request -- advanced os.clock by 0.001s. A budget written in os.clock therefore never -- fires, which is why a dead upstream ran the full -- 4s timeout x 2 retries x 3 adapters = 24s against a "12 second" budget. local t1 = os.time() local lim1 = PROBE_BUDGET_MS / 1000 local function spent() return (os.time() - t1) > lim1 end local budget = { exhausted = spent } local out = {} for _, s in ipairs(srcs) do if type(s) == "table" then local rep = collectSource(s, budget) if rep ~= nil then out[#out + 1] = rep end end end PLUGIN.state.probe_memo = HOST_MEMO return out, nil end -- collect_source probes one named source. PLUGIN.collect_source = function(payload) if type(payload) ~= "table" then return nil, "expected table" end if not next(HOST_MEMO) then local st = PLUGIN.state if type(st) == "table" and type(st.probe_memo) == "table" then for k, v in pairs(st.probe_memo) do HOST_MEMO[k] = v end end end local srcs = payload.sources if type(srcs) ~= "table" then return nil, "missing sources" end -- A closure over the deadline, not a table method. `exhausted = function(self)` -- reads correctly but breaks the moment the call site is `budget.exhausted()` -- from inside a pcall-guarded hook where the upvalue chain differs; the -- explicit capture below cannot go wrong that way. local t0 = os.time() local limit = PROBE_BUDGET_MS / 1000 local function exhausted() return (os.time() - t0) > limit end local budget = { exhausted = exhausted } for _, s in ipairs(srcs) do if type(s) == "table" and s.name == payload.name then local rep = collectSource(s, budget) -- Persist the host→adapter map so the next PROCESS skips the dead ends, -- not just the next page open. PLUGIN.state.probe_memo = HOST_MEMO return rep, nil end end return nil, "source not found: " .. tostring(payload.name) end -- ---------------------------------------------------------------- page PLUGIN.ui = { -- Every row names an upstream host and what it has spent. That is deployment -- information, not per-caller information, so it is admin-only by default; an -- operator who wants a shared read-only view can widen it from config without -- editing this file. visibility = "admin", page = { page_id = "quota", title = "Quota", order = 45, mount = [==[