JianFeeeee
49ec22f916
实测（今天全 Agent 演练时撞到的）：pi 的 `maxWorkers=3` 被**三个正在等人工授权**的 worker 吃满，于是新邮件只能排队 —— 而人可能十分钟后才看邮箱。清掉卡住的请求后队列 立即排空，机制本身没错，错的是"等待"被当成了"在干活"。 两处改动（`src/pool.mjs`）： 1. **等待期间让出并发额度**。worker 发 `permission_pending` 时把它的 entry 标为 parked，`pump()` 只数"在干活"的（`activeCount()`）。停放另有上限 `maxParked` （默认 5，防止内存无界：每 worker 约 140MB）；超出后仍占额度并打日志说明。 决策到达（`routePermission`）时解除停放，回到额度里。 2. **等待期间暂停硬超时**。硬超时的用途是回收**卡死**的进程，而等人点头不是卡死： 停放时清掉计时器，决策到达后重新起一个完整窗口。否则 worker 会在人还在读邮件时 被 SIGKILL —— 那次工具调用直接消失，人后来批了也没人接（这类"批了没反应"的现象 与此吻合）。 判据（`test/pool.test.mjs` 新增 3 条）： · ★ 等待授权的 worker 让出额度：另一个会话的邮件必须能开跑 · ★ 等人点头期间（800ms > 300ms 硬超时）不得被强杀，且恢复后能跑完 · 停放有上限：超出后仍占额度（不无限超发） **扰动验证**：整块回退到 HEAD → 3 条全红；修复后 3/3。全量 pi 套件 420/420。 已部署（快照 20260913-131216，桥重启并重新心跳）。
482 lines
21 KiB
JavaScript
482 lines
21 KiB
JavaScript
/**
|
||
* 工作进程池的调度不变量。
|
||
*
|
||
* 这些用例**真的 fork 子进程**，用一个极小的桩 worker（不装 pi SDK）：
|
||
* 要验的是调度（并发上限、同会话串行、排队、硬超时、状态跨 worker 存活），
|
||
* 而不是模型怎么跑。用桩让每个用例在几百毫秒内完成。
|
||
*
|
||
* 桩 worker 的协议与真 worker 一致：`ready` → 收 `job` → 按 `__hold` 停一会儿
|
||
* → 发 `done`。它还把收到的 job 载荷原样回声成一行 `JOB {…}` 日志 ——
|
||
* 判据因此能落在「主进程真的把 sessionFile / grants / 命名指纹传下来了」上，
|
||
* 而不是一个间接的计数（计数在字段被丢掉时依然会给出绿色）。
|
||
*/
|
||
|
||
import { test } from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||
import { tmpdir } from 'node:os';
|
||
import { join } from 'node:path';
|
||
|
||
// ─── 桩 worker ───
|
||
//
|
||
// 落到临时目录而不是仓库里：它是测试脚手架，不该被 check-shared-libs 之类的
|
||
// 一致性脚本看到，也不该让人误以为是第二个真 worker。
|
||
const STUB_DIR = mkdtempSync(join(tmpdir(), 'pi-pool-test-'));
|
||
const STUB = join(STUB_DIR, 'stub-worker.mjs');
|
||
writeFileSync(STUB, `
|
||
process.on('message', (msg) => {
|
||
if (msg?.type === 'job') {
|
||
const hold = msg.data?.__hold ?? 30;
|
||
process.send({ type: 'log', line: 'JOB ' + JSON.stringify({
|
||
mailID: msg.data.mail_id,
|
||
kind: msg.kind,
|
||
sessionFile: msg.session?.sessionFile || '',
|
||
cwd: msg.session?.cwd || '',
|
||
grants: msg.grants || [],
|
||
lastSyncedName: msg.lastSyncedName || '',
|
||
turnTimeoutMs: msg.config?.turnTimeoutMs ?? null,
|
||
}) });
|
||
process.send({ type: 'session_opened', piSessionId: 'pi-' + msg.data.mail_id,
|
||
sessionFile: '/tmp/f-' + msg.data.mail_id + '.jsonl', cwd: '/tmp', reused: false });
|
||
if (msg.data?.__grant) {
|
||
process.send({ type: 'permission_grant', toolName: msg.data.__grant });
|
||
}
|
||
if (msg.data?.__name) {
|
||
process.send({ type: 'name_synced', signature: msg.data.__name });
|
||
}
|
||
if (msg.data?.__reopen) {
|
||
// 模型降级换会话：同一个 worker 里第二次 session_opened
|
||
process.send({ type: 'session_opened', piSessionId: msg.data.__reopen,
|
||
sessionFile: '/tmp/f2.jsonl', cwd: '/tmp', reused: false });
|
||
}
|
||
if (msg.data?.__pending) {
|
||
process.send({ type: 'permission_pending', relayKey: msg.data.__pending });
|
||
return; // 等决策，见下面的分支
|
||
}
|
||
if (msg.data?.__crash) {
|
||
// 未回报 done 就退出：验证主进程会重投（而不是静默丢信）
|
||
process.exit(1);
|
||
}
|
||
// 每 40ms 报一次心跳：并发的判据必须是「两个进程真的同时在干活」，
|
||
// 而不是「running map 里有两个条目」—— fork 返回后立即就有两个条目了。
|
||
const beat = setInterval(() => process.send({ type: 'log', line: 'TICK ' + msg.data.mail_id }), 40);
|
||
if (hold < 0) return; // 永不结束，用来验硬超时
|
||
setTimeout(() => {
|
||
clearInterval(beat);
|
||
process.send({ type: 'done', ok: true, error: '' });
|
||
setTimeout(() => process.exit(0), 20);
|
||
}, hold);
|
||
return;
|
||
}
|
||
if (msg?.type === 'permission_decision') {
|
||
process.send({ type: 'log', line: 'DECISION ' + msg.relayKey + '=' + msg.decision });
|
||
process.send({ type: 'done', ok: true, error: '' });
|
||
setTimeout(() => process.exit(0), 20);
|
||
return;
|
||
}
|
||
if (msg?.type === 'shutdown') {
|
||
process.send({ type: 'log', line: 'SHUTDOWN' });
|
||
setTimeout(() => process.exit(0), 10);
|
||
}
|
||
});
|
||
process.send({ type: 'ready' });
|
||
`);
|
||
|
||
const { createWorkerPool } = await import('../src/pool.mjs');
|
||
|
||
/** 建一个用桩 worker 的池。 */
|
||
function makePool(opts = {}) {
|
||
const lines = [];
|
||
const pool = createWorkerPool({
|
||
log: (...a) => lines.push(a.join(' ')),
|
||
config: () => ({ turnTimeoutMs: 1000, ...(opts.config || {}) }),
|
||
onReconfigure: opts.onReconfigure || (() => {}),
|
||
maxWorkers: opts.maxWorkers ?? 2,
|
||
maxParked: opts.maxParked ?? 5,
|
||
workerMaxMs: opts.workerMaxMs ?? 5000,
|
||
maxAttempts: opts.maxAttempts,
|
||
workerPath: opts.workerPath || STUB,
|
||
});
|
||
return { pool, lines };
|
||
}
|
||
|
||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||
|
||
/** 轮询到条件成立或超时 —— 比固定 sleep 稳。 */
|
||
async function until(fn, timeoutMs = 4000) {
|
||
const t0 = Date.now();
|
||
while (Date.now() - t0 < timeoutMs) {
|
||
if (fn()) return true;
|
||
await sleep(20);
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/** 从日志行里取出桩 worker 回声的 job 载荷。 */
|
||
function jobs(lines) {
|
||
const out = [];
|
||
for (const l of lines) {
|
||
const i = l.indexOf('JOB ');
|
||
if (i === -1) continue;
|
||
try { out.push(JSON.parse(l.slice(i + 4))); } catch { /* 不是完整一行 */ }
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/** 某个 mailID 的心跳出现过几次。 */
|
||
const ticks = (lines, id) => lines.filter((l) => l.includes(`TICK ${id}`)).length;
|
||
|
||
test('并发上限被遵守：第三条会话要等前面空出来', async () => {
|
||
const { pool } = makePool({ maxWorkers: 2 });
|
||
for (const id of ['a', 'b', 'c']) {
|
||
pool.submit('mail', { mail_id: id, session_id: `S-${id}`, __hold: 250 });
|
||
}
|
||
|
||
let peak = 0;
|
||
const t = setInterval(() => { peak = Math.max(peak, pool.stats().running); }, 15);
|
||
const sawQueue = await until(() => pool.stats().queued > 0, 1000);
|
||
await until(() => pool.stats().running === 0 && pool.stats().queued === 0);
|
||
clearInterval(t);
|
||
pool.stop();
|
||
|
||
assert.ok(peak <= 2, `同时跑的 worker 峰值 ${peak}，不该超过 maxWorkers=2`);
|
||
assert.ok(sawQueue, '满载时第三封该进队列而不是被丢掉');
|
||
});
|
||
|
||
test('同一会话串行：两个 worker 的心跳不得重叠', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 3 });
|
||
pool.submit('mail', { mail_id: 'm1', session_id: 'SAME', __hold: 220 });
|
||
pool.submit('mail', { mail_id: 'm2', session_id: 'SAME', __hold: 60 });
|
||
|
||
// 判据一：任一时刻只有一个 worker 在跑。
|
||
let everTwo = false;
|
||
const t = setInterval(() => { if (pool.stats().running > 1) everTwo = true; }, 10);
|
||
await until(() => jobs(lines).length === 2 && pool.stats().running === 0, 5000);
|
||
clearInterval(t);
|
||
pool.stop();
|
||
|
||
assert.equal(everTwo, false, '同一条会话不得有两个 worker 同时装载会话文件');
|
||
// 判据二：m2 一次心跳都没能在 m1 结束前发出 —— m1 的心跳数应当远多于 m2。
|
||
assert.ok(ticks(lines, 'm1') >= 3, `m1 该跑满 220ms，实际心跳 ${ticks(lines, 'm1')} 次`);
|
||
assert.equal(jobs(lines).length, 2, '两封都要被处理，不能因为串行而丢掉一封');
|
||
});
|
||
|
||
test('不同会话真并发：两个进程的心跳在同一段时间里交错', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 3 });
|
||
pool.submit('mail', { mail_id: 'p', session_id: 'S-P', __hold: 300 });
|
||
pool.submit('mail', { mail_id: 'q', session_id: 'S-Q', __hold: 300 });
|
||
|
||
const interleaved = await until(() => ticks(lines, 'p') >= 2 && ticks(lines, 'q') >= 2, 2500);
|
||
pool.stop();
|
||
assert.ok(interleaved,
|
||
`两条不同会话应当并发，实际 p=${ticks(lines, 'p')} q=${ticks(lines, 'q')} 次心跳`);
|
||
});
|
||
|
||
test('sessionFile 与 cwd 跨 worker 传下去：第二封接着第一封的会话谈', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', { mail_id: 'first', session_id: 'KEEP', __hold: 30 });
|
||
await until(() => jobs(lines).length === 1 && pool.stats().running === 0);
|
||
|
||
pool.submit('mail', { mail_id: 'second', session_id: 'KEEP', __hold: 30 });
|
||
await until(() => jobs(lines).length === 2 && pool.stats().running === 0);
|
||
pool.stop();
|
||
|
||
const [j1, j2] = jobs(lines);
|
||
assert.equal(j1.sessionFile, '', '第一封时还没有会话文件');
|
||
assert.equal(j2.sessionFile, '/tmp/f-first.jsonl',
|
||
'第二封必须带上第一封开出来的会话文件，否则每封邮件都从零开始、上下文全丢');
|
||
assert.equal(j2.cwd, '/tmp', 'cwd 也要传下去');
|
||
});
|
||
|
||
test('「一直同意」与命名指纹跨 worker 存活', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', {
|
||
mail_id: 'g1', session_id: 'GRANT', __hold: 30,
|
||
__grant: 'bash', __name: 'platform:某名字|某名字',
|
||
});
|
||
await until(() => jobs(lines).length === 1 && pool.stats().running === 0);
|
||
|
||
pool.submit('mail', { mail_id: 'g2', session_id: 'GRANT', __hold: 30 });
|
||
await until(() => jobs(lines).length === 2 && pool.stats().running === 0);
|
||
pool.stop();
|
||
|
||
const [j1, j2] = jobs(lines);
|
||
assert.deepEqual(j1.grants, [], '第一封时还没人点过「一直同意」');
|
||
assert.deepEqual(j2.grants, ['bash'],
|
||
'「一直同意」不跨 worker 存活的话，下一封邮件又问一遍 —— 那个选项就是在骗人');
|
||
assert.equal(j2.lastSyncedName, 'platform:某名字|某名字',
|
||
'命名指纹要传下去，否则每封邮件都重新 sync 一次');
|
||
});
|
||
|
||
test('config() 每次派活时重取：allowedModels 随心跳变，不能用快照', async () => {
|
||
let turnTimeoutMs = 111;
|
||
const { pool, lines } = makePool({ maxWorkers: 1, config: {} });
|
||
// makePool 的 config 是固定值，这里换成动态的
|
||
pool.stop();
|
||
|
||
const lines2 = [];
|
||
const p2 = createWorkerPool({
|
||
log: (...a) => lines2.push(a.join(' ')),
|
||
config: () => ({ turnTimeoutMs }),
|
||
onReconfigure: () => {},
|
||
maxWorkers: 1,
|
||
workerMaxMs: 5000,
|
||
workerPath: STUB,
|
||
});
|
||
p2.submit('mail', { mail_id: 'c1', session_id: 'C1', __hold: 20 });
|
||
await until(() => jobs(lines2).length === 1 && p2.stats().running === 0);
|
||
turnTimeoutMs = 222;
|
||
p2.submit('mail', { mail_id: 'c2', session_id: 'C2', __hold: 20 });
|
||
await until(() => jobs(lines2).length === 2 && p2.stats().running === 0);
|
||
p2.stop();
|
||
|
||
const [j1, j2] = jobs(lines2);
|
||
assert.equal(j1.turnTimeoutMs, 111);
|
||
assert.equal(j2.turnTimeoutMs, 222, 'config() 必须每次重取，否则 worker 用的是上一轮的模型范围');
|
||
assert.equal(lines.length >= 0, true);
|
||
});
|
||
|
||
test('硬超时回收卡死的 worker，且不堵住同会话后续邮件', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2, workerMaxMs: 400 });
|
||
pool.submit('mail', { mail_id: 'stuck', session_id: 'STUCK', __hold: -1 });
|
||
await until(() => pool.stats().running === 1, 1500);
|
||
|
||
const freed = await until(() => pool.stats().running === 0, 3000);
|
||
assert.ok(freed, '卡死的 worker 必须被硬超时回收，否则那条会话的后续邮件永远排队');
|
||
assert.ok(lines.some((l) => l.includes('强杀')), `应记下强杀日志，实际:\n${lines.join('\n')}`);
|
||
|
||
pool.submit('mail', { mail_id: 'after', session_id: 'STUCK', __hold: 30 });
|
||
const ran = await until(() => jobs(lines).some((j) => j.mailID === 'after'), 2000);
|
||
await until(() => pool.stats().running === 0);
|
||
pool.stop();
|
||
assert.ok(ran, '硬超时后同一会话的后续邮件必须能被处理');
|
||
});
|
||
|
||
test('权限决策路由到发起询问的那个 worker', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', { mail_id: 'perm', session_id: 'PERM', __pending: 'rk-1' });
|
||
await until(() => pool.stats().running === 1, 1500);
|
||
await sleep(150); // 等 permission_pending 到主进程
|
||
|
||
assert.equal(pool.routePermission('rk-1', '同意'), true, '应当路由成功');
|
||
await until(() => pool.stats().running === 0, 2000);
|
||
pool.stop();
|
||
|
||
assert.ok(lines.some((l) => l.includes('DECISION rk-1=同意')),
|
||
`worker 应收到决策原文，实际:\n${lines.join('\n')}`);
|
||
});
|
||
|
||
test('决策发的是选项原文而不是归一化的 allow/deny', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', { mail_id: 'p2', session_id: 'P2', __pending: 'rk-2' });
|
||
await until(() => pool.stats().running === 1, 1500);
|
||
await sleep(150);
|
||
pool.routePermission('rk-2', '一直同意');
|
||
await until(() => pool.stats().running === 0, 2000);
|
||
pool.stop();
|
||
|
||
// 「同意」与「一直同意」语义不同，归一化会让后者退化成单次授权
|
||
assert.ok(lines.some((l) => l.includes('DECISION rk-2=一直同意')),
|
||
`必须原文透传，实际:\n${lines.join('\n')}`);
|
||
});
|
||
|
||
test('决策找不到 worker 时返回 false（调用方据此走 B-4.2 降级）', async () => {
|
||
const { pool } = makePool();
|
||
assert.equal(pool.routePermission('never-seen', '同意'), false);
|
||
pool.stop();
|
||
});
|
||
|
||
test('worker 退出后它的权限路由被清掉，不会误投给下一个 worker', async () => {
|
||
const { pool } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', { mail_id: 'gone', session_id: 'GONE', __pending: 'rk-gone' });
|
||
await until(() => pool.stats().running === 1, 1500);
|
||
await sleep(150);
|
||
// 不给决策，直接停掉它
|
||
pool.stop();
|
||
await until(() => pool.stats().running === 0, 4000);
|
||
|
||
assert.equal(pool.routePermission('rk-gone', '同意'), false,
|
||
'worker 已退出，路由必须返回 false 让调用方走降级路径');
|
||
});
|
||
|
||
test('mailDrivenIDs 报出所有跑过的 pi 会话，且不随 worker 退出而清', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', { mail_id: 'd1', session_id: 'D1', __hold: 30 });
|
||
pool.submit('mail', { mail_id: 'd2', session_id: 'D2', __hold: 30 });
|
||
await until(() => jobs(lines).length === 2 && pool.stats().running === 0);
|
||
pool.stop();
|
||
|
||
const ids = pool.mailDrivenIDs();
|
||
assert.ok(ids.has('pi-d1'), 'D1 的 pi 会话该被标记为邮件驱动');
|
||
assert.ok(ids.has('pi-d2'), 'D2 的 pi 会话该被标记为邮件驱动');
|
||
});
|
||
|
||
test('模型降级换掉的旧 pi 会话仍算邮件驱动', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', { mail_id: 'r', session_id: 'RETIRE', __hold: 40, __reopen: 'pi-new' });
|
||
await until(() => jobs(lines).length === 1 && pool.stats().running === 0);
|
||
pool.stop();
|
||
|
||
const ids = pool.mailDrivenIDs();
|
||
assert.ok(ids.has('pi-new'), '新会话要在');
|
||
assert.ok(ids.has('pi-r'), '被换掉的旧会话也参与过邮件往来，磁盘上的文件还在，快照该报它');
|
||
});
|
||
|
||
test('hasSession 只对跑过的邮件会话为真', async () => {
|
||
const { pool } = makePool();
|
||
assert.equal(pool.hasSession('NOPE'), false);
|
||
pool.submit('mail', { mail_id: 'h1', session_id: 'HAS', __hold: 30 });
|
||
await until(() => pool.hasSession('HAS'), 2000);
|
||
await until(() => pool.stats().running === 0);
|
||
pool.stop();
|
||
assert.equal(pool.hasSession('HAS'), true, 'worker 退出后仍该记着这条会话');
|
||
});
|
||
|
||
test('kind 透传：权限通知走 permission 而不是 mail', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('permission', { mail_id: 'k1', session_id: 'K1', __hold: 20 });
|
||
await until(() => jobs(lines).length === 1 && pool.stats().running === 0);
|
||
pool.stop();
|
||
assert.equal(jobs(lines)[0].kind, 'permission',
|
||
'kind 决定 worker 用哪套提示词，传错会让模型以为收到一封新邮件');
|
||
});
|
||
|
||
test('stop 之后不再派活', async () => {
|
||
const { pool } = makePool();
|
||
pool.stop();
|
||
pool.submit('mail', { mail_id: 'late', session_id: 'LATE', __hold: 30 });
|
||
await sleep(200);
|
||
assert.equal(pool.stats().running, 0, '关停后不该再起 worker');
|
||
assert.equal(pool.stats().queued, 0, '关停后队列应为空');
|
||
});
|
||
|
||
test('stop 会先给 worker 发 shutdown（让它 fail closed）再杀', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 2 });
|
||
pool.submit('mail', { mail_id: 's1', session_id: 'S1', __hold: -1 });
|
||
await until(() => pool.stats().running === 1, 1500);
|
||
pool.stop();
|
||
const gotShutdown = await until(() => lines.some((l) => l.includes('SHUTDOWN')), 2000);
|
||
assert.ok(gotShutdown,
|
||
'必须先发 shutdown：直接 SIGKILL 会让 pi 侧那些等权限的 await 永不返回');
|
||
});
|
||
|
||
test('没有 session_id 的事件各占一个 key，不会互相串行', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 3 });
|
||
pool.submit('mail', { mail_id: 'n1', __hold: 300 });
|
||
pool.submit('mail', { mail_id: 'n2', __hold: 300 });
|
||
const both = await until(() => ticks(lines, 'n1') >= 2 && ticks(lines, 'n2') >= 2, 2500);
|
||
pool.stop();
|
||
assert.ok(both, '无 session_id 的两封不属于同一条会话，应能并发');
|
||
});
|
||
|
||
test('reconfigure 上报被转达给主进程', async () => {
|
||
const STUB2 = join(STUB_DIR, 'stub-reconf.mjs');
|
||
writeFileSync(STUB2, `
|
||
process.on('message', (msg) => {
|
||
if (msg?.type === 'job') {
|
||
process.send({ type: 'reconfigure', url: 'http://new:9999', agentKey: 'k2' });
|
||
process.send({ type: 'done', ok: true, error: '' });
|
||
setTimeout(() => process.exit(0), 20);
|
||
}
|
||
});
|
||
process.send({ type: 'ready' });
|
||
`);
|
||
let got = null;
|
||
const { pool } = makePool({
|
||
maxWorkers: 1,
|
||
workerPath: STUB2,
|
||
onReconfigure: (url, key) => { got = { url, key }; },
|
||
});
|
||
pool.submit('mail', { mail_id: 'r1', session_id: 'R1' });
|
||
await until(() => got !== null, 3000);
|
||
pool.stop();
|
||
assert.deepEqual(got, { url: 'http://new:9999', key: 'k2' },
|
||
'worker 里 connect_to_server 换的坐标必须回到主进程 —— worker 马上就退了，改在它自己身上等于没改');
|
||
});
|
||
|
||
test('worker 未回报 done 就退出：有界重投而不是静默丢信', async () => {
|
||
// maxAttempts=2：首次 + 一次重投，然后放弃。
|
||
// 这封邮件必定崩溃，重投就是在验证「有界」——不然它会变成永久活锁。
|
||
const { pool, lines } = makePool({ maxWorkers: 1, maxAttempts: 2 });
|
||
pool.submit('mail', { mail_id: 'crashy', session_id: 'CRASH', __crash: true });
|
||
|
||
const gaveUp = await until(() => lines.some((l) => l.includes('放弃')), 6000);
|
||
pool.stop();
|
||
|
||
assert.ok(gaveUp, `重投到上限后应记下「放弃」，实际:\n${lines.join('\n')}`);
|
||
assert.equal(jobs(lines).length, 2,
|
||
`应当尝试 2 次（首次 + 1 次重投），实际 ${jobs(lines).length} 次`);
|
||
assert.ok(lines.some((l) => l.includes('未回报 done 就退出')),
|
||
'必须明说是「未回报 done 就退出」——否则看到 exit code 会误以为是普通崩溃');
|
||
});
|
||
|
||
test('重投上限之下不会无限重投（maxAttempts=1 就是不重投）', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 1, maxAttempts: 1 });
|
||
pool.submit('mail', { mail_id: 'once', session_id: 'ONCE', __crash: true });
|
||
await until(() => lines.some((l) => l.includes('放弃')), 4000);
|
||
await sleep(300); // 再等一会儿，确认没有额外重投
|
||
pool.stop();
|
||
|
||
assert.equal(jobs(lines).length, 1, 'maxAttempts=1 时只跑一次');
|
||
});
|
||
|
||
// ─── 等人点头 ≠ 卡死（2026-09-13 实测后加的）────────────────────────
|
||
//
|
||
// 实测故障：maxWorkers=3 被三个"正在等人工授权"的 worker 吃满，于是新邮件只能
|
||
// 排队 —— 而人可能十分钟后才看邮箱。等待不是故障：它既不占内存活动量也不打上游，
|
||
// 不该占并发额度；同理它也不是"卡死"，不该被硬超时回收（否则人还在读邮件，
|
||
// 那次工具调用就被 SIGKILL 了，人后来批了也没人接）。
|
||
|
||
test('★ 等人工授权的 worker 让出并发额度：别的会话不再被它堵住', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 1 });
|
||
pool.submit('mail', { mail_id: 'waiting', session_id: 'WAIT', __pending: 'rk-wait' });
|
||
await until(() => pool.stats().running === 1, 1500);
|
||
|
||
// 另一个会话的邮件：修复前额度被等待者占着，它永远不会开跑
|
||
pool.submit('mail', { mail_id: 'other', session_id: 'OTHER', __hold: 30 });
|
||
const started = await until(() => jobs(lines).some((j) => j.mailID === 'other'), 2000);
|
||
assert.ok(started, '等待授权的 worker 不该占着额度 —— 另一个会话必须能开跑');
|
||
|
||
// 决策到达 → 等待者恢复、继续跑完
|
||
assert.equal(pool.routePermission('rk-wait', '同意'), true);
|
||
const drained = await until(() => pool.stats().running === 0, 4000);
|
||
pool.stop();
|
||
assert.ok(drained, '收到决策后应正常结束');
|
||
assert.ok(lines.some((l) => l.includes('DECISION rk-wait=同意')), `worker 应收到决策，实际:\n${lines.join('\n')}`);
|
||
});
|
||
|
||
test('★ 等人点头期间不被硬超时杀掉（等待不是卡死）', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 1, workerMaxMs: 300 });
|
||
pool.submit('mail', { mail_id: 'hold-perm', session_id: 'HP', __pending: 'rk-hp' });
|
||
await until(() => pool.stats().running === 1, 1000);
|
||
|
||
await sleep(800); // 远超 300ms 硬超时
|
||
assert.equal(pool.stats().running, 1, '等待决策的 worker 不该被硬超时回收');
|
||
assert.ok(!lines.join('\n').includes('强杀'), '等待期间的强杀不该发生');
|
||
|
||
pool.routePermission('rk-hp', '同意');
|
||
const drained = await until(() => pool.stats().running === 0, 3000);
|
||
pool.stop();
|
||
assert.ok(drained, '恢复后应能正常跑完（超时窗口重启）');
|
||
});
|
||
|
||
test('停放有上限：超出后仍占用额度（不无限超发）', async () => {
|
||
const { pool, lines } = makePool({ maxWorkers: 3, maxParked: 1 });
|
||
try {
|
||
pool.submit('mail', { mail_id: 'p1', session_id: 'P1', __pending: 'rk-p1' });
|
||
pool.submit('mail', { mail_id: 'p2', session_id: 'P2', __pending: 'rk-p2' });
|
||
await until(() => jobs(lines).length >= 2, 2500);
|
||
await sleep(250);
|
||
|
||
assert.ok(
|
||
lines.some((l) => l.includes('停放额度已满')),
|
||
`第二个等待者应记录"额度已满"并继续占额度，实际:\n${lines.join('\n')}`
|
||
);
|
||
for (const rk of ['rk-p1', 'rk-p2']) pool.routePermission(rk, '同意');
|
||
await until(() => pool.stats().running === 0, 3000);
|
||
} finally {
|
||
// 断言失败时也要收掉子进程：否则挂住的 worker 会拖住整个测试文件（实测过）
|
||
pool.stop();
|
||
}
|
||
});
|