fix(deploy): 故障通知的三处缺陷 —— 死信目录、隔夜补投、失败无原因
起因：用户邮箱里「[dsh] 桥服务异常终止」反复出现。查下来有两层。
**第一层：崩溃本身（已修，是历史）**
dsh 在 2026-09-12 11:40 起崩溃循环，根因是当时那次插件快照切换后
`/root/.dsh/profiles/web/node_modules/dsh-mail-bridge/cordis.patch.yml` 不存在
（`failed to read overlay … ENOENT`）→ 起不来 → systemd 反复重启。
现在快照里该文件在、dsh `NRestarts=0`、今天 0 次失败。
**第二层：通知管线本身坏了（本次修）**
1. ★ **死信目录**：`zcode.service`（应用单元）既没有 `AGENTMAIL_AGENT_NAME` 也没有
密钥，报告就落进 `unknown-agent/` —— 而 flush **只读自己那个 Agent 的目录**，
于是 25 份 zcode 崩溃告警永久投不出去。修法是两件事：
· `resolveIdentity()`：身份按「单元 env → 单元名推导 → /etc/agentmail/<agent>.env」
解析；`zcode.service`→zcode、`pi-mail-bridge.service`→pi……
· 支持 `AGENTMAIL_AGENT_SECRET`：zcode 只配了 secret 没有 key，而脚本原先只认
Bearer key ⇒ 就算目录对了也发不出去（网关的 AgentAuth 两种都认）。
2. ★ **隔夜补投**：补投原先只在**同一个单元**的下次 `ExecStartPost` 跑，于是
网关不可达时攒下的报告要等到那个服务自己重启才补投 —— 实测 4 封 Sep-12 的
告警在 Sep-13 10:35 才到。现在新增 `agentmail-failure-flush.timer`（每 10 分钟
`--flush-all`），它遍历所有 Agent 目录、按目录名逐个解析身份后补投。
过时报告还会在主题与正文上标 **「补投：这是 N 分钟前的故障报告，不代表现在仍在
故障」**（原先正文里只有昨天的时间戳，读起来像刚崩）。
3. **补投失败只报数不报因**：`catch { failed++ }` → 日志只有 `spool: sent=0 failed=4`，
没人知道卡在哪。现在每条失败都带回原因（HTTP 状态码/网关不可达/身份未配置）。
4. 顺带两处准确性问题：
· 主题写**单元名**而不是笼统的「桥服务」—— 25 封标题写着"桥服务异常终止"，
实际崩的是 zcode **应用**单元，照标题去查桥方向就错了。
· `created_at_ms` 是我们的元数据，但 `/mail/send` 是**严格解码**的（实测 400
不认识的字段）→ 发送前剥离，线格式保持干净。
判据：新增 `deploy/service-failure-notify.test.mjs`（12 条，含假网关做真实投递、
严格解码断言、补投标记的正反两向）。端到端验证：模拟"没有身份的 zcode.service
崩溃"——修复前落 `unknown-agent/` 永久死信；现在**真的投出去了**，且
`from_name=zcode`、主题 `[zcode] zcode.service 异常终止 #…`。
积压清理：27 份（dsh 2 + unknown-agent 25）全部来自 09-12 那两轮崩溃、事件已在
邮箱与 journal 里出现过，按**不再补投**处理（避免把隔夜告警灌进邮箱），
原始文件留档 /root/gotmp/failure-spool-backlog-20260913.tar.gz（600），
spool 目录留 README.md 说明。
This commit is contained in:
237
deploy/service-failure-notify.test.mjs
Normal file
237
deploy/service-failure-notify.test.mjs
Normal file
@ -0,0 +1,237 @@
|
||||
/**
|
||||
* 服务故障通知脚本的判据。
|
||||
*
|
||||
* 每一条都对应一次**实测故障**（见各 case 的注释），不是凭想象加的：
|
||||
* · zcode.service 的告警落进 `unknown-agent/` 且永远发不出去（25 份死信）
|
||||
* · 隔夜补投的报告读起来像"刚刚又崩了"
|
||||
* · 补投失败只报数不报因（`spool: sent=0 failed=4`，没人知道卡在哪）
|
||||
*/
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createServer } from 'node:http';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import {
|
||||
agentNameFromService,
|
||||
readAgentSecretFromFile,
|
||||
resolveIdentity,
|
||||
buildFailurePayload,
|
||||
spoolPayload,
|
||||
flushSpool,
|
||||
isAbnormalExit
|
||||
} from './service-failure-notify.mjs';
|
||||
|
||||
const tmp = () => mkdtempSync(join(tmpdir(), 'sfn-'));
|
||||
|
||||
/** 起一个假网关，记录收到的请求体，返回 200。 */
|
||||
async function fakeGateway(handler = null) {
|
||||
const got = [];
|
||||
const server = createServer((req, res) => {
|
||||
let body = '';
|
||||
req.on('data', c => (body += c));
|
||||
req.on('end', () => {
|
||||
const status = handler ? handler({ headers: req.headers, body }, got) : 200;
|
||||
got.push({ headers: req.headers, body });
|
||||
res.writeHead(status, { 'Content-Type': 'application/json' });
|
||||
res.end('{"ok":true}');
|
||||
});
|
||||
});
|
||||
await new Promise(r => server.listen(0, '127.0.0.1', r));
|
||||
const port = server.address().port;
|
||||
return { got, url: `http://127.0.0.1:${port}`, close: () => new Promise(r => server.close(r)) };
|
||||
}
|
||||
|
||||
// ─── 身份推导 ────────────────────────────────────────────────────────
|
||||
|
||||
test('★ 从单元名推导 Agent 名（zcode.service 不再落 unknown-agent）', () => {
|
||||
const cases = {
|
||||
'dsh.service': 'dsh',
|
||||
'zcode.service': 'zcode',
|
||||
'zcode-mail-bridge.service': 'zcode',
|
||||
'pi-mail-bridge.service': 'pi',
|
||||
'opencode-serve.service': 'opencode',
|
||||
'homeagent.service': 'homeagent'
|
||||
};
|
||||
for (const [unit, want] of Object.entries(cases)) {
|
||||
assert.equal(agentNameFromService(unit), want, unit);
|
||||
}
|
||||
assert.equal(agentNameFromService('/etc/systemd/system/dsh.service'), 'dsh');
|
||||
assert.equal(agentNameFromService(''), '');
|
||||
});
|
||||
|
||||
test('★ 只有 secret 的单元也能解析出身份（zcode 就是这种）', () => {
|
||||
const dir = tmp();
|
||||
writeFileSync(join(dir, 'zcode.env'), 'AGENTMAIL_AGENT_SECRET=sk-zcode-test\n');
|
||||
const id = resolveIdentity({ AGENTMAIL_ENV_DIR: dir }, 'zcode.service');
|
||||
assert.equal(id.name, 'zcode');
|
||||
assert.equal(id.key, '');
|
||||
assert.equal(id.secret, 'sk-zcode-test', '必须能从 <agent>.env 里读出 secret');
|
||||
});
|
||||
|
||||
test('单元自己的 env 优先于推导与文件', () => {
|
||||
const dir = tmp();
|
||||
writeFileSync(join(dir, 'dsh.env'), 'AGENTMAIL_AGENT_KEY=from-file\n');
|
||||
const id = resolveIdentity({ AGENTMAIL_ENV_DIR: dir, AGENTMAIL_AGENT_NAME: 'pi', AGENTMAIL_AGENT_KEY: 'k1' }, 'dsh.service');
|
||||
assert.equal(id.name, 'pi');
|
||||
assert.equal(id.key, 'k1');
|
||||
assert.equal(id.derived, false);
|
||||
});
|
||||
|
||||
test('两种变量名都能读，引号会被剥掉', () => {
|
||||
const dir = tmp();
|
||||
writeFileSync(join(dir, 'a.env'), 'AGENTMAIL_AGENT_KEY="q1"\n');
|
||||
writeFileSync(join(dir, 'b.env'), "AGENTMAIL_AGENT_SECRET='q2'\n");
|
||||
assert.equal(readAgentSecretFromFile(join(dir, 'a.env'), 'AGENTMAIL_AGENT_KEY'), 'q1');
|
||||
assert.equal(readAgentSecretFromFile(join(dir, 'b.env'), 'AGENTMAIL_AGENT_SECRET'), 'q2');
|
||||
assert.equal(readAgentSecretFromFile(join(dir, 'missing.env')), '');
|
||||
});
|
||||
|
||||
// ─── payload ─────────────────────────────────────────────────────────
|
||||
|
||||
test('payload 带机器可读时间与 invocation 短号（补投判据靠它）', () => {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', INVOCATION_ID: 'abc123def456', SERVICE_RESULT: 'exit-code', EXIT_CODE: 'exited', EXIT_STATUS: '1' },
|
||||
serviceName: 'dsh.service',
|
||||
now: new Date('2026-09-12T03:42:35.000Z')
|
||||
});
|
||||
assert.match(p.subject, /^\[dsh\] /);
|
||||
assert.match(p.subject, /#abc123de/);
|
||||
assert.equal(p.created_at_ms, Date.parse('2026-09-12T03:42:35.000Z'));
|
||||
assert.match(p.body, /2026-09-12T03:42:35/);
|
||||
});
|
||||
|
||||
test('secret 会被脱敏', () => {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'z', AGENTMAIL_AGENT_SECRET: 'super-secret-token', SERVICE_RESULT: 'exit-code' },
|
||||
error: new Error('boom super-secret-token'),
|
||||
serviceName: 'z.service'
|
||||
});
|
||||
assert.ok(!p.body.includes('super-secret-token'));
|
||||
});
|
||||
|
||||
test('正常退出不算异常', () => {
|
||||
assert.equal(isAbnormalExit({ SERVICE_RESULT: 'success' }), false);
|
||||
assert.equal(isAbnormalExit({ SERVICE_RESULT: 'exit-code' }), true);
|
||||
assert.equal(isAbnormalExit({ EXIT_CODE: 'exited', EXIT_STATUS: '0' }), false);
|
||||
assert.equal(isAbnormalExit({ EXIT_CODE: 'exited', EXIT_STATUS: '1' }), true);
|
||||
});
|
||||
|
||||
// ─── spool 落点 ──────────────────────────────────────────────────────
|
||||
|
||||
test('★ 没给 AGENTMAIL_AGENT_NAME 时，报告要落进推导出的 Agent 目录（不是 unknown-agent）', async () => {
|
||||
const root = tmp();
|
||||
const payload = buildFailurePayload({
|
||||
env: { SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'zcode.service'
|
||||
});
|
||||
await spoolPayload(payload, { AGENTMAIL_FAILURE_SPOOL: root }, 'zcode.service');
|
||||
assert.deepEqual(readdirSync(root), ['zcode'], '落错目录 = 没有任何 flush 会读它（实测 25 份死信）');
|
||||
// 反向对照：完全无身份信息时只能落 unknown-agent（并且 flush-all 会警告）
|
||||
await spoolPayload(payload, { AGENTMAIL_FAILURE_SPOOL: root }, '');
|
||||
assert.ok(readdirSync(root).includes('unknown-agent'));
|
||||
});
|
||||
|
||||
// ─── 补投 ────────────────────────────────────────────────────────────
|
||||
|
||||
test('★ 补投陈旧报告要标「补投」并说明不代表现在仍故障', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway();
|
||||
try {
|
||||
const old = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'dsh.service',
|
||||
now: new Date(Date.now() - 26 * 3600 * 1000) // 26 小时前
|
||||
});
|
||||
mkdirSync(join(root, 'dsh'), { recursive: true });
|
||||
writeFileSync(join(root, 'dsh', 'a.json'), JSON.stringify({ payload: old }));
|
||||
|
||||
const r = await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
|
||||
'dsh.service'
|
||||
);
|
||||
assert.equal(r.sent, 1);
|
||||
assert.equal(r.failed, 0);
|
||||
const sent = JSON.parse(gw.got[0].body);
|
||||
assert.match(sent.subject, /^\[补投\] /);
|
||||
assert.match(sent.body, /补投/);
|
||||
assert.match(sent.body, /不代表现在仍在故障/);
|
||||
assert.ok(!existsSync(join(root, 'dsh', 'a.json')), '投递成功要删掉 spool 文件');
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('新鲜的报告不加「补投」标记', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway();
|
||||
try {
|
||||
const fresh = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'dsh.service'
|
||||
});
|
||||
mkdirSync(join(root, 'dsh'), { recursive: true });
|
||||
writeFileSync(join(root, 'dsh', 'b.json'), JSON.stringify({ payload: fresh }));
|
||||
await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
|
||||
'dsh.service'
|
||||
);
|
||||
assert.ok(!JSON.parse(gw.got[0].body).subject.startsWith('[补投]'));
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('★ 补投失败必须带回原因（不能只报数）', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway(() => 403);
|
||||
try {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'dsh.service'
|
||||
});
|
||||
mkdirSync(join(root, 'dsh'), { recursive: true });
|
||||
writeFileSync(join(root, 'dsh', 'c.json'), JSON.stringify({ payload: p }));
|
||||
const r = await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
|
||||
'dsh.service'
|
||||
);
|
||||
assert.equal(r.sent, 0);
|
||||
assert.equal(r.failed, 1);
|
||||
assert.equal(r.errors.length, 1);
|
||||
assert.match(r.errors[0].reason, /403/, '原因里要能看到 HTTP 状态码');
|
||||
assert.ok(existsSync(join(root, 'dsh', 'c.json')), '失败要保留文件以便下次重试');
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('★ 只有 secret 时用 X-Agent-Name/Secret 投递（zcode 的唯一出路）', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway();
|
||||
try {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'zcode', AGENTMAIL_AGENT_SECRET: 'sec-1', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'zcode.service'
|
||||
});
|
||||
mkdirSync(join(root, 'zcode'), { recursive: true });
|
||||
writeFileSync(join(root, 'zcode', 'd.json'), JSON.stringify({ payload: p }));
|
||||
const envDir = tmp();
|
||||
writeFileSync(join(envDir, 'zcode.env'), 'AGENTMAIL_AGENT_SECRET=sec-1\n');
|
||||
const r = await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_SECRET: '', AGENTMAIL_AGENT_KEY: '', AGENTMAIL_ENV_DIR: envDir },
|
||||
'zcode.service'
|
||||
);
|
||||
assert.equal(r.sent, 1, `应投递成功：${JSON.stringify(r.errors)}`);
|
||||
// ★ 端点是严格解码的：带我们自己的元数据会被 400 拒掉（实测踩过）
|
||||
assert.equal(JSON.parse(gw.got[0].body).created_at_ms, undefined);
|
||||
const h = gw.got[0].headers;
|
||||
assert.equal(h['x-agent-name'], 'zcode');
|
||||
assert.equal(h['x-agent-secret'], 'sec-1');
|
||||
assert.equal(h.authorization, undefined);
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user