MailUI4Agents/deploy/service-failure-notify.test.mjs

/**
 * 服务故障通知脚本的判据。
 *
 * 每一条都对应一次**实测故障**（见各 case 的注释），不是凭想象加的：
 *   · zcode.service 的告警落进 `unknown-agent/` 且永远发不出去（25 份死信）
 *   · 隔夜补投的报告读起来像"刚刚又崩了"
 *   · 补投失败只报数不报因（`spool: sent=0 failed=4`，没人知道卡在哪）
 */

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { createServer } from 'node:http';
import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

import {
  agentNameFromService,
  readAgentSecretFromFile,
  resolveIdentity,
  buildFailurePayload,
  spoolPayload,
  flushSpool,
  isAbnormalExit
} from './service-failure-notify.mjs';

const tmp = () => mkdtempSync(join(tmpdir(), 'sfn-'));

/** 起一个假网关，记录收到的请求体，返回 200。 */
async function fakeGateway(handler = null) {
  const got = [];
  const server = createServer((req, res) => {
    let body = '';
    req.on('data', c => (body += c));
    req.on('end', () => {
      const status = handler ? handler({ headers: req.headers, body }, got) : 200;
      got.push({ headers: req.headers, body });
      res.writeHead(status, { 'Content-Type': 'application/json' });
      res.end('{"ok":true}');
    });
  });
  await new Promise(r => server.listen(0, '127.0.0.1', r));
  const port = server.address().port;
  return { got, url: `http://127.0.0.1:${port}`, close: () => new Promise(r => server.close(r)) };
}

// ─── 身份推导 ────────────────────────────────────────────────────────

test('★ 从单元名推导 Agent 名（zcode.service 不再落 unknown-agent）', () => {
  const cases = {
    'dsh.service': 'dsh',
    'zcode.service': 'zcode',
    'zcode-mail-bridge.service': 'zcode',
    'pi-mail-bridge.service': 'pi',
    'opencode-serve.service': 'opencode',
    'homeagent.service': 'homeagent'
  };
  for (const [unit, want] of Object.entries(cases)) {
    assert.equal(agentNameFromService(unit), want, unit);
  }
  assert.equal(agentNameFromService('/etc/systemd/system/dsh.service'), 'dsh');
  assert.equal(agentNameFromService(''), '');
});

test('★ 只有 secret 的单元也能解析出身份（zcode 就是这种）', () => {
  const dir = tmp();
  writeFileSync(join(dir, 'zcode.env'), 'AGENTMAIL_AGENT_SECRET=sk-zcode-test\n');
  const id = resolveIdentity({ AGENTMAIL_ENV_DIR: dir }, 'zcode.service');
  assert.equal(id.name, 'zcode');
  assert.equal(id.key, '');
  assert.equal(id.secret, 'sk-zcode-test', '必须能从 <agent>.env 里读出 secret');
});

test('单元自己的 env 优先于推导与文件', () => {
  const dir = tmp();
  writeFileSync(join(dir, 'dsh.env'), 'AGENTMAIL_AGENT_KEY=from-file\n');
  const id = resolveIdentity({ AGENTMAIL_ENV_DIR: dir, AGENTMAIL_AGENT_NAME: 'pi', AGENTMAIL_AGENT_KEY: 'k1' }, 'dsh.service');
  assert.equal(id.name, 'pi');
  assert.equal(id.key, 'k1');
  assert.equal(id.derived, false);
});

test('两种变量名都能读，引号会被剥掉', () => {
  const dir = tmp();
  writeFileSync(join(dir, 'a.env'), 'AGENTMAIL_AGENT_KEY="q1"\n');
  writeFileSync(join(dir, 'b.env'), "AGENTMAIL_AGENT_SECRET='q2'\n");
  assert.equal(readAgentSecretFromFile(join(dir, 'a.env'), 'AGENTMAIL_AGENT_KEY'), 'q1');
  assert.equal(readAgentSecretFromFile(join(dir, 'b.env'), 'AGENTMAIL_AGENT_SECRET'), 'q2');
  assert.equal(readAgentSecretFromFile(join(dir, 'missing.env')), '');
});

// ─── payload ─────────────────────────────────────────────────────────

test('payload 带机器可读时间与 invocation 短号（补投判据靠它）', () => {
  const p = buildFailurePayload({
    env: { AGENTMAIL_AGENT_NAME: 'dsh', INVOCATION_ID: 'abc123def456', SERVICE_RESULT: 'exit-code', EXIT_CODE: 'exited', EXIT_STATUS: '1' },
    serviceName: 'dsh.service',
    now: new Date('2026-09-12T03:42:35.000Z')
  });
  assert.match(p.subject, /^\[dsh\] /);
  assert.match(p.subject, /#abc123de/);
  assert.equal(p.created_at_ms, Date.parse('2026-09-12T03:42:35.000Z'));
  assert.match(p.body, /2026-09-12T03:42:35/);
});

test('secret 会被脱敏', () => {
  const p = buildFailurePayload({
    env: { AGENTMAIL_AGENT_NAME: 'z', AGENTMAIL_AGENT_SECRET: 'super-secret-token', SERVICE_RESULT: 'exit-code' },
    error: new Error('boom super-secret-token'),
    serviceName: 'z.service'
  });
  assert.ok(!p.body.includes('super-secret-token'));
});

test('正常退出不算异常', () => {
  assert.equal(isAbnormalExit({ SERVICE_RESULT: 'success' }), false);
  assert.equal(isAbnormalExit({ SERVICE_RESULT: 'exit-code' }), true);
  assert.equal(isAbnormalExit({ EXIT_CODE: 'exited', EXIT_STATUS: '0' }), false);
  assert.equal(isAbnormalExit({ EXIT_CODE: 'exited', EXIT_STATUS: '1' }), true);
});

// ─── spool 落点 ──────────────────────────────────────────────────────

test('★ 没给 AGENTMAIL_AGENT_NAME 时，报告要落进推导出的 Agent 目录（不是 unknown-agent）', async () => {
  const root = tmp();
  const payload = buildFailurePayload({
    env: { SERVICE_RESULT: 'exit-code' },
    serviceName: 'zcode.service'
  });
  await spoolPayload(payload, { AGENTMAIL_FAILURE_SPOOL: root }, 'zcode.service');
  assert.deepEqual(readdirSync(root), ['zcode'], '落错目录 = 没有任何 flush 会读它（实测 25 份死信）');
  // 反向对照：完全无身份信息时只能落 unknown-agent（并且 flush-all 会警告）
  await spoolPayload(payload, { AGENTMAIL_FAILURE_SPOOL: root }, '');
  assert.ok(readdirSync(root).includes('unknown-agent'));
});

// ─── 补投 ────────────────────────────────────────────────────────────

test('★ 补投陈旧报告要标「补投」并说明不代表现在仍故障', async () => {
  const root = tmp();
  const gw = await fakeGateway();
  try {
    const old = buildFailurePayload({
      env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
      serviceName: 'dsh.service',
      now: new Date(Date.now() - 26 * 3600 * 1000)   // 26 小时前
    });
    mkdirSync(join(root, 'dsh'), { recursive: true });
    writeFileSync(join(root, 'dsh', 'a.json'), JSON.stringify({ payload: old }));

    const r = await flushSpool(
      { AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
      'dsh.service'
    );
    assert.equal(r.sent, 1);
    assert.equal(r.failed, 0);
    const sent = JSON.parse(gw.got[0].body);
    assert.match(sent.subject, /^\[补投\] /);
    assert.match(sent.body, /补投/);
    assert.match(sent.body, /不代表现在仍在故障/);
    assert.ok(!existsSync(join(root, 'dsh', 'a.json')), '投递成功要删掉 spool 文件');
  } finally {
    await gw.close();
  }
});

test('新鲜的报告不加「补投」标记', async () => {
  const root = tmp();
  const gw = await fakeGateway();
  try {
    const fresh = buildFailurePayload({
      env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
      serviceName: 'dsh.service'
    });
    mkdirSync(join(root, 'dsh'), { recursive: true });
    writeFileSync(join(root, 'dsh', 'b.json'), JSON.stringify({ payload: fresh }));
    await flushSpool(
      { AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
      'dsh.service'
    );
    assert.ok(!JSON.parse(gw.got[0].body).subject.startsWith('[补投]'));
  } finally {
    await gw.close();
  }
});

test('★ 补投失败必须带回原因（不能只报数）', async () => {
  const root = tmp();
  const gw = await fakeGateway(() => 403);
  try {
    const p = buildFailurePayload({
      env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
      serviceName: 'dsh.service'
    });
    mkdirSync(join(root, 'dsh'), { recursive: true });
    writeFileSync(join(root, 'dsh', 'c.json'), JSON.stringify({ payload: p }));
    const r = await flushSpool(
      { AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
      'dsh.service'
    );
    assert.equal(r.sent, 0);
    assert.equal(r.failed, 1);
    assert.equal(r.errors.length, 1);
    assert.match(r.errors[0].reason, /403/, '原因里要能看到 HTTP 状态码');
    assert.ok(existsSync(join(root, 'dsh', 'c.json')), '失败要保留文件以便下次重试');
  } finally {
    await gw.close();
  }
});

test('★ 只有 secret 时用 X-Agent-Name/Secret 投递（zcode 的唯一出路）', async () => {
  const root = tmp();
  const gw = await fakeGateway();
  try {
    const p = buildFailurePayload({
      env: { AGENTMAIL_AGENT_NAME: 'zcode', AGENTMAIL_AGENT_SECRET: 'sec-1', SERVICE_RESULT: 'exit-code' },
      serviceName: 'zcode.service'
    });
    mkdirSync(join(root, 'zcode'), { recursive: true });
    writeFileSync(join(root, 'zcode', 'd.json'), JSON.stringify({ payload: p }));
    const envDir = tmp();
    writeFileSync(join(envDir, 'zcode.env'), 'AGENTMAIL_AGENT_SECRET=sec-1\n');
    const r = await flushSpool(
      { AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_SECRET: '', AGENTMAIL_AGENT_KEY: '', AGENTMAIL_ENV_DIR: envDir },
      'zcode.service'
    );
    assert.equal(r.sent, 1, `应投递成功：${JSON.stringify(r.errors)}`);
    // ★ 端点是严格解码的：带我们自己的元数据会被 400 拒掉（实测踩过）
    assert.equal(JSON.parse(gw.got[0].body).created_at_ms, undefined);
    const h = gw.got[0].headers;
    assert.equal(h['x-agent-name'], 'zcode');
    assert.equal(h['x-agent-secret'], 'sec-1');
    assert.equal(h.authorization, undefined);
  } finally {
    await gw.close();
  }
});