/** * Plugin-wiring tests against **real** dsh services. * * These are the assertions that would silently rot if dsh changed a hook, so they * run the actual `SystemPrompt` and `ToolRuntime` rather than stubs. The * checkpoint script covers the same ground with human-readable output; this file * is what CI should run. * * Run: node --test test/ * * @module dsh-memory/test/plugin */ import assert from 'node:assert/strict' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import test from 'node:test' import { Context } from '@deepseek-ai/cordis' import { createUserMessage } from '@deepseek-ai/dsh-llm' import { SystemPrompt } from '@deepseek-ai/dsh-system-prompt' import { ToolRuntime } from '@deepseek-ai/dsh-tools' import { apply as applyMemory, lastUserText, name as pluginName } from '../lib/index.js' /** * Build a real cordis app carrying the services this plugin injects. * `ToolRuntime` declares `inject = ['systemPrompt']`, so prompt comes first. * @returns {Promise<{ctx: Context, dir: string, done: () => void}>} */ async function app() { const dir = mkdtempSync(join(tmpdir(), 'dsh-memory-plugin-')) const ctx = new Context() new SystemPrompt(ctx, {}) new ToolRuntime(ctx, {}) return { ctx, dir, done: async () => { await ctx.fiber.dispose() rmSync(dir, { recursive: true, force: true }) }, } } /** A realistic dsh step batch: question, then two user-role injected contexts. */ function realBatch(question) { return [ { id: 'm1', role: 'user', source: { kind: 'user' }, content: [{ type: 'text', text: question }], }, { id: 'm2', role: 'user', source: { kind: 'runtime-context', form: 'snapshot' }, content: [{ type: 'text', text: '[Working directory]\n/home/agent/project' }], }, { id: 'm3', role: 'user', source: { kind: 'skill-catalog', form: 'catalog' }, content: [{ type: 'text', text: '[Skills] git-release-discipline, android-cli, shared-browser' }], }, ] } test('lastUserText picks the newest HUMAN prompt and joins only text blocks', () => { // Every real message carries `source.kind`; the fixtures must too, or the test // does not exercise the actual discrimination (see the REGRESSION block below). const messages = [ { role: 'user', source: { kind: 'user' }, content: [{ type: 'text', text: 'first' }] }, { role: 'assistant', source: { kind: 'model' }, content: [{ type: 'text', text: 'reply' }] }, { role: 'user', source: { kind: 'user' }, content: [{ type: 'tool-result' }, { type: 'text', text: 'second' }], }, ] assert.equal(lastUserText(messages), 'second') assert.equal(lastUserText([]), '') assert.equal( lastUserText([{ role: 'assistant', source: { kind: 'model' }, content: [{ type: 'text', text: 'x' }] }]), '', ) }) // ── G6: the allowlist must be proven against REAL injected kinds ─────────── // // Scope note (important): this table is **not** claimed to be exhaustive. It is // "every user-role injected kind declared in the currently installed harness". // The full `MessageSourceMap` surface is larger (25 source keys at the time of // writing), and more will appear. That is fine — `HUMAN_PROMPT_KINDS` is an // allowlist, so an unlisted future kind fails closed by construction, which is // the property that actually matters at runtime. // // What this table DOES prove: the names we think are injected really are, and a // rename in the harness breaks a test here instead of silently killing recall. // // Sources (each verified as a literal declaration, not inferred): // agent-instructions dsh-agent-instructions/lib/types/state.d.ts:27 // agent-message dsh-subagent/lib/types/continuation-messages.d.ts:37 // compact-checkpoint dsh-compaction/lib/types/checkpoint.d.ts:18 // cordis-host-runner dsh-cordis-host-runner/lib/types/index.d.ts:13 // dsh-session-title-llm dsh-session-title-llm/lib/types/index.d.ts:14 // model-selection dsh-agent/lib/types/model-selection.d.ts:11 // plan-mode dsh-plan-mode/lib/types/index.d.ts:31 // ptc-mode dsh-tools/lib/types/ptc.d.ts:14 // repeat-tool-reminder dsh-repeat-tool-reminder/lib/types/index.d.ts:14 // runtime-context dsh-agent-loop/lib/types/runtime-context.d.ts:13 // schedule dsh-schedule/lib/types/runtime.d.ts:5 // session-reference dsh-session-reference/lib/types/types.d.ts:33 // skill-catalog dsh-tool-skill/lib/types/index.d.ts:29 // skill-invocation dsh-skill/lib/types/index.d.ts:131 // subagent-settled dsh-subagent/lib/types/continuation-messages.d.ts:38 // team-message dsh-experimental-agent-team/lib/types/types.d.ts:123 // time-context dsh-time-context/lib/types/index.d.ts:14 // tmux-context dsh-tmux-context/lib/types/index.d.ts // tool-goal dsh-tool-goal/lib/types/index.d.ts:12 // tool-jobs dsh-tool-jobs/lib/types/index.d.ts:16 // tool-registry dsh-tools/lib/types/index.d.ts:20 // user-approval dsh-user-approval/lib/types/index.d.ts:14 // working-directory dsh-working-directory/lib/types/index.d.ts const REAL_INJECTED_KINDS = [ 'agent-instructions', 'agent-message', 'compact-checkpoint', 'cordis-host-runner', 'dsh-session-title-llm', 'model-selection', 'plan-mode', 'ptc-mode', 'repeat-tool-reminder', 'runtime-context', 'schedule', 'session-reference', 'skill-catalog', 'skill-invocation', 'subagent-settled', 'team-message', 'time-context', 'tmux-context', 'tool-goal', 'tool-jobs', 'tool-registry', 'user-approval', 'working-directory', ] test('G6: every user-role injected kind in the installed harness is rejected as a query', () => { for (const kind of REAL_INJECTED_KINDS) { const batch = [ { role: 'user', source: { kind: 'user' }, content: [{ type: 'text', text: '真正的问题' }] }, { role: 'user', source: { kind }, content: [{ type: 'text', text: `[${kind}] injected body` }] }, ] assert.equal( lastUserText(batch), '真正的问题', `kind "${kind}" must be treated as injected, not as the human prompt`, ) } }) test('G6 mutation: "role is user ⇒ accept" must fail on every real injected kind', () => { // The mutation is the exact first implementation: no kind filter at all. const roleOnly = (messages) => { for (let i = messages.length - 1; i >= 0; i -= 1) { if (messages[i]?.role !== 'user') continue const t = (messages[i].content ?? []) .filter((b) => b?.type === 'text' && typeof b.text === 'string') .map((b) => b.text) .join('\n') .trim() if (t !== '') return t } return '' } let mutatedFailures = 0 for (const kind of REAL_INJECTED_KINDS) { const batch = [ { role: 'user', source: { kind: 'user' }, content: [{ type: 'text', text: '真正的问题' }] }, { role: 'user', source: { kind }, content: [{ type: 'text', text: `[${kind}] injected body` }] }, ] assert.equal(lastUserText(batch), '真正的问题') if (roleOnly(batch) !== '真正的问题') mutatedFailures += 1 } assert.equal( mutatedFailures, REAL_INJECTED_KINDS.length, `the mutation must fail on ALL ${REAL_INJECTED_KINDS.length} kinds, failed on ${mutatedFailures}`, ) }) // ── G6b: the allowlist must NOT exclude a real human prompt ─────────────── // // This is the safety half of G6, and it is the reason a strict allowlist is // acceptable at all. A remote prompt arrives with a DIFFERENT source key // (`user-rpc`) but the SAME kind (`user`): // // dsh-api-session-controller/lib/types/types.d.ts:395-400 // 'user-rpc': { kind: 'user'; rpcId: SessionRequestId; clientTimeZone?: string; principal?: Principal } // // So `kind: 'user'` covers both the local prompt and the browser/RPC prompt. // Without this assertion, a future tightening of the allowlist (e.g. switching // to key-based filtering) could silently stop recalling for web users only. test('G6b: a user-rpc prompt (kind "user", extra fields) IS accepted', () => { const rpc = { role: 'user', source: { kind: 'user', rpcId: 'rpc-123', clientTimeZone: 'Asia/Shanghai', principal: { id: 'other-caller' }, }, content: [{ type: 'text', text: '远程提问:值班室分机号是多少' }], } assert.equal( lastUserText([rpc]), '远程提问:值班室分机号是多少', 'user-rpc carries kind "user" and must be treated as a human prompt', ) // And it still wins over an injected message that arrives after it. const batch = [ rpc, { role: 'user', source: { kind: 'runtime-context' }, content: [{ type: 'text', text: '[wd]' }] }, ] assert.equal(lastUserText(batch), '远程提问:值班室分机号是多少') }) // ── G7: a message with no source at all must yield '' ───────────────────── // // Pinned explicitly rather than left to inference. Reachability note: the // current harness makes this state unreachable — `assertMessageEventShape` // rejects `user/message` whose `source.kind` is not a non-empty string, and it // runs on BOTH the append path (dsh-session/lib/index.js:1089) and the // seed/restore path (:1165). So this is defense in depth against a future // format/migration change, not a live bug. The assertion still earns its keep: // it is the guard that keeps a missing `source` from silently degrading to // "use whatever the last user-role message says". test('G7: kind === undefined (and any non-string kind) yields ""', () => { const cases = [ { name: 'source missing entirely', message: { role: 'user', content: [{ type: 'text', text: 'q' }] } }, { name: 'source is null', message: { role: 'user', source: null, content: [{ type: 'text', text: 'q' }] } }, { name: 'source.kind missing', message: { role: 'user', source: {}, content: [{ type: 'text', text: 'q' }] } }, { name: 'source.kind not a string', message: { role: 'user', source: { kind: 42 }, content: [{ type: 'text', text: 'q' }] }, }, { name: 'source.kind empty string', message: { role: 'user', source: { kind: '' }, content: [{ type: 'text', text: 'q' }] }, }, ] for (const { name, message } of cases) { assert.equal(lastUserText([message]), '', `${name} must yield "" (fail closed)`) } }) test('G7 mutation: dropping the typeof guard admits a sourceless message', () => { // The mutation: `HUMAN_PROMPT_KINDS.has(undefined)` is false, so removing the // typeof check would NOT by itself admit undefined — the real risk is a // future refactor to `kind !== 'user'` negation or a role-only fallback. // Pin the behaviour that the guard exists to produce: const mutated = (messages) => { for (let i = messages.length - 1; i >= 0; i -= 1) { if (messages[i]?.role !== 'user') continue // Mutation: no source check at all. const t = (messages[i].content ?? []) .filter((b) => b?.type === 'text' && typeof b.text === 'string') .map((b) => b.text) .join('\n') .trim() if (t !== '') return t } return '' } const sourceless = [{ role: 'user', content: [{ type: 'text', text: 'q' }] }] assert.equal(lastUserText(sourceless), '', 'guarded implementation rejects it') assert.equal(mutated(sourceless), 'q', 'unguarded implementation admits it — G7 catches this') }) test('V3: systemPrompt.context exposes agent but no step messages', async () => { const { ctx, done } = await app() try { let keys = [] ctx.systemPrompt.context({ name: 'probe:v3', order: 210, text: (context) => { keys = Object.keys(context).sort() return 'probe' }, }) await ctx.systemPrompt.assemble({ agent: { id: 'a', session: {} } }) assert.ok(keys.includes('agent'), `AssembleContext must carry agent; got ${keys.join(',')}`) // The negative assertion is the load-bearing one: if dsh ever adds the step // messages here, recall could move to this hook and this slice's whole // design note would be wrong. for (const forbidden of ['messages', 'input', 'text']) { assert.ok(!keys.includes(forbidden), `AssembleContext must NOT expose ${forbidden}`) } } finally { await done() } }) test('V3b: pre-step injection adds ONE attributed message after a real batch', async () => { const { ctx, dir, done } = await app() try { const mem = applyMemory(ctx, { path: join(dir, 'm.db'), topK: 3 }) await mem.commit({ text: '值班室分机号 4324,值班人 老周', source: 'seed' }) // Uses realBatch, not a hand-built single message: the whole lesson of the // regression above is that a 1- or 2-message batch is a fake batch. const batch = realBatch('值班室分机号 4324 是谁在值班') const decision = await ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: batch, turn: 1, step: 1, signal: new AbortController().signal, }, () => Promise.resolve({ kind: 'enter', messages: batch }), ) assert.equal(decision.kind, 'enter') assert.equal(decision.messages.length, batch.length + 1, 'batch + exactly one injected recall') const injected = decision.messages[batch.length] assert.equal(injected.role, 'user', 'injected context is a user message') assert.equal(injected.source.kind, pluginName) assert.equal(injected.source.form, 'snapshot') assert.ok(injected.content.map((b) => b.text ?? '').join('').includes('4324')) } finally { await done() } }) test('V3b mutation: with recall silenced the batch stays un-augmented', async () => { const { ctx, dir, done } = await app() try { const mem = applyMemory(ctx, { path: join(dir, 'm.db'), topK: 3 }) await mem.commit({ text: '值班室分机号 4324,值班人 老周' }) mem.contextForStep = () => '' const input = createUserMessage({ content: [{ type: 'text', text: '值班室分机号 4324' }], source: { kind: 'user' }, }) const decision = await ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal, }, () => Promise.resolve({ kind: 'enter', messages: [input] }), ) assert.equal(decision.messages.length, 1, 'no injection when recall yields nothing') } finally { await done() } }) test('V3b: a rejected step is passed through untouched', async () => { const { ctx, dir, done } = await app() try { applyMemory(ctx, { path: join(dir, 'm.db') }) const decision = await ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: [], turn: 1, step: 1, signal: new AbortController().signal, }, () => Promise.resolve({ kind: 'reject' }), ) assert.equal(decision.kind, 'reject', 'a rejection must not be converted into an enter') } finally { await done() } }) test('V4: stablemem_commit registers with a schema and executes end to end', async () => { const { ctx, dir, done } = await app() try { const mem = applyMemory(ctx, { path: join(dir, 'm.db') }) const schemas = ctx.tools.schemas() const tool = schemas.find((entry) => entry.name === 'stablemem_commit') assert.ok(tool, 'tool must be visible to the model') assert.ok(tool.description.length > 0, 'tool must carry a description') assert.ok(tool.parameters?.properties?.text, 'tool must declare its text parameter') const result = await ctx.tools.execute({ callId: 'c1', name: 'stablemem_commit', arguments: { text: 'etcd 集群端口 2379', source: 'test' }, signal: new AbortController().signal, }) assert.equal(result.isError, false, JSON.stringify(result)) assert.ok(String(result.value).includes('Committed'), result.value) assert.equal(result.content[0].type, 'text', 'render must produce a text block') assert.equal(mem.store.countBlocks(), 1) } finally { await done() } }) test('V5: disposing the fiber closes the store and releases the file', async () => { const { ctx, dir, done } = await app() const path = join(dir, 'm.db') const mem = applyMemory(ctx, { path }) await mem.commit({ text: 'V5 句柄释放块' }) assert.equal(mem.store.isOpen, true) await ctx.fiber.dispose() assert.equal(mem.store.isOpen, false, 'fiber disposal must close the handle') // Strongest proof: reopen the same file for writing. const { BlockStore } = await import('../lib/store.js') const reopened = new BlockStore({ path }).open() assert.equal(reopened.countBlocks(), 1) reopened.close() rmSync(dir, { recursive: true, force: true }) void done }) test('config.inject:false registers the tool but no pre-step listener', async () => { const { ctx, dir, done } = await app() try { applyMemory(ctx, { path: join(dir, 'm.db'), inject: false }) const input = createUserMessage({ content: [{ type: 'text', text: 'anything' }], source: { kind: 'user' }, }) const decision = await ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal, }, () => Promise.resolve({ kind: 'enter', messages: [input] }), ) assert.equal(decision.messages.length, 1, 'injection must be off') assert.ok(ctx.tools.schemas().some((s) => s.name === 'stablemem_commit'), 'tool still present') } finally { await done() } }) // ── REGRESSION: auto-recall died on a realistic dsh batch ────────────────── // // Found by running the plugin in a real headless profile, not by unit test. // The unit test above passes because it builds a 2-message batch; a REAL step // batch has additional user-role messages appended AFTER the question: // // [ user question ][ runtime-context snapshot ][ skill catalog reminder ] // // `lastUserText` used to return the last `role === 'user'` message, which is the // skill catalog — so every real turn used a list of skill names as its recall // query and matched nothing. Auto-recall was 100% dead in production while all // checkpoints were green. // // The fix: filter by `source.kind`, not by role alone. test('REGRESSION: lastUserText picks the real question out of a 3-part batch', () => { const extracted = lastUserText(realBatch('值班室分机号是多少')) assert.equal( extracted, '值班室分机号是多少', `must return the human question, not the injected context; got ${JSON.stringify(extracted)}`, ) }) test('REGRESSION: auto-recall hits on a realistic batch (the bug was 0 hits)', async () => { const { ctx, dir, done } = await app() try { const mem = applyMemory(ctx, { path: join(dir, 'm.db'), topK: 3 }) await mem.commit({ text: '值班室分机号 4324,值班人 老周', source: 'seed' }) const decision = await ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: realBatch('值班室分机号是多少'), turn: 1, step: 1, signal: new AbortController().signal, }, () => Promise.resolve({ kind: 'enter', messages: realBatch('值班室分机号是多少') }), ) assert.equal(decision.messages.length, 4, '3 batch messages + 1 injected recall') const injected = decision.messages[3] const text = injected.content.map((b) => b.text ?? '').join('') assert.ok( text.includes('4324'), `recall must find the note; injected was ${JSON.stringify(text.slice(0, 100))}`, ) } finally { await done() } }) test('REGRESSION mutation: falling back to "last role=user" must drop recall to 0', async () => { const { ctx, dir, done } = await app() try { const mem = applyMemory(ctx, { path: join(dir, 'm.db'), topK: 3 }) await mem.commit({ text: '值班室分机号 4324,值班人 老周', source: 'seed' }) // The mutation: reintroduce the old extraction rule. const oldRule = (messages) => { for (let i = messages.length - 1; i >= 0; i -= 1) { if (messages[i]?.role !== 'user') continue const t = (messages[i].content ?? []) .filter((b) => b?.type === 'text' && typeof b.text === 'string') .map((b) => b.text) .join('\n') .trim() if (t !== '') return t } return '' } const batch = realBatch('值班室分机号是多少') const mutatedQuery = oldRule(batch) // `recall` became async in phase B group A. Await it: a bare Promise has no // `.length`, and `undefined < undefined` is false, so this assertion would // silently pass while measuring nothing. const mutatedHits = await mem.recall(mutatedQuery) const realHits = await mem.recall(lastUserText(batch)) // Numbers on the record: the old rule must lose every hit. assert.ok( mutatedHits.length < realHits.length, `mutation must lose hits: old-rule=${mutatedHits.length}, fixed=${realHits.length}`, ) assert.equal(mutatedHits.length, 0, `old rule must score 0 hits, got ${mutatedHits.length}`) assert.ok(realHits.length >= 1, 'the fixed rule must find at least one') } finally { await done() } })