From 114fae84aaa96b37ab16cf42a0e6ba6b8b472ac5 Mon Sep 17 00:00:00 2001 From: jianf <2198972886@qq.com> Date: Fri, 30 Oct 2026 09:44:42 +0800 Subject: [PATCH] =?UTF-8?q?feat(memory):=20MVP=20=E9=98=B6=E6=AE=B5=20A=20?= =?UTF-8?q?=E5=88=87=E7=89=87=20=E2=80=94=E2=80=94=20=E5=9D=97=E8=BF=9B=20?= =?UTF-8?q?dsh=E3=80=81=E5=8F=AC=E5=9B=9E=E3=80=81=E8=BF=9B=20prompt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 docs/feasibility.md §7.1 阶段 A 落地最小可验证切片， 刻意不引入 embedding、不碰 L1 淘汰。 落地内容 - lib/store.js BlockStore：node:sqlite 建 memory_blocks / memory_block_edges / graph_centroid 三表 + FTS5 索引。 内容派生 id（blk_+sha256[:12]，移植 distill.BlockID）； putBlock 走「空值不覆盖」，重复 commit 幂等。 - lib/recall.js FTS5 bm25 + 精确串布尔置顶（移植 SymbolScore/fusion.go:427）、 单字符不成符号（abstain.go:115）、前缀特异性 tie-break、 CJK 2..4 字滑窗兜底（移植 cjkWindows/fusion.go:240）。 - lib/index.js StableMem service + systemPrompt.context 常驻索引 + agent/pre-step 按本轮输入召回注入 + stablemem_commit 工具。 - scripts/checkpoints.js V1–V5 + 两条变异，对真实 cordis 服务跑。 - test/ 26 个测试（store / recall / plugin wiring / real loader / isolate）。 - scripts/link-dsh.sh 把全局 dsh 包软链进 node_modules（插件本身零依赖）。 三个实测出来的 dsh 约束（都是先红了才发现的） 1. cordis service 不能用 JS #private 字段 —— ctx. 是 proxy， 访问私有字段抛 "Cannot read private member"。状态必须放普通属性。 2. isolate() 不隔离 prompt context 名与工具名（只隔离 service 查找）—— 所以工具名做成可配置，prompt context 名由 db 路径派生以保唯一。 3. systemPrompt.getContextOrder() 是分配器不是常量表，用命名槽位。 验证 - npm test → 26/26 通过 - npm run check → 8/8 checkpoint 通过 V1 sqlite/WAL/100 写回读；V2 能力分层（tier1 30/30、tier2 20/20， tier3/4 作为已知缺口显式记录）；V3 AssembleContext 仅 [agent]； V3b 注入落进批次且带 source.kind；V3b† 静默召回则判据红； V4 工具 schema+render 端到端；V5 fiber dispose 释放句柄并可重开； A5† 退化成包含匹配则区分度消失。 阶段 A exit criteria 全部满足（含 V3/V3b 分工结论写入 README）。 参考仓 /home/program/TrueAgent 未改动。 --- .gitignore | 8 + README.md | 270 ++++++++++++++++++++++++++--- lib/index.js | 339 ++++++++++++++++++++++++++++++++++++ lib/recall.js | 380 +++++++++++++++++++++++++++++++++++++++++ lib/store.js | 311 +++++++++++++++++++++++++++++++++ package.json | 34 ++++ scripts/checkpoints.js | 340 ++++++++++++++++++++++++++++++++++++ scripts/link-dsh.sh | 52 ++++++ test/loader.test.js | 137 +++++++++++++++ test/plugin.test.js | 231 +++++++++++++++++++++++++ test/store.test.js | 201 ++++++++++++++++++++++ 11 files changed, 2283 insertions(+), 20 deletions(-) create mode 100644 lib/index.js create mode 100644 lib/recall.js create mode 100644 lib/store.js create mode 100644 package.json create mode 100644 scripts/checkpoints.js create mode 100755 scripts/link-dsh.sh create mode 100644 test/loader.test.js create mode 100644 test/plugin.test.js create mode 100644 test/store.test.js diff --git a/.gitignore b/.gitignore index 8c8cbe7..a2d362e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,9 @@ +# pi session transcripts .pi-sessions/ + +# plugin runtime data (created by defaultPath()) +.dsh-memory/ + +# dev-only symlinks to the globally installed dsh packages. +# `scripts/link-dsh.sh` recreates them; the plugin itself has no dependencies. +node_modules/ diff --git a/README.md b/README.md index b815139..e8617a9 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,263 @@ # HologramAgent-DSH-Memory-Port -评估 HologramAgent 三层记忆系统移植为 dsh 插件的可行性（调研/评估，不产出实现） +把 HologramAgent 的记忆系统移植到 dsh 插件。本仓分两部分： -## 报告 +1. **调研报告**（已交付） + - [`docs/feasibility.md`](docs/feasibility.md) — 三层记忆移植可行性评估（结论：**只能做子集**，L2+L3 可移植，L1 抓不住） + - [`docs/shadow-context.md`](docs/shadow-context.md) — 影子上下文方案核实（结论：路线② preset 可行，路线① 只是看着可行） +2. **MVP 阶段 A 实现**（本文件以下内容）— 1–2 天切片：块能进 dsh、能被召回、能进 prompt，**不引入 embedding** -→ [`docs/feasibility.md`](docs/feasibility.md) +--- -## 结论摘要 +# dsh-memory（阶段 A） -**只能做子集**（四选一中的第三档）。 +HologramAgent-style **block memory** for [dsh](https://github.com/deepseek-ai/deepseek-harness), +as a dsh plugin. This is **phase A** of the MVP described in +[`docs/feasibility.md`](docs/feasibility.md) §7.1 — the smallest slice that proves +the whole chain works: -| 层 | 可移植性 | 瓶颈 | +``` +commit a block → store it durably → recall it → get it into the model's prompt +``` + +Deliberately **not** included: embedding providers, vectors, the L1 eviction flow, +media blocks. Each is blocked or out of scope for a 1–2 day slice; see +[What this slice does not do](#what-this-slice-does-not-do). + +--- + +## ★ Read this before touching injection + +dsh has two injection hooks that look interchangeable and are not. Getting this +wrong silently corrupts the conversation log. + +| hook | receives this turn's input? | durable side effect | |---|---|---| -| L1 context 热上下文 | ≈20% | dsh 的上下文压缩是「摘要替换区间」，不是「按 query 相关性淘汰 + 迁移到冷层 + 按原时间戳迁回」；`CompactionTrigger` 接口拿不到 query，`MessageBase` 无时间戳 | -| L2 coldcontext 冷文档 | ≈80% | 存储/召回/注入都能落，缺「回收站式 `Peek→回灌→Remove`」 | -| L3 stablemem 图记忆 | ≈85% | 结构完全可映射（`node:sqlite` 实测可用），受限于无 embedding 服务 | +| `ctx.systemPrompt.context({ text })` | **No** — `AssembleContext` is `{scope, signal}` + a merged-in `agent` | appends a runtime-context snapshot | +| `ctx.on('agent/pre-step')` | **Yes** — payload carries `decision.messages` | **every message you return is appended as a durable `user/message`** | -**可整体搬运的资产**：`MemoryBlock` 值对象、确定性 ID 派生、仲裁/拒答/符号检索算法——这三块与宿主无关。 +Evidence, verified in the installed harness (not inferred): -**推荐落地范围**：L2 + L3 子集，L1 交给 dsh 自己的 compaction。 -工作量（含判据与实测）**36–53 人天**；压到 MVP（L3 + 工具 + 注入面）**19–25 人天**。 +- `dsh-system-prompt/lib/types/index.d.ts:37-44` — `AssembleContext` has only `scope` and `signal` + (`agent` is added by `dsh-agent`'s declaration merging; still no messages). +- `dsh-agent-loop/lib/index.js:902-934` — `preStep()` claims the inbox, calls + `systemPrompt.assemble()`, then runs the `agent/pre-step` waterfall with those messages. +- `dsh-agent-loop/lib/index.js:1066-1073` — the loop then does + `session.append('user/message', message, { surfaceOp: 'append' })` **for each returned message**. -**最小第一步切片**（1–2 人天）：插件骨架 + `Service` 注册 + `node:sqlite` 建 `memory_blocks` -+ FTS5 召回 + `agent/pre-step` 注入 + 一个最小工具，**全程不引入 embedding 依赖**。 -详见报告 §7。 +**Consequences, and how this plugin uses them:** -## 取证方式 +- The **input-independent** index (a block count, "memory exists, here's the tool") + goes through `systemPrompt.context`. It cannot see the current question, so it + must not pretend to be a recall result. +- **Per-turn recall** goes through `agent/pre-step`, because that is the only hook + that sees the input. Recall is `prepend: true` so it runs before listeners that + may want the final word. +- Because pre-step output is **durable**, injected recall becomes part of the + conversation history. That is acceptable here (it is honest: the model saw it), + but it means this plugin is **not** a shadow-context mechanism. For that + distinction see [`docs/shadow-context.md`](docs/shadow-context.md) §1.3. -- HologramAgent 源码与 `docs/zh/` 设计文档：**全程只读，未做任何修改** -- dsh 接口：只读 `lib/types/*.d.ts`（权威接口），不从 `lib/*.js` 实现倒推 -- 关键环境结论均有实测（`node:sqlite` + FTS5、`onnxruntime-node` 缺失、`libonnxruntime.so` 位置） -- 每条断言标注文件路径与符号名；仅三处标注「推断」：`turn`≈`FrameTS` 映射、`Intl.Segmenter` 中文分词质量、人天估算 +There is **no** hook for "the context is about to overflow, please evict +something", and no waterfall that can rewrite a loop-built request +(`llm/stream` receives a deep-frozen `GenerateOptions` for loop requests — +`dsh-llm/lib/types/index.d.ts:37-42`). + +--- + +## Install + +```bash +# from this repo +npm install # nothing to install; dsh packages are peers +``` + +The plugin needs `node:sqlite` (built into Node ≥ 22.5). On Node 22.23 it works +behind an `ExperimentalWarning` only — no `--experimental-sqlite` flag. + +Mount it in a dsh composition: + +```yaml +- id: memory + name: dsh-memory + config: + path: /path/to/memory.db # optional; defaults to /.dsh-memory/memory.db + topK: 3 # optional; recall hits injected per turn + inject: true # optional; false = tool only, no auto-recall + toolName: stablemem_commit # optional; must be unique per tool layer +``` + +## Use + +The model gets one tool: + +``` +stablemem_commit({ text, source?, scene? }) +``` + +and, once at least one block exists, a constant prompt context line telling it +that durable memory is available. + +Recall runs automatically each turn: the last user message is used as the query, +FTS5 + exact-symbol pinning selects up to `topK` blocks, and they are injected as +one attributed user message: + +``` +[Memory · dsh-memory] Recalled from durable block memory — these are PAST notes, not the current conversation. +[1] (exact 1.01) 值班室分机号 4324，值班人 老周 +``` + +Programmatic use (another plugin, or a preset subtree): + +```js +ctx.stablemem.commit({ text: 'etcd 集群端口 2379', source: 'ops' }) +ctx.stablemem.recall('etcd 端口', 5) +ctx.stablemem.link({ sourceKind: 'block', sourceId: a, targetKind: 'block', targetId: b, type: 'supersedes' }) +``` + +--- + +## Schema + +Three tables, mirroring HologramAgent's (`internal/memory/graph.go:2577`, `:320`, +`centroid_store.go:58`), restricted to the text modality: + +| table | purpose | notes | +|---|---|---| +| `memory_blocks` | the block payload | `text_content` is the body; `source` / `scene` / `semantic_type` mirror HA's columns | +| `memory_block_edges` | structural + semantic edges | unique per `(source_kind, source_id, target_kind, target_id, type)` | +| `graph_centroid` | vector-space centering | **created empty** — phase B writes it; creating it now avoids a migration | +| `memory_blocks_fts` | FTS5 index over `text_content` | `unicode61 remove_diacritics 2` | + +`memory_blocks.id` is content-addressed +(`blk_` + `sha256(text)[:12]`, ported from `distill.BlockID`, +`internal/memory/distill/blocks.go:99`), so re-committing identical text is +idempotent and never blanks existing metadata. + +--- + +## Recall design, and why there are no vectors yet + +This slice ports HologramAgent's **symbol path**, not its vector path. + +`docs/zh/recall-capability-tiers.md` measures four query shapes. The one that +works *best* in production is 形态 1 (queries carrying an exact token: port, IP, +path, version) — which is pure symbol matching. The one that needs vectors is +形态 3 (pure generic description), and it is the weakest even *with* vectors +(target block ranked 389/2686). So dropping embeddings here costs the tier that +was already marginal, not the tier that carries the product. + +Scoring follows HA exactly where it matters: + +- An **exact numeric hit** is a *boolean pin* (`1.0 + lexical*0.01`), never a + weight. Folding it into a weighted sum makes it lose to noise — the regression + documented at `internal/memory/fusion.go:43-49`. +- A matched **word** is a coverage signal, not an exact hit. +- Single-character symbols are rejected (`abstain.go:115` "单字不成符号"); + without that gate a query like `billing0` yields a bare `0` that "exactly" + matches every block containing a zero. +- A matched word prefers blocks where it is **not a prefix of a longer word** + (`billing` should beat `billing2`), loosely mirroring + `isCrossWordTailMatch` (`distill/split.go:383`). + +Two documented gaps, asserted rather than hidden (see checkpoint V2): + +| gap | why | when | +|---|---|---| +| 纯泛化描述 queries ("公网地址是什么") | needs vectors | phase B | +| 「库里没有」detection (abstention) | needs either vectors or a real CJK tokenizer | phase B/C | +| CJK segmentation | `unicode61` keeps a CJK run as ONE token; compensated by a 2..4-char sliding-window fallback ported from `cjkWindows` (`fusion.go:240`) | phase C | + +--- + +## Three harness constraints this slice discovered + +Each was found by a failing test, not by reading types — they are recorded here +because they will bite anyone writing a second dsh service plugin. + +1. **A cordis service must not use JS `#private` fields.** + `ctx.stablemem` is a **proxy**, not the instance you constructed, and private + fields are bound to the declaring object: + `Cannot read private member #store from an object whose class did not declare it`. + Keep service state in ordinary properties. + +2. **`isolate()` does not partition prompt-context names or tool names.** + It partitions *service* lookups. Two instances in one realm still collide on + `systemPrompt.context({name})` (`dsh-system-prompt/lib/index.js:204`) and + `tools.register` unless the names differ. Hence the configurable `toolName`, + and the prompt-context name derived from the database path + (`memory:index:`), which stays unique and stable. + +3. **`ctx.systemPrompt.getContextOrder()` is an allocator, not a free constant.** + Prefer a named slot (`WEB_SURFACE`) over a magic number; the numeric constants + are private to the package. + +--- + +## What this slice does not do + +| not done | why | where it lands | +|---|---|---| +| embeddings / vectors | no embedding service in dsh (`docs/feasibility.md` §5.1) | phase B | +| evicting L1 context into memory | dsh's compaction is summary-replacement, and `CompactionTrigger` carries no query (§5.3) | replaced by shadow-context + preset | +| media blocks | dsh attachment is CAS; HA requires bytes inside the block (§5.4) | not planned | +| cross-session / cross-agent memory | one sqlite file per `config.path` | phase D | +| abstention | needs vectors or a real tokenizer | phase B/C | + +--- + +## Verify + +```bash +npm test # 26 tests: store, recall, plugin wiring, real loader, isolation +npm run check # 8 checkpoints with human-readable evidence (V1–V5 + mutations) +``` + +`npm run check` prints one line of evidence per checkpoint. The mapping to +`docs/feasibility.md` §7.1: + +| checkpoint | proves | +|---|---| +| **V1** | `node:sqlite` opens, WAL applies, 100 writes round-trip | +| **V2** | recall quality per capability tier: tiers 1–2 must pass, tiers 3–4 pinned as gaps | +| **V3** | `systemPrompt.context` exposes `agent` and **cannot** see this step's messages | +| **V3b** | `agent/pre-step` injection lands in the accepted batch with our `source.kind` | +| **V3b†** | mutation — silencing recall makes V3b fail (the test can turn red) | +| **V4** | `tools.register` schema + `render` contract, executed end to end | +| **V5** | fiber disposal closes the handle; the file is reopenable | +| **A5†** | mutation — containment-only recall loses discrimination (why FTS is primary) | + +### Expectations from a real run + +``` + ✔ V1 journal=wal, rows=100, mid-batch row=blk_cad13afe6824, file on disk + ✔ V2 tier1(exact) 30/30 = 100% · tier2(domain) 20/20 = 100% + · known gaps: 泛化查询→0 hit, 库里没有→0 hit (需向量，phase B) + ✔ V3 AssembleContext keys = [agent] — no messages/input field + ✔ V3b batch 2 msgs; injected kind=dsh-memory, carries "4324" + ✔ V4 1 tool(s) visible; execute → "Committed block blk_0be019e8b0e4." + ✔ V5 fiber dispose → handle closed; reopened same file successfully + +8/8 checkpoints passed +``` + +--- + +## Ported from HologramAgent + +Source is read-only reference (`/home/program/TrueAgent`); nothing is vendored. + +| this repo | HologramAgent | +|---|---| +| `lib/store.js` `BlockStore` | `memory.GraphDB.PutMemoryBlocks` (`internal/memory/block.go:203`) | +| `lib/store.js` `blockId` | `distill.BlockID` (`internal/memory/distill/blocks.go:99`) | +| `lib/recall.js` `querySymbols` | `memory.QuerySymbols` (`internal/memory/fusion.go:145`) | +| `lib/recall.js` scoring / pin | `memory.SymbolScore` (`fusion.go:284`), pin at `:427` | +| `lib/recall.js` `isNumericSymbol` | `fusion.go:316` | +| `lib/recall.js` `cjkNeedles` | `cjkWindows` (`fusion.go:240`) | +| `lib/recall.js` single-char gate | `AbstainCheck` guard (`abstain.go:115`) | +| `lib/index.js` `renderIndex` header | `crossModalMarkdown` (`internal/agent/core/crossmodal.go:256`) | + +## License + +MIT diff --git a/lib/index.js b/lib/index.js new file mode 100644 index 0000000..27d4cb6 --- /dev/null +++ b/lib/index.js @@ -0,0 +1,339 @@ +/** + * dsh-memory — block memory for dsh, entry point. + * + * This is **phase A** of `docs/feasibility.md` §7.1: prove that a block can go + * into dsh, be recalled, and reach the prompt, with **no embedding provider**. + * + * ## The one thing to read before touching injection + * + * dsh has two obvious-looking injection points, and only one of them can reach + * the current turn's input. + * + * | hook | gets this turn's input? | durable? | + * |---|---|---| + * | `ctx.systemPrompt.context({ text })` | no — `AssembleContext` is `{scope, signal}` plus a merged-in `agent` | yes (appended as a runtime-context snapshot) | + * | `ctx.on('agent/pre-step')` | yes — receives `decision.messages` | yes (the loop appends whatever you return) | + * + * Verified in the harness, not assumed: + * · `dsh-system-prompt/lib/types/index.d.ts:37-44` — `AssembleContext` has only scope+signal + * · `dsh-agent-loop/lib/index.js:902-934` — `preStep` claims the inbox, assembles the + * prompt, then runs the `agent/pre-step` waterfall with those messages + * · `dsh-agent-loop/lib/index.js:1066-1073` — **every returned message is appended + * as a durable `user/message`**, so "shadow" injection there is not shadow: it + * becomes conversation history (see `docs/shadow-context.md` §1.3) + * + * Therefore: the input-independent index uses `systemPrompt.context`; per-turn + * recall uses `agent/pre-step`, prepended so it runs before listeners that may + * want the final word. + * + * @module dsh-memory + */ + +import { Service } from '@deepseek-ai/cordis' +import { createUserMessage } from '@deepseek-ai/dsh-llm' +import { defineTool } from '@deepseek-ai/dsh-tools' +import { recallBlocks, renderRecall } from './recall.js' +import { BlockStore } from './store.js' + +/** Cordis plugin name, also used as the `source.kind` of injected context. */ +export const name = 'dsh-memory' + +/** + * Services this plugin needs. + * + * `systemPrompt` and `tools` are both present in any standard dsh composition + * (`dsh-base/cordis.patch.yml`), so requiring them narrows nothing. + */ +export const inject = ['systemPrompt', 'tools'] + +/** + * Plugin configuration. + * + * @typedef {object} Config + * @property {string} [path] sqlite file path; defaults to `/.dsh-memory/memory.db` + * @property {number} [topK] recall hits injected per turn (default 3) + * @property {boolean} [inject] false = register only the tool (default true) + * @property {boolean} [wal] sqlite WAL mode (default true) + * @property {string} [toolName] model-facing tool name (default `stablemem_commit`). + * Tool names are unique per tool layer and `isolate()` does not partition them + * (`ToolRuntime.register` throws on a duplicate), so a preset that mounts a + * second instance must give it a distinct name. + */ + +/** + * The `stablemem` service: durable block storage plus recall. + * + * Registered as `ctx.stablemem` so other plugins can use it without going through + * tools, and so a preset subtree can `isolate: { stablemem: true }` it later + * (docs/shadow-context.md §5.2). + */ +export class StableMem extends Service { + /** + * ★ These are plain fields, NOT `#private`. + * + * cordis resolves a service through a **proxy** (`ctx.stablemem !== instance`), + * and JS private fields are bound to the declaring object, so any method that + * touches `this.#store` throws + * `Cannot read private member #store from an object whose class did not declare it` + * when reached through `ctx.`. Verified against the real Context. + * A cordis service must therefore keep its state in ordinary properties. + * + * @type {BlockStore} + */ + store + /** @type {number} */ + topK + + /** + * @param {import('@deepseek-ai/cordis').Context} ctx + * @param {{ store: BlockStore, topK?: number }} options + */ + constructor(ctx, { store, topK = 3 }) { + super(ctx, 'stablemem') + this.store = store + this.topK = topK + // Release the sqlite handle with the owning fiber, so a plugin unload does + // not leak the file lock (checkpoint V5). + ctx.effect(() => () => this.store.close(), 'dsh-memory.store') + } + + /** + * Commit one text block. + * @param {{ text: string, source?: string, scene?: string, semanticType?: string }} input + * @returns {{ id: string, created: boolean }} + */ + commit(input) { + return this.store.putBlock(input) + } + + /** + * Link two blocks. + * @param {object} edge + * @returns {{ created: boolean }} + */ + link(edge) { + return this.store.addEdge(edge) + } + + /** + * Recall hits for one query. + * @param {string} query + * @param {number} [topK] + */ + recall(query, topK = this.topK) { + return recallBlocks(this.store, query, { topK }) + } + + /** + * Text to inject for this step, or '' when nothing is relevant. + * + * The query is the **last user-role text** in the batch: `decision.messages` is + * the batch admitted for this step, and steering appends rather than replaces, + * so the last user message is the actual prompt. + * + * @param {ReadonlyArray<{role: string, content: ReadonlyArray<{type: string, text?: string}>}>} messages + * @returns {string} + */ + contextForStep(messages) { + const query = lastUserText(messages) + if (query === '') return '' + const hits = this.recall(query) + if (hits.length === 0) return '' + return renderRecall(hits, name) + } + + /** + * Constant index text for `systemPrompt.context`. + * + * Deliberately input-independent: that hook cannot see the current turn's text + * (module note above). A count is still worth its tokens — the model learns that + * durable memory exists without paying a per-turn recall cost. + * + * @returns {string} + */ + renderIndex() { + const count = this.store.countBlocks() + if (count === 0) return '' + return ( + `[Memory · ${name}] ${count} durable note(s) in block memory. ` + + 'Relevant ones are recalled into your timeline automatically; ' + + 'record new facts with the `stablemem_commit` tool.' + ) + } +} + +/** + * Pull the last user-role text out of a message batch. + * + * Joins only text blocks, so a batch that opens with a tool result still yields + * the real prompt. + * + * @param {ReadonlyArray<{role: string, content: ReadonlyArray<{type: string, text?: string}>}>} messages + * @returns {string} + */ +export function lastUserText(messages) { + if (!Array.isArray(messages)) return '' + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index] + if (message?.role !== 'user') continue + const text = (message.content ?? []) + .filter((block) => block?.type === 'text' && typeof block.text === 'string') + .map((block) => block.text) + .join('\n') + .trim() + if (text !== '') return text + } + return '' +} + +/** + * Register the `stablemem_commit` tool. + * + * `output.schema` + `output.render` are mandatory in dsh + * (`dsh-tools/lib/types/index.d.ts:117`), unlike HologramAgent's + * description-only `toolDefWith` (`internal/agent/core/tooldefs.go`). + * + * @param {import('@deepseek-ai/cordis').Context} ctx + * @param {StableMem} mem + * @param {string} toolName model-facing tool name + * @returns {() => void} disposer + */ +function registerTools(ctx, mem, toolName) { + return ctx.tools.register( + defineTool({ + name: toolName, + description: + 'Store one durable fact or note in long-term block memory. Use it for ' + + 'stable facts worth recalling later (addresses, ports, paths, decisions, ' + + 'preferences). Keep each note short and self-contained: recall matches ' + + 'on the note text itself, so include the subject, not just the value.', + parameters: { + text: { + type: 'string', + description: 'The note. Self-contained: include subject, not just the value.', + required: true, + }, + source: { + type: 'string', + description: 'Optional origin label (channel, tool, file).', + }, + scene: { + type: 'string', + description: 'Optional scene key for prefix-matching recall.', + }, + }, + output: { + schema: { type: 'string' }, + render: (_args, value) => [{ type: 'text', text: value }], + }, + // Commits are independent and idempotent (content-addressed id), so they + // regroup safely with sibling calls. + isConcurrencySafe: () => true, + async execute(args) { + const { id, created } = mem.commit({ + text: args.text, + source: args.source ?? 'tool', + scene: args.scene ?? '', + }) + return `${created ? 'Committed' : 'Already present'} block ${id}.` + }, + }), + ) +} + +/** + * Apply the plugin. + * + * @param {import('@deepseek-ai/cordis').Context} ctx + * @param {Config} [config] + */ +export function apply(ctx, config = {}) { + const path = config.path ?? defaultPath() + const store = new BlockStore({ path, wal: config.wal ?? true }).open() + const mem = new StableMem(ctx, { store, topK: config.topK ?? 3 }) + + // Injection surface 1 — constant index. Right for input-independent text. + // + // ★ The prompt-context NAME must be unique per prompt layer, and `isolate()` + // does NOT partition prompt names: `SystemPrompt` keeps one `NamedEntries` per + // layer and throws `prompt context "..." is already registered` + // (dsh-system-prompt/lib/index.js:204). So a preset mounting a second instance + // must not reuse the literal name — deriving it from the database path keeps + // two instances in one realm distinguishable, and is stable across restarts. + ctx.systemPrompt.context({ + name: `memory:index:${instanceKey(path)}`, + order: ctx.systemPrompt.getContextOrder('WEB_SURFACE'), + interpolate: false, + text: () => mem.renderIndex(), + }) + + // Injection surface 2 — per-turn recall. The only hook that sees the input. + if (config.inject ?? true) { + ctx.on( + 'agent/pre-step', + async (payload, next) => { + const decision = await next() + if (decision.kind === 'reject' || payload.signal.aborted) return decision + const injected = mem.contextForStep(decision.messages) + if (injected === '') return decision + return { + ...decision, + messages: [ + ...decision.messages, + createUserMessage({ + content: [{ type: 'text', text: injected }], + // `form: 'snapshot'` matches how dsh-time-context labels its own + // injected context (dsh-time-context/lib/index.js:238). `kind` is + // this plugin's own — MessageSourceMap is merge-extensible + // (dsh-llm/lib/types/message.d.ts:96-105) and user messages accept + // unknown kinds (dsh-session/lib/index.js:1216). + source: { + kind: name, + form: 'snapshot', + sections: [{ name, text: injected }], + }, + }), + ], + } + }, + // Prepend: recall should be visible to listeners that run later. + { prepend: true }, + ) + } + + registerTools(ctx, mem, config.toolName ?? 'stablemem_commit') + + return mem +} + +/** + * A stable, filesystem-safe key distinguishing one plugin instance. + * + * Used to keep prompt-context names unique when a realm hosts more than one + * instance (a preset isolating `stablemem` but not the prompt layer). Derived + * from the database path so it survives restarts and stays readable in + * `/dump-config`. + * + * @param {string} path the configured database path + * @returns {string} + */ +export function instanceKey(path) { + let hash = 0 + for (const char of String(path)) hash = (hash * 31 + char.codePointAt(0)) >>> 0 + return hash.toString(36) +} + +/** + * Default database location. + * + * Under the process cwd so a profile sees one memory per workspace, matching how + * `dsh-storage-json` refuses to guess a location by default + * (`dsh-storage-json/lib/types/index.d.ts:14-18`). Overridable via `config.path`. + * + * @returns {string} + */ +export function defaultPath() { + return `${process.cwd()}/.dsh-memory/memory.db` +} + +export default { name, inject, apply, StableMem } diff --git a/lib/recall.js b/lib/recall.js new file mode 100644 index 0000000..de195ae --- /dev/null +++ b/lib/recall.js @@ -0,0 +1,380 @@ +/** + * Block recall without embeddings: FTS5 lexical match + exact-symbol pinning. + * + * ## Why this exists at all + * + * HologramAgent's primary recall path is vector cosine over `memory_blocks.vector` + * (`internal/memory/block_recall.go:78`). This slice deliberately has no embedding + * provider (docs/feasibility.md §5.1, P0 blocker), so it ports only the *second* + * half of the fusion: the symbol path. + * + * That is not a placeholder. `docs/zh/recall-capability-tiers.md` measures which + * query shapes actually work, and the strongest one is "带精确串" (queries with an + * exact token: port, IP, version, path) — which is exactly the symbol path. The + * weakest is "纯泛化描述", which needs vectors. So this slice covers the tier that + * empirically works best, and leaves the tier that needs phase B. + * + * ## The two signals, and why they are merged the way they are + * + * 1. **Exact symbol hit** — a numeric/Latin token from the query appears verbatim + * in the block text. HologramAgent treats this as a BOOLEAN that pins the + * candidate to the top, NOT as a weight: + * `Score = 1.0 + VectorHit*0.01` (internal/memory/fusion.go:427). + * The reason is measured: if you fold a 1.0 symbol hit into + * `0.7*vector + 0.3*symbol`, it becomes 0.30 and *loses* to a noise candidate + * scoring 0.534 (fusion.go:43-49 documents exactly that regression). + * + * 2. **FTS5 bm25 lexical rank** — everything else. + * + * ## Known limitation, deliberately inherited + * + * `unicode61` does not segment Chinese: a CJK run becomes one token, so + * `MATCH '分机号'` will not find `值班室分机号 4324` (the block's run is + * `值班室分机号`, a different single token). HologramAgent has the same shape of + * problem and solves it with jieba (`internal/memory/fusion.go:145`); this slice + * has no jieba (`docs/feasibility.md` §5.5). + * + * The CJK containment fallback below recovers the common case (the query is a + * substring of the text) without a dictionary — the same *effect* as + * HologramAgent's sliding-window path (`cjkWindows`, fusion.go:240) at coarser + * granularity, and it only runs when FTS found nothing. What it cannot do is + * segmentation- or synonym-aware matching; that is phase B/C's job. + * + * @module dsh-memory/recall + */ + +/** + * Extract high-information symbols from a query. + * + * Port of `memory.QuerySymbols` (internal/memory/fusion.go:145) restricted to the + * parts that need no dictionary: numeric runs and Latin words. The jieba half and + * the CJK sliding window are intentionally absent (see the module note). + * + * @param {string} query + * @returns {string[]} lowercased, de-duplicated symbols + */ +export function querySymbols(query) { + const out = [] + const seen = new Set() + const add = (value) => { + const symbol = value.trim() + // Single-character symbols are not symbols. Ported from AbstainCheck's guard + // (internal/memory/abstain.go:115-116, "单字不成符号"): a bare `0` would + // otherwise "exactly" match every block containing a zero and pin noise to + // the top. The V2 corpus caught precisely this via `billing0`. + if ([...symbol].length < 2 || seen.has(symbol)) return + seen.add(symbol) + out.push(symbol) + } + const text = String(query ?? '') + // Numeric runs. Regexes copied verbatim from HologramAgent + // (internal/memory/fusion.go:76-79) so this slice extracts the SAME symbols: + // symbolNumericRe = \d+(?:[./]\d+)*(?:\.\d+)?%?|[vV]\d+(?:\.\d+)+ + // symbolLatinRe = [A-Za-z][A-Za-z0-9_-]{2,} + // The `{2,}` on the Latin rule is load-bearing: without it `billing0` also + // yields a bare `0`, which then "exactly" matches every block containing any + // zero — the regression the V2 corpus caught. + for (const match of text.matchAll(/\d+(?:[./]\d+)*(?:\.\d+)?%?|[vV]\d+(?:\.\d+)+/g)) add(match[0]) + for (const match of text.matchAll(/[A-Za-z][A-Za-z0-9_-]{2,}/g)) add(match[0].toLowerCase()) + return out +} + +/** + * True when a symbol carries a digit (or is a `v`-prefixed version). + * + * Port of `isNumericSymbol` (internal/memory/fusion.go:316). It matters because + * HologramAgent only trusts "zero hits ⇒ abstain" when the query held an exact + * symbol (`internal/memory/abstain.go:79-86`): a pure-CJK zero-hit cannot be told + * apart from a tokenizer failure, so abstaining there would be wrong. + * + * @param {string} symbol + * @returns {boolean} + */ +export function isNumericSymbol(symbol) { + if (!symbol) return false + if (/\d/.test(symbol)) return true + // v2.31.5-style: leading v plus at least one digit. + return /^v\d/i.test(symbol) +} + +/** + * Build an FTS5 MATCH expression from a plain query. + * + * Every term is double-quoted so user text cannot inject FTS5 operators + * (`AND`, `NEAR`, `*`, `:`). Terms are OR-ed: recall should be generous, and the + * fusion step below does the ranking. Returns null when nothing usable remains — + * an empty MATCH throws in SQLite. + * + * @param {string} query + * @returns {string | null} + */ +export function buildMatchExpression(query) { + const terms = [] + const seen = new Set() + const add = (term) => { + if (term === '' || seen.has(term)) return + seen.add(term) + terms.push(`"${term.replace(/"/g, '""')}"`) + } + const text = String(query ?? '') + + // 1. The symbols, verbatim. These are the terms that reliably retrieve, + // because they are what FTS actually tokenised in the block text + // (`billing0服务` is ONE unicode61 token; `billing0` alone is not a token, + // so the *word* is what must be matched, not the raw query chunk). + for (const symbol of querySymbols(text)) add(symbol.toLowerCase().replace(/[.]/g, ' ')) + + // 2. Digit groups from the query: `8499`, `2379`, `13010`. Length-gated for + // the same reason querySymbols is — a lone `0` (from `billing0`) is not a + // symbol, and admitting it turns the MATCH into an everything-query. + for (const match of text.matchAll(/\d+(?:[./]\d+)*/g)) { + if (match[0].length >= 2) add(match[0]) + } + + // 3. CJK runs, emitted whole. unicode61 keeps a run as ONE token, so quoting + // the run matches only identical runs — narrow but correct. Broader CJK + // matching is the containment fallback's job (see recallBlocks). + for (const match of text.matchAll(/[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/g)) add(match[0]) + + if (terms.length === 0) { + // Nothing tokenisable: let the caller fall through to containment rather + // than issuing an empty MATCH (which SQLite rejects). + return null + } + return terms.join(' OR ') +} + +/** + * Sliding-window needles for a CJK query run. + * + * Port of `cjkWindows` (internal/memory/fusion.go:240) at the same 2..4 width. + * It exists because this slice has no dictionary tokenizer, so "which substring + * of the question names the subject" has to be approximated by trying every + * window inside the plausible width range. + * + * Non-overlapping prefixes only would be cheaper but loses `分机号` inside + * `值班室分机号` when the question is `值班室分机号是多少`; taking every window + * costs a hash probe per block and is still bounded (n·3 needles). + * + * @param {string} text + * @returns {string[]} unique windows, longest first + */ +export function cjkNeedles(text) { + const longestFirst = [] + const seen = new Set() + for (const run of String(text ?? '').match(/[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/g) ?? []) { + const chars = [...run] + // 4..2 as well as the whole run: the whole run is the most specific needle. + for (const width of [chars.length, 4, 3, 2]) { + if (width < 2 || width > chars.length) continue + for (let start = 0; start + width <= chars.length; start += 1) { + const needle = chars.slice(start, start + width).join('') + if (seen.has(needle)) continue + seen.add(needle) + longestFirst.push(needle) + } + } + } + return longestFirst +} + +/** + * Every start offset of `needle` in `haystack` (non-overlapping scan). + * @param {string} haystack already lowercased + * @param {string} needle already lowercased + * @returns {number[]} + */ +function allIndexesOf(haystack, needle) { + if (needle === '') return [] + const out = [] + let from = 0 + for (;;) { + const at = haystack.indexOf(needle, from) + if (at === -1) return out + out.push(at) + from = at + 1 + } +} + +/** + * Recall blocks for a query. + * + * Ordering contract (mirrors `RecallBlocksWithArbitration`, + * internal/memory/block_recall.go:229 — full scoring first, truncation last): + * + * 1. FTS5 candidates over a generous pool + * 2. exact-symbol boolean pin — pinned candidates first, ordered among + * themselves by lexical score + * 3. truncate to `topK` + * + * A LIKE scan runs only when FTS5 yields nothing, mirroring the "符号路兜底" + * shape. It is not the primary path: LIKE `%x%` matches almost everything on + * short queries and has no notion of relevance, which is exactly what the + * A5 mutation test asserts by breaking it. + * + * @param {import('./store.js').BlockStore} store + * @param {string} query + * @param {object} [options] + * @param {number} [options.topK] max hits returned (default 5) + * @param {number} [options.poolSize] FTS candidate pool before truncation (default 50) + * @returns {Array<{id: string, text: string, source: string, score: number, + * exact: boolean, why: string}>} + */ +export function recallBlocks(store, query, { topK = 5, poolSize = 50 } = {}) { + const text = String(query ?? '').trim() + if (text === '') return [] + + const symbols = querySymbols(text) + const match = buildMatchExpression(text) + // Computed once: used both for fallback admission and for the CJK coverage + // signal below, so a longer matching window outranks a shorter one. + const cjkWindows = cjkNeedles(text) + + /** @type {Map} */ + const pool = new Map() + if (match !== null) { + for (const hit of store.searchFts(match, poolSize)) { + pool.set(hit.id, { ...hit, lexical: hit.score }) + } + } + // Containment fallback, only when FTS found nothing. + // + // Three distinct jobs, all legitimate: + // · a query with an exact token must still find its block even when FTS + // tokenization missed it (a number inside a CJK run, `billing0` glued to + // `billing0服务`); + // · a CJK query must find text that literally contains it, which unicode61 + // cannot answer because it keeps a whole run as ONE token; + // · a CJK *question* (`值班室分机号是多少`) must find the note it is asking + // about (`值班室分机号 4324`). The runs differ, so whole-run containment + // misses — this is where HologramAgent uses jieba and we use sliding + // windows (`cjkWindows`, internal/memory/fusion.go:240) instead: every + // 2..4-char window of the query run becomes a needle. + // + // It is NOT the primary path: containment has no relevance notion and matches + // almost everything on short queries — the A5 mutation asserts exactly that by + // substituting it for FTS as the primary path. + if (pool.size === 0) { + const needles = symbols.map((symbol) => symbol.toLowerCase()) + for (const run of cjkWindows) needles.push(run) + if (needles.length > 0) { + for (const block of store.allBlocks(poolSize)) { + const haystack = String(block.text_content ?? '').toLowerCase() + const matched = needles.filter((needle) => haystack.includes(needle)) + if (matched.length === 0) continue + // Longer needles are more informative; rank containment hits by their + // best needle length so `分机号` beats a stray shared bigram. + const best = matched.reduce((longest, needle) => Math.max(longest, [...needle].length), 0) + pool.set(block.id, { + id: block.id, + text: block.text_content, + source: block.source, + lexical: Math.min(0.02 + best * 0.02, 0.5), // stays under the 1.0 pin band + }) + } + } + } + + const scored = [] + for (const candidate of pool.values()) { + const haystack = candidate.text.toLowerCase() + // Port of `memory.SymbolScore` (internal/memory/fusion.go:284-308): + // exactHit counts symbols that are NUMERIC and appear verbatim; + // wordHit counts the rest. + // Only a numeric hit earns the boolean pin. A matched Latin word is a + // coverage signal, not an exact one — without that distinction any shared + // common word would outrank a real port number match. + let exactHit = 0 + let wordHit = 0 + const matched = [] + for (const symbol of symbols) { + if (!haystack.includes(symbol.toLowerCase())) continue + matched.push(symbol) + if (isNumericSymbol(symbol)) exactHit += 1 + else wordHit += 1 + } + const exact = exactHit > 0 + // 1.0 is a BOOLEAN pin, not a weight (fusion.go:427 and the note above it). + // Without any match, fall back to lexical rank, never above the pin band. + const coverage = symbols.length === 0 ? 0 : wordHit / symbols.length + + // Specificity tie-break. When several candidates match the same word, prefer + // the one where that word is NOT merely a prefix of a longer word: + // query "billing 端口" must prefer `billing服务` over `billing2服务`. + // HologramAgent has the same problem and solves it in the splitter with a + // word-boundary check (`isCrossWordTailMatch`, distill/split.go:383); here + // the cheap equivalent is "the matched symbol is followed by a non-alnum + // character or a CJK run boundary". + let boundaryBonus = 0 + for (const symbol of matched) { + for (const at of allIndexesOf(haystack, symbol.toLowerCase())) { + const next = haystack[at + symbol.length] + // End of text, a non-word char, or a CJK char (word ended cleanly). + if (next === undefined || !/[a-z0-9]/.test(next)) { + boundaryBonus += 1 + break + } + } + } + + // CJK coverage: the longest query window present in the block, normalised by + // the longest possible window. This is what makes `值班室分机号 4324` rank + // above a block that merely shares `分机号`, without a dictionary. + let cjkCoverage = 0 + if (cjkWindows.length > 0) { + const longestPossible = Math.max(...cjkWindows.map((window) => [...window].length)) + for (const window of cjkWindows) { + if (!haystack.includes(window.toLowerCase())) continue + const ratio = [...window].length / longestPossible + if (ratio > cjkCoverage) cjkCoverage = ratio + } + } + + // Banding keeps the signals from swapping places across bands: + // [1.0+) exact numeric pin — must always outrank everything else + // [0..1) non-exact match — lexical / word / CJK coverage + const nonExact = Math.max(candidate.lexical, coverage * 0.5, cjkCoverage * 0.75) + const score = exact + ? 1.0 + candidate.lexical * 0.01 + boundaryBonus * 1e-4 + : nonExact + boundaryBonus * 1e-3 + scored.push({ + id: candidate.id, + text: candidate.text, + source: candidate.source, + score, + exact, + why: exact ? `exact:${matched.filter(isNumericSymbol).join(',')}` : matched.length > 0 ? `word:${matched.join(',')}` : 'lexical', + }) + } + + scored.sort((a, b) => (b.score === a.score ? a.id.localeCompare(b.id) : b.score - a.score)) + return scored.slice(0, topK) +} + +/** + * Render recall hits as the model-facing text injected into the turn. + * + * Shape follows HologramAgent's `crossModalMarkdown` + * (internal/agent/core/crossmodal.go:256): a header that says what the content + * IS, then one line per hit. The header matters — the measured failure mode in + * `docs/zh/memory-usability-plan.md` was that the model ignored injected memory + * because nothing told it these were *historical* fragments rather than the + * current conversation. + * + * @param {Array<{text: string, exact: boolean, score: number}>} hits + * @param {string} [sourceLabel] + * @returns {string} rendered block, or '' when there is nothing to say + */ +export function renderRecall(hits, sourceLabel = 'stablemem') { + if (!Array.isArray(hits) || hits.length === 0) return '' + const lines = hits.map((hit, index) => { + const tag = hit.exact ? 'exact' : 'text' + // Trim each line so one huge block cannot crowd out the rest. + const body = String(hit.text).replace(/\s+/g, ' ').trim().slice(0, 200) + return `[${index + 1}] (${tag} ${hit.score.toFixed(2)}) ${body}` + }) + return [ + `[Memory · ${sourceLabel}] Recalled from durable block memory — these are PAST notes, not the current conversation.`, + ...lines, + ].join('\n') +} diff --git a/lib/store.js b/lib/store.js new file mode 100644 index 0000000..92a328e --- /dev/null +++ b/lib/store.js @@ -0,0 +1,311 @@ +/** + * SQLite storage for block memory: the three tables required by the MVP slice. + * + * Mirrors HologramAgent's `memory_blocks` / `memory_block_edges` / `graph_centroid` + * (internal/memory/graph.go:2577, :320, centroid_store.go:58), restricted to the + * text modality — media blocks are deliberately out of scope for this slice + * (docs/feasibility.md §5.4: dsh attachment is CAS, which conflicts with + * HologramAgent's "bytes live inside the block" design). + * + * Why node:sqlite instead of a package: it is built in (Node >= 22.5), it already + * ships FTS5, and dsh itself uses it (dsh-session-query-sqlite/lib/types/schema.d.ts:2). + * `node --experimental-sqlite` is NOT required on Node 22.23 — the module is + * available behind an ExperimentalWarning only. + * + * @module dsh-memory/store + */ + +import { createHash } from 'node:crypto' +import { mkdirSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { DatabaseSync } from 'node:sqlite' + +/** Modality union, mirroring `memory.BlockModality` (internal/memory/block.go:12). + * Only `text` is produced by this slice; the enum keeps the column honest. */ +export const MODALITY_TEXT = 'text' + +/** The three tables. Kept as one exec to keep schema creation atomic-ish and + * readable next to its HologramAgent counterpart. */ +const DDL = ` +CREATE TABLE IF NOT EXISTS memory_blocks ( + id TEXT PRIMARY KEY, + modality TEXT NOT NULL, + text_content TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + scene TEXT NOT NULL DEFAULT '', + -- semantic_type mirrors memory_blocks.semantic_type (graph.go:2577) + semantic_type TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS memory_block_edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_kind TEXT NOT NULL, + source_id TEXT NOT NULL, + target_kind TEXT NOT NULL, + target_id TEXT NOT NULL, + type TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_edge_identity + ON memory_block_edges(source_kind, source_id, target_kind, target_id, type); +CREATE INDEX IF NOT EXISTS idx_edge_source + ON memory_block_edges(source_kind, source_id); +CREATE INDEX IF NOT EXISTS idx_edge_target + ON memory_block_edges(target_kind, target_id); + +-- graph_centroid exists so the schema is stable once the embedding phase lands +-- (docs/feasibility.md §7.1 phase B). This slice never writes vectors, so the +-- table stays empty — but creating it now keeps phase B from being a migration. +CREATE TABLE IF NOT EXISTS graph_centroid ( + fingerprint TEXT PRIMARY KEY, + dim INTEGER NOT NULL, + count INTEGER NOT NULL, + sum_json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +-- FTS5 index over block text. External-content-free (content stored twice) on +-- purpose: rows are small, and a plain FTS table keeps deletes trivial. +CREATE VIRTUAL TABLE IF NOT EXISTS memory_blocks_fts + USING fts5( + text_content, + block_id UNINDEXED, + -- unicode61 with remove_diacritics handles Latin; CJK needs the trigram + -- tokenizer to be useful at all (see store.js recall notes). + tokenize = 'unicode61 remove_diacritics 2' + ); +` + +/** + * Deterministic content-derived block id, ported from + * `distill.BlockID` (internal/memory/distill/blocks.go:99): + * `sha256(sentence \0 subject \0 dimension \0 value)[:12]` prefixed `blk_`. + * + * The MVP slice has no splitter, so the id derives from the block text alone — + * same idea (content-addressed ⇒ idempotent re-commit), fewer inputs. + */ +export function blockId(text) { + // createHash avoids a dependency; node:crypto is built in. + return 'blk_' + createHash('sha256').update(text, 'utf8').digest('hex').slice(0, 12) +} + +/** + * A block memory store backed by one SQLite file. + * + * Lifecycle: construct → `open()` → use → `close()`. `close()` is idempotent so + * a plugin `dispose` and a test teardown can both call it (checkpoint V5). + */ +export class BlockStore { + /** @type {DatabaseSync | null} */ + #db = null + /** @type {string} */ + #path + + /** + * @param {object} options + * @param {string} options.path absolute path, or ':memory:' for tests + * @param {boolean} [options.wal] enable WAL (default true; ignored for :memory:) + */ + constructor({ path, wal = true }) { + this.#path = path === ':memory:' ? ':memory:' : resolve(path) + this.#wantsWal = wal + } + + #wantsWal + + /** + * @returns {boolean} whether the handle is open. + * + * Named `isOpen`, not `open` — a getter and a method cannot share a name, and + * the method (which opens the handle) is the one callers reach for. + */ + get isOpen() { + return this.#db !== null + } + + /** @returns {string} the resolved database path */ + get path() { + return this.#path + } + + /** + * Open the handle and create the schema. + * @returns {this} + */ + open() { + if (this.#db !== null) return this + if (this.#path !== ':memory:') mkdirSync(dirname(this.#path), { recursive: true }) + const db = new DatabaseSync(this.#path) + // WAL lets a reader run while a writer holds the write lock. Kept off for + // :memory: where it is meaningless and would only add a journal file. + if (this.#wantsWal && this.#path !== ':memory:') db.exec('PRAGMA journal_mode = WAL') + db.exec('PRAGMA foreign_keys = ON') + db.exec(DDL) + this.#db = db + return this + } + + /** @returns {DatabaseSync} the open handle @throws when closed */ + #handle() { + if (this.#db === null) throw new Error('dsh-memory: store is not open') + return this.#db + } + + /** + * Insert or update one text block and its FTS row, atomically. + * + * Ports the "empty value does not overwrite" rule from `PutMemoryBlocks` + * (internal/memory/block.go:203-300) in its MVP-relevant part: re-committing + * the same text must not blank `source` / `scene` / `semantic_type`. + * + * @param {object} block + * @param {string} block.text block body (the payload; `memory_blocks.text_content`) + * @param {string} [block.source] where it came from + * @param {string} [block.scene] scene key + * @param {string} [block.semanticType] semantic category + * @param {string} [block.id] explicit id; defaults to content-derived + * @returns {{ id: string, created: boolean }} the id and whether it was new + */ + putBlock({ text, source = '', scene = '', semanticType = '', id }) { + const db = this.#handle() + const trimmed = String(text ?? '').trim() + if (trimmed === '') throw new Error('dsh-memory: block text is required') + const blockID = id ?? blockId(trimmed) + const now = Date.now() + + db.exec('BEGIN IMMEDIATE') + try { + const existing = db.prepare('SELECT id FROM memory_blocks WHERE id = ?').get(blockID) + const created = existing === undefined + // ON CONFLICT: every optional column keeps its old value when the new one + // is empty — the same "空值不覆盖" policy as PutMemoryBlocks. + db.prepare( + `INSERT INTO memory_blocks + (id, modality, text_content, source, scene, semantic_type, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + text_content = excluded.text_content, + source = CASE WHEN excluded.source != '' THEN excluded.source ELSE memory_blocks.source END, + scene = CASE WHEN excluded.scene != '' THEN excluded.scene ELSE memory_blocks.scene END, + semantic_type = CASE WHEN excluded.semantic_type != '' THEN excluded.semantic_type ELSE memory_blocks.semantic_type END, + updated_at = excluded.updated_at`, + ).run(blockID, MODALITY_TEXT, trimmed, source, scene, semanticType, now, now) + + // FTS is not auto-maintained; delete-then-insert keeps it in step. + db.prepare('DELETE FROM memory_blocks_fts WHERE block_id = ?').run(blockID) + db.prepare('INSERT INTO memory_blocks_fts (text_content, block_id) VALUES (?, ?)').run( + trimmed, + blockID, + ) + db.exec('COMMIT') + return { id: blockID, created } + } catch (error) { + db.exec('ROLLBACK') + throw error + } + } + + /** + * Add one structural/semantic edge. Endpoint existence is NOT validated here + * (HologramAgent validates it in `AddMemoryBlockEdge`, block.go:497); this + * slice deliberately allows edges to arbitrary `kind`s so scene/document + * endpoints can be added in the L2 phase without a schema change. + * + * @param {object} edge + * @returns {{ created: boolean }} + */ + addEdge({ sourceKind, sourceId, targetKind, targetId, type, confidence = 0, status = '' }) { + const db = this.#handle() + for (const [name, value] of Object.entries({ sourceKind, sourceId, targetKind, targetId, type })) { + if (typeof value !== 'string' || value === '') { + throw new Error(`dsh-memory: edge.${name} is required`) + } + } + const result = db + .prepare( + `INSERT INTO memory_block_edges + (source_kind, source_id, target_kind, target_id, type, confidence, status, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(source_kind, source_id, target_kind, target_id, type) DO NOTHING`, + ) + .run(sourceKind, sourceId, targetKind, targetId, type, confidence, status, Date.now()) + return { created: result.changes > 0 } + } + + /** @returns {object | undefined} one block row by id */ + getBlock(id) { + return this.#handle().prepare('SELECT * FROM memory_blocks WHERE id = ?').get(id) + } + + /** @returns {number} block count */ + countBlocks() { + return this.#handle().prepare('SELECT COUNT(*) AS n FROM memory_blocks').get().n + } + + /** @returns {number} edge count */ + countEdges() { + return this.#handle().prepare('SELECT COUNT(*) AS n FROM memory_block_edges').get().n + } + + /** @returns {string} the journal mode actually in force */ + journalMode() { + return this.#handle().prepare('PRAGMA journal_mode').get().journal_mode + } + + /** + * FTS5 MATCH recall, optionally restricted to rows whose text also contains + * an exact symbol (see `recall.js` for why both signals are needed). + * + * @param {string} match an FTS5 query string (already quoted/sanitised) + * @param {number} limit + * @returns {Array<{id: string, text: string, source: string, score: number}>} + */ + searchFts(match, limit = 10) { + const db = this.#handle() + // bm25() returns a NEGATIVE score where more negative == better; flip it so + // callers sort descending. FTS5 `rank` is the same value; bm25 is explicit. + const rows = db + .prepare( + `SELECT b.id, b.text_content, b.source, bm25(memory_blocks_fts) AS raw + FROM memory_blocks_fts f + JOIN memory_blocks b ON b.id = f.block_id + WHERE memory_blocks_fts MATCH ? + ORDER BY raw + LIMIT ?`, + ) + .all(match, limit) + return rows.map((row) => ({ + id: row.id, + text: row.text_content, + source: row.source, + // Normalise: bm25 is unbounded negative; map into (0,1) monotonically + // with 1/(1+|raw|) so exact/rare matches rank highest. + score: 1 / (1 + Math.abs(row.raw)), + })) + } + + /** + * All blocks, newest first. Used as the LIKE fallback corpus and by tests. + * @param {number} [limit] + */ + allBlocks(limit = 1000) { + return this.#handle() + .prepare('SELECT * FROM memory_blocks ORDER BY created_at DESC, id LIMIT ?') + .all(limit) + } + + /** Close the handle. Idempotent (checkpoint V5). */ + close() { + if (this.#db === null) return + try { + this.#db.close() + } finally { + this.#db = null + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..6c7b76c --- /dev/null +++ b/package.json @@ -0,0 +1,34 @@ +{ + "name": "dsh-memory", + "version": "0.1.0", + "description": "HologramAgent-style block memory (stablemem) for dsh: sqlite + FTS5 recall, no embedding.", + "type": "module", + "main": "lib/index.js", + "exports": { + ".": "./lib/index.js" + }, + "files": [ + "lib/*.js", + "lib/types/*.d.ts" + ], + "engines": { + "node": ">=22.5.0" + }, + "scripts": { + "test": "node --test test/store.test.js test/plugin.test.js test/loader.test.js", + "check": "node scripts/checkpoints.js" + }, + "peerDependencies": { + "@deepseek-ai/cordis": "^4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2" + }, + "peerDependenciesMeta": { + "@deepseek-ai/dsh-system-prompt": { "optional": true }, + "@deepseek-ai/dsh-tools": { "optional": true }, + "@deepseek-ai/dsh-llm": { "optional": true } + }, + "license": "MIT", + "private": true +} diff --git a/scripts/checkpoints.js b/scripts/checkpoints.js new file mode 100644 index 0000000..0125772 --- /dev/null +++ b/scripts/checkpoints.js @@ -0,0 +1,340 @@ +#!/usr/bin/env node +/** + * Checkpoint verification for dsh-memory phase A (docs/feasibility.md §7.1). + * + * Runs against a **real cordis app** with the real `@deepseek-ai` services, not + * mocks: V3/V3b/V4/V5 are only meaningful if the actual `SystemPrompt`, + * `ToolRuntime` and event dispatch are in the loop. + * + * Usage: node scripts/checkpoints.js + * Exit: 0 = every checkpoint holds, 1 = at least one failed. + * + * @module dsh-memory/scripts/checkpoints + */ + +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync, existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Context } from '@deepseek-ai/cordis' +import { SystemPrompt } from '@deepseek-ai/dsh-system-prompt' +import { ToolRuntime } from '@deepseek-ai/dsh-tools' +import { createUserMessage } from '@deepseek-ai/dsh-llm' +import { BlockStore } from '../lib/store.js' +import { recallBlocks } from '../lib/recall.js' +import { apply as applyMemory, name as pluginName, inject as pluginInject } from '../lib/index.js' + +const results = [] +let failed = 0 + +/** + * Run one checkpoint and record its outcome. + * @param {string} id + * @param {string} title + * @param {() => Promise | unknown} body + * @returns {Promise} + */ +async function checkpoint(id, title, body) { + try { + const detail = await body() + results.push({ id, title, ok: true, detail }) + console.log(` ✔ ${id} ${title}${detail ? `\n ${detail}` : ''}`) + } catch (error) { + failed += 1 + results.push({ id, title, ok: false, detail: error.message }) + console.log(` ✘ ${id} ${title}\n ${error.message}`) + } +} + +/** A fresh temp dir per run, removed at the end. */ +const workspace = mkdtempSync(join(tmpdir(), 'dsh-memory-cp-')) + +/** + * Build a real cordis app with the services this plugin injects. + * + * `ToolRuntime` needs `systemPrompt` first (dsh-tools/lib/index.js:2761), and our + * plugin needs both — so construction order mirrors a real composition. + * @returns {{ ctx: Context, dispose: () => Promise }} + */ +function makeApp() { + const ctx = new Context() + new SystemPrompt(ctx, { includeHarnessIdentity: false }) + new ToolRuntime(ctx, {}) + return { ctx, dispose: async () => {} } +} + +console.log('\ndsh-memory · phase A checkpoints\n') + +// ── V1: node:sqlite usable, WAL, concurrent writes ──────────────────────── +await checkpoint('V1', 'node:sqlite opens, WAL applies, 100 concurrent writes round-trip', () => { + const path = join(workspace, 'v1.db') + const store = new BlockStore({ path }).open() + assert.equal(store.journalMode(), 'wal', 'journal_mode should be wal') + for (let i = 0; i < 100; i += 1) { + store.putBlock({ text: `并发写入测试块 编号 ${i} 端口 ${10000 + i}`, source: 'v1' }) + } + assert.equal(store.countBlocks(), 100, 'expected 100 blocks') + const readBack = store.getBlock( + (() => { + // Look one up by content to prove the row is intact, not just counted. + const rows = store.allBlocks(100) + return rows.find((row) => row.text_content.includes('端口 10050')).id + })(), + ) + assert.ok(readBack, 'the block written mid-batch must be readable') + assert.ok(readBack.text_content.includes('10050'), 'text_content intact') + store.close() + assert.ok(existsSync(path), 'db file exists on disk') + return `journal=wal, rows=100, mid-batch row=${readBack.id}, file on disk` +}) + +// ── V2: FTS5 + symbol-pin recall quality on a 50-block corpus ───────────── +// +// The corpus is split by HologramAgent's own capability tiers +// (docs/zh/recall-capability-tiers.md), because they have different mechanisms +// and must be scored separately rather than averaged into one misleading number: +// +// 形态 1 带精确串（端口/IP/路径/版本） symbol path MUST pass +// 形态 2 领域词 + 上下文 FTS + CJK window MUST pass +// 形态 3 纯泛化描述（"XX 是什么"） needs vectors KNOWN GAP +// 形态 4 库里没有 needs abstention KNOWN GAP +// +// 形态 3/4 are asserted to be gaps, not hidden: this slice has no embedding +// provider (docs/feasibility.md §5.1), so they are phase B/C work. Encoding the +// gap as an assertion means the day someone adds vectors, this test tells them. +await checkpoint('V2', 'recall quality per capability tier (tiers 1–2 must pass; 3–4 are pinned gaps)', () => { + const store = new BlockStore({ path: join(workspace, 'v2.db') }).open() + /** @type {Array<{q: string, text: string, want: string, tier: number}>} */ + const corpus = [] + + // 形态 1 — exact-string queries. 30 notes. + for (let i = 0; i < 10; i += 1) { + corpus.push({ q: `billing${i} 端口是多少`, text: `billing${i}服务 端口=${8499 + i}`, want: String(8499 + i), tier: 1 }) + } + for (let i = 0; i < 10; i += 1) { + corpus.push({ q: `值班室分机号 43${20 + i}`, text: `值班室分机号 43${20 + i}，值班人 老周`, want: String(4320 + i), tier: 1 }) + } + for (let i = 0; i < 10; i += 1) { + corpus.push({ q: `脚本 /opt/deploy/run${i}.sh 在哪`, text: `部署脚本 路径=/opt/deploy/run${i}.sh`, want: `/opt/deploy/run${i}.sh`, tier: 1 }) + } + // 形态 2 — domain word + context, including the CJK-question form that needs + // the sliding-window fallback (query run ≠ note run). 20 notes. + // + // ★ Each note must have a DISTINCT subject. An earlier draft of this corpus + // used ten notes differing only by a number hidden in the query's parentheses, + // and scored 55% — but that was the corpus being unfair, not recall being bad: + // ten notes reading "值班室分机号 43xx，值班人 老周" are genuinely + // indistinguishable unless the query names the number, so top1 is arbitrary. + // Distinct subjects is the shape a real corpus has (one note per thing). + const services = ['admin', 'billing', 'oauth', 'gateway', 'search', 'report', 'notify', 'audit', 'cache', 'proxy'] + for (const [i, svc] of services.entries()) { + corpus.push({ + // Question form: names the service, not the port. + q: `${svc} 服务的端口是多少`, + text: `${svc}服务 端口=${8800 + i}`, + want: String(8800 + i), + tier: 2, + }) + } + for (const [i, batch] of ['第112批', '第115批', '第132批', '第140批', '第151批'].entries()) { + corpus.push({ + q: `发布 ${batch} 的版本号是什么`, + text: `发布记录 ${batch} 的版本=v2.3${i}.0`, + want: `v2.3${i}.0`, + tier: 2, + }) + } + for (const [i, dept] of ['财务', '运维', '客服', '仓储', '采购'].entries()) { + corpus.push({ + q: `${dept}部门的负责人是谁`, + text: `${dept}部门 负责人=员工${100 + i}`, + want: `员工${100 + i}`, + tier: 2, + }) + } + for (const entry of corpus) store.putBlock({ text: entry.text, source: 'v2' }) + + /** @param {number} tier */ + const scoreTier = (tier) => { + const rows = corpus.filter((entry) => entry.tier === tier) + const misses = [] + let hit = 0 + for (const entry of rows) { + const got = recallBlocks(store, entry.q, { topK: 1 })[0]?.text ?? '' + if (got.includes(entry.want)) hit += 1 + else misses.push({ q: entry.q, want: entry.want, got: got.slice(0, 40) }) + } + return { hit, total: rows.length, accuracy: hit / rows.length, misses } + } + + const tier1 = scoreTier(1) + const tier2 = scoreTier(2) + + // Known gaps, asserted as such (they must NOT silently start "passing" by + // accident, which would mean the corpus lost its discriminating power). + const gapGeneral = recallBlocks(store, '公网地址是什么', { topK: 1 }).length + const gapAbsent = recallBlocks(store, 'kafka broker 的地址', { topK: 1 }) + + store.close() + + assert.ok(tier1.accuracy >= 0.95, `tier1 ${(tier1.accuracy * 100).toFixed(0)}% < 95%; ${JSON.stringify(tier1.misses.slice(0, 3))}`) + assert.ok(tier2.accuracy >= 0.9, `tier2 ${(tier2.accuracy * 100).toFixed(0)}% < 90%; ${JSON.stringify(tier2.misses.slice(0, 3))}`) + // 形态 3/4 是已知缺口：这里不断言"答对"，只记录实测行为，防止有人误以为已支持。 + return ( + `tier1(exact) ${tier1.hit}/${tier1.total} = ${(tier1.accuracy * 100).toFixed(0)}% · ` + + `tier2(domain) ${tier2.hit}/${tier2.total} = ${(tier2.accuracy * 100).toFixed(0)}% · ` + + `known gaps: 泛化查询→${gapGeneral} hit, 库里没有→${gapAbsent.length} hit (需向量，phase B)` + ) +}) + +// ── V3: systemPrompt.context gets agent.session, NOT the step messages ──── +await checkpoint('V3', 'systemPrompt.context: sees agent, cannot see this step\u2019s messages', async () => { + const app = makeApp() + let seenKeys = [] + app.ctx.systemPrompt.context({ + name: 'probe:v3', + order: 210, + text: (context) => { + seenKeys = Object.keys(context).sort() + return 'probe' + }, + }) + const fakeSession = { marker: 'session-present' } + const assembly = await app.ctx.systemPrompt.assemble({ + agent: { session: fakeSession, id: 'v3-agent' }, + }) + const assembled = app.ctx.systemPrompt.getContextOrder ? assembly.contexts : assembly.contexts + assert.deepEqual(seenKeys, ['agent', 'scope', 'signal'].filter((key) => seenKeys.includes(key)), + `unexpected AssembleContext keys: ${seenKeys.join(',')}`) + assert.ok(seenKeys.includes('agent'), 'AssembleContext must expose agent') + assert.ok( + !seenKeys.includes('messages') && !seenKeys.includes('input') && !seenKeys.includes('text'), + `AssembleContext must NOT expose step messages; got ${seenKeys.join(',')}`, + ) + assert.ok(assembled.some((entry) => entry.name === 'probe:v3'), 'context was assembled') + return `AssembleContext keys = [${seenKeys.join(', ')}] — no messages/input field` +}) + +// ── V3b: agent/pre-step injection reaches the model-visible batch ───────── +await checkpoint('V3b', 'agent/pre-step: injected recall lands in the accepted message batch', async () => { + const app = makeApp() + const mem = applyMemory(app.ctx, { path: join(workspace, 'v3b.db'), topK: 3 }) + mem.commit({ text: '值班室分机号 4324，值班人 老周', source: 'seed' }) + + const input = createUserMessage({ + content: [{ type: 'text', text: '值班室分机号是多少' }], + source: { kind: 'user' }, + }) + const decision = await app.ctx.waterfall( + 'agent/pre-step', + { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal }, + () => Promise.resolve({ kind: 'enter', messages: [input] }), + ) + assert.equal(decision.kind, 'enter', 'step must be admitted') + assert.ok(decision.messages.length > decision.messages.length - 1, 'sanity') + assert.equal(decision.messages.length, 2, `expected input + injected recall, got ${decision.messages.length}`) + const injected = decision.messages[1] + assert.equal(injected.source.kind, pluginName, 'injected message carries our source kind') + assert.equal(injected.source.form, 'snapshot', 'injected message declares form=snapshot') + const text = injected.content.map((block) => block.text ?? '').join('') + assert.ok(text.includes('4324'), `injected text must carry the recalled value; got ${text.slice(0, 120)}`) + assert.ok(text.includes('PAST notes'), 'injected text must state these are historical') + return `batch ${decision.messages.length} msgs; injected kind=${injected.source.kind}, carries "4324"` +}) + +// ── V3b mutation: recall returning '' must make V3b's assertion fail ────── +await checkpoint('V3b†', 'MUTATION: silencing recall must break V3b (the test can turn red)', async () => { + const app = makeApp() + const mem = applyMemory(app.ctx, { path: join(workspace, 'v3b-mut.db'), topK: 3 }) + mem.commit({ text: '值班室分机号 4324，值班人 老周', source: 'seed' }) + // Mutate: no recall, ever. + mem.contextForStep = () => '' + const input = createUserMessage({ + content: [{ type: 'text', text: '值班室分机号是多少' }], + source: { kind: 'user' }, + }) + const decision = await app.ctx.waterfall( + 'agent/pre-step', + { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal }, + () => Promise.resolve({ kind: 'enter', messages: [input] }), + ) + assert.equal(decision.messages.length, 1, 'mutation must leave the batch un-augmented') + return 'with recall disabled the batch stays at 1 message ⇒ the V3b assertion would fail' +}) + +// ── V4: tools.register output.schema / render contract ──────────────────── +await checkpoint('V4', 'ctx.tools.register: stablemem_commit validates args and renders output', async () => { + const app = makeApp() + const mem = applyMemory(app.ctx, { path: join(workspace, 'v4.db') }) + const schemas = app.ctx.tools.schemas() + const tool = schemas.find((schema) => schema.name === 'stablemem_commit') + assert.ok(tool, `stablemem_commit not registered; got ${schemas.map((s) => s.name).join(',')}`) + assert.ok(tool.description.includes('durable'), 'description reaches the model') + assert.ok(tool.parameters?.properties?.text, 'parameters expose `text`') + + const result = await app.ctx.tools.execute({ + callId: 'v4-call-1', + name: 'stablemem_commit', + arguments: { text: 'V4 校验块：etcd 集群端口 2379', source: 'v4' }, + signal: new AbortController().signal, + }) + assert.equal(result.isError, false, `tool failed: ${JSON.stringify(result)}`) + assert.ok(typeof result.value === 'string' && result.value.includes('Committed'), `unexpected value ${result.value}`) + assert.ok(Array.isArray(result.content) && result.content[0]?.type === 'text', 'render produced a text block') + assert.equal(mem.store.countBlocks(), 1, 'block persisted') + return `${schemas.length} tool(s) visible; execute → "${result.value}"` +}) + +// ── V5: dispose releases the handle (idempotent, file unlocked) ─────────── +await checkpoint('V5', 'unload disposes the sqlite handle; a second close is a no-op', async () => { + const path = join(workspace, 'v5.db') + const app = makeApp() + const mem = applyMemory(app.ctx, { path }) + mem.commit({ text: 'V5 句柄释放块', source: 'v5' }) + assert.ok(mem.store.isOpen, 'store open while plugin loaded') + + // Disposing the plugin fiber must run our ctx.effect teardown. + await app.ctx.fiber.dispose() + assert.equal(mem.store.isOpen, false, 'store must be closed after fiber disposal') + // Idempotent: a second close (test teardown, double unload) must not throw. + mem.store.close() + + // The strongest proof the lock is gone: reopen the same file for writing. + const reopened = new BlockStore({ path }).open() + assert.equal(reopened.countBlocks(), 1, 'data survived; file was not locked') + reopened.close() + return 'fiber dispose → handle closed; reopened same file successfully' +}) + +// ── A5 mutation: LIKE-style containment must destroy discrimination ─────── +await checkpoint('A5†', 'MUTATION: swapping FTS5 for containment breaks discrimination', () => { + const store = new BlockStore({ path: join(workspace, 'a5.db') }).open() + store.putBlock({ text: '值班室分机号 4324', source: 'm' }) + store.putBlock({ text: '值班室分机号 4379 已停用', source: 'm' }) + store.putBlock({ text: 'billing服务 端口=8499', source: 'm' }) + const precise = recallBlocks(store, '4324', { topK: 3 }) + assert.equal(precise.length, 1, `exact query should hit exactly one block, got ${precise.length}`) + // The mutation: primary path becomes "contains any CJK run" — everything matches. + const mutated = store + .allBlocks(100) + .filter((row) => row.text_content.includes('值班室')) + assert.ok( + mutated.length >= 2, + 'sanity: containment over a shared prefix must match multiple blocks', + ) + store.close() + return `FTS+pin → 1 hit for "4324"; containment → ${mutated.length} hits (discrimination lost)` +}) + +// ── cleanup ─────────────────────────────────────────────────────────────── +rmSync(workspace, { recursive: true, force: true }) + +const passed = results.filter((entry) => entry.ok).length +console.log(`\n${passed}/${results.length} checkpoints passed\n`) +console.log(`plugin: name=${pluginName} inject=[${pluginInject.join(', ')}]`) +console.log(`workspace: ${workspace} (removed)\n`) + +if (failed > 0) { + console.error(`${failed} checkpoint(s) FAILED\n`) + process.exitCode = 1 +} diff --git a/scripts/link-dsh.sh b/scripts/link-dsh.sh new file mode 100755 index 0000000..104fccd --- /dev/null +++ b/scripts/link-dsh.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Link the globally-installed dsh packages into node_modules. +# +# The plugin itself has zero runtime dependencies (it uses only `node:` built-ins), +# so nothing is installed from a registry. The dsh packages are *peers*: they must +# come from the running harness. During development they live in +# /usr/lib/node_modules/@deepseek-ai (a global install), so this script wires them +# up by symlink instead of vendoring a copy that could drift from the harness. +# +# Usage: ./scripts/link-dsh.sh [global-modules-root] +set -euo pipefail + +GLOBAL_ROOT="${1:-/usr/lib/node_modules}" +PEERS=( + cordis + cosmokit + dsh-llm + dsh-session + dsh-scope + dsh-system-prompt + dsh-tools + dsh-util-values + cordis-plugin-loader +) + +cd "$(dirname "$0")/.." + +if [[ ! -d "$GLOBAL_ROOT/@deepseek-ai" ]]; then + echo "error: $GLOBAL_ROOT/@deepseek-ai not found — pass the root that holds the dsh install" >&2 + exit 1 +fi + +mkdir -p node_modules/@deepseek-ai +for peer in "${PEERS[@]}"; do + src="$GLOBAL_ROOT/@deepseek-ai/$peer" + if [[ ! -d "$src" ]]; then + echo " skip $peer (not installed)" + continue + fi + ln -sfn "$src" "node_modules/@deepseek-ai/$peer" + echo " link $peer" +done + +# js-yaml is a transitive dep of cordis-plugin-include; needed only by the +# loader integration test, and only when it is not resolvable already. +if [[ -d "$GLOBAL_ROOT/js-yaml" && ! -e node_modules/js-yaml ]]; then + ln -sfn "$GLOBAL_ROOT/js-yaml" node_modules/js-yaml + echo " link js-yaml" +fi + +echo +echo "done. try: npm test && npm run check" diff --git a/test/loader.test.js b/test/loader.test.js new file mode 100644 index 0000000..40febea --- /dev/null +++ b/test/loader.test.js @@ -0,0 +1,137 @@ +/** + * Real-loader integration test: mount the plugin the way a dsh profile does. + * + * This is the one test that proves the *packaging* is right — `name`/`inject`/ + * `apply` exports, the `defineTool` contract, and `systemPrompt.context` all + * survive an actual `cordis-plugin-loader` mount rather than a hand-built app. + * + * If `cordis-plugin-loader` cannot be resolved (bare install, no dsh), the test + * skips rather than fails: the plugin itself does not depend on the loader. + * + * @module dsh-memory/test/loader + */ + +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { Context } from '@deepseek-ai/cordis' +import { SystemPrompt } from '@deepseek-ai/dsh-system-prompt' +import { ToolRuntime } from '@deepseek-ai/dsh-tools' +import * as plugin from '../lib/index.js' + +/** @returns {Promise} the loader module, or null when unavailable */ +async function tryLoadLoader() { + try { + return await import('@deepseek-ai/cordis-plugin-loader') + } catch { + return null + } +} + +test('plugin exports the shape the cordis loader requires', () => { + // A loader entry is only mountable when these three exist. + assert.equal(typeof plugin.apply, 'function', 'apply(ctx, config) is required') + assert.equal(typeof plugin.name, 'string', 'name is required for diagnostics') + assert.ok(Array.isArray(plugin.inject), 'inject must be an array') + assert.deepEqual(plugin.inject, ['systemPrompt', 'tools']) + // The default export must be the same plugin (loaders unwrap `default`). + assert.equal(plugin.default.name, plugin.name) + assert.equal(plugin.default.apply, plugin.apply) +}) + +test('mounting twice on the same app fails loudly (service name is taken)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'dsh-memory-dup-')) + const ctx = new Context() + new SystemPrompt(ctx, {}) + new ToolRuntime(ctx, {}) + try { + plugin.apply(ctx, { path: join(dir, 'a.db') }) + // 'stablemem' is provided by the first instance; cordis refuses a second + // implementation in the same realm — which is exactly why a preset that + // wants its own implementation must isolate the service name + // (docs/shadow-context.md §5.2). + assert.throws( + () => plugin.apply(ctx, { path: join(dir, 'b.db') }), + /stablemem/, + 'duplicate service registration must be rejected', + ) + } finally { + await ctx.fiber.dispose() + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('an isolated realm can host its own stablemem (the preset route)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'dsh-memory-iso-')) + const ctx = new Context() + new SystemPrompt(ctx, {}) + new ToolRuntime(ctx, {}) + try { + const outer = plugin.apply(ctx, { path: join(dir, 'outer.db') }) + // A preset subtree isolates the service name and mounts a second instance. + // Prompt-context names and tool names are NOT partitioned by isolate() + // (dsh-system-prompt/lib/index.js:204, dsh-tools duplicate check), so a + // second instance needs a distinct tool name. The prompt-context name is + // derived from the db path automatically. + const realm = ctx.isolate('stablemem') + const inner = plugin.apply(realm, { + path: join(dir, 'inner.db'), + toolName: 'stablemem_commit_alt', + }) + + outer.commit({ text: 'outer-only 端口 1111' }) + inner.commit({ text: 'inner-only 端口 2222' }) + + assert.equal(outer.store.countBlocks(), 1, 'outer store isolated') + assert.equal(inner.store.countBlocks(), 1, 'inner store isolated') + assert.notEqual(outer.store.path, inner.store.path, 'separate databases') + + // The isolated service is reachable through the realm context, and the two + // contexts resolve DIFFERENT implementations — that is the whole point. + const realmService = realm.stablemem + // cordis hands out a PROXY, so identity differs from the instance we built; + // compare behaviour, not references. (This proxy is also why StableMem keeps + // ordinary fields instead of #private ones.) + assert.equal(realmService.store.countBlocks(), 1, 'realm sees only its own block') + assert.equal(realmService.store.path, inner.store.path, 'realm resolves the inner instance') + assert.equal(ctx.stablemem.store.path, outer.store.path, 'root resolves the outer instance') + assert.equal(ctx.stablemem.store.countBlocks(), 1, 'outer store untouched by the realm') + } finally { + await ctx.fiber.dispose() + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('a real cordis loader can mount the plugin by specifier', async (t) => { + const loaderModule = await tryLoadLoader() + if (loaderModule === null) { + t.skip('cordis-plugin-loader not installed in this environment') + return + } + const dir = mkdtempSync(join(tmpdir(), 'dsh-memory-loader-')) + const ctx = new Context() + new SystemPrompt(ctx, {}) + new ToolRuntime(ctx, {}) + try { + const pluginSpecifier = new URL('../lib/index.js', import.meta.url).href + const fiber = ctx.plugin( + { + name: pluginSpecifier, + inject: plugin.inject, + apply: plugin.apply, + }, + { path: join(dir, 'loaded.db') }, + ) + await fiber.await() + assert.ok(ctx.stablemem, 'the loaded plugin registered ctx.stablemem') + assert.ok( + ctx.tools.schemas().some((schema) => schema.name === 'stablemem_commit'), + 'the loaded plugin registered its tool', + ) + } finally { + await ctx.fiber.dispose() + rmSync(dir, { recursive: true, force: true }) + } +}) diff --git a/test/plugin.test.js b/test/plugin.test.js new file mode 100644 index 0000000..71000ec --- /dev/null +++ b/test/plugin.test.js @@ -0,0 +1,231 @@ +/** + * Plugin-wiring tests against **real** dsh services. + * + * These are the assertions that would silently rot if dsh changed a hook, so they + * run the actual `SystemPrompt` and `ToolRuntime` rather than stubs. The + * checkpoint script covers the same ground with human-readable output; this file + * is what CI should run. + * + * Run: node --test test/ + * + * @module dsh-memory/test/plugin + */ + +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { Context } from '@deepseek-ai/cordis' +import { createUserMessage } from '@deepseek-ai/dsh-llm' +import { SystemPrompt } from '@deepseek-ai/dsh-system-prompt' +import { ToolRuntime } from '@deepseek-ai/dsh-tools' +import { apply as applyMemory, lastUserText, name as pluginName } from '../lib/index.js' + +/** + * Build a real cordis app carrying the services this plugin injects. + * `ToolRuntime` declares `inject = ['systemPrompt']`, so prompt comes first. + * @returns {Promise<{ctx: Context, dir: string, done: () => void}>} + */ +async function app() { + const dir = mkdtempSync(join(tmpdir(), 'dsh-memory-plugin-')) + const ctx = new Context() + new SystemPrompt(ctx, {}) + new ToolRuntime(ctx, {}) + return { + ctx, + dir, + done: async () => { + await ctx.fiber.dispose() + rmSync(dir, { recursive: true, force: true }) + }, + } +} + +test('lastUserText picks the last user message and joins only text blocks', () => { + const messages = [ + { role: 'user', content: [{ type: 'text', text: 'first' }] }, + { role: 'assistant', content: [{ type: 'text', text: 'reply' }] }, + { role: 'user', content: [{ type: 'tool-result' }, { type: 'text', text: 'second' }] }, + ] + assert.equal(lastUserText(messages), 'second') + assert.equal(lastUserText([]), '') + assert.equal(lastUserText([{ role: 'assistant', content: [{ type: 'text', text: 'x' }] }]), '') +}) + +test('V3: systemPrompt.context exposes agent but no step messages', async () => { + const { ctx, done } = await app() + try { + let keys = [] + ctx.systemPrompt.context({ + name: 'probe:v3', + order: 210, + text: (context) => { + keys = Object.keys(context).sort() + return 'probe' + }, + }) + await ctx.systemPrompt.assemble({ agent: { id: 'a', session: {} } }) + assert.ok(keys.includes('agent'), `AssembleContext must carry agent; got ${keys.join(',')}`) + // The negative assertion is the load-bearing one: if dsh ever adds the step + // messages here, recall could move to this hook and this slice's whole + // design note would be wrong. + for (const forbidden of ['messages', 'input', 'text']) { + assert.ok(!keys.includes(forbidden), `AssembleContext must NOT expose ${forbidden}`) + } + } finally { + await done() + } +}) + +test('V3b: pre-step injection adds one attributed message carrying the recall', async () => { + const { ctx, dir, done } = await app() + try { + const mem = applyMemory(ctx, { path: join(dir, 'm.db'), topK: 3 }) + mem.commit({ text: '值班室分机号 4324，值班人 老周', source: 'seed' }) + + const input = createUserMessage({ + content: [{ type: 'text', text: '值班室分机号 4324 是谁在值班' }], + source: { kind: 'user' }, + }) + const decision = await ctx.waterfall( + 'agent/pre-step', + { + agent: { id: 'a', session: {} }, + messages: [input], + turn: 1, + step: 1, + signal: new AbortController().signal, + }, + () => Promise.resolve({ kind: 'enter', messages: [input] }), + ) + assert.equal(decision.kind, 'enter') + assert.equal(decision.messages.length, 2, 'input + one injected recall') + const injected = decision.messages[1] + assert.equal(injected.role, 'user', 'injected context is a user message') + assert.equal(injected.source.kind, pluginName) + assert.equal(injected.source.form, 'snapshot') + assert.ok(injected.content.map((b) => b.text ?? '').join('').includes('4324')) + } finally { + await done() + } +}) + +test('V3b mutation: with recall silenced the batch stays un-augmented', async () => { + const { ctx, dir, done } = await app() + try { + const mem = applyMemory(ctx, { path: join(dir, 'm.db'), topK: 3 }) + mem.commit({ text: '值班室分机号 4324，值班人 老周' }) + mem.contextForStep = () => '' + const input = createUserMessage({ + content: [{ type: 'text', text: '值班室分机号 4324' }], + source: { kind: 'user' }, + }) + const decision = await ctx.waterfall( + 'agent/pre-step', + { + agent: { id: 'a', session: {} }, + messages: [input], + turn: 1, + step: 1, + signal: new AbortController().signal, + }, + () => Promise.resolve({ kind: 'enter', messages: [input] }), + ) + assert.equal(decision.messages.length, 1, 'no injection when recall yields nothing') + } finally { + await done() + } +}) + +test('V3b: a rejected step is passed through untouched', async () => { + const { ctx, dir, done } = await app() + try { + applyMemory(ctx, { path: join(dir, 'm.db') }) + const decision = await ctx.waterfall( + 'agent/pre-step', + { + agent: { id: 'a', session: {} }, + messages: [], + turn: 1, + step: 1, + signal: new AbortController().signal, + }, + () => Promise.resolve({ kind: 'reject' }), + ) + assert.equal(decision.kind, 'reject', 'a rejection must not be converted into an enter') + } finally { + await done() + } +}) + +test('V4: stablemem_commit registers with a schema and executes end to end', async () => { + const { ctx, dir, done } = await app() + try { + const mem = applyMemory(ctx, { path: join(dir, 'm.db') }) + const schemas = ctx.tools.schemas() + const tool = schemas.find((entry) => entry.name === 'stablemem_commit') + assert.ok(tool, 'tool must be visible to the model') + assert.ok(tool.description.length > 0, 'tool must carry a description') + assert.ok(tool.parameters?.properties?.text, 'tool must declare its text parameter') + + const result = await ctx.tools.execute({ + callId: 'c1', + name: 'stablemem_commit', + arguments: { text: 'etcd 集群端口 2379', source: 'test' }, + signal: new AbortController().signal, + }) + assert.equal(result.isError, false, JSON.stringify(result)) + assert.ok(String(result.value).includes('Committed'), result.value) + assert.equal(result.content[0].type, 'text', 'render must produce a text block') + assert.equal(mem.store.countBlocks(), 1) + } finally { + await done() + } +}) + +test('V5: disposing the fiber closes the store and releases the file', async () => { + const { ctx, dir, done } = await app() + const path = join(dir, 'm.db') + const mem = applyMemory(ctx, { path }) + mem.commit({ text: 'V5 句柄释放块' }) + assert.equal(mem.store.isOpen, true) + + await ctx.fiber.dispose() + assert.equal(mem.store.isOpen, false, 'fiber disposal must close the handle') + + // Strongest proof: reopen the same file for writing. + const { BlockStore } = await import('../lib/store.js') + const reopened = new BlockStore({ path }).open() + assert.equal(reopened.countBlocks(), 1) + reopened.close() + + rmSync(dir, { recursive: true, force: true }) + void done +}) + +test('config.inject:false registers the tool but no pre-step listener', async () => { + const { ctx, dir, done } = await app() + try { + applyMemory(ctx, { path: join(dir, 'm.db'), inject: false }) + const input = createUserMessage({ + content: [{ type: 'text', text: 'anything' }], + source: { kind: 'user' }, + }) + const decision = await ctx.waterfall( + 'agent/pre-step', + { + agent: { id: 'a', session: {} }, + messages: [input], + turn: 1, + step: 1, + signal: new AbortController().signal, + }, + () => Promise.resolve({ kind: 'enter', messages: [input] }), + ) + assert.equal(decision.messages.length, 1, 'injection must be off') + assert.ok(ctx.tools.schemas().some((s) => s.name === 'stablemem_commit'), 'tool still present') + } finally { + await done() + } +}) diff --git a/test/store.test.js b/test/store.test.js new file mode 100644 index 0000000..b7e8b69 --- /dev/null +++ b/test/store.test.js @@ -0,0 +1,201 @@ +/** + * Unit tests for dsh-memory store + recall (phase A). + * + * Complements `scripts/checkpoints.js`: the checkpoints prove the harness wiring + * (real cordis, real SystemPrompt/ToolRuntime); these prove the algorithm and + * carry the mutation assertions that must be able to turn red. + * + * Run: node --test test/ + * + * @module dsh-memory/test/store + */ + +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { BlockStore, blockId } from '../lib/store.js' +import { + buildMatchExpression, + cjkNeedles, + isNumericSymbol, + querySymbols, + recallBlocks, + renderRecall, +} from '../lib/recall.js' + +/** @returns {{store: BlockStore, done: () => void}} */ +function tempStore() { + const dir = mkdtempSync(join(tmpdir(), 'dsh-memory-test-')) + const store = new BlockStore({ path: join(dir, 'x.db') }).open() + return { + store, + done: () => { + store.close() + rmSync(dir, { recursive: true, force: true }) + }, + } +} + +test('blockId is content-addressed and stable', () => { + assert.equal(blockId('值班室分机号 4324'), blockId('值班室分机号 4324')) + assert.notEqual(blockId('a'), blockId('b')) + assert.match(blockId('x'), /^blk_[0-9a-f]{12}$/) +}) + +test('putBlock is idempotent and keeps existing optional columns', () => { + const { store, done } = tempStore() + try { + const first = store.putBlock({ text: 'a note', source: 'qq', scene: 'chan:qq', semanticType: 'Fact' }) + assert.equal(first.created, true) + // Re-commit with no optional fields: text updates, metadata must NOT be blanked. + const second = store.putBlock({ text: 'a note' }) + assert.equal(second.created, false) + assert.equal(second.id, first.id) + const row = store.getBlock(first.id) + assert.equal(row.source, 'qq', 'source must survive a bare re-commit') + assert.equal(row.scene, 'chan:qq', 'scene must survive a bare re-commit') + assert.equal(row.semantic_type, 'Fact', 'semantic_type must survive a bare re-commit') + assert.equal(store.countBlocks(), 1) + } finally { + done() + } +}) + +test('edges are unique per (source,target,type)', () => { + const { store, done } = tempStore() + try { + const a = store.putBlock({ text: '值班室分机号 4324' }).id + const b = store.putBlock({ text: '4324 已停用' }).id + const edge = { sourceKind: 'block', sourceId: a, targetKind: 'block', targetId: b, type: 'superseded_by' } + assert.equal(store.addEdge(edge).created, true) + assert.equal(store.addEdge(edge).created, false, 'duplicate edge must not double') + assert.equal(store.countEdges(), 1) + } finally { + done() + } +}) + +test('querySymbols mirrors HologramAgent regexes and drops single chars', () => { + assert.deepEqual(querySymbols('本机 13010 端口'), ['13010']) + assert.deepEqual(querySymbols('版本 v2.31.5'), ['v2.31.5']) + assert.deepEqual(querySymbols('billing0 端口'), ['billing0'], 'no bare 0 from billing0') + // Single-character symbols are rejected (abstain.go:115 "单字不成符号"). + assert.deepEqual(querySymbols('a 1 中'), []) +}) + +test('isNumericSymbol identifies digit-bearing symbols', () => { + assert.equal(isNumericSymbol('13010'), true) + assert.equal(isNumericSymbol('v2.31.5'), true) + assert.equal(isNumericSymbol('billing'), false) +}) + +test('buildMatchExpression quotes terms and refuses an empty query', () => { + assert.equal(buildMatchExpression(''), null) + assert.equal(buildMatchExpression(' '), null) + const expr = buildMatchExpression('billing 端口') + assert.ok(expr.includes('"billing"'), expr) + // Injected FTS5 operators must be neutralised, never passed through. + const hostile = buildMatchExpression('a OR b NEAR(c)') + assert.ok(!/\bNEAR\(/.test(hostile ?? ''), `FTS operators must be quoted, got ${hostile}`) +}) + +test('cjkNeedles yields 2..4-char windows plus the whole run', () => { + const needles = cjkNeedles('值班室分机号') + assert.ok(needles.includes('值班室分机号'), 'whole run present') + assert.ok(needles.includes('分机号'), '3-char window present') + assert.ok(needles.includes('分机'), '2-char window present') + assert.ok(!needles.includes('值'), 'single chars excluded') +}) + +test('exact numeric hit pins to the top above plain lexical matches', () => { + const { store, done } = tempStore() + try { + store.putBlock({ text: '值班室分机号 4379 已停用' }) + store.putBlock({ text: '值班室分机号 4324，值班人 老周' }) + const hits = recallBlocks(store, '4324 是多少', { topK: 5 }) + assert.equal(hits[0].text.includes('4324'), true, `expected 4324 first, got ${hits[0].text}`) + assert.equal(hits[0].exact, true) + } finally { + done() + } +}) + +test('a matched word prefers the block where it is not a prefix of a longer word', () => { + const { store, done } = tempStore() + try { + store.putBlock({ text: 'billing2服务 端口=8501' }) + store.putBlock({ text: 'billing服务 端口=8801' }) + const hits = recallBlocks(store, 'billing 服务的端口是多少', { topK: 2 }) + assert.equal( + hits[0].text.includes('8801'), + true, + `expected the exact "billing" note first, got ${hits[0].text}`, + ) + } finally { + done() + } +}) + +test('unrelated query yields nothing (no silent everything-match)', () => { + const { store, done } = tempStore() + try { + store.putBlock({ text: '值班室分机号 4324' }) + store.putBlock({ text: 'billing服务 端口=8499' }) + assert.deepEqual(recallBlocks(store, 'zzz-不存在-qqq'), []) + } finally { + done() + } +}) + +test('renderRecall states the content is historical and yields "" for no hits', () => { + assert.equal(renderRecall([]), '') + const text = renderRecall([{ text: '值班室分机号 4324', exact: true, score: 1.01 }]) + assert.ok(text.includes('PAST notes'), 'the model must be told these are historical') + assert.ok(text.includes('4324')) +}) + +// ── mutations: each must turn a specific assertion red ──────────────────── +test('MUTATION: removing the numeric-gate makes a bare 0 pin noise', () => { + const { store, done } = tempStore() + try { + store.putBlock({ text: '发布批次 第8批|版本=v2.38.0' }) + store.putBlock({ text: 'billing0服务 端口=8499' }) + // Real behaviour: `billing0` is the symbol, so the billing note wins. + const real = recallBlocks(store, 'billing0 端口是多少', { topK: 1 }) + assert.equal(real[0].text.includes('8499'), true, `expected billing note, got ${real[0].text}`) + // Mutation: pretend a bare `0` were a symbol — it appears in v2.38.0, so the + // unrelated note would pin to the top. Demonstrated directly: + const bare = querySymbols('billing0') + assert.ok(!bare.includes('0'), 'the single-char gate is what prevents this') + } finally { + done() + } +}) + +test('MUTATION: containment-only recall loses discrimination (the reason FTS is primary)', () => { + const { store, done } = tempStore() + try { + store.putBlock({ text: '值班室分机号 4324' }) + store.putBlock({ text: '值班室分机号 4379 已停用' }) + // Primary path: precise. One hit for an exact query. + assert.equal(recallBlocks(store, '4324', { topK: 5 }).length, 1) + // Mutated path: a shared CJK prefix matches both, so the answer becomes + // ambiguous. This is why FTS + symbol pin, not LIKE, is the primary path. + const contained = store.allBlocks(100).filter((row) => row.text_content.includes('值班室')) + assert.equal(contained.length, 2) + } finally { + done() + } +}) + +test('close is idempotent and a closed store refuses work', () => { + const { store, done } = tempStore() + store.putBlock({ text: 'x' }) + store.close() + store.close() // must not throw + assert.equal(store.isOpen, false) + assert.throws(() => store.countBlocks(), /not open/) + done() +})