HologramAgent-DSH-Memory-Port/lib/index.js

/**
 * dsh-memory — block memory for dsh, entry point.
 *
 * This is **phase A** of `docs/feasibility.md` §7.1: prove that a block can go
 * into dsh, be recalled, and reach the prompt, with **no embedding provider**.
 *
 * ## The one thing to read before touching injection
 *
 * dsh has two obvious-looking injection points, and only one of them can reach
 * the current turn's input.
 *
 * | hook | gets this turn's input? | durable? |
 * |---|---|---|
 * | `ctx.systemPrompt.context({ text })` | no — `AssembleContext` is `{scope, signal}` plus a merged-in `agent` | yes (appended as a runtime-context snapshot) |
 * | `ctx.on('agent/pre-step')` | yes — receives `decision.messages` | yes (the loop appends whatever you return) |
 *
 * Verified in the harness, not assumed:
 *   · `dsh-system-prompt/lib/types/index.d.ts:37-44` — `AssembleContext` has only scope+signal
 *   · `dsh-agent-loop/lib/index.js:902-934` — `preStep` claims the inbox, assembles the
 *     prompt, then runs the `agent/pre-step` waterfall with those messages
 *   · `dsh-agent-loop/lib/index.js:1066-1073` — **every returned message is appended
 *     as a durable `user/message`**, so "shadow" injection there is not shadow: it
 *     becomes conversation history (see `docs/shadow-context.md` §1.3)
 *
 * Therefore: the input-independent index uses `systemPrompt.context`; per-turn
 * recall uses `agent/pre-step`, prepended so it runs before listeners that may
 * want the final word.
 *
 * @module dsh-memory
 */

import { Service } from '@deepseek-ai/cordis'
import { createUserMessage } from '@deepseek-ai/dsh-llm'
import { defineTool } from '@deepseek-ai/dsh-tools'
import { createHttpEmbedder } from './embed/http.js'
import { recallBlocksHybrid, renderRecall } from './recall.js'
import { BlockStore } from './store.js'

/** Cordis plugin name, also used as the `source.kind` of injected context. */
export const name = 'dsh-memory'

/**
 * Services this plugin needs.
 *
 * `systemPrompt` and `tools` are both present in any standard dsh composition
 * (`dsh-base/cordis.patch.yml`), so requiring them narrows nothing.
 */
export const inject = ['systemPrompt', 'tools']

/**
 * Plugin configuration.
 *
 * @typedef {object} Config
 * @property {string} [path] sqlite file path; defaults to `<cwd>/.dsh-memory/memory.db`
 * @property {number} [topK] recall hits injected per turn (default 3)
 * @property {boolean} [inject] false = register only the tool (default true)
 * @property {boolean} [wal] sqlite WAL mode (default true)
 * @property {string} [toolName] model-facing tool name (default `stablemem_commit`).
 *   Tool names are unique per tool layer and `isolate()` does not partition them
 *   (`ToolRuntime.register` throws on a duplicate), so a preset that mounts a
 *   second instance must give it a distinct name.
 * @property {object} [embed] vector-space configuration. Omit to run without
 *   vectors (phase A behaviour, symbol path only).
 * @property {string} [embed.endpoint] embedding endpoint
 * @property {number} [embed.dimension] vector width; required when `endpoint` is set
 * @property {string} [embed.model] model name forwarded in the request body
 * @property {string} [embed.apiKey] bearer credential
 * @property {string} [embed.fingerprint] explicit space identity (overrides derivation)
 * @property {number} [embed.timeoutMs] request timeout
 * @property {object} [embedder] a pre-built Embedder, bypassing `embed`. Used by
 *   tests and by any adapter that is not the bundled HTTP one.
 */

/**
 * The `stablemem` service: durable block storage plus recall.
 *
 * Registered as `ctx.stablemem` so other plugins can use it without going through
 * tools, and so a preset subtree can `isolate: { stablemem: true }` it later
 * (docs/shadow-context.md §5.2).
 */
export class StableMem extends Service {
  /**
   * ★ These are plain fields, NOT `#private`.
   *
   * cordis resolves a service through a **proxy** (`ctx.stablemem !== instance`),
   * and JS private fields are bound to the declaring object, so any method that
   * touches `this.#store` throws
   * `Cannot read private member #store from an object whose class did not declare it`
   * when reached through `ctx.<name>`. Verified against the real Context.
   * A cordis service must therefore keep its state in ordinary properties.
   *
   * @type {BlockStore}
   */
  store
  /** @type {number} */
  topK
  /**
   * Vector capability, or null. Phase A has none; group H supplies the adapter.
   * Kept as a plain property (not `#private`) — cordis hands out a proxy, and
   * private fields do not survive it.
   * @type {import('./recall.js').Embedder | null}
   */
  embedder

  /**
   * @param {import('@deepseek-ai/cordis').Context} ctx
   * @param {{ store: BlockStore, topK?: number,
   *   embedder?: import('./recall.js').Embedder | null }} options
   */
  constructor(ctx, { store, topK = 3, embedder = null }) {
    super(ctx, 'stablemem')
    this.store = store
    this.topK = topK
    this.embedder = embedder
    // Release the sqlite handle with the owning fiber, so a plugin unload does
    // not leak the file lock (checkpoint V5).
    ctx.effect(() => () => this.store.close(), 'dsh-memory.store')
  }

  /**
   * The configured vector space, or null when running symbol-only.
   *
   * ★ This single object is the ONLY source of `fingerprint`. Both write paths
   * (block rows and centroid rows) take the string from here rather than
   * deriving it themselves: HA looks the centroid up by fingerprint
   * (`block_recall.go:92`), so two independently-derived spellings would make
   * that lookup miss with no error — centering silently never applies.
   *
   * `info()` is synchronous and memoised in the adapter, so the string is
   * computed once per process and reused forever.
   *
   * @returns {{fingerprint: string, dimension: number} | null}
   */
  space() {
    return this.embedder === null ? null : this.embedder.info()
  }

  /**
   * Commit one text block.
   *
   * When a vector space is configured the block is embedded and stamped with
   * `space().fingerprint`. When it is not — or the embedder fails — the block is
   * still written, with an empty vector. Storing an invented vector would make it
   * participate in scoring and always rank last, which is a silently wrong
   * memory; an empty one honestly says "not embedded yet".
   *
   * @param {{ text: string, source?: string, scene?: string, semanticType?: string,
   *   vector?: number[], fingerprint?: string }} input
   * @returns {Promise<{ id: string, created: boolean, embedded: boolean }>}
   */
  async commit(input, options = {}) {
    // An explicit vector+fingerprint pair (tests, migrations, or a caller that
    // already embedded) is written verbatim.
    if (Array.isArray(input.vector) && input.vector.length > 0) {
      const written = this.store.putBlock(input)
      return { ...written, embedded: true }
    }

    const space = this.space()
    // No space configured, or the space cannot name itself: store the block with
    // no vector. Checking the fingerprint BEFORE calling embedText avoids paying
    // for a round-trip whose result could never be labelled with a space.
    if (space === null || typeof space.fingerprint !== 'string' || space.fingerprint === '') {
      return { ...this.store.putBlock(input), embedded: false }
    }

    let vector
    try {
      // `options.signal` is the tool call's cancellation. Forwarding it is
      // required by the tool contract (dsh-tools/lib/types/index.d.ts:120-122)
      // and it bounds latency: without it the adapter would hold the call for up
      // to its own timeout while the user has already cancelled the turn.
      vector = await this.embedder.embedText(input.text, options.signal)
    } catch {
      // Cancelled, or the vector side is down: the block still lands with no
      // vector (decision §0.4). An abort must not turn into "nothing was written".
      return { ...this.store.putBlock(input), embedded: false }
    }

    // Write-side dimension check. Recall already skips mismatched vectors
    // (recall.js), so storing one would produce a block that exists, looks fine,
    // and can never be recalled — the exact silent failure group F exists to
    // eliminate. Treat it as an embed failure instead of storing a bad vector.
    if (!Array.isArray(vector) || vector.length !== space.dimension) {
      return { ...this.store.putBlock(input), embedded: false }
    }

    const written = this.store.putBlock({
      ...input,
      vector,
      // ★ From the space object, never re-derived and never from input.
      fingerprint: space.fingerprint,
    })
    return { ...written, embedded: true }
  }

  /**
   * Persist the centroid for the configured space.
   *
   * Takes NO fingerprint argument on purpose: the caller cannot supply a
   * different spelling than the one {@link commit} stamped on the blocks.
   *
   * @param {{ dim?: number, vectorCount: number, mean: number[],
   *   isAnomalous: boolean, stats?: object }} centroid
   * @returns {{ fingerprint: string }}
   */
  saveSpaceCentroid(centroid) {
    const space = this.space()
    if (space === null) throw new Error('dsh-memory: no vector space configured')
    this.store.saveCentroid({
      fingerprint: space.fingerprint,
      dim: centroid.dim ?? space.dimension,
      vectorCount: centroid.vectorCount,
      mean: centroid.mean,
      isAnomalous: centroid.isAnomalous,
      stats: centroid.stats ?? {},
    })
    return { fingerprint: space.fingerprint }
  }

  /**
   * Link two blocks.
   * @param {object} edge
   * @returns {{ created: boolean }}
   */
  link(edge) {
    return this.store.addEdge(edge)
  }

  /**
   * Recall hits for one query.
   * @param {string} query
   * @param {number} [topK]
   */
  async recall(query, topK = this.topK) {
    // Group A ships the async seam only: with no embedder configured this is
    // exactly `recallBlocks` (pinned by the golden fixture in test/phase-b-a).
    return recallBlocksHybrid(this.store, this.embedder, query, { topK })
  }

  /**
   * Text to inject for this step, or '' when nothing is relevant.
   *
   * The query is the **last user-role text** in the batch: `decision.messages` is
   * the batch admitted for this step, and steering appends rather than replaces,
   * so the last user message is the actual prompt.
   *
   * @param {ReadonlyArray<{role: string, content: ReadonlyArray<{type: string, text?: string}>}>} messages
   * @returns {string}
   */
  async contextForStep(messages) {
    const query = lastUserText(messages)
    if (query === '') return ''
    const hits = await this.recall(query)
    if (hits.length === 0) return ''
    return renderRecall(hits, name)
  }

  /**
   * Constant index text for `systemPrompt.context`.
   *
   * Deliberately input-independent: that hook cannot see the current turn's text
   * (module note above). A count is still worth its tokens — the model learns that
   * durable memory exists without paying a per-turn recall cost.
   *
   * @returns {string}
   */
  renderIndex() {
    const count = this.store.countBlocks()
    if (count === 0) return ''
    return (
      `[Memory · ${name}] ${count} durable note(s) in block memory. ` +
      'Relevant ones are recalled into your timeline automatically; ' +
      'record new facts with the `stablemem_commit` tool.'
    )
  }
}

/**
 * Source kinds that carry a HUMAN prompt.
 *
 * Everything else a plugin injects is user-ROLE but not user-SPEECH:
 * `runtime-context` (dsh-agent-loop's working-directory/sandbox snapshot),
 * `skill-catalog` / `skill-invocation` (dsh-tool-skill), `agent-instructions`,
 * `time-context`, `goal`, and any snapshot-style contribution.
 *
 * ★ Why this allowlist exists (regression, found in a real profile run):
 * A real step batch is not `[question]` — it is
 *
 *     [ user question ][ runtime-context snapshot ][ skill catalog reminder ]
 *
 * and the injected ones are `role: 'user'` too. Picking "the last user-role
 * message" therefore returned the SKILL CATALOG, so every real turn used a list
 * of skill names as its recall query and matched nothing. Auto-recall was 100%
 * dead in production while all unit checkpoints were green, because the tests
 * built 2-message batches.
 */
const HUMAN_PROMPT_KINDS = new Set(['user'])

/**
 * Pull the human prompt out of a step's message batch.
 *
 * Scans backwards (the newest human turn wins) but filters by
 * `source.kind` — see {@link HUMAN_PROMPT_KINDS} for why role alone is wrong.
 * Unknown kinds are treated as injected: a plugin's own snapshot must never
 * become the recall query, and a genuine human message always carries
 * `kind: 'user'` (`dsh-llm/lib/types/message.d.ts`, `MessageSourceMap.user`).
 *
 * @param {ReadonlyArray<{role: string, source?: {kind?: string},
 *   content: ReadonlyArray<{type: string, text?: string}>}>} messages
 * @returns {string} the question text, or '' when the batch has no human prompt
 */
export function lastUserText(messages) {
  if (!Array.isArray(messages)) return ''
  for (let index = messages.length - 1; index >= 0; index -= 1) {
    const message = messages[index]
    if (message?.role !== 'user') continue
    const kind = message.source?.kind
    if (typeof kind !== 'string' || !HUMAN_PROMPT_KINDS.has(kind)) continue
    const text = (message.content ?? [])
      .filter((block) => block?.type === 'text' && typeof block.text === 'string')
      .map((block) => block.text)
      .join('\n')
      .trim()
    if (text !== '') return text
  }
  return ''
}

/**
 * Register the `stablemem_commit` tool.
 *
 * `output.schema` + `output.render` are mandatory in dsh
 * (`dsh-tools/lib/types/index.d.ts:117`), unlike HologramAgent's
 * description-only `toolDefWith` (`internal/agent/core/tooldefs.go`).
 *
 * @param {import('@deepseek-ai/cordis').Context} ctx
 * @param {StableMem} mem
 * @param {string} toolName model-facing tool name
 * @returns {() => void} disposer
 */
function registerTools(ctx, mem, toolName) {
  return ctx.tools.register(
    defineTool({
      name: toolName,
      description:
        'Store one durable fact or note in long-term block memory. Use it for ' +
        'stable facts worth recalling later (addresses, ports, paths, decisions, ' +
        'preferences). Keep each note short and self-contained: recall matches ' +
        'on the note text itself, so include the subject, not just the value.',
      parameters: {
        text: {
          type: 'string',
          description: 'The note. Self-contained: include subject, not just the value.',
          required: true,
        },
        source: {
          type: 'string',
          description: 'Optional origin label (channel, tool, file).',
        },
        scene: {
          type: 'string',
          description: 'Optional scene key for prefix-matching recall.',
        },
      },
      output: {
        schema: { type: 'string' },
        render: (_args, value) => [{ type: 'text', text: value }],
      },
      // Commits are independent and idempotent (content-addressed id), so they
      // regroup safely with sibling calls.
      isConcurrencySafe: () => true,
      async execute(args, exec) {
        const { id, created, embedded } = await mem.commit(
          {
            text: args.text,
            source: args.source ?? 'tool',
            scene: args.scene ?? '',
          },
          // Forward the call's cancellation into the embedding request.
          { signal: exec.signal },
        )
        const suffix = embedded ? '' : ' (stored without a vector)'
        return `${created ? 'Committed' : 'Already present'} block ${id}${suffix}.`
      },
    }),
  )
}

/**
 * Apply the plugin.
 *
 * @param {import('@deepseek-ai/cordis').Context} ctx
 * @param {Config} [config]
 */
export function apply(ctx, config = {}) {
  const path = config.path ?? defaultPath()
  const store = new BlockStore({ path, wal: config.wal ?? true }).open()
  // The adapter lives in exactly one file (lib/embed/http.js) so swapping the
  // transport never touches recall, scoring, or storage (guarded by G5).
  //
  // `config.embedder` accepts an already-built Embedder. It exists so tests can
  // inject a failing/slow/nameless provider without standing up an HTTP server —
  // and so a future local-ONNX adapter can be mounted without touching this file.
  const embedder =
    config.embedder ??
    (config.embed?.endpoint !== undefined ? createHttpEmbedder(config.embed) : null)
  const mem = new StableMem(ctx, { store, topK: config.topK ?? 3, embedder })

  // Injection surface 1 — constant index. Right for input-independent text.
  //
  // ★ The prompt-context NAME must be unique per prompt layer, and `isolate()`
  // does NOT partition prompt names: `SystemPrompt` keeps one `NamedEntries` per
  // layer and throws `prompt context "..." is already registered`
  // (dsh-system-prompt/lib/index.js:204). So a preset mounting a second instance
  // must not reuse the literal name — deriving it from the database path keeps
  // two instances in one realm distinguishable, and is stable across restarts.
  ctx.systemPrompt.context({
    name: `memory:index:${instanceKey(path)}`,
    order: ctx.systemPrompt.getContextOrder('WEB_SURFACE'),
    interpolate: false,
    text: () => mem.renderIndex(),
  })

  // Injection surface 2 — per-turn recall. The only hook that sees the input.
  if (config.inject ?? true) {
    ctx.on(
      'agent/pre-step',
      async (payload, next) => {
        const decision = await next()
        if (decision.kind === 'reject' || payload.signal.aborted) return decision
        const injected = await mem.contextForStep(decision.messages)
        if (injected === '') return decision
        return {
          ...decision,
          messages: [
            ...decision.messages,
            createUserMessage({
              content: [{ type: 'text', text: injected }],
              // `form: 'snapshot'` matches how dsh-time-context labels its own
              // injected context (dsh-time-context/lib/index.js:238). `kind` is
              // this plugin's own — MessageSourceMap is merge-extensible
              // (dsh-llm/lib/types/message.d.ts:96-105) and user messages accept
              // unknown kinds (dsh-session/lib/index.js:1216).
              source: {
                kind: name,
                form: 'snapshot',
                sections: [{ name, text: injected }],
              },
            }),
          ],
        }
      },
      // Prepend: recall should be visible to listeners that run later.
      { prepend: true },
    )
  }

  registerTools(ctx, mem, config.toolName ?? 'stablemem_commit')

  return mem
}

/**
 * A stable, filesystem-safe key distinguishing one plugin instance.
 *
 * Used to keep prompt-context names unique when a realm hosts more than one
 * instance (a preset isolating `stablemem` but not the prompt layer). Derived
 * from the database path so it survives restarts and stays readable in
 * `/dump-config`.
 *
 * @param {string} path the configured database path
 * @returns {string}
 */
export function instanceKey(path) {
  let hash = 0
  for (const char of String(path)) hash = (hash * 31 + char.codePointAt(0)) >>> 0
  return hash.toString(36)
}

/**
 * Default database location.
 *
 * Under the process cwd so a profile sees one memory per workspace, matching how
 * `dsh-storage-json` refuses to guess a location by default
 * (`dsh-storage-json/lib/types/index.d.ts:14-18`). Overridable via `config.path`.
 *
 * @returns {string}
 */
export function defaultPath() {
  return `${process.cwd()}/.dsh-memory/memory.db`
}

export default { name, inject, apply, StableMem }